xref: /linux/drivers/pci/pci.c (revision 8e1bb4a41aa78d6105e59186af3dcd545fc66e70)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI Bus Services, see include/linux/pci.h for further explanation.
4  *
5  * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
6  * David Mosberger-Tang
7  *
8  * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
9  */
10 
11 #include <linux/acpi.h>
12 #include <linux/kernel.h>
13 #include <linux/delay.h>
14 #include <linux/dmi.h>
15 #include <linux/init.h>
16 #include <linux/msi.h>
17 #include <linux/of.h>
18 #include <linux/pci.h>
19 #include <linux/pm.h>
20 #include <linux/slab.h>
21 #include <linux/module.h>
22 #include <linux/spinlock.h>
23 #include <linux/string.h>
24 #include <linux/log2.h>
25 #include <linux/logic_pio.h>
26 #include <linux/pm_wakeup.h>
27 #include <linux/device.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/pci_hotplug.h>
30 #include <linux/vmalloc.h>
31 #include <asm/dma.h>
32 #include <linux/aer.h>
33 #include <linux/bitfield.h>
34 #include "pci.h"
35 
36 DEFINE_MUTEX(pci_slot_mutex);
37 
38 const char *pci_power_names[] = {
39 	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
40 };
41 EXPORT_SYMBOL_GPL(pci_power_names);
42 
43 #ifdef CONFIG_X86_32
44 int isa_dma_bridge_buggy;
45 EXPORT_SYMBOL(isa_dma_bridge_buggy);
46 #endif
47 
48 int pci_pci_problems;
49 EXPORT_SYMBOL(pci_pci_problems);
50 
51 unsigned int pci_pm_d3hot_delay;
52 
53 static void pci_pme_list_scan(struct work_struct *work);
54 
55 static LIST_HEAD(pci_pme_list);
56 static DEFINE_MUTEX(pci_pme_list_mutex);
57 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
58 
59 struct pci_pme_device {
60 	struct list_head list;
61 	struct pci_dev *dev;
62 };
63 
64 #define PME_TIMEOUT 1000 /* How long between PME checks */
65 
66 /*
67  * Following exit from Conventional Reset, devices must be ready within 1 sec
68  * (PCIe r6.0 sec 6.6.1).  A D3cold to D0 transition implies a Conventional
69  * Reset (PCIe r6.0 sec 5.8).
70  */
71 #define PCI_RESET_WAIT 1000 /* msec */
72 
73 /*
74  * Devices may extend the 1 sec period through Request Retry Status
75  * completions (PCIe r6.0 sec 2.3.1).  The spec does not provide an upper
76  * limit, but 60 sec ought to be enough for any device to become
77  * responsive.
78  */
79 #define PCIE_RESET_READY_POLL_MS 60000 /* msec */
80 
81 static void pci_dev_d3_sleep(struct pci_dev *dev)
82 {
83 	unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay);
84 	unsigned int upper;
85 
86 	if (delay_ms) {
87 		/* Use a 20% upper bound, 1ms minimum */
88 		upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U);
89 		usleep_range(delay_ms * USEC_PER_MSEC,
90 			     (delay_ms + upper) * USEC_PER_MSEC);
91 	}
92 }
93 
94 bool pci_reset_supported(struct pci_dev *dev)
95 {
96 	return dev->reset_methods[0] != 0;
97 }
98 
99 #ifdef CONFIG_PCI_DOMAINS
100 int pci_domains_supported = 1;
101 #endif
102 
103 #define DEFAULT_CARDBUS_IO_SIZE		(256)
104 #define DEFAULT_CARDBUS_MEM_SIZE	(64*1024*1024)
105 /* pci=cbmemsize=nnM,cbiosize=nn can override this */
106 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
107 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
108 
109 #define DEFAULT_HOTPLUG_IO_SIZE		(256)
110 #define DEFAULT_HOTPLUG_MMIO_SIZE	(2*1024*1024)
111 #define DEFAULT_HOTPLUG_MMIO_PREF_SIZE	(2*1024*1024)
112 /* hpiosize=nn can override this */
113 unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
114 /*
115  * pci=hpmmiosize=nnM overrides non-prefetchable MMIO size,
116  * pci=hpmmioprefsize=nnM overrides prefetchable MMIO size;
117  * pci=hpmemsize=nnM overrides both
118  */
119 unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE;
120 unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE;
121 
122 #define DEFAULT_HOTPLUG_BUS_SIZE	1
123 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
124 
125 
126 /* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */
127 #ifdef CONFIG_PCIE_BUS_TUNE_OFF
128 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF;
129 #elif defined CONFIG_PCIE_BUS_SAFE
130 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE;
131 #elif defined CONFIG_PCIE_BUS_PERFORMANCE
132 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE;
133 #elif defined CONFIG_PCIE_BUS_PEER2PEER
134 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER;
135 #else
136 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
137 #endif
138 
139 /*
140  * The default CLS is used if arch didn't set CLS explicitly and not
141  * all pci devices agree on the same value.  Arch can override either
142  * the dfl or actual value as it sees fit.  Don't forget this is
143  * measured in 32-bit words, not bytes.
144  */
145 u8 pci_dfl_cache_line_size __ro_after_init = L1_CACHE_BYTES >> 2;
146 u8 pci_cache_line_size __ro_after_init ;
147 
148 /*
149  * If we set up a device for bus mastering, we need to check the latency
150  * timer as certain BIOSes forget to set it properly.
151  */
152 unsigned int pcibios_max_latency = 255;
153 
154 /* If set, the PCIe ARI capability will not be used. */
155 static bool pcie_ari_disabled;
156 
157 /* If set, the PCIe ATS capability will not be used. */
158 static bool pcie_ats_disabled;
159 
160 /* If set, the PCI config space of each device is printed during boot. */
161 bool pci_early_dump;
162 
163 bool pci_ats_disabled(void)
164 {
165 	return pcie_ats_disabled;
166 }
167 EXPORT_SYMBOL_GPL(pci_ats_disabled);
168 
169 /* Disable bridge_d3 for all PCIe ports */
170 static bool pci_bridge_d3_disable;
171 /* Force bridge_d3 for all PCIe ports */
172 static bool pci_bridge_d3_force;
173 
174 static int __init pcie_port_pm_setup(char *str)
175 {
176 	if (!strcmp(str, "off"))
177 		pci_bridge_d3_disable = true;
178 	else if (!strcmp(str, "force"))
179 		pci_bridge_d3_force = true;
180 	return 1;
181 }
182 __setup("pcie_port_pm=", pcie_port_pm_setup);
183 
184 /**
185  * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
186  * @bus: pointer to PCI bus structure to search
187  *
188  * Given a PCI bus, returns the highest PCI bus number present in the set
189  * including the given PCI bus and its list of child PCI buses.
190  */
191 unsigned char pci_bus_max_busnr(struct pci_bus *bus)
192 {
193 	struct pci_bus *tmp;
194 	unsigned char max, n;
195 
196 	max = bus->busn_res.end;
197 	list_for_each_entry(tmp, &bus->children, node) {
198 		n = pci_bus_max_busnr(tmp);
199 		if (n > max)
200 			max = n;
201 	}
202 	return max;
203 }
204 EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
205 
206 /**
207  * pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS
208  * @pdev: the PCI device
209  *
210  * Returns error bits set in PCI_STATUS and clears them.
211  */
212 int pci_status_get_and_clear_errors(struct pci_dev *pdev)
213 {
214 	u16 status;
215 	int ret;
216 
217 	ret = pci_read_config_word(pdev, PCI_STATUS, &status);
218 	if (ret != PCIBIOS_SUCCESSFUL)
219 		return -EIO;
220 
221 	status &= PCI_STATUS_ERROR_BITS;
222 	if (status)
223 		pci_write_config_word(pdev, PCI_STATUS, status);
224 
225 	return status;
226 }
227 EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors);
228 
229 #ifdef CONFIG_HAS_IOMEM
230 static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar,
231 					    bool write_combine)
232 {
233 	struct resource *res = &pdev->resource[bar];
234 	resource_size_t start = res->start;
235 	resource_size_t size = resource_size(res);
236 
237 	/*
238 	 * Make sure the BAR is actually a memory resource, not an IO resource
239 	 */
240 	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
241 		pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res);
242 		return NULL;
243 	}
244 
245 	if (write_combine)
246 		return ioremap_wc(start, size);
247 
248 	return ioremap(start, size);
249 }
250 
251 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
252 {
253 	return __pci_ioremap_resource(pdev, bar, false);
254 }
255 EXPORT_SYMBOL_GPL(pci_ioremap_bar);
256 
257 void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
258 {
259 	return __pci_ioremap_resource(pdev, bar, true);
260 }
261 EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
262 #endif
263 
264 /**
265  * pci_dev_str_match_path - test if a path string matches a device
266  * @dev: the PCI device to test
267  * @path: string to match the device against
268  * @endptr: pointer to the string after the match
269  *
270  * Test if a string (typically from a kernel parameter) formatted as a
271  * path of device/function addresses matches a PCI device. The string must
272  * be of the form:
273  *
274  *   [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
275  *
276  * A path for a device can be obtained using 'lspci -t'.  Using a path
277  * is more robust against bus renumbering than using only a single bus,
278  * device and function address.
279  *
280  * Returns 1 if the string matches the device, 0 if it does not and
281  * a negative error code if it fails to parse the string.
282  */
283 static int pci_dev_str_match_path(struct pci_dev *dev, const char *path,
284 				  const char **endptr)
285 {
286 	int ret;
287 	unsigned int seg, bus, slot, func;
288 	char *wpath, *p;
289 	char end;
290 
291 	*endptr = strchrnul(path, ';');
292 
293 	wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC);
294 	if (!wpath)
295 		return -ENOMEM;
296 
297 	while (1) {
298 		p = strrchr(wpath, '/');
299 		if (!p)
300 			break;
301 		ret = sscanf(p, "/%x.%x%c", &slot, &func, &end);
302 		if (ret != 2) {
303 			ret = -EINVAL;
304 			goto free_and_exit;
305 		}
306 
307 		if (dev->devfn != PCI_DEVFN(slot, func)) {
308 			ret = 0;
309 			goto free_and_exit;
310 		}
311 
312 		/*
313 		 * Note: we don't need to get a reference to the upstream
314 		 * bridge because we hold a reference to the top level
315 		 * device which should hold a reference to the bridge,
316 		 * and so on.
317 		 */
318 		dev = pci_upstream_bridge(dev);
319 		if (!dev) {
320 			ret = 0;
321 			goto free_and_exit;
322 		}
323 
324 		*p = 0;
325 	}
326 
327 	ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot,
328 		     &func, &end);
329 	if (ret != 4) {
330 		seg = 0;
331 		ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end);
332 		if (ret != 3) {
333 			ret = -EINVAL;
334 			goto free_and_exit;
335 		}
336 	}
337 
338 	ret = (seg == pci_domain_nr(dev->bus) &&
339 	       bus == dev->bus->number &&
340 	       dev->devfn == PCI_DEVFN(slot, func));
341 
342 free_and_exit:
343 	kfree(wpath);
344 	return ret;
345 }
346 
347 /**
348  * pci_dev_str_match - test if a string matches a device
349  * @dev: the PCI device to test
350  * @p: string to match the device against
351  * @endptr: pointer to the string after the match
352  *
353  * Test if a string (typically from a kernel parameter) matches a specified
354  * PCI device. The string may be of one of the following formats:
355  *
356  *   [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
357  *   pci:<vendor>:<device>[:<subvendor>:<subdevice>]
358  *
359  * The first format specifies a PCI bus/device/function address which
360  * may change if new hardware is inserted, if motherboard firmware changes,
361  * or due to changes caused in kernel parameters. If the domain is
362  * left unspecified, it is taken to be 0.  In order to be robust against
363  * bus renumbering issues, a path of PCI device/function numbers may be used
364  * to address the specific device.  The path for a device can be determined
365  * through the use of 'lspci -t'.
366  *
367  * The second format matches devices using IDs in the configuration
368  * space which may match multiple devices in the system. A value of 0
369  * for any field will match all devices. (Note: this differs from
370  * in-kernel code that uses PCI_ANY_ID which is ~0; this is for
371  * legacy reasons and convenience so users don't have to specify
372  * FFFFFFFFs on the command line.)
373  *
374  * Returns 1 if the string matches the device, 0 if it does not and
375  * a negative error code if the string cannot be parsed.
376  */
377 static int pci_dev_str_match(struct pci_dev *dev, const char *p,
378 			     const char **endptr)
379 {
380 	int ret;
381 	int count;
382 	unsigned short vendor, device, subsystem_vendor, subsystem_device;
383 
384 	if (strncmp(p, "pci:", 4) == 0) {
385 		/* PCI vendor/device (subvendor/subdevice) IDs are specified */
386 		p += 4;
387 		ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device,
388 			     &subsystem_vendor, &subsystem_device, &count);
389 		if (ret != 4) {
390 			ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count);
391 			if (ret != 2)
392 				return -EINVAL;
393 
394 			subsystem_vendor = 0;
395 			subsystem_device = 0;
396 		}
397 
398 		p += count;
399 
400 		if ((!vendor || vendor == dev->vendor) &&
401 		    (!device || device == dev->device) &&
402 		    (!subsystem_vendor ||
403 			    subsystem_vendor == dev->subsystem_vendor) &&
404 		    (!subsystem_device ||
405 			    subsystem_device == dev->subsystem_device))
406 			goto found;
407 	} else {
408 		/*
409 		 * PCI Bus, Device, Function IDs are specified
410 		 * (optionally, may include a path of devfns following it)
411 		 */
412 		ret = pci_dev_str_match_path(dev, p, &p);
413 		if (ret < 0)
414 			return ret;
415 		else if (ret)
416 			goto found;
417 	}
418 
419 	*endptr = p;
420 	return 0;
421 
422 found:
423 	*endptr = p;
424 	return 1;
425 }
426 
427 static u8 __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
428 				  u8 pos, int cap, int *ttl)
429 {
430 	u8 id;
431 	u16 ent;
432 
433 	pci_bus_read_config_byte(bus, devfn, pos, &pos);
434 
435 	while ((*ttl)--) {
436 		if (pos < 0x40)
437 			break;
438 		pos &= ~3;
439 		pci_bus_read_config_word(bus, devfn, pos, &ent);
440 
441 		id = ent & 0xff;
442 		if (id == 0xff)
443 			break;
444 		if (id == cap)
445 			return pos;
446 		pos = (ent >> 8);
447 	}
448 	return 0;
449 }
450 
451 static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
452 			      u8 pos, int cap)
453 {
454 	int ttl = PCI_FIND_CAP_TTL;
455 
456 	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
457 }
458 
459 u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
460 {
461 	return __pci_find_next_cap(dev->bus, dev->devfn,
462 				   pos + PCI_CAP_LIST_NEXT, cap);
463 }
464 EXPORT_SYMBOL_GPL(pci_find_next_capability);
465 
466 static u8 __pci_bus_find_cap_start(struct pci_bus *bus,
467 				    unsigned int devfn, u8 hdr_type)
468 {
469 	u16 status;
470 
471 	pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
472 	if (!(status & PCI_STATUS_CAP_LIST))
473 		return 0;
474 
475 	switch (hdr_type) {
476 	case PCI_HEADER_TYPE_NORMAL:
477 	case PCI_HEADER_TYPE_BRIDGE:
478 		return PCI_CAPABILITY_LIST;
479 	case PCI_HEADER_TYPE_CARDBUS:
480 		return PCI_CB_CAPABILITY_LIST;
481 	}
482 
483 	return 0;
484 }
485 
486 /**
487  * pci_find_capability - query for devices' capabilities
488  * @dev: PCI device to query
489  * @cap: capability code
490  *
491  * Tell if a device supports a given PCI capability.
492  * Returns the address of the requested capability structure within the
493  * device's PCI configuration space or 0 in case the device does not
494  * support it.  Possible values for @cap include:
495  *
496  *  %PCI_CAP_ID_PM           Power Management
497  *  %PCI_CAP_ID_AGP          Accelerated Graphics Port
498  *  %PCI_CAP_ID_VPD          Vital Product Data
499  *  %PCI_CAP_ID_SLOTID       Slot Identification
500  *  %PCI_CAP_ID_MSI          Message Signalled Interrupts
501  *  %PCI_CAP_ID_CHSWP        CompactPCI HotSwap
502  *  %PCI_CAP_ID_PCIX         PCI-X
503  *  %PCI_CAP_ID_EXP          PCI Express
504  */
505 u8 pci_find_capability(struct pci_dev *dev, int cap)
506 {
507 	u8 pos;
508 
509 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
510 	if (pos)
511 		pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);
512 
513 	return pos;
514 }
515 EXPORT_SYMBOL(pci_find_capability);
516 
517 /**
518  * pci_bus_find_capability - query for devices' capabilities
519  * @bus: the PCI bus to query
520  * @devfn: PCI device to query
521  * @cap: capability code
522  *
523  * Like pci_find_capability() but works for PCI devices that do not have a
524  * pci_dev structure set up yet.
525  *
526  * Returns the address of the requested capability structure within the
527  * device's PCI configuration space or 0 in case the device does not
528  * support it.
529  */
530 u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
531 {
532 	u8 hdr_type, pos;
533 
534 	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);
535 
536 	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & PCI_HEADER_TYPE_MASK);
537 	if (pos)
538 		pos = __pci_find_next_cap(bus, devfn, pos, cap);
539 
540 	return pos;
541 }
542 EXPORT_SYMBOL(pci_bus_find_capability);
543 
544 /**
545  * pci_find_next_ext_capability - Find an extended capability
546  * @dev: PCI device to query
547  * @start: address at which to start looking (0 to start at beginning of list)
548  * @cap: capability code
549  *
550  * Returns the address of the next matching extended capability structure
551  * within the device's PCI configuration space or 0 if the device does
552  * not support it.  Some capabilities can occur several times, e.g., the
553  * vendor-specific capability, and this provides a way to find them all.
554  */
555 u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap)
556 {
557 	u32 header;
558 	int ttl;
559 	u16 pos = PCI_CFG_SPACE_SIZE;
560 
561 	/* minimum 8 bytes per capability */
562 	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
563 
564 	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
565 		return 0;
566 
567 	if (start)
568 		pos = start;
569 
570 	if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
571 		return 0;
572 
573 	/*
574 	 * If we have no capabilities, this is indicated by cap ID,
575 	 * cap version and next pointer all being 0.
576 	 */
577 	if (header == 0)
578 		return 0;
579 
580 	while (ttl-- > 0) {
581 		if (PCI_EXT_CAP_ID(header) == cap && pos != start)
582 			return pos;
583 
584 		pos = PCI_EXT_CAP_NEXT(header);
585 		if (pos < PCI_CFG_SPACE_SIZE)
586 			break;
587 
588 		if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
589 			break;
590 	}
591 
592 	return 0;
593 }
594 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
595 
596 /**
597  * pci_find_ext_capability - Find an extended capability
598  * @dev: PCI device to query
599  * @cap: capability code
600  *
601  * Returns the address of the requested extended capability structure
602  * within the device's PCI configuration space or 0 if the device does
603  * not support it.  Possible values for @cap include:
604  *
605  *  %PCI_EXT_CAP_ID_ERR		Advanced Error Reporting
606  *  %PCI_EXT_CAP_ID_VC		Virtual Channel
607  *  %PCI_EXT_CAP_ID_DSN		Device Serial Number
608  *  %PCI_EXT_CAP_ID_PWR		Power Budgeting
609  */
610 u16 pci_find_ext_capability(struct pci_dev *dev, int cap)
611 {
612 	return pci_find_next_ext_capability(dev, 0, cap);
613 }
614 EXPORT_SYMBOL_GPL(pci_find_ext_capability);
615 
616 /**
617  * pci_get_dsn - Read and return the 8-byte Device Serial Number
618  * @dev: PCI device to query
619  *
620  * Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial
621  * Number.
622  *
623  * Returns the DSN, or zero if the capability does not exist.
624  */
625 u64 pci_get_dsn(struct pci_dev *dev)
626 {
627 	u32 dword;
628 	u64 dsn;
629 	int pos;
630 
631 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
632 	if (!pos)
633 		return 0;
634 
635 	/*
636 	 * The Device Serial Number is two dwords offset 4 bytes from the
637 	 * capability position. The specification says that the first dword is
638 	 * the lower half, and the second dword is the upper half.
639 	 */
640 	pos += 4;
641 	pci_read_config_dword(dev, pos, &dword);
642 	dsn = (u64)dword;
643 	pci_read_config_dword(dev, pos + 4, &dword);
644 	dsn |= ((u64)dword) << 32;
645 
646 	return dsn;
647 }
648 EXPORT_SYMBOL_GPL(pci_get_dsn);
649 
650 static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap)
651 {
652 	int rc, ttl = PCI_FIND_CAP_TTL;
653 	u8 cap, mask;
654 
655 	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
656 		mask = HT_3BIT_CAP_MASK;
657 	else
658 		mask = HT_5BIT_CAP_MASK;
659 
660 	pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
661 				      PCI_CAP_ID_HT, &ttl);
662 	while (pos) {
663 		rc = pci_read_config_byte(dev, pos + 3, &cap);
664 		if (rc != PCIBIOS_SUCCESSFUL)
665 			return 0;
666 
667 		if ((cap & mask) == ht_cap)
668 			return pos;
669 
670 		pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
671 					      pos + PCI_CAP_LIST_NEXT,
672 					      PCI_CAP_ID_HT, &ttl);
673 	}
674 
675 	return 0;
676 }
677 
678 /**
679  * pci_find_next_ht_capability - query a device's HyperTransport capabilities
680  * @dev: PCI device to query
681  * @pos: Position from which to continue searching
682  * @ht_cap: HyperTransport capability code
683  *
684  * To be used in conjunction with pci_find_ht_capability() to search for
685  * all capabilities matching @ht_cap. @pos should always be a value returned
686  * from pci_find_ht_capability().
687  *
688  * NB. To be 100% safe against broken PCI devices, the caller should take
689  * steps to avoid an infinite loop.
690  */
691 u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap)
692 {
693 	return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
694 }
695 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
696 
697 /**
698  * pci_find_ht_capability - query a device's HyperTransport capabilities
699  * @dev: PCI device to query
700  * @ht_cap: HyperTransport capability code
701  *
702  * Tell if a device supports a given HyperTransport capability.
703  * Returns an address within the device's PCI configuration space
704  * or 0 in case the device does not support the request capability.
705  * The address points to the PCI capability, of type PCI_CAP_ID_HT,
706  * which has a HyperTransport capability matching @ht_cap.
707  */
708 u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
709 {
710 	u8 pos;
711 
712 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
713 	if (pos)
714 		pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
715 
716 	return pos;
717 }
718 EXPORT_SYMBOL_GPL(pci_find_ht_capability);
719 
720 /**
721  * pci_find_vsec_capability - Find a vendor-specific extended capability
722  * @dev: PCI device to query
723  * @vendor: Vendor ID for which capability is defined
724  * @cap: Vendor-specific capability ID
725  *
726  * If @dev has Vendor ID @vendor, search for a VSEC capability with
727  * VSEC ID @cap. If found, return the capability offset in
728  * config space; otherwise return 0.
729  */
730 u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap)
731 {
732 	u16 vsec = 0;
733 	u32 header;
734 	int ret;
735 
736 	if (vendor != dev->vendor)
737 		return 0;
738 
739 	while ((vsec = pci_find_next_ext_capability(dev, vsec,
740 						     PCI_EXT_CAP_ID_VNDR))) {
741 		ret = pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, &header);
742 		if (ret != PCIBIOS_SUCCESSFUL)
743 			continue;
744 
745 		if (PCI_VNDR_HEADER_ID(header) == cap)
746 			return vsec;
747 	}
748 
749 	return 0;
750 }
751 EXPORT_SYMBOL_GPL(pci_find_vsec_capability);
752 
753 /**
754  * pci_find_dvsec_capability - Find DVSEC for vendor
755  * @dev: PCI device to query
756  * @vendor: Vendor ID to match for the DVSEC
757  * @dvsec: Designated Vendor-specific capability ID
758  *
759  * If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability
760  * offset in config space; otherwise return 0.
761  */
762 u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec)
763 {
764 	int pos;
765 
766 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC);
767 	if (!pos)
768 		return 0;
769 
770 	while (pos) {
771 		u16 v, id;
772 
773 		pci_read_config_word(dev, pos + PCI_DVSEC_HEADER1, &v);
774 		pci_read_config_word(dev, pos + PCI_DVSEC_HEADER2, &id);
775 		if (vendor == v && dvsec == id)
776 			return pos;
777 
778 		pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC);
779 	}
780 
781 	return 0;
782 }
783 EXPORT_SYMBOL_GPL(pci_find_dvsec_capability);
784 
785 /**
786  * pci_find_parent_resource - return resource region of parent bus of given
787  *			      region
788  * @dev: PCI device structure contains resources to be searched
789  * @res: child resource record for which parent is sought
790  *
791  * For given resource region of given device, return the resource region of
792  * parent bus the given region is contained in.
793  */
794 struct resource *pci_find_parent_resource(const struct pci_dev *dev,
795 					  struct resource *res)
796 {
797 	const struct pci_bus *bus = dev->bus;
798 	struct resource *r;
799 
800 	pci_bus_for_each_resource(bus, r) {
801 		if (!r)
802 			continue;
803 		if (resource_contains(r, res)) {
804 
805 			/*
806 			 * If the window is prefetchable but the BAR is
807 			 * not, the allocator made a mistake.
808 			 */
809 			if (r->flags & IORESOURCE_PREFETCH &&
810 			    !(res->flags & IORESOURCE_PREFETCH))
811 				return NULL;
812 
813 			/*
814 			 * If we're below a transparent bridge, there may
815 			 * be both a positively-decoded aperture and a
816 			 * subtractively-decoded region that contain the BAR.
817 			 * We want the positively-decoded one, so this depends
818 			 * on pci_bus_for_each_resource() giving us those
819 			 * first.
820 			 */
821 			return r;
822 		}
823 	}
824 	return NULL;
825 }
826 EXPORT_SYMBOL(pci_find_parent_resource);
827 
828 /**
829  * pci_find_resource - Return matching PCI device resource
830  * @dev: PCI device to query
831  * @res: Resource to look for
832  *
833  * Goes over standard PCI resources (BARs) and checks if the given resource
834  * is partially or fully contained in any of them. In that case the
835  * matching resource is returned, %NULL otherwise.
836  */
837 struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
838 {
839 	int i;
840 
841 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
842 		struct resource *r = &dev->resource[i];
843 
844 		if (r->start && resource_contains(r, res))
845 			return r;
846 	}
847 
848 	return NULL;
849 }
850 EXPORT_SYMBOL(pci_find_resource);
851 
852 /**
853  * pci_resource_name - Return the name of the PCI resource
854  * @dev: PCI device to query
855  * @i: index of the resource
856  *
857  * Return the standard PCI resource (BAR) name according to their index.
858  */
859 const char *pci_resource_name(struct pci_dev *dev, unsigned int i)
860 {
861 	static const char * const bar_name[] = {
862 		"BAR 0",
863 		"BAR 1",
864 		"BAR 2",
865 		"BAR 3",
866 		"BAR 4",
867 		"BAR 5",
868 		"ROM",
869 #ifdef CONFIG_PCI_IOV
870 		"VF BAR 0",
871 		"VF BAR 1",
872 		"VF BAR 2",
873 		"VF BAR 3",
874 		"VF BAR 4",
875 		"VF BAR 5",
876 #endif
877 		"bridge window",	/* "io" included in %pR */
878 		"bridge window",	/* "mem" included in %pR */
879 		"bridge window",	/* "mem pref" included in %pR */
880 	};
881 	static const char * const cardbus_name[] = {
882 		"BAR 1",
883 		"unknown",
884 		"unknown",
885 		"unknown",
886 		"unknown",
887 		"unknown",
888 #ifdef CONFIG_PCI_IOV
889 		"unknown",
890 		"unknown",
891 		"unknown",
892 		"unknown",
893 		"unknown",
894 		"unknown",
895 #endif
896 		"CardBus bridge window 0",	/* I/O */
897 		"CardBus bridge window 1",	/* I/O */
898 		"CardBus bridge window 0",	/* mem */
899 		"CardBus bridge window 1",	/* mem */
900 	};
901 
902 	if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS &&
903 	    i < ARRAY_SIZE(cardbus_name))
904 		return cardbus_name[i];
905 
906 	if (i < ARRAY_SIZE(bar_name))
907 		return bar_name[i];
908 
909 	return "unknown";
910 }
911 
912 /**
913  * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
914  * @dev: the PCI device to operate on
915  * @pos: config space offset of status word
916  * @mask: mask of bit(s) to care about in status word
917  *
918  * Return 1 when mask bit(s) in status word clear, 0 otherwise.
919  */
920 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
921 {
922 	int i;
923 
924 	/* Wait for Transaction Pending bit clean */
925 	for (i = 0; i < 4; i++) {
926 		u16 status;
927 		if (i)
928 			msleep((1 << (i - 1)) * 100);
929 
930 		pci_read_config_word(dev, pos, &status);
931 		if (!(status & mask))
932 			return 1;
933 	}
934 
935 	return 0;
936 }
937 
938 static int pci_acs_enable;
939 
940 /**
941  * pci_request_acs - ask for ACS to be enabled if supported
942  */
943 void pci_request_acs(void)
944 {
945 	pci_acs_enable = 1;
946 }
947 
948 static const char *disable_acs_redir_param;
949 static const char *config_acs_param;
950 
951 struct pci_acs {
952 	u16 cap;
953 	u16 ctrl;
954 	u16 fw_ctrl;
955 };
956 
957 static void __pci_config_acs(struct pci_dev *dev, struct pci_acs *caps,
958 			     const char *p, u16 mask, u16 flags)
959 {
960 	char *delimit;
961 	int ret = 0;
962 
963 	if (!p)
964 		return;
965 
966 	while (*p) {
967 		if (!mask) {
968 			/* Check for ACS flags */
969 			delimit = strstr(p, "@");
970 			if (delimit) {
971 				int end;
972 				u32 shift = 0;
973 
974 				end = delimit - p - 1;
975 
976 				while (end > -1) {
977 					if (*(p + end) == '0') {
978 						mask |= 1 << shift;
979 						shift++;
980 						end--;
981 					} else if (*(p + end) == '1') {
982 						mask |= 1 << shift;
983 						flags |= 1 << shift;
984 						shift++;
985 						end--;
986 					} else if ((*(p + end) == 'x') || (*(p + end) == 'X')) {
987 						shift++;
988 						end--;
989 					} else {
990 						pci_err(dev, "Invalid ACS flags... Ignoring\n");
991 						return;
992 					}
993 				}
994 				p = delimit + 1;
995 			} else {
996 				pci_err(dev, "ACS Flags missing\n");
997 				return;
998 			}
999 		}
1000 
1001 		if (mask & ~(PCI_ACS_SV | PCI_ACS_TB | PCI_ACS_RR | PCI_ACS_CR |
1002 			    PCI_ACS_UF | PCI_ACS_EC | PCI_ACS_DT)) {
1003 			pci_err(dev, "Invalid ACS flags specified\n");
1004 			return;
1005 		}
1006 
1007 		ret = pci_dev_str_match(dev, p, &p);
1008 		if (ret < 0) {
1009 			pr_info_once("PCI: Can't parse ACS command line parameter\n");
1010 			break;
1011 		} else if (ret == 1) {
1012 			/* Found a match */
1013 			break;
1014 		}
1015 
1016 		if (*p != ';' && *p != ',') {
1017 			/* End of param or invalid format */
1018 			break;
1019 		}
1020 		p++;
1021 	}
1022 
1023 	if (ret != 1)
1024 		return;
1025 
1026 	if (!pci_dev_specific_disable_acs_redir(dev))
1027 		return;
1028 
1029 	pci_dbg(dev, "ACS mask  = %#06x\n", mask);
1030 	pci_dbg(dev, "ACS flags = %#06x\n", flags);
1031 
1032 	/* If mask is 0 then we copy the bit from the firmware setting. */
1033 	caps->ctrl = (caps->ctrl & ~mask) | (caps->fw_ctrl & mask);
1034 	caps->ctrl |= flags;
1035 
1036 	pci_info(dev, "Configured ACS to %#06x\n", caps->ctrl);
1037 }
1038 
1039 /**
1040  * pci_std_enable_acs - enable ACS on devices using standard ACS capabilities
1041  * @dev: the PCI device
1042  * @caps: default ACS controls
1043  */
1044 static void pci_std_enable_acs(struct pci_dev *dev, struct pci_acs *caps)
1045 {
1046 	/* Source Validation */
1047 	caps->ctrl |= (caps->cap & PCI_ACS_SV);
1048 
1049 	/* P2P Request Redirect */
1050 	caps->ctrl |= (caps->cap & PCI_ACS_RR);
1051 
1052 	/* P2P Completion Redirect */
1053 	caps->ctrl |= (caps->cap & PCI_ACS_CR);
1054 
1055 	/* Upstream Forwarding */
1056 	caps->ctrl |= (caps->cap & PCI_ACS_UF);
1057 
1058 	/* Enable Translation Blocking for external devices and noats */
1059 	if (pci_ats_disabled() || dev->external_facing || dev->untrusted)
1060 		caps->ctrl |= (caps->cap & PCI_ACS_TB);
1061 }
1062 
1063 /**
1064  * pci_enable_acs - enable ACS if hardware support it
1065  * @dev: the PCI device
1066  */
1067 static void pci_enable_acs(struct pci_dev *dev)
1068 {
1069 	struct pci_acs caps;
1070 	int pos;
1071 
1072 	pos = dev->acs_cap;
1073 	if (!pos)
1074 		return;
1075 
1076 	pci_read_config_word(dev, pos + PCI_ACS_CAP, &caps.cap);
1077 	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &caps.ctrl);
1078 	caps.fw_ctrl = caps.ctrl;
1079 
1080 	/* If an iommu is present we start with kernel default caps */
1081 	if (pci_acs_enable) {
1082 		if (pci_dev_specific_enable_acs(dev))
1083 			pci_std_enable_acs(dev, &caps);
1084 	}
1085 
1086 	/*
1087 	 * Always apply caps from the command line, even if there is no iommu.
1088 	 * Trust that the admin has a reason to change the ACS settings.
1089 	 */
1090 	__pci_config_acs(dev, &caps, disable_acs_redir_param,
1091 			 PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC,
1092 			 ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC));
1093 	__pci_config_acs(dev, &caps, config_acs_param, 0, 0);
1094 
1095 	pci_write_config_word(dev, pos + PCI_ACS_CTRL, caps.ctrl);
1096 }
1097 
1098 /**
1099  * pcie_read_tlp_log - read TLP Header Log
1100  * @dev: PCIe device
1101  * @where: PCI Config offset of TLP Header Log
1102  * @tlp_log: TLP Log structure to fill
1103  *
1104  * Fill @tlp_log from TLP Header Log registers, e.g., AER or DPC.
1105  *
1106  * Return: 0 on success and filled TLP Log structure, <0 on error.
1107  */
1108 int pcie_read_tlp_log(struct pci_dev *dev, int where,
1109 		      struct pcie_tlp_log *tlp_log)
1110 {
1111 	int i, ret;
1112 
1113 	memset(tlp_log, 0, sizeof(*tlp_log));
1114 
1115 	for (i = 0; i < 4; i++) {
1116 		ret = pci_read_config_dword(dev, where + i * 4,
1117 					    &tlp_log->dw[i]);
1118 		if (ret)
1119 			return pcibios_err_to_errno(ret);
1120 	}
1121 
1122 	return 0;
1123 }
1124 EXPORT_SYMBOL_GPL(pcie_read_tlp_log);
1125 
1126 /**
1127  * pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
1128  * @dev: PCI device to have its BARs restored
1129  *
1130  * Restore the BAR values for a given device, so as to make it
1131  * accessible by its driver.
1132  */
1133 static void pci_restore_bars(struct pci_dev *dev)
1134 {
1135 	int i;
1136 
1137 	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
1138 		pci_update_resource(dev, i);
1139 }
1140 
1141 static inline bool platform_pci_power_manageable(struct pci_dev *dev)
1142 {
1143 	if (pci_use_mid_pm())
1144 		return true;
1145 
1146 	return acpi_pci_power_manageable(dev);
1147 }
1148 
1149 static inline int platform_pci_set_power_state(struct pci_dev *dev,
1150 					       pci_power_t t)
1151 {
1152 	if (pci_use_mid_pm())
1153 		return mid_pci_set_power_state(dev, t);
1154 
1155 	return acpi_pci_set_power_state(dev, t);
1156 }
1157 
1158 static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
1159 {
1160 	if (pci_use_mid_pm())
1161 		return mid_pci_get_power_state(dev);
1162 
1163 	return acpi_pci_get_power_state(dev);
1164 }
1165 
1166 static inline void platform_pci_refresh_power_state(struct pci_dev *dev)
1167 {
1168 	if (!pci_use_mid_pm())
1169 		acpi_pci_refresh_power_state(dev);
1170 }
1171 
1172 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
1173 {
1174 	if (pci_use_mid_pm())
1175 		return PCI_POWER_ERROR;
1176 
1177 	return acpi_pci_choose_state(dev);
1178 }
1179 
1180 static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
1181 {
1182 	if (pci_use_mid_pm())
1183 		return PCI_POWER_ERROR;
1184 
1185 	return acpi_pci_wakeup(dev, enable);
1186 }
1187 
1188 static inline bool platform_pci_need_resume(struct pci_dev *dev)
1189 {
1190 	if (pci_use_mid_pm())
1191 		return false;
1192 
1193 	return acpi_pci_need_resume(dev);
1194 }
1195 
1196 static inline bool platform_pci_bridge_d3(struct pci_dev *dev)
1197 {
1198 	if (pci_use_mid_pm())
1199 		return false;
1200 
1201 	return acpi_pci_bridge_d3(dev);
1202 }
1203 
1204 /**
1205  * pci_update_current_state - Read power state of given device and cache it
1206  * @dev: PCI device to handle.
1207  * @state: State to cache in case the device doesn't have the PM capability
1208  *
1209  * The power state is read from the PMCSR register, which however is
1210  * inaccessible in D3cold.  The platform firmware is therefore queried first
1211  * to detect accessibility of the register.  In case the platform firmware
1212  * reports an incorrect state or the device isn't power manageable by the
1213  * platform at all, we try to detect D3cold by testing accessibility of the
1214  * vendor ID in config space.
1215  */
1216 void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
1217 {
1218 	if (platform_pci_get_power_state(dev) == PCI_D3cold) {
1219 		dev->current_state = PCI_D3cold;
1220 	} else if (dev->pm_cap) {
1221 		u16 pmcsr;
1222 
1223 		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1224 		if (PCI_POSSIBLE_ERROR(pmcsr)) {
1225 			dev->current_state = PCI_D3cold;
1226 			return;
1227 		}
1228 		dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1229 	} else {
1230 		dev->current_state = state;
1231 	}
1232 }
1233 
1234 /**
1235  * pci_refresh_power_state - Refresh the given device's power state data
1236  * @dev: Target PCI device.
1237  *
1238  * Ask the platform to refresh the devices power state information and invoke
1239  * pci_update_current_state() to update its current PCI power state.
1240  */
1241 void pci_refresh_power_state(struct pci_dev *dev)
1242 {
1243 	platform_pci_refresh_power_state(dev);
1244 	pci_update_current_state(dev, dev->current_state);
1245 }
1246 
1247 /**
1248  * pci_platform_power_transition - Use platform to change device power state
1249  * @dev: PCI device to handle.
1250  * @state: State to put the device into.
1251  */
1252 int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
1253 {
1254 	int error;
1255 
1256 	error = platform_pci_set_power_state(dev, state);
1257 	if (!error)
1258 		pci_update_current_state(dev, state);
1259 	else if (!dev->pm_cap) /* Fall back to PCI_D0 */
1260 		dev->current_state = PCI_D0;
1261 
1262 	return error;
1263 }
1264 EXPORT_SYMBOL_GPL(pci_platform_power_transition);
1265 
1266 static int pci_resume_one(struct pci_dev *pci_dev, void *ign)
1267 {
1268 	pm_request_resume(&pci_dev->dev);
1269 	return 0;
1270 }
1271 
1272 /**
1273  * pci_resume_bus - Walk given bus and runtime resume devices on it
1274  * @bus: Top bus of the subtree to walk.
1275  */
1276 void pci_resume_bus(struct pci_bus *bus)
1277 {
1278 	if (bus)
1279 		pci_walk_bus(bus, pci_resume_one, NULL);
1280 }
1281 
1282 static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
1283 {
1284 	int delay = 1;
1285 	bool retrain = false;
1286 	struct pci_dev *bridge;
1287 
1288 	if (pci_is_pcie(dev)) {
1289 		bridge = pci_upstream_bridge(dev);
1290 		if (bridge)
1291 			retrain = true;
1292 	}
1293 
1294 	/*
1295 	 * After reset, the device should not silently discard config
1296 	 * requests, but it may still indicate that it needs more time by
1297 	 * responding to them with CRS completions.  The Root Port will
1298 	 * generally synthesize ~0 (PCI_ERROR_RESPONSE) data to complete
1299 	 * the read (except when CRS SV is enabled and the read was for the
1300 	 * Vendor ID; in that case it synthesizes 0x0001 data).
1301 	 *
1302 	 * Wait for the device to return a non-CRS completion.  Read the
1303 	 * Command register instead of Vendor ID so we don't have to
1304 	 * contend with the CRS SV value.
1305 	 */
1306 	for (;;) {
1307 		u32 id;
1308 
1309 		if (pci_dev_is_disconnected(dev)) {
1310 			pci_dbg(dev, "disconnected; not waiting\n");
1311 			return -ENOTTY;
1312 		}
1313 
1314 		pci_read_config_dword(dev, PCI_COMMAND, &id);
1315 		if (!PCI_POSSIBLE_ERROR(id))
1316 			break;
1317 
1318 		if (delay > timeout) {
1319 			pci_warn(dev, "not ready %dms after %s; giving up\n",
1320 				 delay - 1, reset_type);
1321 			return -ENOTTY;
1322 		}
1323 
1324 		if (delay > PCI_RESET_WAIT) {
1325 			if (retrain) {
1326 				retrain = false;
1327 				if (pcie_failed_link_retrain(bridge)) {
1328 					delay = 1;
1329 					continue;
1330 				}
1331 			}
1332 			pci_info(dev, "not ready %dms after %s; waiting\n",
1333 				 delay - 1, reset_type);
1334 		}
1335 
1336 		msleep(delay);
1337 		delay *= 2;
1338 	}
1339 
1340 	if (delay > PCI_RESET_WAIT)
1341 		pci_info(dev, "ready %dms after %s\n", delay - 1,
1342 			 reset_type);
1343 	else
1344 		pci_dbg(dev, "ready %dms after %s\n", delay - 1,
1345 			reset_type);
1346 
1347 	return 0;
1348 }
1349 
1350 /**
1351  * pci_power_up - Put the given device into D0
1352  * @dev: PCI device to power up
1353  *
1354  * On success, return 0 or 1, depending on whether or not it is necessary to
1355  * restore the device's BARs subsequently (1 is returned in that case).
1356  *
1357  * On failure, return a negative error code.  Always return failure if @dev
1358  * lacks a Power Management Capability, even if the platform was able to
1359  * put the device in D0 via non-PCI means.
1360  */
1361 int pci_power_up(struct pci_dev *dev)
1362 {
1363 	bool need_restore;
1364 	pci_power_t state;
1365 	u16 pmcsr;
1366 
1367 	platform_pci_set_power_state(dev, PCI_D0);
1368 
1369 	if (!dev->pm_cap) {
1370 		state = platform_pci_get_power_state(dev);
1371 		if (state == PCI_UNKNOWN)
1372 			dev->current_state = PCI_D0;
1373 		else
1374 			dev->current_state = state;
1375 
1376 		return -EIO;
1377 	}
1378 
1379 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1380 	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1381 		pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n",
1382 			pci_power_name(dev->current_state));
1383 		dev->current_state = PCI_D3cold;
1384 		return -EIO;
1385 	}
1386 
1387 	state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1388 
1389 	need_restore = (state == PCI_D3hot || dev->current_state >= PCI_D3hot) &&
1390 			!(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
1391 
1392 	if (state == PCI_D0)
1393 		goto end;
1394 
1395 	/*
1396 	 * Force the entire word to 0. This doesn't affect PME_Status, disables
1397 	 * PME_En, and sets PowerState to 0.
1398 	 */
1399 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, 0);
1400 
1401 	/* Mandatory transition delays; see PCI PM 1.2. */
1402 	if (state == PCI_D3hot)
1403 		pci_dev_d3_sleep(dev);
1404 	else if (state == PCI_D2)
1405 		udelay(PCI_PM_D2_DELAY);
1406 
1407 end:
1408 	dev->current_state = PCI_D0;
1409 	if (need_restore)
1410 		return 1;
1411 
1412 	return 0;
1413 }
1414 
1415 /**
1416  * pci_set_full_power_state - Put a PCI device into D0 and update its state
1417  * @dev: PCI device to power up
1418  * @locked: whether pci_bus_sem is held
1419  *
1420  * Call pci_power_up() to put @dev into D0, read from its PCI_PM_CTRL register
1421  * to confirm the state change, restore its BARs if they might be lost and
1422  * reconfigure ASPM in accordance with the new power state.
1423  *
1424  * If pci_restore_state() is going to be called right after a power state change
1425  * to D0, it is more efficient to use pci_power_up() directly instead of this
1426  * function.
1427  */
1428 static int pci_set_full_power_state(struct pci_dev *dev, bool locked)
1429 {
1430 	u16 pmcsr;
1431 	int ret;
1432 
1433 	ret = pci_power_up(dev);
1434 	if (ret < 0) {
1435 		if (dev->current_state == PCI_D0)
1436 			return 0;
1437 
1438 		return ret;
1439 	}
1440 
1441 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1442 	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1443 	if (dev->current_state != PCI_D0) {
1444 		pci_info_ratelimited(dev, "Refused to change power state from %s to D0\n",
1445 				     pci_power_name(dev->current_state));
1446 	} else if (ret > 0) {
1447 		/*
1448 		 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
1449 		 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
1450 		 * from D3hot to D0 _may_ perform an internal reset, thereby
1451 		 * going to "D0 Uninitialized" rather than "D0 Initialized".
1452 		 * For example, at least some versions of the 3c905B and the
1453 		 * 3c556B exhibit this behaviour.
1454 		 *
1455 		 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
1456 		 * devices in a D3hot state at boot.  Consequently, we need to
1457 		 * restore at least the BARs so that the device will be
1458 		 * accessible to its driver.
1459 		 */
1460 		pci_restore_bars(dev);
1461 	}
1462 
1463 	if (dev->bus->self)
1464 		pcie_aspm_pm_state_change(dev->bus->self, locked);
1465 
1466 	return 0;
1467 }
1468 
1469 /**
1470  * __pci_dev_set_current_state - Set current state of a PCI device
1471  * @dev: Device to handle
1472  * @data: pointer to state to be set
1473  */
1474 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
1475 {
1476 	pci_power_t state = *(pci_power_t *)data;
1477 
1478 	dev->current_state = state;
1479 	return 0;
1480 }
1481 
1482 /**
1483  * pci_bus_set_current_state - Walk given bus and set current state of devices
1484  * @bus: Top bus of the subtree to walk.
1485  * @state: state to be set
1486  */
1487 void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
1488 {
1489 	if (bus)
1490 		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
1491 }
1492 
1493 static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state, bool locked)
1494 {
1495 	if (!bus)
1496 		return;
1497 
1498 	if (locked)
1499 		pci_walk_bus_locked(bus, __pci_dev_set_current_state, &state);
1500 	else
1501 		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
1502 }
1503 
1504 /**
1505  * pci_set_low_power_state - Put a PCI device into a low-power state.
1506  * @dev: PCI device to handle.
1507  * @state: PCI power state (D1, D2, D3hot) to put the device into.
1508  * @locked: whether pci_bus_sem is held
1509  *
1510  * Use the device's PCI_PM_CTRL register to put it into a low-power state.
1511  *
1512  * RETURN VALUE:
1513  * -EINVAL if the requested state is invalid.
1514  * -EIO if device does not support PCI PM or its PM capabilities register has a
1515  * wrong version, or device doesn't support the requested state.
1516  * 0 if device already is in the requested state.
1517  * 0 if device's power state has been successfully changed.
1518  */
1519 static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool locked)
1520 {
1521 	u16 pmcsr;
1522 
1523 	if (!dev->pm_cap)
1524 		return -EIO;
1525 
1526 	/*
1527 	 * Validate transition: We can enter D0 from any state, but if
1528 	 * we're already in a low-power state, we can only go deeper.  E.g.,
1529 	 * we can go from D1 to D3, but we can't go directly from D3 to D1;
1530 	 * we'd have to go from D3 to D0, then to D1.
1531 	 */
1532 	if (dev->current_state <= PCI_D3cold && dev->current_state > state) {
1533 		pci_dbg(dev, "Invalid power transition (from %s to %s)\n",
1534 			pci_power_name(dev->current_state),
1535 			pci_power_name(state));
1536 		return -EINVAL;
1537 	}
1538 
1539 	/* Check if this device supports the desired state */
1540 	if ((state == PCI_D1 && !dev->d1_support)
1541 	   || (state == PCI_D2 && !dev->d2_support))
1542 		return -EIO;
1543 
1544 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1545 	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1546 		pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n",
1547 			pci_power_name(dev->current_state),
1548 			pci_power_name(state));
1549 		dev->current_state = PCI_D3cold;
1550 		return -EIO;
1551 	}
1552 
1553 	pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1554 	pmcsr |= state;
1555 
1556 	/* Enter specified state */
1557 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
1558 
1559 	/* Mandatory power management transition delays; see PCI PM 1.2. */
1560 	if (state == PCI_D3hot)
1561 		pci_dev_d3_sleep(dev);
1562 	else if (state == PCI_D2)
1563 		udelay(PCI_PM_D2_DELAY);
1564 
1565 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1566 	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1567 	if (dev->current_state != state)
1568 		pci_info_ratelimited(dev, "Refused to change power state from %s to %s\n",
1569 				     pci_power_name(dev->current_state),
1570 				     pci_power_name(state));
1571 
1572 	if (dev->bus->self)
1573 		pcie_aspm_pm_state_change(dev->bus->self, locked);
1574 
1575 	return 0;
1576 }
1577 
1578 static int __pci_set_power_state(struct pci_dev *dev, pci_power_t state, bool locked)
1579 {
1580 	int error;
1581 
1582 	/* Bound the state we're entering */
1583 	if (state > PCI_D3cold)
1584 		state = PCI_D3cold;
1585 	else if (state < PCI_D0)
1586 		state = PCI_D0;
1587 	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
1588 
1589 		/*
1590 		 * If the device or the parent bridge do not support PCI
1591 		 * PM, ignore the request if we're doing anything other
1592 		 * than putting it into D0 (which would only happen on
1593 		 * boot).
1594 		 */
1595 		return 0;
1596 
1597 	/* Check if we're already there */
1598 	if (dev->current_state == state)
1599 		return 0;
1600 
1601 	if (state == PCI_D0)
1602 		return pci_set_full_power_state(dev, locked);
1603 
1604 	/*
1605 	 * This device is quirked not to be put into D3, so don't put it in
1606 	 * D3
1607 	 */
1608 	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
1609 		return 0;
1610 
1611 	if (state == PCI_D3cold) {
1612 		/*
1613 		 * To put the device in D3cold, put it into D3hot in the native
1614 		 * way, then put it into D3cold using platform ops.
1615 		 */
1616 		error = pci_set_low_power_state(dev, PCI_D3hot, locked);
1617 
1618 		if (pci_platform_power_transition(dev, PCI_D3cold))
1619 			return error;
1620 
1621 		/* Powering off a bridge may power off the whole hierarchy */
1622 		if (dev->current_state == PCI_D3cold)
1623 			__pci_bus_set_current_state(dev->subordinate, PCI_D3cold, locked);
1624 	} else {
1625 		error = pci_set_low_power_state(dev, state, locked);
1626 
1627 		if (pci_platform_power_transition(dev, state))
1628 			return error;
1629 	}
1630 
1631 	return 0;
1632 }
1633 
1634 /**
1635  * pci_set_power_state - Set the power state of a PCI device
1636  * @dev: PCI device to handle.
1637  * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
1638  *
1639  * Transition a device to a new power state, using the platform firmware and/or
1640  * the device's PCI PM registers.
1641  *
1642  * RETURN VALUE:
1643  * -EINVAL if the requested state is invalid.
1644  * -EIO if device does not support PCI PM or its PM capabilities register has a
1645  * wrong version, or device doesn't support the requested state.
1646  * 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
1647  * 0 if device already is in the requested state.
1648  * 0 if the transition is to D3 but D3 is not supported.
1649  * 0 if device's power state has been successfully changed.
1650  */
1651 int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
1652 {
1653 	return __pci_set_power_state(dev, state, false);
1654 }
1655 EXPORT_SYMBOL(pci_set_power_state);
1656 
1657 int pci_set_power_state_locked(struct pci_dev *dev, pci_power_t state)
1658 {
1659 	lockdep_assert_held(&pci_bus_sem);
1660 
1661 	return __pci_set_power_state(dev, state, true);
1662 }
1663 EXPORT_SYMBOL(pci_set_power_state_locked);
1664 
1665 #define PCI_EXP_SAVE_REGS	7
1666 
1667 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
1668 						       u16 cap, bool extended)
1669 {
1670 	struct pci_cap_saved_state *tmp;
1671 
1672 	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
1673 		if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
1674 			return tmp;
1675 	}
1676 	return NULL;
1677 }
1678 
1679 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
1680 {
1681 	return _pci_find_saved_cap(dev, cap, false);
1682 }
1683 
1684 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
1685 {
1686 	return _pci_find_saved_cap(dev, cap, true);
1687 }
1688 
1689 static int pci_save_pcie_state(struct pci_dev *dev)
1690 {
1691 	int i = 0;
1692 	struct pci_cap_saved_state *save_state;
1693 	u16 *cap;
1694 
1695 	if (!pci_is_pcie(dev))
1696 		return 0;
1697 
1698 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1699 	if (!save_state) {
1700 		pci_err(dev, "buffer not found in %s\n", __func__);
1701 		return -ENOMEM;
1702 	}
1703 
1704 	cap = (u16 *)&save_state->cap.data[0];
1705 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]);
1706 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]);
1707 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]);
1708 	pcie_capability_read_word(dev, PCI_EXP_RTCTL,  &cap[i++]);
1709 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]);
1710 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]);
1711 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]);
1712 
1713 	pci_save_aspm_l1ss_state(dev);
1714 	pci_save_ltr_state(dev);
1715 
1716 	return 0;
1717 }
1718 
1719 static void pci_restore_pcie_state(struct pci_dev *dev)
1720 {
1721 	int i = 0;
1722 	struct pci_cap_saved_state *save_state;
1723 	u16 *cap;
1724 
1725 	/*
1726 	 * Restore max latencies (in the LTR capability) before enabling
1727 	 * LTR itself in PCI_EXP_DEVCTL2.
1728 	 */
1729 	pci_restore_ltr_state(dev);
1730 	pci_restore_aspm_l1ss_state(dev);
1731 
1732 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1733 	if (!save_state)
1734 		return;
1735 
1736 	/*
1737 	 * Downstream ports reset the LTR enable bit when link goes down.
1738 	 * Check and re-configure the bit here before restoring device.
1739 	 * PCIe r5.0, sec 7.5.3.16.
1740 	 */
1741 	pci_bridge_reconfigure_ltr(dev);
1742 
1743 	cap = (u16 *)&save_state->cap.data[0];
1744 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]);
1745 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]);
1746 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]);
1747 	pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]);
1748 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]);
1749 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]);
1750 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]);
1751 }
1752 
1753 static int pci_save_pcix_state(struct pci_dev *dev)
1754 {
1755 	int pos;
1756 	struct pci_cap_saved_state *save_state;
1757 
1758 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1759 	if (!pos)
1760 		return 0;
1761 
1762 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1763 	if (!save_state) {
1764 		pci_err(dev, "buffer not found in %s\n", __func__);
1765 		return -ENOMEM;
1766 	}
1767 
1768 	pci_read_config_word(dev, pos + PCI_X_CMD,
1769 			     (u16 *)save_state->cap.data);
1770 
1771 	return 0;
1772 }
1773 
1774 static void pci_restore_pcix_state(struct pci_dev *dev)
1775 {
1776 	int i = 0, pos;
1777 	struct pci_cap_saved_state *save_state;
1778 	u16 *cap;
1779 
1780 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1781 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1782 	if (!save_state || !pos)
1783 		return;
1784 	cap = (u16 *)&save_state->cap.data[0];
1785 
1786 	pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]);
1787 }
1788 
1789 /**
1790  * pci_save_state - save the PCI configuration space of a device before
1791  *		    suspending
1792  * @dev: PCI device that we're dealing with
1793  */
1794 int pci_save_state(struct pci_dev *dev)
1795 {
1796 	int i;
1797 	/* XXX: 100% dword access ok here? */
1798 	for (i = 0; i < 16; i++) {
1799 		pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]);
1800 		pci_dbg(dev, "save config %#04x: %#010x\n",
1801 			i * 4, dev->saved_config_space[i]);
1802 	}
1803 	dev->state_saved = true;
1804 
1805 	i = pci_save_pcie_state(dev);
1806 	if (i != 0)
1807 		return i;
1808 
1809 	i = pci_save_pcix_state(dev);
1810 	if (i != 0)
1811 		return i;
1812 
1813 	pci_save_dpc_state(dev);
1814 	pci_save_aer_state(dev);
1815 	pci_save_ptm_state(dev);
1816 	return pci_save_vc_state(dev);
1817 }
1818 EXPORT_SYMBOL(pci_save_state);
1819 
1820 static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1821 				     u32 saved_val, int retry, bool force)
1822 {
1823 	u32 val;
1824 
1825 	pci_read_config_dword(pdev, offset, &val);
1826 	if (!force && val == saved_val)
1827 		return;
1828 
1829 	for (;;) {
1830 		pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n",
1831 			offset, val, saved_val);
1832 		pci_write_config_dword(pdev, offset, saved_val);
1833 		if (retry-- <= 0)
1834 			return;
1835 
1836 		pci_read_config_dword(pdev, offset, &val);
1837 		if (val == saved_val)
1838 			return;
1839 
1840 		mdelay(1);
1841 	}
1842 }
1843 
1844 static void pci_restore_config_space_range(struct pci_dev *pdev,
1845 					   int start, int end, int retry,
1846 					   bool force)
1847 {
1848 	int index;
1849 
1850 	for (index = end; index >= start; index--)
1851 		pci_restore_config_dword(pdev, 4 * index,
1852 					 pdev->saved_config_space[index],
1853 					 retry, force);
1854 }
1855 
1856 static void pci_restore_config_space(struct pci_dev *pdev)
1857 {
1858 	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1859 		pci_restore_config_space_range(pdev, 10, 15, 0, false);
1860 		/* Restore BARs before the command register. */
1861 		pci_restore_config_space_range(pdev, 4, 9, 10, false);
1862 		pci_restore_config_space_range(pdev, 0, 3, 0, false);
1863 	} else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1864 		pci_restore_config_space_range(pdev, 12, 15, 0, false);
1865 
1866 		/*
1867 		 * Force rewriting of prefetch registers to avoid S3 resume
1868 		 * issues on Intel PCI bridges that occur when these
1869 		 * registers are not explicitly written.
1870 		 */
1871 		pci_restore_config_space_range(pdev, 9, 11, 0, true);
1872 		pci_restore_config_space_range(pdev, 0, 8, 0, false);
1873 	} else {
1874 		pci_restore_config_space_range(pdev, 0, 15, 0, false);
1875 	}
1876 }
1877 
1878 static void pci_restore_rebar_state(struct pci_dev *pdev)
1879 {
1880 	unsigned int pos, nbars, i;
1881 	u32 ctrl;
1882 
1883 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
1884 	if (!pos)
1885 		return;
1886 
1887 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1888 	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
1889 
1890 	for (i = 0; i < nbars; i++, pos += 8) {
1891 		struct resource *res;
1892 		int bar_idx, size;
1893 
1894 		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1895 		bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
1896 		res = pdev->resource + bar_idx;
1897 		size = pci_rebar_bytes_to_size(resource_size(res));
1898 		ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
1899 		ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
1900 		pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
1901 	}
1902 }
1903 
1904 /**
1905  * pci_restore_state - Restore the saved state of a PCI device
1906  * @dev: PCI device that we're dealing with
1907  */
1908 void pci_restore_state(struct pci_dev *dev)
1909 {
1910 	if (!dev->state_saved)
1911 		return;
1912 
1913 	pci_restore_pcie_state(dev);
1914 	pci_restore_pasid_state(dev);
1915 	pci_restore_pri_state(dev);
1916 	pci_restore_ats_state(dev);
1917 	pci_restore_vc_state(dev);
1918 	pci_restore_rebar_state(dev);
1919 	pci_restore_dpc_state(dev);
1920 	pci_restore_ptm_state(dev);
1921 
1922 	pci_aer_clear_status(dev);
1923 	pci_restore_aer_state(dev);
1924 
1925 	pci_restore_config_space(dev);
1926 
1927 	pci_restore_pcix_state(dev);
1928 	pci_restore_msi_state(dev);
1929 
1930 	/* Restore ACS and IOV configuration state */
1931 	pci_enable_acs(dev);
1932 	pci_restore_iov_state(dev);
1933 
1934 	dev->state_saved = false;
1935 }
1936 EXPORT_SYMBOL(pci_restore_state);
1937 
1938 struct pci_saved_state {
1939 	u32 config_space[16];
1940 	struct pci_cap_saved_data cap[];
1941 };
1942 
1943 /**
1944  * pci_store_saved_state - Allocate and return an opaque struct containing
1945  *			   the device saved state.
1946  * @dev: PCI device that we're dealing with
1947  *
1948  * Return NULL if no state or error.
1949  */
1950 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
1951 {
1952 	struct pci_saved_state *state;
1953 	struct pci_cap_saved_state *tmp;
1954 	struct pci_cap_saved_data *cap;
1955 	size_t size;
1956 
1957 	if (!dev->state_saved)
1958 		return NULL;
1959 
1960 	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
1961 
1962 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1963 		size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1964 
1965 	state = kzalloc(size, GFP_KERNEL);
1966 	if (!state)
1967 		return NULL;
1968 
1969 	memcpy(state->config_space, dev->saved_config_space,
1970 	       sizeof(state->config_space));
1971 
1972 	cap = state->cap;
1973 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1974 		size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1975 		memcpy(cap, &tmp->cap, len);
1976 		cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
1977 	}
1978 	/* Empty cap_save terminates list */
1979 
1980 	return state;
1981 }
1982 EXPORT_SYMBOL_GPL(pci_store_saved_state);
1983 
1984 /**
1985  * pci_load_saved_state - Reload the provided save state into struct pci_dev.
1986  * @dev: PCI device that we're dealing with
1987  * @state: Saved state returned from pci_store_saved_state()
1988  */
1989 int pci_load_saved_state(struct pci_dev *dev,
1990 			 struct pci_saved_state *state)
1991 {
1992 	struct pci_cap_saved_data *cap;
1993 
1994 	dev->state_saved = false;
1995 
1996 	if (!state)
1997 		return 0;
1998 
1999 	memcpy(dev->saved_config_space, state->config_space,
2000 	       sizeof(state->config_space));
2001 
2002 	cap = state->cap;
2003 	while (cap->size) {
2004 		struct pci_cap_saved_state *tmp;
2005 
2006 		tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended);
2007 		if (!tmp || tmp->cap.size != cap->size)
2008 			return -EINVAL;
2009 
2010 		memcpy(tmp->cap.data, cap->data, tmp->cap.size);
2011 		cap = (struct pci_cap_saved_data *)((u8 *)cap +
2012 		       sizeof(struct pci_cap_saved_data) + cap->size);
2013 	}
2014 
2015 	dev->state_saved = true;
2016 	return 0;
2017 }
2018 EXPORT_SYMBOL_GPL(pci_load_saved_state);
2019 
2020 /**
2021  * pci_load_and_free_saved_state - Reload the save state pointed to by state,
2022  *				   and free the memory allocated for it.
2023  * @dev: PCI device that we're dealing with
2024  * @state: Pointer to saved state returned from pci_store_saved_state()
2025  */
2026 int pci_load_and_free_saved_state(struct pci_dev *dev,
2027 				  struct pci_saved_state **state)
2028 {
2029 	int ret = pci_load_saved_state(dev, *state);
2030 	kfree(*state);
2031 	*state = NULL;
2032 	return ret;
2033 }
2034 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
2035 
2036 int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
2037 {
2038 	return pci_enable_resources(dev, bars);
2039 }
2040 
2041 static int do_pci_enable_device(struct pci_dev *dev, int bars)
2042 {
2043 	int err;
2044 	struct pci_dev *bridge;
2045 	u16 cmd;
2046 	u8 pin;
2047 
2048 	err = pci_set_power_state(dev, PCI_D0);
2049 	if (err < 0 && err != -EIO)
2050 		return err;
2051 
2052 	bridge = pci_upstream_bridge(dev);
2053 	if (bridge)
2054 		pcie_aspm_powersave_config_link(bridge);
2055 
2056 	err = pcibios_enable_device(dev, bars);
2057 	if (err < 0)
2058 		return err;
2059 	pci_fixup_device(pci_fixup_enable, dev);
2060 
2061 	if (dev->msi_enabled || dev->msix_enabled)
2062 		return 0;
2063 
2064 	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
2065 	if (pin) {
2066 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
2067 		if (cmd & PCI_COMMAND_INTX_DISABLE)
2068 			pci_write_config_word(dev, PCI_COMMAND,
2069 					      cmd & ~PCI_COMMAND_INTX_DISABLE);
2070 	}
2071 
2072 	return 0;
2073 }
2074 
2075 /**
2076  * pci_reenable_device - Resume abandoned device
2077  * @dev: PCI device to be resumed
2078  *
2079  * NOTE: This function is a backend of pci_default_resume() and is not supposed
2080  * to be called by normal code, write proper resume handler and use it instead.
2081  */
2082 int pci_reenable_device(struct pci_dev *dev)
2083 {
2084 	if (pci_is_enabled(dev))
2085 		return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1);
2086 	return 0;
2087 }
2088 EXPORT_SYMBOL(pci_reenable_device);
2089 
2090 static void pci_enable_bridge(struct pci_dev *dev)
2091 {
2092 	struct pci_dev *bridge;
2093 	int retval;
2094 
2095 	bridge = pci_upstream_bridge(dev);
2096 	if (bridge)
2097 		pci_enable_bridge(bridge);
2098 
2099 	if (pci_is_enabled(dev)) {
2100 		if (!dev->is_busmaster)
2101 			pci_set_master(dev);
2102 		return;
2103 	}
2104 
2105 	retval = pci_enable_device(dev);
2106 	if (retval)
2107 		pci_err(dev, "Error enabling bridge (%d), continuing\n",
2108 			retval);
2109 	pci_set_master(dev);
2110 }
2111 
2112 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
2113 {
2114 	struct pci_dev *bridge;
2115 	int err;
2116 	int i, bars = 0;
2117 
2118 	/*
2119 	 * Power state could be unknown at this point, either due to a fresh
2120 	 * boot or a device removal call.  So get the current power state
2121 	 * so that things like MSI message writing will behave as expected
2122 	 * (e.g. if the device really is in D0 at enable time).
2123 	 */
2124 	pci_update_current_state(dev, dev->current_state);
2125 
2126 	if (atomic_inc_return(&dev->enable_cnt) > 1)
2127 		return 0;		/* already enabled */
2128 
2129 	bridge = pci_upstream_bridge(dev);
2130 	if (bridge)
2131 		pci_enable_bridge(bridge);
2132 
2133 	/* only skip sriov related */
2134 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
2135 		if (dev->resource[i].flags & flags)
2136 			bars |= (1 << i);
2137 	for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
2138 		if (dev->resource[i].flags & flags)
2139 			bars |= (1 << i);
2140 
2141 	err = do_pci_enable_device(dev, bars);
2142 	if (err < 0)
2143 		atomic_dec(&dev->enable_cnt);
2144 	return err;
2145 }
2146 
2147 /**
2148  * pci_enable_device_mem - Initialize a device for use with Memory space
2149  * @dev: PCI device to be initialized
2150  *
2151  * Initialize device before it's used by a driver. Ask low-level code
2152  * to enable Memory resources. Wake up the device if it was suspended.
2153  * Beware, this function can fail.
2154  */
2155 int pci_enable_device_mem(struct pci_dev *dev)
2156 {
2157 	return pci_enable_device_flags(dev, IORESOURCE_MEM);
2158 }
2159 EXPORT_SYMBOL(pci_enable_device_mem);
2160 
2161 /**
2162  * pci_enable_device - Initialize device before it's used by a driver.
2163  * @dev: PCI device to be initialized
2164  *
2165  * Initialize device before it's used by a driver. Ask low-level code
2166  * to enable I/O and memory. Wake up the device if it was suspended.
2167  * Beware, this function can fail.
2168  *
2169  * Note we don't actually enable the device many times if we call
2170  * this function repeatedly (we just increment the count).
2171  */
2172 int pci_enable_device(struct pci_dev *dev)
2173 {
2174 	return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
2175 }
2176 EXPORT_SYMBOL(pci_enable_device);
2177 
2178 /*
2179  * pcibios_device_add - provide arch specific hooks when adding device dev
2180  * @dev: the PCI device being added
2181  *
2182  * Permits the platform to provide architecture specific functionality when
2183  * devices are added. This is the default implementation. Architecture
2184  * implementations can override this.
2185  */
2186 int __weak pcibios_device_add(struct pci_dev *dev)
2187 {
2188 	return 0;
2189 }
2190 
2191 /**
2192  * pcibios_release_device - provide arch specific hooks when releasing
2193  *			    device dev
2194  * @dev: the PCI device being released
2195  *
2196  * Permits the platform to provide architecture specific functionality when
2197  * devices are released. This is the default implementation. Architecture
2198  * implementations can override this.
2199  */
2200 void __weak pcibios_release_device(struct pci_dev *dev) {}
2201 
2202 /**
2203  * pcibios_disable_device - disable arch specific PCI resources for device dev
2204  * @dev: the PCI device to disable
2205  *
2206  * Disables architecture specific PCI resources for the device. This
2207  * is the default implementation. Architecture implementations can
2208  * override this.
2209  */
2210 void __weak pcibios_disable_device(struct pci_dev *dev) {}
2211 
2212 static void do_pci_disable_device(struct pci_dev *dev)
2213 {
2214 	u16 pci_command;
2215 
2216 	pci_read_config_word(dev, PCI_COMMAND, &pci_command);
2217 	if (pci_command & PCI_COMMAND_MASTER) {
2218 		pci_command &= ~PCI_COMMAND_MASTER;
2219 		pci_write_config_word(dev, PCI_COMMAND, pci_command);
2220 	}
2221 
2222 	pcibios_disable_device(dev);
2223 }
2224 
2225 /**
2226  * pci_disable_enabled_device - Disable device without updating enable_cnt
2227  * @dev: PCI device to disable
2228  *
2229  * NOTE: This function is a backend of PCI power management routines and is
2230  * not supposed to be called drivers.
2231  */
2232 void pci_disable_enabled_device(struct pci_dev *dev)
2233 {
2234 	if (pci_is_enabled(dev))
2235 		do_pci_disable_device(dev);
2236 }
2237 
2238 /**
2239  * pci_disable_device - Disable PCI device after use
2240  * @dev: PCI device to be disabled
2241  *
2242  * Signal to the system that the PCI device is not in use by the system
2243  * anymore.  This only involves disabling PCI bus-mastering, if active.
2244  *
2245  * Note we don't actually disable the device until all callers of
2246  * pci_enable_device() have called pci_disable_device().
2247  */
2248 void pci_disable_device(struct pci_dev *dev)
2249 {
2250 	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
2251 		      "disabling already-disabled device");
2252 
2253 	if (atomic_dec_return(&dev->enable_cnt) != 0)
2254 		return;
2255 
2256 	do_pci_disable_device(dev);
2257 
2258 	dev->is_busmaster = 0;
2259 }
2260 EXPORT_SYMBOL(pci_disable_device);
2261 
2262 /**
2263  * pcibios_set_pcie_reset_state - set reset state for device dev
2264  * @dev: the PCIe device reset
2265  * @state: Reset state to enter into
2266  *
2267  * Set the PCIe reset state for the device. This is the default
2268  * implementation. Architecture implementations can override this.
2269  */
2270 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
2271 					enum pcie_reset_state state)
2272 {
2273 	return -EINVAL;
2274 }
2275 
2276 /**
2277  * pci_set_pcie_reset_state - set reset state for device dev
2278  * @dev: the PCIe device reset
2279  * @state: Reset state to enter into
2280  *
2281  * Sets the PCI reset state for the device.
2282  */
2283 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
2284 {
2285 	return pcibios_set_pcie_reset_state(dev, state);
2286 }
2287 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
2288 
2289 #ifdef CONFIG_PCIEAER
2290 void pcie_clear_device_status(struct pci_dev *dev)
2291 {
2292 	u16 sta;
2293 
2294 	pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta);
2295 	pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta);
2296 }
2297 #endif
2298 
2299 /**
2300  * pcie_clear_root_pme_status - Clear root port PME interrupt status.
2301  * @dev: PCIe root port or event collector.
2302  */
2303 void pcie_clear_root_pme_status(struct pci_dev *dev)
2304 {
2305 	pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME);
2306 }
2307 
2308 /**
2309  * pci_check_pme_status - Check if given device has generated PME.
2310  * @dev: Device to check.
2311  *
2312  * Check the PME status of the device and if set, clear it and clear PME enable
2313  * (if set).  Return 'true' if PME status and PME enable were both set or
2314  * 'false' otherwise.
2315  */
2316 bool pci_check_pme_status(struct pci_dev *dev)
2317 {
2318 	int pmcsr_pos;
2319 	u16 pmcsr;
2320 	bool ret = false;
2321 
2322 	if (!dev->pm_cap)
2323 		return false;
2324 
2325 	pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
2326 	pci_read_config_word(dev, pmcsr_pos, &pmcsr);
2327 	if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
2328 		return false;
2329 
2330 	/* Clear PME status. */
2331 	pmcsr |= PCI_PM_CTRL_PME_STATUS;
2332 	if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
2333 		/* Disable PME to avoid interrupt flood. */
2334 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2335 		ret = true;
2336 	}
2337 
2338 	pci_write_config_word(dev, pmcsr_pos, pmcsr);
2339 
2340 	return ret;
2341 }
2342 
2343 /**
2344  * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
2345  * @dev: Device to handle.
2346  * @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
2347  *
2348  * Check if @dev has generated PME and queue a resume request for it in that
2349  * case.
2350  */
2351 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
2352 {
2353 	if (pme_poll_reset && dev->pme_poll)
2354 		dev->pme_poll = false;
2355 
2356 	if (pci_check_pme_status(dev)) {
2357 		pci_wakeup_event(dev);
2358 		pm_request_resume(&dev->dev);
2359 	}
2360 	return 0;
2361 }
2362 
2363 /**
2364  * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
2365  * @bus: Top bus of the subtree to walk.
2366  */
2367 void pci_pme_wakeup_bus(struct pci_bus *bus)
2368 {
2369 	if (bus)
2370 		pci_walk_bus(bus, pci_pme_wakeup, (void *)true);
2371 }
2372 
2373 
2374 /**
2375  * pci_pme_capable - check the capability of PCI device to generate PME#
2376  * @dev: PCI device to handle.
2377  * @state: PCI state from which device will issue PME#.
2378  */
2379 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
2380 {
2381 	if (!dev->pm_cap)
2382 		return false;
2383 
2384 	return !!(dev->pme_support & (1 << state));
2385 }
2386 EXPORT_SYMBOL(pci_pme_capable);
2387 
2388 static void pci_pme_list_scan(struct work_struct *work)
2389 {
2390 	struct pci_pme_device *pme_dev, *n;
2391 
2392 	mutex_lock(&pci_pme_list_mutex);
2393 	list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
2394 		struct pci_dev *pdev = pme_dev->dev;
2395 
2396 		if (pdev->pme_poll) {
2397 			struct pci_dev *bridge = pdev->bus->self;
2398 			struct device *dev = &pdev->dev;
2399 			struct device *bdev = bridge ? &bridge->dev : NULL;
2400 			int bref = 0;
2401 
2402 			/*
2403 			 * If we have a bridge, it should be in an active/D0
2404 			 * state or the configuration space of subordinate
2405 			 * devices may not be accessible or stable over the
2406 			 * course of the call.
2407 			 */
2408 			if (bdev) {
2409 				bref = pm_runtime_get_if_active(bdev);
2410 				if (!bref)
2411 					continue;
2412 
2413 				if (bridge->current_state != PCI_D0)
2414 					goto put_bridge;
2415 			}
2416 
2417 			/*
2418 			 * The device itself should be suspended but config
2419 			 * space must be accessible, therefore it cannot be in
2420 			 * D3cold.
2421 			 */
2422 			if (pm_runtime_suspended(dev) &&
2423 			    pdev->current_state != PCI_D3cold)
2424 				pci_pme_wakeup(pdev, NULL);
2425 
2426 put_bridge:
2427 			if (bref > 0)
2428 				pm_runtime_put(bdev);
2429 		} else {
2430 			list_del(&pme_dev->list);
2431 			kfree(pme_dev);
2432 		}
2433 	}
2434 	if (!list_empty(&pci_pme_list))
2435 		queue_delayed_work(system_freezable_wq, &pci_pme_work,
2436 				   msecs_to_jiffies(PME_TIMEOUT));
2437 	mutex_unlock(&pci_pme_list_mutex);
2438 }
2439 
2440 static void __pci_pme_active(struct pci_dev *dev, bool enable)
2441 {
2442 	u16 pmcsr;
2443 
2444 	if (!dev->pme_support)
2445 		return;
2446 
2447 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
2448 	/* Clear PME_Status by writing 1 to it and enable PME# */
2449 	pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
2450 	if (!enable)
2451 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2452 
2453 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
2454 }
2455 
2456 /**
2457  * pci_pme_restore - Restore PME configuration after config space restore.
2458  * @dev: PCI device to update.
2459  */
2460 void pci_pme_restore(struct pci_dev *dev)
2461 {
2462 	u16 pmcsr;
2463 
2464 	if (!dev->pme_support)
2465 		return;
2466 
2467 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
2468 	if (dev->wakeup_prepared) {
2469 		pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2470 		pmcsr &= ~PCI_PM_CTRL_PME_STATUS;
2471 	} else {
2472 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2473 		pmcsr |= PCI_PM_CTRL_PME_STATUS;
2474 	}
2475 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
2476 }
2477 
2478 /**
2479  * pci_pme_active - enable or disable PCI device's PME# function
2480  * @dev: PCI device to handle.
2481  * @enable: 'true' to enable PME# generation; 'false' to disable it.
2482  *
2483  * The caller must verify that the device is capable of generating PME# before
2484  * calling this function with @enable equal to 'true'.
2485  */
2486 void pci_pme_active(struct pci_dev *dev, bool enable)
2487 {
2488 	__pci_pme_active(dev, enable);
2489 
2490 	/*
2491 	 * PCI (as opposed to PCIe) PME requires that the device have
2492 	 * its PME# line hooked up correctly. Not all hardware vendors
2493 	 * do this, so the PME never gets delivered and the device
2494 	 * remains asleep. The easiest way around this is to
2495 	 * periodically walk the list of suspended devices and check
2496 	 * whether any have their PME flag set. The assumption is that
2497 	 * we'll wake up often enough anyway that this won't be a huge
2498 	 * hit, and the power savings from the devices will still be a
2499 	 * win.
2500 	 *
2501 	 * Although PCIe uses in-band PME message instead of PME# line
2502 	 * to report PME, PME does not work for some PCIe devices in
2503 	 * reality.  For example, there are devices that set their PME
2504 	 * status bits, but don't really bother to send a PME message;
2505 	 * there are PCI Express Root Ports that don't bother to
2506 	 * trigger interrupts when they receive PME messages from the
2507 	 * devices below.  So PME poll is used for PCIe devices too.
2508 	 */
2509 
2510 	if (dev->pme_poll) {
2511 		struct pci_pme_device *pme_dev;
2512 		if (enable) {
2513 			pme_dev = kmalloc(sizeof(struct pci_pme_device),
2514 					  GFP_KERNEL);
2515 			if (!pme_dev) {
2516 				pci_warn(dev, "can't enable PME#\n");
2517 				return;
2518 			}
2519 			pme_dev->dev = dev;
2520 			mutex_lock(&pci_pme_list_mutex);
2521 			list_add(&pme_dev->list, &pci_pme_list);
2522 			if (list_is_singular(&pci_pme_list))
2523 				queue_delayed_work(system_freezable_wq,
2524 						   &pci_pme_work,
2525 						   msecs_to_jiffies(PME_TIMEOUT));
2526 			mutex_unlock(&pci_pme_list_mutex);
2527 		} else {
2528 			mutex_lock(&pci_pme_list_mutex);
2529 			list_for_each_entry(pme_dev, &pci_pme_list, list) {
2530 				if (pme_dev->dev == dev) {
2531 					list_del(&pme_dev->list);
2532 					kfree(pme_dev);
2533 					break;
2534 				}
2535 			}
2536 			mutex_unlock(&pci_pme_list_mutex);
2537 		}
2538 	}
2539 
2540 	pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled");
2541 }
2542 EXPORT_SYMBOL(pci_pme_active);
2543 
2544 /**
2545  * __pci_enable_wake - enable PCI device as wakeup event source
2546  * @dev: PCI device affected
2547  * @state: PCI state from which device will issue wakeup events
2548  * @enable: True to enable event generation; false to disable
2549  *
2550  * This enables the device as a wakeup event source, or disables it.
2551  * When such events involves platform-specific hooks, those hooks are
2552  * called automatically by this routine.
2553  *
2554  * Devices with legacy power management (no standard PCI PM capabilities)
2555  * always require such platform hooks.
2556  *
2557  * RETURN VALUE:
2558  * 0 is returned on success
2559  * -EINVAL is returned if device is not supposed to wake up the system
2560  * Error code depending on the platform is returned if both the platform and
2561  * the native mechanism fail to enable the generation of wake-up events
2562  */
2563 static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
2564 {
2565 	int ret = 0;
2566 
2567 	/*
2568 	 * Bridges that are not power-manageable directly only signal
2569 	 * wakeup on behalf of subordinate devices which is set up
2570 	 * elsewhere, so skip them. However, bridges that are
2571 	 * power-manageable may signal wakeup for themselves (for example,
2572 	 * on a hotplug event) and they need to be covered here.
2573 	 */
2574 	if (!pci_power_manageable(dev))
2575 		return 0;
2576 
2577 	/* Don't do the same thing twice in a row for one device. */
2578 	if (!!enable == !!dev->wakeup_prepared)
2579 		return 0;
2580 
2581 	/*
2582 	 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
2583 	 * Anderson we should be doing PME# wake enable followed by ACPI wake
2584 	 * enable.  To disable wake-up we call the platform first, for symmetry.
2585 	 */
2586 
2587 	if (enable) {
2588 		int error;
2589 
2590 		/*
2591 		 * Enable PME signaling if the device can signal PME from
2592 		 * D3cold regardless of whether or not it can signal PME from
2593 		 * the current target state, because that will allow it to
2594 		 * signal PME when the hierarchy above it goes into D3cold and
2595 		 * the device itself ends up in D3cold as a result of that.
2596 		 */
2597 		if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold))
2598 			pci_pme_active(dev, true);
2599 		else
2600 			ret = 1;
2601 		error = platform_pci_set_wakeup(dev, true);
2602 		if (ret)
2603 			ret = error;
2604 		if (!ret)
2605 			dev->wakeup_prepared = true;
2606 	} else {
2607 		platform_pci_set_wakeup(dev, false);
2608 		pci_pme_active(dev, false);
2609 		dev->wakeup_prepared = false;
2610 	}
2611 
2612 	return ret;
2613 }
2614 
2615 /**
2616  * pci_enable_wake - change wakeup settings for a PCI device
2617  * @pci_dev: Target device
2618  * @state: PCI state from which device will issue wakeup events
2619  * @enable: Whether or not to enable event generation
2620  *
2621  * If @enable is set, check device_may_wakeup() for the device before calling
2622  * __pci_enable_wake() for it.
2623  */
2624 int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable)
2625 {
2626 	if (enable && !device_may_wakeup(&pci_dev->dev))
2627 		return -EINVAL;
2628 
2629 	return __pci_enable_wake(pci_dev, state, enable);
2630 }
2631 EXPORT_SYMBOL(pci_enable_wake);
2632 
2633 /**
2634  * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
2635  * @dev: PCI device to prepare
2636  * @enable: True to enable wake-up event generation; false to disable
2637  *
2638  * Many drivers want the device to wake up the system from D3_hot or D3_cold
2639  * and this function allows them to set that up cleanly - pci_enable_wake()
2640  * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
2641  * ordering constraints.
2642  *
2643  * This function only returns error code if the device is not allowed to wake
2644  * up the system from sleep or it is not capable of generating PME# from both
2645  * D3_hot and D3_cold and the platform is unable to enable wake-up power for it.
2646  */
2647 int pci_wake_from_d3(struct pci_dev *dev, bool enable)
2648 {
2649 	return pci_pme_capable(dev, PCI_D3cold) ?
2650 			pci_enable_wake(dev, PCI_D3cold, enable) :
2651 			pci_enable_wake(dev, PCI_D3hot, enable);
2652 }
2653 EXPORT_SYMBOL(pci_wake_from_d3);
2654 
2655 /**
2656  * pci_target_state - find an appropriate low power state for a given PCI dev
2657  * @dev: PCI device
2658  * @wakeup: Whether or not wakeup functionality will be enabled for the device.
2659  *
2660  * Use underlying platform code to find a supported low power state for @dev.
2661  * If the platform can't manage @dev, return the deepest state from which it
2662  * can generate wake events, based on any available PME info.
2663  */
2664 static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
2665 {
2666 	if (platform_pci_power_manageable(dev)) {
2667 		/*
2668 		 * Call the platform to find the target state for the device.
2669 		 */
2670 		pci_power_t state = platform_pci_choose_state(dev);
2671 
2672 		switch (state) {
2673 		case PCI_POWER_ERROR:
2674 		case PCI_UNKNOWN:
2675 			return PCI_D3hot;
2676 
2677 		case PCI_D1:
2678 		case PCI_D2:
2679 			if (pci_no_d1d2(dev))
2680 				return PCI_D3hot;
2681 		}
2682 
2683 		return state;
2684 	}
2685 
2686 	/*
2687 	 * If the device is in D3cold even though it's not power-manageable by
2688 	 * the platform, it may have been powered down by non-standard means.
2689 	 * Best to let it slumber.
2690 	 */
2691 	if (dev->current_state == PCI_D3cold)
2692 		return PCI_D3cold;
2693 	else if (!dev->pm_cap)
2694 		return PCI_D0;
2695 
2696 	if (wakeup && dev->pme_support) {
2697 		pci_power_t state = PCI_D3hot;
2698 
2699 		/*
2700 		 * Find the deepest state from which the device can generate
2701 		 * PME#.
2702 		 */
2703 		while (state && !(dev->pme_support & (1 << state)))
2704 			state--;
2705 
2706 		if (state)
2707 			return state;
2708 		else if (dev->pme_support & 1)
2709 			return PCI_D0;
2710 	}
2711 
2712 	return PCI_D3hot;
2713 }
2714 
2715 /**
2716  * pci_prepare_to_sleep - prepare PCI device for system-wide transition
2717  *			  into a sleep state
2718  * @dev: Device to handle.
2719  *
2720  * Choose the power state appropriate for the device depending on whether
2721  * it can wake up the system and/or is power manageable by the platform
2722  * (PCI_D3hot is the default) and put the device into that state.
2723  */
2724 int pci_prepare_to_sleep(struct pci_dev *dev)
2725 {
2726 	bool wakeup = device_may_wakeup(&dev->dev);
2727 	pci_power_t target_state = pci_target_state(dev, wakeup);
2728 	int error;
2729 
2730 	if (target_state == PCI_POWER_ERROR)
2731 		return -EIO;
2732 
2733 	pci_enable_wake(dev, target_state, wakeup);
2734 
2735 	error = pci_set_power_state(dev, target_state);
2736 
2737 	if (error)
2738 		pci_enable_wake(dev, target_state, false);
2739 
2740 	return error;
2741 }
2742 EXPORT_SYMBOL(pci_prepare_to_sleep);
2743 
2744 /**
2745  * pci_back_from_sleep - turn PCI device on during system-wide transition
2746  *			 into working state
2747  * @dev: Device to handle.
2748  *
2749  * Disable device's system wake-up capability and put it into D0.
2750  */
2751 int pci_back_from_sleep(struct pci_dev *dev)
2752 {
2753 	int ret = pci_set_power_state(dev, PCI_D0);
2754 
2755 	if (ret)
2756 		return ret;
2757 
2758 	pci_enable_wake(dev, PCI_D0, false);
2759 	return 0;
2760 }
2761 EXPORT_SYMBOL(pci_back_from_sleep);
2762 
2763 /**
2764  * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
2765  * @dev: PCI device being suspended.
2766  *
2767  * Prepare @dev to generate wake-up events at run time and put it into a low
2768  * power state.
2769  */
2770 int pci_finish_runtime_suspend(struct pci_dev *dev)
2771 {
2772 	pci_power_t target_state;
2773 	int error;
2774 
2775 	target_state = pci_target_state(dev, device_can_wakeup(&dev->dev));
2776 	if (target_state == PCI_POWER_ERROR)
2777 		return -EIO;
2778 
2779 	__pci_enable_wake(dev, target_state, pci_dev_run_wake(dev));
2780 
2781 	error = pci_set_power_state(dev, target_state);
2782 
2783 	if (error)
2784 		pci_enable_wake(dev, target_state, false);
2785 
2786 	return error;
2787 }
2788 
2789 /**
2790  * pci_dev_run_wake - Check if device can generate run-time wake-up events.
2791  * @dev: Device to check.
2792  *
2793  * Return true if the device itself is capable of generating wake-up events
2794  * (through the platform or using the native PCIe PME) or if the device supports
2795  * PME and one of its upstream bridges can generate wake-up events.
2796  */
2797 bool pci_dev_run_wake(struct pci_dev *dev)
2798 {
2799 	struct pci_bus *bus = dev->bus;
2800 
2801 	if (!dev->pme_support)
2802 		return false;
2803 
2804 	/* PME-capable in principle, but not from the target power state */
2805 	if (!pci_pme_capable(dev, pci_target_state(dev, true)))
2806 		return false;
2807 
2808 	if (device_can_wakeup(&dev->dev))
2809 		return true;
2810 
2811 	while (bus->parent) {
2812 		struct pci_dev *bridge = bus->self;
2813 
2814 		if (device_can_wakeup(&bridge->dev))
2815 			return true;
2816 
2817 		bus = bus->parent;
2818 	}
2819 
2820 	/* We have reached the root bus. */
2821 	if (bus->bridge)
2822 		return device_can_wakeup(bus->bridge);
2823 
2824 	return false;
2825 }
2826 EXPORT_SYMBOL_GPL(pci_dev_run_wake);
2827 
2828 /**
2829  * pci_dev_need_resume - Check if it is necessary to resume the device.
2830  * @pci_dev: Device to check.
2831  *
2832  * Return 'true' if the device is not runtime-suspended or it has to be
2833  * reconfigured due to wakeup settings difference between system and runtime
2834  * suspend, or the current power state of it is not suitable for the upcoming
2835  * (system-wide) transition.
2836  */
2837 bool pci_dev_need_resume(struct pci_dev *pci_dev)
2838 {
2839 	struct device *dev = &pci_dev->dev;
2840 	pci_power_t target_state;
2841 
2842 	if (!pm_runtime_suspended(dev) || platform_pci_need_resume(pci_dev))
2843 		return true;
2844 
2845 	target_state = pci_target_state(pci_dev, device_may_wakeup(dev));
2846 
2847 	/*
2848 	 * If the earlier platform check has not triggered, D3cold is just power
2849 	 * removal on top of D3hot, so no need to resume the device in that
2850 	 * case.
2851 	 */
2852 	return target_state != pci_dev->current_state &&
2853 		target_state != PCI_D3cold &&
2854 		pci_dev->current_state != PCI_D3hot;
2855 }
2856 
2857 /**
2858  * pci_dev_adjust_pme - Adjust PME setting for a suspended device.
2859  * @pci_dev: Device to check.
2860  *
2861  * If the device is suspended and it is not configured for system wakeup,
2862  * disable PME for it to prevent it from waking up the system unnecessarily.
2863  *
2864  * Note that if the device's power state is D3cold and the platform check in
2865  * pci_dev_need_resume() has not triggered, the device's configuration need not
2866  * be changed.
2867  */
2868 void pci_dev_adjust_pme(struct pci_dev *pci_dev)
2869 {
2870 	struct device *dev = &pci_dev->dev;
2871 
2872 	spin_lock_irq(&dev->power.lock);
2873 
2874 	if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) &&
2875 	    pci_dev->current_state < PCI_D3cold)
2876 		__pci_pme_active(pci_dev, false);
2877 
2878 	spin_unlock_irq(&dev->power.lock);
2879 }
2880 
2881 /**
2882  * pci_dev_complete_resume - Finalize resume from system sleep for a device.
2883  * @pci_dev: Device to handle.
2884  *
2885  * If the device is runtime suspended and wakeup-capable, enable PME for it as
2886  * it might have been disabled during the prepare phase of system suspend if
2887  * the device was not configured for system wakeup.
2888  */
2889 void pci_dev_complete_resume(struct pci_dev *pci_dev)
2890 {
2891 	struct device *dev = &pci_dev->dev;
2892 
2893 	if (!pci_dev_run_wake(pci_dev))
2894 		return;
2895 
2896 	spin_lock_irq(&dev->power.lock);
2897 
2898 	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold)
2899 		__pci_pme_active(pci_dev, true);
2900 
2901 	spin_unlock_irq(&dev->power.lock);
2902 }
2903 
2904 /**
2905  * pci_choose_state - Choose the power state of a PCI device.
2906  * @dev: Target PCI device.
2907  * @state: Target state for the whole system.
2908  *
2909  * Returns PCI power state suitable for @dev and @state.
2910  */
2911 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
2912 {
2913 	if (state.event == PM_EVENT_ON)
2914 		return PCI_D0;
2915 
2916 	return pci_target_state(dev, false);
2917 }
2918 EXPORT_SYMBOL(pci_choose_state);
2919 
2920 void pci_config_pm_runtime_get(struct pci_dev *pdev)
2921 {
2922 	struct device *dev = &pdev->dev;
2923 	struct device *parent = dev->parent;
2924 
2925 	if (parent)
2926 		pm_runtime_get_sync(parent);
2927 	pm_runtime_get_noresume(dev);
2928 	/*
2929 	 * pdev->current_state is set to PCI_D3cold during suspending,
2930 	 * so wait until suspending completes
2931 	 */
2932 	pm_runtime_barrier(dev);
2933 	/*
2934 	 * Only need to resume devices in D3cold, because config
2935 	 * registers are still accessible for devices suspended but
2936 	 * not in D3cold.
2937 	 */
2938 	if (pdev->current_state == PCI_D3cold)
2939 		pm_runtime_resume(dev);
2940 }
2941 
2942 void pci_config_pm_runtime_put(struct pci_dev *pdev)
2943 {
2944 	struct device *dev = &pdev->dev;
2945 	struct device *parent = dev->parent;
2946 
2947 	pm_runtime_put(dev);
2948 	if (parent)
2949 		pm_runtime_put_sync(parent);
2950 }
2951 
2952 static const struct dmi_system_id bridge_d3_blacklist[] = {
2953 #ifdef CONFIG_X86
2954 	{
2955 		/*
2956 		 * Gigabyte X299 root port is not marked as hotplug capable
2957 		 * which allows Linux to power manage it.  However, this
2958 		 * confuses the BIOS SMI handler so don't power manage root
2959 		 * ports on that system.
2960 		 */
2961 		.ident = "X299 DESIGNARE EX-CF",
2962 		.matches = {
2963 			DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."),
2964 			DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"),
2965 		},
2966 	},
2967 	{
2968 		/*
2969 		 * Downstream device is not accessible after putting a root port
2970 		 * into D3cold and back into D0 on Elo Continental Z2 board
2971 		 */
2972 		.ident = "Elo Continental Z2",
2973 		.matches = {
2974 			DMI_MATCH(DMI_BOARD_VENDOR, "Elo Touch Solutions"),
2975 			DMI_MATCH(DMI_BOARD_NAME, "Geminilake"),
2976 			DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"),
2977 		},
2978 	},
2979 	{
2980 		/*
2981 		 * Changing power state of root port dGPU is connected fails
2982 		 * https://gitlab.freedesktop.org/drm/amd/-/issues/3229
2983 		 */
2984 		.ident = "Hewlett-Packard HP Pavilion 17 Notebook PC/1972",
2985 		.matches = {
2986 			DMI_MATCH(DMI_BOARD_VENDOR, "Hewlett-Packard"),
2987 			DMI_MATCH(DMI_BOARD_NAME, "1972"),
2988 			DMI_MATCH(DMI_BOARD_VERSION, "95.33"),
2989 		},
2990 	},
2991 #endif
2992 	{ }
2993 };
2994 
2995 /**
2996  * pci_bridge_d3_possible - Is it possible to put the bridge into D3
2997  * @bridge: Bridge to check
2998  *
2999  * This function checks if it is possible to move the bridge to D3.
3000  * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt.
3001  */
3002 bool pci_bridge_d3_possible(struct pci_dev *bridge)
3003 {
3004 	if (!pci_is_pcie(bridge))
3005 		return false;
3006 
3007 	switch (pci_pcie_type(bridge)) {
3008 	case PCI_EXP_TYPE_ROOT_PORT:
3009 	case PCI_EXP_TYPE_UPSTREAM:
3010 	case PCI_EXP_TYPE_DOWNSTREAM:
3011 		if (pci_bridge_d3_disable)
3012 			return false;
3013 
3014 		/*
3015 		 * Hotplug ports handled by firmware in System Management Mode
3016 		 * may not be put into D3 by the OS (Thunderbolt on non-Macs).
3017 		 */
3018 		if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge))
3019 			return false;
3020 
3021 		if (pci_bridge_d3_force)
3022 			return true;
3023 
3024 		/* Even the oldest 2010 Thunderbolt controller supports D3. */
3025 		if (bridge->is_thunderbolt)
3026 			return true;
3027 
3028 		/* Platform might know better if the bridge supports D3 */
3029 		if (platform_pci_bridge_d3(bridge))
3030 			return true;
3031 
3032 		/*
3033 		 * Hotplug ports handled natively by the OS were not validated
3034 		 * by vendors for runtime D3 at least until 2018 because there
3035 		 * was no OS support.
3036 		 */
3037 		if (bridge->is_hotplug_bridge)
3038 			return false;
3039 
3040 		if (dmi_check_system(bridge_d3_blacklist))
3041 			return false;
3042 
3043 		/*
3044 		 * It should be safe to put PCIe ports from 2015 or newer
3045 		 * to D3.
3046 		 */
3047 		if (dmi_get_bios_year() >= 2015)
3048 			return true;
3049 		break;
3050 	}
3051 
3052 	return false;
3053 }
3054 
3055 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
3056 {
3057 	bool *d3cold_ok = data;
3058 
3059 	if (/* The device needs to be allowed to go D3cold ... */
3060 	    dev->no_d3cold || !dev->d3cold_allowed ||
3061 
3062 	    /* ... and if it is wakeup capable to do so from D3cold. */
3063 	    (device_may_wakeup(&dev->dev) &&
3064 	     !pci_pme_capable(dev, PCI_D3cold)) ||
3065 
3066 	    /* If it is a bridge it must be allowed to go to D3. */
3067 	    !pci_power_manageable(dev))
3068 
3069 		*d3cold_ok = false;
3070 
3071 	return !*d3cold_ok;
3072 }
3073 
3074 /*
3075  * pci_bridge_d3_update - Update bridge D3 capabilities
3076  * @dev: PCI device which is changed
3077  *
3078  * Update upstream bridge PM capabilities accordingly depending on if the
3079  * device PM configuration was changed or the device is being removed.  The
3080  * change is also propagated upstream.
3081  */
3082 void pci_bridge_d3_update(struct pci_dev *dev)
3083 {
3084 	bool remove = !device_is_registered(&dev->dev);
3085 	struct pci_dev *bridge;
3086 	bool d3cold_ok = true;
3087 
3088 	bridge = pci_upstream_bridge(dev);
3089 	if (!bridge || !pci_bridge_d3_possible(bridge))
3090 		return;
3091 
3092 	/*
3093 	 * If D3 is currently allowed for the bridge, removing one of its
3094 	 * children won't change that.
3095 	 */
3096 	if (remove && bridge->bridge_d3)
3097 		return;
3098 
3099 	/*
3100 	 * If D3 is currently allowed for the bridge and a child is added or
3101 	 * changed, disallowance of D3 can only be caused by that child, so
3102 	 * we only need to check that single device, not any of its siblings.
3103 	 *
3104 	 * If D3 is currently not allowed for the bridge, checking the device
3105 	 * first may allow us to skip checking its siblings.
3106 	 */
3107 	if (!remove)
3108 		pci_dev_check_d3cold(dev, &d3cold_ok);
3109 
3110 	/*
3111 	 * If D3 is currently not allowed for the bridge, this may be caused
3112 	 * either by the device being changed/removed or any of its siblings,
3113 	 * so we need to go through all children to find out if one of them
3114 	 * continues to block D3.
3115 	 */
3116 	if (d3cold_ok && !bridge->bridge_d3)
3117 		pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold,
3118 			     &d3cold_ok);
3119 
3120 	if (bridge->bridge_d3 != d3cold_ok) {
3121 		bridge->bridge_d3 = d3cold_ok;
3122 		/* Propagate change to upstream bridges */
3123 		pci_bridge_d3_update(bridge);
3124 	}
3125 }
3126 
3127 /**
3128  * pci_d3cold_enable - Enable D3cold for device
3129  * @dev: PCI device to handle
3130  *
3131  * This function can be used in drivers to enable D3cold from the device
3132  * they handle.  It also updates upstream PCI bridge PM capabilities
3133  * accordingly.
3134  */
3135 void pci_d3cold_enable(struct pci_dev *dev)
3136 {
3137 	if (dev->no_d3cold) {
3138 		dev->no_d3cold = false;
3139 		pci_bridge_d3_update(dev);
3140 	}
3141 }
3142 EXPORT_SYMBOL_GPL(pci_d3cold_enable);
3143 
3144 /**
3145  * pci_d3cold_disable - Disable D3cold for device
3146  * @dev: PCI device to handle
3147  *
3148  * This function can be used in drivers to disable D3cold from the device
3149  * they handle.  It also updates upstream PCI bridge PM capabilities
3150  * accordingly.
3151  */
3152 void pci_d3cold_disable(struct pci_dev *dev)
3153 {
3154 	if (!dev->no_d3cold) {
3155 		dev->no_d3cold = true;
3156 		pci_bridge_d3_update(dev);
3157 	}
3158 }
3159 EXPORT_SYMBOL_GPL(pci_d3cold_disable);
3160 
3161 /**
3162  * pci_pm_init - Initialize PM functions of given PCI device
3163  * @dev: PCI device to handle.
3164  */
3165 void pci_pm_init(struct pci_dev *dev)
3166 {
3167 	int pm;
3168 	u16 status;
3169 	u16 pmc;
3170 
3171 	pm_runtime_forbid(&dev->dev);
3172 	pm_runtime_set_active(&dev->dev);
3173 	pm_runtime_enable(&dev->dev);
3174 	device_enable_async_suspend(&dev->dev);
3175 	dev->wakeup_prepared = false;
3176 
3177 	dev->pm_cap = 0;
3178 	dev->pme_support = 0;
3179 
3180 	/* find PCI PM capability in list */
3181 	pm = pci_find_capability(dev, PCI_CAP_ID_PM);
3182 	if (!pm)
3183 		return;
3184 	/* Check device's ability to generate PME# */
3185 	pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc);
3186 
3187 	if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
3188 		pci_err(dev, "unsupported PM cap regs version (%u)\n",
3189 			pmc & PCI_PM_CAP_VER_MASK);
3190 		return;
3191 	}
3192 
3193 	dev->pm_cap = pm;
3194 	dev->d3hot_delay = PCI_PM_D3HOT_WAIT;
3195 	dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
3196 	dev->bridge_d3 = pci_bridge_d3_possible(dev);
3197 	dev->d3cold_allowed = true;
3198 
3199 	dev->d1_support = false;
3200 	dev->d2_support = false;
3201 	if (!pci_no_d1d2(dev)) {
3202 		if (pmc & PCI_PM_CAP_D1)
3203 			dev->d1_support = true;
3204 		if (pmc & PCI_PM_CAP_D2)
3205 			dev->d2_support = true;
3206 
3207 		if (dev->d1_support || dev->d2_support)
3208 			pci_info(dev, "supports%s%s\n",
3209 				   dev->d1_support ? " D1" : "",
3210 				   dev->d2_support ? " D2" : "");
3211 	}
3212 
3213 	pmc &= PCI_PM_CAP_PME_MASK;
3214 	if (pmc) {
3215 		pci_info(dev, "PME# supported from%s%s%s%s%s\n",
3216 			 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
3217 			 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
3218 			 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
3219 			 (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "",
3220 			 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
3221 		dev->pme_support = FIELD_GET(PCI_PM_CAP_PME_MASK, pmc);
3222 		dev->pme_poll = true;
3223 		/*
3224 		 * Make device's PM flags reflect the wake-up capability, but
3225 		 * let the user space enable it to wake up the system as needed.
3226 		 */
3227 		device_set_wakeup_capable(&dev->dev, true);
3228 		/* Disable the PME# generation functionality */
3229 		pci_pme_active(dev, false);
3230 	}
3231 
3232 	pci_read_config_word(dev, PCI_STATUS, &status);
3233 	if (status & PCI_STATUS_IMM_READY)
3234 		dev->imm_ready = 1;
3235 }
3236 
3237 static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop)
3238 {
3239 	unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI;
3240 
3241 	switch (prop) {
3242 	case PCI_EA_P_MEM:
3243 	case PCI_EA_P_VF_MEM:
3244 		flags |= IORESOURCE_MEM;
3245 		break;
3246 	case PCI_EA_P_MEM_PREFETCH:
3247 	case PCI_EA_P_VF_MEM_PREFETCH:
3248 		flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
3249 		break;
3250 	case PCI_EA_P_IO:
3251 		flags |= IORESOURCE_IO;
3252 		break;
3253 	default:
3254 		return 0;
3255 	}
3256 
3257 	return flags;
3258 }
3259 
3260 static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
3261 					    u8 prop)
3262 {
3263 	if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
3264 		return &dev->resource[bei];
3265 #ifdef CONFIG_PCI_IOV
3266 	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 &&
3267 		 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH))
3268 		return &dev->resource[PCI_IOV_RESOURCES +
3269 				      bei - PCI_EA_BEI_VF_BAR0];
3270 #endif
3271 	else if (bei == PCI_EA_BEI_ROM)
3272 		return &dev->resource[PCI_ROM_RESOURCE];
3273 	else
3274 		return NULL;
3275 }
3276 
3277 /* Read an Enhanced Allocation (EA) entry */
3278 static int pci_ea_read(struct pci_dev *dev, int offset)
3279 {
3280 	struct resource *res;
3281 	const char *res_name;
3282 	int ent_size, ent_offset = offset;
3283 	resource_size_t start, end;
3284 	unsigned long flags;
3285 	u32 dw0, bei, base, max_offset;
3286 	u8 prop;
3287 	bool support_64 = (sizeof(resource_size_t) >= 8);
3288 
3289 	pci_read_config_dword(dev, ent_offset, &dw0);
3290 	ent_offset += 4;
3291 
3292 	/* Entry size field indicates DWORDs after 1st */
3293 	ent_size = (FIELD_GET(PCI_EA_ES, dw0) + 1) << 2;
3294 
3295 	if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
3296 		goto out;
3297 
3298 	bei = FIELD_GET(PCI_EA_BEI, dw0);
3299 	prop = FIELD_GET(PCI_EA_PP, dw0);
3300 
3301 	/*
3302 	 * If the Property is in the reserved range, try the Secondary
3303 	 * Property instead.
3304 	 */
3305 	if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
3306 		prop = FIELD_GET(PCI_EA_SP, dw0);
3307 	if (prop > PCI_EA_P_BRIDGE_IO)
3308 		goto out;
3309 
3310 	res = pci_ea_get_resource(dev, bei, prop);
3311 	res_name = pci_resource_name(dev, bei);
3312 	if (!res) {
3313 		pci_err(dev, "Unsupported EA entry BEI: %u\n", bei);
3314 		goto out;
3315 	}
3316 
3317 	flags = pci_ea_flags(dev, prop);
3318 	if (!flags) {
3319 		pci_err(dev, "Unsupported EA properties: %#x\n", prop);
3320 		goto out;
3321 	}
3322 
3323 	/* Read Base */
3324 	pci_read_config_dword(dev, ent_offset, &base);
3325 	start = (base & PCI_EA_FIELD_MASK);
3326 	ent_offset += 4;
3327 
3328 	/* Read MaxOffset */
3329 	pci_read_config_dword(dev, ent_offset, &max_offset);
3330 	ent_offset += 4;
3331 
3332 	/* Read Base MSBs (if 64-bit entry) */
3333 	if (base & PCI_EA_IS_64) {
3334 		u32 base_upper;
3335 
3336 		pci_read_config_dword(dev, ent_offset, &base_upper);
3337 		ent_offset += 4;
3338 
3339 		flags |= IORESOURCE_MEM_64;
3340 
3341 		/* entry starts above 32-bit boundary, can't use */
3342 		if (!support_64 && base_upper)
3343 			goto out;
3344 
3345 		if (support_64)
3346 			start |= ((u64)base_upper << 32);
3347 	}
3348 
3349 	end = start + (max_offset | 0x03);
3350 
3351 	/* Read MaxOffset MSBs (if 64-bit entry) */
3352 	if (max_offset & PCI_EA_IS_64) {
3353 		u32 max_offset_upper;
3354 
3355 		pci_read_config_dword(dev, ent_offset, &max_offset_upper);
3356 		ent_offset += 4;
3357 
3358 		flags |= IORESOURCE_MEM_64;
3359 
3360 		/* entry too big, can't use */
3361 		if (!support_64 && max_offset_upper)
3362 			goto out;
3363 
3364 		if (support_64)
3365 			end += ((u64)max_offset_upper << 32);
3366 	}
3367 
3368 	if (end < start) {
3369 		pci_err(dev, "EA Entry crosses address boundary\n");
3370 		goto out;
3371 	}
3372 
3373 	if (ent_size != ent_offset - offset) {
3374 		pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n",
3375 			ent_size, ent_offset - offset);
3376 		goto out;
3377 	}
3378 
3379 	res->name = pci_name(dev);
3380 	res->start = start;
3381 	res->end = end;
3382 	res->flags = flags;
3383 
3384 	if (bei <= PCI_EA_BEI_BAR5)
3385 		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3386 			 res_name, res, prop);
3387 	else if (bei == PCI_EA_BEI_ROM)
3388 		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3389 			 res_name, res, prop);
3390 	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5)
3391 		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3392 			 res_name, res, prop);
3393 	else
3394 		pci_info(dev, "BEI %d %pR: from Enhanced Allocation, properties %#02x\n",
3395 			   bei, res, prop);
3396 
3397 out:
3398 	return offset + ent_size;
3399 }
3400 
3401 /* Enhanced Allocation Initialization */
3402 void pci_ea_init(struct pci_dev *dev)
3403 {
3404 	int ea;
3405 	u8 num_ent;
3406 	int offset;
3407 	int i;
3408 
3409 	/* find PCI EA capability in list */
3410 	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
3411 	if (!ea)
3412 		return;
3413 
3414 	/* determine the number of entries */
3415 	pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT,
3416 					&num_ent);
3417 	num_ent &= PCI_EA_NUM_ENT_MASK;
3418 
3419 	offset = ea + PCI_EA_FIRST_ENT;
3420 
3421 	/* Skip DWORD 2 for type 1 functions */
3422 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
3423 		offset += 4;
3424 
3425 	/* parse each EA entry */
3426 	for (i = 0; i < num_ent; ++i)
3427 		offset = pci_ea_read(dev, offset);
3428 }
3429 
3430 static void pci_add_saved_cap(struct pci_dev *pci_dev,
3431 	struct pci_cap_saved_state *new_cap)
3432 {
3433 	hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space);
3434 }
3435 
3436 /**
3437  * _pci_add_cap_save_buffer - allocate buffer for saving given
3438  *			      capability registers
3439  * @dev: the PCI device
3440  * @cap: the capability to allocate the buffer for
3441  * @extended: Standard or Extended capability ID
3442  * @size: requested size of the buffer
3443  */
3444 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
3445 				    bool extended, unsigned int size)
3446 {
3447 	int pos;
3448 	struct pci_cap_saved_state *save_state;
3449 
3450 	if (extended)
3451 		pos = pci_find_ext_capability(dev, cap);
3452 	else
3453 		pos = pci_find_capability(dev, cap);
3454 
3455 	if (!pos)
3456 		return 0;
3457 
3458 	save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL);
3459 	if (!save_state)
3460 		return -ENOMEM;
3461 
3462 	save_state->cap.cap_nr = cap;
3463 	save_state->cap.cap_extended = extended;
3464 	save_state->cap.size = size;
3465 	pci_add_saved_cap(dev, save_state);
3466 
3467 	return 0;
3468 }
3469 
3470 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
3471 {
3472 	return _pci_add_cap_save_buffer(dev, cap, false, size);
3473 }
3474 
3475 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
3476 {
3477 	return _pci_add_cap_save_buffer(dev, cap, true, size);
3478 }
3479 
3480 /**
3481  * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
3482  * @dev: the PCI device
3483  */
3484 void pci_allocate_cap_save_buffers(struct pci_dev *dev)
3485 {
3486 	int error;
3487 
3488 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
3489 					PCI_EXP_SAVE_REGS * sizeof(u16));
3490 	if (error)
3491 		pci_err(dev, "unable to preallocate PCI Express save buffer\n");
3492 
3493 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16));
3494 	if (error)
3495 		pci_err(dev, "unable to preallocate PCI-X save buffer\n");
3496 
3497 	error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR,
3498 					    2 * sizeof(u16));
3499 	if (error)
3500 		pci_err(dev, "unable to allocate suspend buffer for LTR\n");
3501 
3502 	pci_allocate_vc_save_buffers(dev);
3503 }
3504 
3505 void pci_free_cap_save_buffers(struct pci_dev *dev)
3506 {
3507 	struct pci_cap_saved_state *tmp;
3508 	struct hlist_node *n;
3509 
3510 	hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
3511 		kfree(tmp);
3512 }
3513 
3514 /**
3515  * pci_configure_ari - enable or disable ARI forwarding
3516  * @dev: the PCI device
3517  *
3518  * If @dev and its upstream bridge both support ARI, enable ARI in the
3519  * bridge.  Otherwise, disable ARI in the bridge.
3520  */
3521 void pci_configure_ari(struct pci_dev *dev)
3522 {
3523 	u32 cap;
3524 	struct pci_dev *bridge;
3525 
3526 	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
3527 		return;
3528 
3529 	bridge = dev->bus->self;
3530 	if (!bridge)
3531 		return;
3532 
3533 	pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
3534 	if (!(cap & PCI_EXP_DEVCAP2_ARI))
3535 		return;
3536 
3537 	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
3538 		pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,
3539 					 PCI_EXP_DEVCTL2_ARI);
3540 		bridge->ari_enabled = 1;
3541 	} else {
3542 		pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2,
3543 					   PCI_EXP_DEVCTL2_ARI);
3544 		bridge->ari_enabled = 0;
3545 	}
3546 }
3547 
3548 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
3549 {
3550 	int pos;
3551 	u16 cap, ctrl;
3552 
3553 	pos = pdev->acs_cap;
3554 	if (!pos)
3555 		return false;
3556 
3557 	/*
3558 	 * Except for egress control, capabilities are either required
3559 	 * or only required if controllable.  Features missing from the
3560 	 * capability field can therefore be assumed as hard-wired enabled.
3561 	 */
3562 	pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap);
3563 	acs_flags &= (cap | PCI_ACS_EC);
3564 
3565 	pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
3566 	return (ctrl & acs_flags) == acs_flags;
3567 }
3568 
3569 /**
3570  * pci_acs_enabled - test ACS against required flags for a given device
3571  * @pdev: device to test
3572  * @acs_flags: required PCI ACS flags
3573  *
3574  * Return true if the device supports the provided flags.  Automatically
3575  * filters out flags that are not implemented on multifunction devices.
3576  *
3577  * Note that this interface checks the effective ACS capabilities of the
3578  * device rather than the actual capabilities.  For instance, most single
3579  * function endpoints are not required to support ACS because they have no
3580  * opportunity for peer-to-peer access.  We therefore return 'true'
3581  * regardless of whether the device exposes an ACS capability.  This makes
3582  * it much easier for callers of this function to ignore the actual type
3583  * or topology of the device when testing ACS support.
3584  */
3585 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
3586 {
3587 	int ret;
3588 
3589 	ret = pci_dev_specific_acs_enabled(pdev, acs_flags);
3590 	if (ret >= 0)
3591 		return ret > 0;
3592 
3593 	/*
3594 	 * Conventional PCI and PCI-X devices never support ACS, either
3595 	 * effectively or actually.  The shared bus topology implies that
3596 	 * any device on the bus can receive or snoop DMA.
3597 	 */
3598 	if (!pci_is_pcie(pdev))
3599 		return false;
3600 
3601 	switch (pci_pcie_type(pdev)) {
3602 	/*
3603 	 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
3604 	 * but since their primary interface is PCI/X, we conservatively
3605 	 * handle them as we would a non-PCIe device.
3606 	 */
3607 	case PCI_EXP_TYPE_PCIE_BRIDGE:
3608 	/*
3609 	 * PCIe 3.0, 6.12.1 excludes ACS on these devices.  "ACS is never
3610 	 * applicable... must never implement an ACS Extended Capability...".
3611 	 * This seems arbitrary, but we take a conservative interpretation
3612 	 * of this statement.
3613 	 */
3614 	case PCI_EXP_TYPE_PCI_BRIDGE:
3615 	case PCI_EXP_TYPE_RC_EC:
3616 		return false;
3617 	/*
3618 	 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
3619 	 * implement ACS in order to indicate their peer-to-peer capabilities,
3620 	 * regardless of whether they are single- or multi-function devices.
3621 	 */
3622 	case PCI_EXP_TYPE_DOWNSTREAM:
3623 	case PCI_EXP_TYPE_ROOT_PORT:
3624 		return pci_acs_flags_enabled(pdev, acs_flags);
3625 	/*
3626 	 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
3627 	 * implemented by the remaining PCIe types to indicate peer-to-peer
3628 	 * capabilities, but only when they are part of a multifunction
3629 	 * device.  The footnote for section 6.12 indicates the specific
3630 	 * PCIe types included here.
3631 	 */
3632 	case PCI_EXP_TYPE_ENDPOINT:
3633 	case PCI_EXP_TYPE_UPSTREAM:
3634 	case PCI_EXP_TYPE_LEG_END:
3635 	case PCI_EXP_TYPE_RC_END:
3636 		if (!pdev->multifunction)
3637 			break;
3638 
3639 		return pci_acs_flags_enabled(pdev, acs_flags);
3640 	}
3641 
3642 	/*
3643 	 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
3644 	 * to single function devices with the exception of downstream ports.
3645 	 */
3646 	return true;
3647 }
3648 
3649 /**
3650  * pci_acs_path_enabled - test ACS flags from start to end in a hierarchy
3651  * @start: starting downstream device
3652  * @end: ending upstream device or NULL to search to the root bus
3653  * @acs_flags: required flags
3654  *
3655  * Walk up a device tree from start to end testing PCI ACS support.  If
3656  * any step along the way does not support the required flags, return false.
3657  */
3658 bool pci_acs_path_enabled(struct pci_dev *start,
3659 			  struct pci_dev *end, u16 acs_flags)
3660 {
3661 	struct pci_dev *pdev, *parent = start;
3662 
3663 	do {
3664 		pdev = parent;
3665 
3666 		if (!pci_acs_enabled(pdev, acs_flags))
3667 			return false;
3668 
3669 		if (pci_is_root_bus(pdev->bus))
3670 			return (end == NULL);
3671 
3672 		parent = pdev->bus->self;
3673 	} while (pdev != end);
3674 
3675 	return true;
3676 }
3677 
3678 /**
3679  * pci_acs_init - Initialize ACS if hardware supports it
3680  * @dev: the PCI device
3681  */
3682 void pci_acs_init(struct pci_dev *dev)
3683 {
3684 	dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
3685 
3686 	/*
3687 	 * Attempt to enable ACS regardless of capability because some Root
3688 	 * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have
3689 	 * the standard ACS capability but still support ACS via those
3690 	 * quirks.
3691 	 */
3692 	pci_enable_acs(dev);
3693 }
3694 
3695 /**
3696  * pci_rebar_find_pos - find position of resize ctrl reg for BAR
3697  * @pdev: PCI device
3698  * @bar: BAR to find
3699  *
3700  * Helper to find the position of the ctrl register for a BAR.
3701  * Returns -ENOTSUPP if resizable BARs are not supported at all.
3702  * Returns -ENOENT if no ctrl register for the BAR could be found.
3703  */
3704 static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
3705 {
3706 	unsigned int pos, nbars, i;
3707 	u32 ctrl;
3708 
3709 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
3710 	if (!pos)
3711 		return -ENOTSUPP;
3712 
3713 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3714 	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
3715 
3716 	for (i = 0; i < nbars; i++, pos += 8) {
3717 		int bar_idx;
3718 
3719 		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3720 		bar_idx = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, ctrl);
3721 		if (bar_idx == bar)
3722 			return pos;
3723 	}
3724 
3725 	return -ENOENT;
3726 }
3727 
3728 /**
3729  * pci_rebar_get_possible_sizes - get possible sizes for BAR
3730  * @pdev: PCI device
3731  * @bar: BAR to query
3732  *
3733  * Get the possible sizes of a resizable BAR as bitmask defined in the spec
3734  * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable.
3735  */
3736 u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar)
3737 {
3738 	int pos;
3739 	u32 cap;
3740 
3741 	pos = pci_rebar_find_pos(pdev, bar);
3742 	if (pos < 0)
3743 		return 0;
3744 
3745 	pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap);
3746 	cap = FIELD_GET(PCI_REBAR_CAP_SIZES, cap);
3747 
3748 	/* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */
3749 	if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f &&
3750 	    bar == 0 && cap == 0x700)
3751 		return 0x3f00;
3752 
3753 	return cap;
3754 }
3755 EXPORT_SYMBOL(pci_rebar_get_possible_sizes);
3756 
3757 /**
3758  * pci_rebar_get_current_size - get the current size of a BAR
3759  * @pdev: PCI device
3760  * @bar: BAR to set size to
3761  *
3762  * Read the size of a BAR from the resizable BAR config.
3763  * Returns size if found or negative error code.
3764  */
3765 int pci_rebar_get_current_size(struct pci_dev *pdev, int bar)
3766 {
3767 	int pos;
3768 	u32 ctrl;
3769 
3770 	pos = pci_rebar_find_pos(pdev, bar);
3771 	if (pos < 0)
3772 		return pos;
3773 
3774 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3775 	return FIELD_GET(PCI_REBAR_CTRL_BAR_SIZE, ctrl);
3776 }
3777 
3778 /**
3779  * pci_rebar_set_size - set a new size for a BAR
3780  * @pdev: PCI device
3781  * @bar: BAR to set size to
3782  * @size: new size as defined in the spec (0=1MB, 19=512GB)
3783  *
3784  * Set the new size of a BAR as defined in the spec.
3785  * Returns zero if resizing was successful, error code otherwise.
3786  */
3787 int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size)
3788 {
3789 	int pos;
3790 	u32 ctrl;
3791 
3792 	pos = pci_rebar_find_pos(pdev, bar);
3793 	if (pos < 0)
3794 		return pos;
3795 
3796 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3797 	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
3798 	ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
3799 	pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
3800 	return 0;
3801 }
3802 
3803 /**
3804  * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
3805  * @dev: the PCI device
3806  * @cap_mask: mask of desired AtomicOp sizes, including one or more of:
3807  *	PCI_EXP_DEVCAP2_ATOMIC_COMP32
3808  *	PCI_EXP_DEVCAP2_ATOMIC_COMP64
3809  *	PCI_EXP_DEVCAP2_ATOMIC_COMP128
3810  *
3811  * Return 0 if all upstream bridges support AtomicOp routing, egress
3812  * blocking is disabled on all upstream ports, and the root port supports
3813  * the requested completion capabilities (32-bit, 64-bit and/or 128-bit
3814  * AtomicOp completion), or negative otherwise.
3815  */
3816 int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
3817 {
3818 	struct pci_bus *bus = dev->bus;
3819 	struct pci_dev *bridge;
3820 	u32 cap, ctl2;
3821 
3822 	/*
3823 	 * Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit
3824 	 * in Device Control 2 is reserved in VFs and the PF value applies
3825 	 * to all associated VFs.
3826 	 */
3827 	if (dev->is_virtfn)
3828 		return -EINVAL;
3829 
3830 	if (!pci_is_pcie(dev))
3831 		return -EINVAL;
3832 
3833 	/*
3834 	 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be
3835 	 * AtomicOp requesters.  For now, we only support endpoints as
3836 	 * requesters and root ports as completers.  No endpoints as
3837 	 * completers, and no peer-to-peer.
3838 	 */
3839 
3840 	switch (pci_pcie_type(dev)) {
3841 	case PCI_EXP_TYPE_ENDPOINT:
3842 	case PCI_EXP_TYPE_LEG_END:
3843 	case PCI_EXP_TYPE_RC_END:
3844 		break;
3845 	default:
3846 		return -EINVAL;
3847 	}
3848 
3849 	while (bus->parent) {
3850 		bridge = bus->self;
3851 
3852 		pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
3853 
3854 		switch (pci_pcie_type(bridge)) {
3855 		/* Ensure switch ports support AtomicOp routing */
3856 		case PCI_EXP_TYPE_UPSTREAM:
3857 		case PCI_EXP_TYPE_DOWNSTREAM:
3858 			if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
3859 				return -EINVAL;
3860 			break;
3861 
3862 		/* Ensure root port supports all the sizes we care about */
3863 		case PCI_EXP_TYPE_ROOT_PORT:
3864 			if ((cap & cap_mask) != cap_mask)
3865 				return -EINVAL;
3866 			break;
3867 		}
3868 
3869 		/* Ensure upstream ports don't block AtomicOps on egress */
3870 		if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) {
3871 			pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2,
3872 						   &ctl2);
3873 			if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK)
3874 				return -EINVAL;
3875 		}
3876 
3877 		bus = bus->parent;
3878 	}
3879 
3880 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
3881 				 PCI_EXP_DEVCTL2_ATOMIC_REQ);
3882 	return 0;
3883 }
3884 EXPORT_SYMBOL(pci_enable_atomic_ops_to_root);
3885 
3886 /**
3887  * pci_release_region - Release a PCI bar
3888  * @pdev: PCI device whose resources were previously reserved by
3889  *	  pci_request_region()
3890  * @bar: BAR to release
3891  *
3892  * Releases the PCI I/O and memory resources previously reserved by a
3893  * successful call to pci_request_region().  Call this function only
3894  * after all use of the PCI regions has ceased.
3895  */
3896 void pci_release_region(struct pci_dev *pdev, int bar)
3897 {
3898 	/*
3899 	 * This is done for backwards compatibility, because the old PCI devres
3900 	 * API had a mode in which the function became managed if it had been
3901 	 * enabled with pcim_enable_device() instead of pci_enable_device().
3902 	 */
3903 	if (pci_is_managed(pdev)) {
3904 		pcim_release_region(pdev, bar);
3905 		return;
3906 	}
3907 
3908 	if (pci_resource_len(pdev, bar) == 0)
3909 		return;
3910 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
3911 		release_region(pci_resource_start(pdev, bar),
3912 				pci_resource_len(pdev, bar));
3913 	else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
3914 		release_mem_region(pci_resource_start(pdev, bar),
3915 				pci_resource_len(pdev, bar));
3916 }
3917 EXPORT_SYMBOL(pci_release_region);
3918 
3919 /**
3920  * __pci_request_region - Reserved PCI I/O and memory resource
3921  * @pdev: PCI device whose resources are to be reserved
3922  * @bar: BAR to be reserved
3923  * @res_name: Name to be associated with resource.
3924  * @exclusive: whether the region access is exclusive or not
3925  *
3926  * Returns: 0 on success, negative error code on failure.
3927  *
3928  * Mark the PCI region associated with PCI device @pdev BAR @bar as
3929  * being reserved by owner @res_name.  Do not access any
3930  * address inside the PCI regions unless this call returns
3931  * successfully.
3932  *
3933  * If @exclusive is set, then the region is marked so that userspace
3934  * is explicitly not allowed to map the resource via /dev/mem or
3935  * sysfs MMIO access.
3936  *
3937  * Returns 0 on success, or %EBUSY on error.  A warning
3938  * message is also printed on failure.
3939  */
3940 static int __pci_request_region(struct pci_dev *pdev, int bar,
3941 				const char *res_name, int exclusive)
3942 {
3943 	if (pci_is_managed(pdev)) {
3944 		if (exclusive == IORESOURCE_EXCLUSIVE)
3945 			return pcim_request_region_exclusive(pdev, bar, res_name);
3946 
3947 		return pcim_request_region(pdev, bar, res_name);
3948 	}
3949 
3950 	if (pci_resource_len(pdev, bar) == 0)
3951 		return 0;
3952 
3953 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
3954 		if (!request_region(pci_resource_start(pdev, bar),
3955 			    pci_resource_len(pdev, bar), res_name))
3956 			goto err_out;
3957 	} else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
3958 		if (!__request_mem_region(pci_resource_start(pdev, bar),
3959 					pci_resource_len(pdev, bar), res_name,
3960 					exclusive))
3961 			goto err_out;
3962 	}
3963 
3964 	return 0;
3965 
3966 err_out:
3967 	pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar,
3968 		 &pdev->resource[bar]);
3969 	return -EBUSY;
3970 }
3971 
3972 /**
3973  * pci_request_region - Reserve PCI I/O and memory resource
3974  * @pdev: PCI device whose resources are to be reserved
3975  * @bar: BAR to be reserved
3976  * @res_name: Name to be associated with resource
3977  *
3978  * Returns: 0 on success, negative error code on failure.
3979  *
3980  * Mark the PCI region associated with PCI device @pdev BAR @bar as
3981  * being reserved by owner @res_name.  Do not access any
3982  * address inside the PCI regions unless this call returns
3983  * successfully.
3984  *
3985  * Returns 0 on success, or %EBUSY on error.  A warning
3986  * message is also printed on failure.
3987  *
3988  * NOTE:
3989  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
3990  * when pcim_enable_device() has been called in advance. This hybrid feature is
3991  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
3992  */
3993 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
3994 {
3995 	return __pci_request_region(pdev, bar, res_name, 0);
3996 }
3997 EXPORT_SYMBOL(pci_request_region);
3998 
3999 /**
4000  * pci_release_selected_regions - Release selected PCI I/O and memory resources
4001  * @pdev: PCI device whose resources were previously reserved
4002  * @bars: Bitmask of BARs to be released
4003  *
4004  * Release selected PCI I/O and memory resources previously reserved.
4005  * Call this function only after all use of the PCI regions has ceased.
4006  */
4007 void pci_release_selected_regions(struct pci_dev *pdev, int bars)
4008 {
4009 	int i;
4010 
4011 	for (i = 0; i < PCI_STD_NUM_BARS; i++)
4012 		if (bars & (1 << i))
4013 			pci_release_region(pdev, i);
4014 }
4015 EXPORT_SYMBOL(pci_release_selected_regions);
4016 
4017 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
4018 					  const char *res_name, int excl)
4019 {
4020 	int i;
4021 
4022 	for (i = 0; i < PCI_STD_NUM_BARS; i++)
4023 		if (bars & (1 << i))
4024 			if (__pci_request_region(pdev, i, res_name, excl))
4025 				goto err_out;
4026 	return 0;
4027 
4028 err_out:
4029 	while (--i >= 0)
4030 		if (bars & (1 << i))
4031 			pci_release_region(pdev, i);
4032 
4033 	return -EBUSY;
4034 }
4035 
4036 
4037 /**
4038  * pci_request_selected_regions - Reserve selected PCI I/O and memory resources
4039  * @pdev: PCI device whose resources are to be reserved
4040  * @bars: Bitmask of BARs to be requested
4041  * @res_name: Name to be associated with resource
4042  *
4043  * Returns: 0 on success, negative error code on failure.
4044  *
4045  * NOTE:
4046  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4047  * when pcim_enable_device() has been called in advance. This hybrid feature is
4048  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4049  */
4050 int pci_request_selected_regions(struct pci_dev *pdev, int bars,
4051 				 const char *res_name)
4052 {
4053 	return __pci_request_selected_regions(pdev, bars, res_name, 0);
4054 }
4055 EXPORT_SYMBOL(pci_request_selected_regions);
4056 
4057 /**
4058  * pci_request_selected_regions_exclusive - Request regions exclusively
4059  * @pdev: PCI device to request regions from
4060  * @bars: bit mask of BARs to request
4061  * @res_name: name to be associated with the requests
4062  *
4063  * Returns: 0 on success, negative error code on failure.
4064  *
4065  * NOTE:
4066  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4067  * when pcim_enable_device() has been called in advance. This hybrid feature is
4068  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4069  */
4070 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
4071 					   const char *res_name)
4072 {
4073 	return __pci_request_selected_regions(pdev, bars, res_name,
4074 			IORESOURCE_EXCLUSIVE);
4075 }
4076 EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
4077 
4078 /**
4079  * pci_release_regions - Release reserved PCI I/O and memory resources
4080  * @pdev: PCI device whose resources were previously reserved by
4081  *	  pci_request_regions()
4082  *
4083  * Releases all PCI I/O and memory resources previously reserved by a
4084  * successful call to pci_request_regions().  Call this function only
4085  * after all use of the PCI regions has ceased.
4086  */
4087 void pci_release_regions(struct pci_dev *pdev)
4088 {
4089 	pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1);
4090 }
4091 EXPORT_SYMBOL(pci_release_regions);
4092 
4093 /**
4094  * pci_request_regions - Reserve PCI I/O and memory resources
4095  * @pdev: PCI device whose resources are to be reserved
4096  * @res_name: Name to be associated with resource.
4097  *
4098  * Mark all PCI regions associated with PCI device @pdev as
4099  * being reserved by owner @res_name.  Do not access any
4100  * address inside the PCI regions unless this call returns
4101  * successfully.
4102  *
4103  * Returns 0 on success, or %EBUSY on error.  A warning
4104  * message is also printed on failure.
4105  *
4106  * NOTE:
4107  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4108  * when pcim_enable_device() has been called in advance. This hybrid feature is
4109  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4110  */
4111 int pci_request_regions(struct pci_dev *pdev, const char *res_name)
4112 {
4113 	return pci_request_selected_regions(pdev,
4114 			((1 << PCI_STD_NUM_BARS) - 1), res_name);
4115 }
4116 EXPORT_SYMBOL(pci_request_regions);
4117 
4118 /**
4119  * pci_request_regions_exclusive - Reserve PCI I/O and memory resources
4120  * @pdev: PCI device whose resources are to be reserved
4121  * @res_name: Name to be associated with resource.
4122  *
4123  * Returns: 0 on success, negative error code on failure.
4124  *
4125  * Mark all PCI regions associated with PCI device @pdev as being reserved
4126  * by owner @res_name.  Do not access any address inside the PCI regions
4127  * unless this call returns successfully.
4128  *
4129  * pci_request_regions_exclusive() will mark the region so that /dev/mem
4130  * and the sysfs MMIO access will not be allowed.
4131  *
4132  * Returns 0 on success, or %EBUSY on error.  A warning message is also
4133  * printed on failure.
4134  *
4135  * NOTE:
4136  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4137  * when pcim_enable_device() has been called in advance. This hybrid feature is
4138  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4139  */
4140 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name)
4141 {
4142 	return pci_request_selected_regions_exclusive(pdev,
4143 				((1 << PCI_STD_NUM_BARS) - 1), res_name);
4144 }
4145 EXPORT_SYMBOL(pci_request_regions_exclusive);
4146 
4147 /*
4148  * Record the PCI IO range (expressed as CPU physical address + size).
4149  * Return a negative value if an error has occurred, zero otherwise
4150  */
4151 int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr,
4152 			resource_size_t	size)
4153 {
4154 	int ret = 0;
4155 #ifdef PCI_IOBASE
4156 	struct logic_pio_hwaddr *range;
4157 
4158 	if (!size || addr + size < addr)
4159 		return -EINVAL;
4160 
4161 	range = kzalloc(sizeof(*range), GFP_ATOMIC);
4162 	if (!range)
4163 		return -ENOMEM;
4164 
4165 	range->fwnode = fwnode;
4166 	range->size = size;
4167 	range->hw_start = addr;
4168 	range->flags = LOGIC_PIO_CPU_MMIO;
4169 
4170 	ret = logic_pio_register_range(range);
4171 	if (ret)
4172 		kfree(range);
4173 
4174 	/* Ignore duplicates due to deferred probing */
4175 	if (ret == -EEXIST)
4176 		ret = 0;
4177 #endif
4178 
4179 	return ret;
4180 }
4181 
4182 phys_addr_t pci_pio_to_address(unsigned long pio)
4183 {
4184 #ifdef PCI_IOBASE
4185 	if (pio < MMIO_UPPER_LIMIT)
4186 		return logic_pio_to_hwaddr(pio);
4187 #endif
4188 
4189 	return (phys_addr_t) OF_BAD_ADDR;
4190 }
4191 EXPORT_SYMBOL_GPL(pci_pio_to_address);
4192 
4193 unsigned long __weak pci_address_to_pio(phys_addr_t address)
4194 {
4195 #ifdef PCI_IOBASE
4196 	return logic_pio_trans_cpuaddr(address);
4197 #else
4198 	if (address > IO_SPACE_LIMIT)
4199 		return (unsigned long)-1;
4200 
4201 	return (unsigned long) address;
4202 #endif
4203 }
4204 
4205 /**
4206  * pci_remap_iospace - Remap the memory mapped I/O space
4207  * @res: Resource describing the I/O space
4208  * @phys_addr: physical address of range to be mapped
4209  *
4210  * Remap the memory mapped I/O space described by the @res and the CPU
4211  * physical address @phys_addr into virtual address space.  Only
4212  * architectures that have memory mapped IO functions defined (and the
4213  * PCI_IOBASE value defined) should call this function.
4214  */
4215 #ifndef pci_remap_iospace
4216 int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
4217 {
4218 #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4219 	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4220 
4221 	if (!(res->flags & IORESOURCE_IO))
4222 		return -EINVAL;
4223 
4224 	if (res->end > IO_SPACE_LIMIT)
4225 		return -EINVAL;
4226 
4227 	return vmap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
4228 			       pgprot_device(PAGE_KERNEL));
4229 #else
4230 	/*
4231 	 * This architecture does not have memory mapped I/O space,
4232 	 * so this function should never be called
4233 	 */
4234 	WARN_ONCE(1, "This architecture does not support memory mapped I/O\n");
4235 	return -ENODEV;
4236 #endif
4237 }
4238 EXPORT_SYMBOL(pci_remap_iospace);
4239 #endif
4240 
4241 /**
4242  * pci_unmap_iospace - Unmap the memory mapped I/O space
4243  * @res: resource to be unmapped
4244  *
4245  * Unmap the CPU virtual address @res from virtual address space.  Only
4246  * architectures that have memory mapped IO functions defined (and the
4247  * PCI_IOBASE value defined) should call this function.
4248  */
4249 void pci_unmap_iospace(struct resource *res)
4250 {
4251 #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4252 	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4253 
4254 	vunmap_range(vaddr, vaddr + resource_size(res));
4255 #endif
4256 }
4257 EXPORT_SYMBOL(pci_unmap_iospace);
4258 
4259 static void __pci_set_master(struct pci_dev *dev, bool enable)
4260 {
4261 	u16 old_cmd, cmd;
4262 
4263 	pci_read_config_word(dev, PCI_COMMAND, &old_cmd);
4264 	if (enable)
4265 		cmd = old_cmd | PCI_COMMAND_MASTER;
4266 	else
4267 		cmd = old_cmd & ~PCI_COMMAND_MASTER;
4268 	if (cmd != old_cmd) {
4269 		pci_dbg(dev, "%s bus mastering\n",
4270 			enable ? "enabling" : "disabling");
4271 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4272 	}
4273 	dev->is_busmaster = enable;
4274 }
4275 
4276 /**
4277  * pcibios_setup - process "pci=" kernel boot arguments
4278  * @str: string used to pass in "pci=" kernel boot arguments
4279  *
4280  * Process kernel boot arguments.  This is the default implementation.
4281  * Architecture specific implementations can override this as necessary.
4282  */
4283 char * __weak __init pcibios_setup(char *str)
4284 {
4285 	return str;
4286 }
4287 
4288 /**
4289  * pcibios_set_master - enable PCI bus-mastering for device dev
4290  * @dev: the PCI device to enable
4291  *
4292  * Enables PCI bus-mastering for the device.  This is the default
4293  * implementation.  Architecture specific implementations can override
4294  * this if necessary.
4295  */
4296 void __weak pcibios_set_master(struct pci_dev *dev)
4297 {
4298 	u8 lat;
4299 
4300 	/* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
4301 	if (pci_is_pcie(dev))
4302 		return;
4303 
4304 	pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat);
4305 	if (lat < 16)
4306 		lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
4307 	else if (lat > pcibios_max_latency)
4308 		lat = pcibios_max_latency;
4309 	else
4310 		return;
4311 
4312 	pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat);
4313 }
4314 
4315 /**
4316  * pci_set_master - enables bus-mastering for device dev
4317  * @dev: the PCI device to enable
4318  *
4319  * Enables bus-mastering on the device and calls pcibios_set_master()
4320  * to do the needed arch specific settings.
4321  */
4322 void pci_set_master(struct pci_dev *dev)
4323 {
4324 	__pci_set_master(dev, true);
4325 	pcibios_set_master(dev);
4326 }
4327 EXPORT_SYMBOL(pci_set_master);
4328 
4329 /**
4330  * pci_clear_master - disables bus-mastering for device dev
4331  * @dev: the PCI device to disable
4332  */
4333 void pci_clear_master(struct pci_dev *dev)
4334 {
4335 	__pci_set_master(dev, false);
4336 }
4337 EXPORT_SYMBOL(pci_clear_master);
4338 
4339 /**
4340  * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
4341  * @dev: the PCI device for which MWI is to be enabled
4342  *
4343  * Helper function for pci_set_mwi.
4344  * Originally copied from drivers/net/acenic.c.
4345  * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
4346  *
4347  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4348  */
4349 int pci_set_cacheline_size(struct pci_dev *dev)
4350 {
4351 	u8 cacheline_size;
4352 
4353 	if (!pci_cache_line_size)
4354 		return -EINVAL;
4355 
4356 	/* Validate current setting: the PCI_CACHE_LINE_SIZE must be
4357 	   equal to or multiple of the right value. */
4358 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
4359 	if (cacheline_size >= pci_cache_line_size &&
4360 	    (cacheline_size % pci_cache_line_size) == 0)
4361 		return 0;
4362 
4363 	/* Write the correct value. */
4364 	pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size);
4365 	/* Read it back. */
4366 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
4367 	if (cacheline_size == pci_cache_line_size)
4368 		return 0;
4369 
4370 	pci_dbg(dev, "cache line size of %d is not supported\n",
4371 		   pci_cache_line_size << 2);
4372 
4373 	return -EINVAL;
4374 }
4375 EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
4376 
4377 /**
4378  * pci_set_mwi - enables memory-write-invalidate PCI transaction
4379  * @dev: the PCI device for which MWI is enabled
4380  *
4381  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4382  *
4383  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4384  */
4385 int pci_set_mwi(struct pci_dev *dev)
4386 {
4387 #ifdef PCI_DISABLE_MWI
4388 	return 0;
4389 #else
4390 	int rc;
4391 	u16 cmd;
4392 
4393 	rc = pci_set_cacheline_size(dev);
4394 	if (rc)
4395 		return rc;
4396 
4397 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4398 	if (!(cmd & PCI_COMMAND_INVALIDATE)) {
4399 		pci_dbg(dev, "enabling Mem-Wr-Inval\n");
4400 		cmd |= PCI_COMMAND_INVALIDATE;
4401 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4402 	}
4403 	return 0;
4404 #endif
4405 }
4406 EXPORT_SYMBOL(pci_set_mwi);
4407 
4408 /**
4409  * pci_try_set_mwi - enables memory-write-invalidate PCI transaction
4410  * @dev: the PCI device for which MWI is enabled
4411  *
4412  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4413  * Callers are not required to check the return value.
4414  *
4415  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4416  */
4417 int pci_try_set_mwi(struct pci_dev *dev)
4418 {
4419 #ifdef PCI_DISABLE_MWI
4420 	return 0;
4421 #else
4422 	return pci_set_mwi(dev);
4423 #endif
4424 }
4425 EXPORT_SYMBOL(pci_try_set_mwi);
4426 
4427 /**
4428  * pci_clear_mwi - disables Memory-Write-Invalidate for device dev
4429  * @dev: the PCI device to disable
4430  *
4431  * Disables PCI Memory-Write-Invalidate transaction on the device
4432  */
4433 void pci_clear_mwi(struct pci_dev *dev)
4434 {
4435 #ifndef PCI_DISABLE_MWI
4436 	u16 cmd;
4437 
4438 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4439 	if (cmd & PCI_COMMAND_INVALIDATE) {
4440 		cmd &= ~PCI_COMMAND_INVALIDATE;
4441 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4442 	}
4443 #endif
4444 }
4445 EXPORT_SYMBOL(pci_clear_mwi);
4446 
4447 /**
4448  * pci_disable_parity - disable parity checking for device
4449  * @dev: the PCI device to operate on
4450  *
4451  * Disable parity checking for device @dev
4452  */
4453 void pci_disable_parity(struct pci_dev *dev)
4454 {
4455 	u16 cmd;
4456 
4457 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4458 	if (cmd & PCI_COMMAND_PARITY) {
4459 		cmd &= ~PCI_COMMAND_PARITY;
4460 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4461 	}
4462 }
4463 
4464 /**
4465  * pci_intx - enables/disables PCI INTx for device dev
4466  * @pdev: the PCI device to operate on
4467  * @enable: boolean: whether to enable or disable PCI INTx
4468  *
4469  * Enables/disables PCI INTx for device @pdev
4470  *
4471  * NOTE:
4472  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4473  * when pcim_enable_device() has been called in advance. This hybrid feature is
4474  * DEPRECATED! If you want managed cleanup, use pcim_intx() instead.
4475  */
4476 void pci_intx(struct pci_dev *pdev, int enable)
4477 {
4478 	u16 pci_command, new;
4479 
4480 	/* Preserve the "hybrid" behavior for backwards compatibility */
4481 	if (pci_is_managed(pdev)) {
4482 		WARN_ON_ONCE(pcim_intx(pdev, enable) != 0);
4483 		return;
4484 	}
4485 
4486 	pci_read_config_word(pdev, PCI_COMMAND, &pci_command);
4487 
4488 	if (enable)
4489 		new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
4490 	else
4491 		new = pci_command | PCI_COMMAND_INTX_DISABLE;
4492 
4493 	if (new != pci_command)
4494 		pci_write_config_word(pdev, PCI_COMMAND, new);
4495 }
4496 EXPORT_SYMBOL_GPL(pci_intx);
4497 
4498 /**
4499  * pci_wait_for_pending_transaction - wait for pending transaction
4500  * @dev: the PCI device to operate on
4501  *
4502  * Return 0 if transaction is pending 1 otherwise.
4503  */
4504 int pci_wait_for_pending_transaction(struct pci_dev *dev)
4505 {
4506 	if (!pci_is_pcie(dev))
4507 		return 1;
4508 
4509 	return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
4510 				    PCI_EXP_DEVSTA_TRPND);
4511 }
4512 EXPORT_SYMBOL(pci_wait_for_pending_transaction);
4513 
4514 /**
4515  * pcie_flr - initiate a PCIe function level reset
4516  * @dev: device to reset
4517  *
4518  * Initiate a function level reset unconditionally on @dev without
4519  * checking any flags and DEVCAP
4520  */
4521 int pcie_flr(struct pci_dev *dev)
4522 {
4523 	if (!pci_wait_for_pending_transaction(dev))
4524 		pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
4525 
4526 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
4527 
4528 	if (dev->imm_ready)
4529 		return 0;
4530 
4531 	/*
4532 	 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
4533 	 * 100ms, but may silently discard requests while the FLR is in
4534 	 * progress.  Wait 100ms before trying to access the device.
4535 	 */
4536 	msleep(100);
4537 
4538 	return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);
4539 }
4540 EXPORT_SYMBOL_GPL(pcie_flr);
4541 
4542 /**
4543  * pcie_reset_flr - initiate a PCIe function level reset
4544  * @dev: device to reset
4545  * @probe: if true, return 0 if device can be reset this way
4546  *
4547  * Initiate a function level reset on @dev.
4548  */
4549 int pcie_reset_flr(struct pci_dev *dev, bool probe)
4550 {
4551 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4552 		return -ENOTTY;
4553 
4554 	if (!(dev->devcap & PCI_EXP_DEVCAP_FLR))
4555 		return -ENOTTY;
4556 
4557 	if (probe)
4558 		return 0;
4559 
4560 	return pcie_flr(dev);
4561 }
4562 EXPORT_SYMBOL_GPL(pcie_reset_flr);
4563 
4564 static int pci_af_flr(struct pci_dev *dev, bool probe)
4565 {
4566 	int pos;
4567 	u8 cap;
4568 
4569 	pos = pci_find_capability(dev, PCI_CAP_ID_AF);
4570 	if (!pos)
4571 		return -ENOTTY;
4572 
4573 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4574 		return -ENOTTY;
4575 
4576 	pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap);
4577 	if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
4578 		return -ENOTTY;
4579 
4580 	if (probe)
4581 		return 0;
4582 
4583 	/*
4584 	 * Wait for Transaction Pending bit to clear.  A word-aligned test
4585 	 * is used, so we use the control offset rather than status and shift
4586 	 * the test bit to match.
4587 	 */
4588 	if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL,
4589 				 PCI_AF_STATUS_TP << 8))
4590 		pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
4591 
4592 	pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
4593 
4594 	if (dev->imm_ready)
4595 		return 0;
4596 
4597 	/*
4598 	 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
4599 	 * updated 27 July 2006; a device must complete an FLR within
4600 	 * 100ms, but may silently discard requests while the FLR is in
4601 	 * progress.  Wait 100ms before trying to access the device.
4602 	 */
4603 	msleep(100);
4604 
4605 	return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);
4606 }
4607 
4608 /**
4609  * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
4610  * @dev: Device to reset.
4611  * @probe: if true, return 0 if the device can be reset this way.
4612  *
4613  * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
4614  * unset, it will be reinitialized internally when going from PCI_D3hot to
4615  * PCI_D0.  If that's the case and the device is not in a low-power state
4616  * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
4617  *
4618  * NOTE: This causes the caller to sleep for twice the device power transition
4619  * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
4620  * by default (i.e. unless the @dev's d3hot_delay field has a different value).
4621  * Moreover, only devices in D0 can be reset by this function.
4622  */
4623 static int pci_pm_reset(struct pci_dev *dev, bool probe)
4624 {
4625 	u16 csr;
4626 
4627 	if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
4628 		return -ENOTTY;
4629 
4630 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr);
4631 	if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
4632 		return -ENOTTY;
4633 
4634 	if (probe)
4635 		return 0;
4636 
4637 	if (dev->current_state != PCI_D0)
4638 		return -EINVAL;
4639 
4640 	csr &= ~PCI_PM_CTRL_STATE_MASK;
4641 	csr |= PCI_D3hot;
4642 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4643 	pci_dev_d3_sleep(dev);
4644 
4645 	csr &= ~PCI_PM_CTRL_STATE_MASK;
4646 	csr |= PCI_D0;
4647 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4648 	pci_dev_d3_sleep(dev);
4649 
4650 	return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
4651 }
4652 
4653 /**
4654  * pcie_wait_for_link_status - Wait for link status change
4655  * @pdev: Device whose link to wait for.
4656  * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE.
4657  * @active: Waiting for active or inactive?
4658  *
4659  * Return 0 if successful, or -ETIMEDOUT if status has not changed within
4660  * PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4661  */
4662 static int pcie_wait_for_link_status(struct pci_dev *pdev,
4663 				     bool use_lt, bool active)
4664 {
4665 	u16 lnksta_mask, lnksta_match;
4666 	unsigned long end_jiffies;
4667 	u16 lnksta;
4668 
4669 	lnksta_mask = use_lt ? PCI_EXP_LNKSTA_LT : PCI_EXP_LNKSTA_DLLLA;
4670 	lnksta_match = active ? lnksta_mask : 0;
4671 
4672 	end_jiffies = jiffies + msecs_to_jiffies(PCIE_LINK_RETRAIN_TIMEOUT_MS);
4673 	do {
4674 		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
4675 		if ((lnksta & lnksta_mask) == lnksta_match)
4676 			return 0;
4677 		msleep(1);
4678 	} while (time_before(jiffies, end_jiffies));
4679 
4680 	return -ETIMEDOUT;
4681 }
4682 
4683 /**
4684  * pcie_retrain_link - Request a link retrain and wait for it to complete
4685  * @pdev: Device whose link to retrain.
4686  * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE, for status.
4687  *
4688  * Retrain completion status is retrieved from the Link Status Register
4689  * according to @use_lt.  It is not verified whether the use of the DLLLA
4690  * bit is valid.
4691  *
4692  * Return 0 if successful, or -ETIMEDOUT if training has not completed
4693  * within PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4694  */
4695 int pcie_retrain_link(struct pci_dev *pdev, bool use_lt)
4696 {
4697 	int rc;
4698 
4699 	/*
4700 	 * Ensure the updated LNKCTL parameters are used during link
4701 	 * training by checking that there is no ongoing link training that
4702 	 * may have started before link parameters were changed, so as to
4703 	 * avoid LTSSM race as recommended in Implementation Note at the end
4704 	 * of PCIe r6.1 sec 7.5.3.7.
4705 	 */
4706 	rc = pcie_wait_for_link_status(pdev, true, false);
4707 	if (rc)
4708 		return rc;
4709 
4710 	pcie_capability_set_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4711 	if (pdev->clear_retrain_link) {
4712 		/*
4713 		 * Due to an erratum in some devices the Retrain Link bit
4714 		 * needs to be cleared again manually to allow the link
4715 		 * training to succeed.
4716 		 */
4717 		pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4718 	}
4719 
4720 	return pcie_wait_for_link_status(pdev, use_lt, !use_lt);
4721 }
4722 
4723 /**
4724  * pcie_wait_for_link_delay - Wait until link is active or inactive
4725  * @pdev: Bridge device
4726  * @active: waiting for active or inactive?
4727  * @delay: Delay to wait after link has become active (in ms)
4728  *
4729  * Use this to wait till link becomes active or inactive.
4730  */
4731 static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active,
4732 				     int delay)
4733 {
4734 	int rc;
4735 
4736 	/*
4737 	 * Some controllers might not implement link active reporting. In this
4738 	 * case, we wait for 1000 ms + any delay requested by the caller.
4739 	 */
4740 	if (!pdev->link_active_reporting) {
4741 		msleep(PCIE_LINK_RETRAIN_TIMEOUT_MS + delay);
4742 		return true;
4743 	}
4744 
4745 	/*
4746 	 * PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms,
4747 	 * after which we should expect an link active if the reset was
4748 	 * successful. If so, software must wait a minimum 100ms before sending
4749 	 * configuration requests to devices downstream this port.
4750 	 *
4751 	 * If the link fails to activate, either the device was physically
4752 	 * removed or the link is permanently failed.
4753 	 */
4754 	if (active)
4755 		msleep(20);
4756 	rc = pcie_wait_for_link_status(pdev, false, active);
4757 	if (active) {
4758 		if (rc)
4759 			rc = pcie_failed_link_retrain(pdev);
4760 		if (rc)
4761 			return false;
4762 
4763 		msleep(delay);
4764 		return true;
4765 	}
4766 
4767 	if (rc)
4768 		return false;
4769 
4770 	return true;
4771 }
4772 
4773 /**
4774  * pcie_wait_for_link - Wait until link is active or inactive
4775  * @pdev: Bridge device
4776  * @active: waiting for active or inactive?
4777  *
4778  * Use this to wait till link becomes active or inactive.
4779  */
4780 bool pcie_wait_for_link(struct pci_dev *pdev, bool active)
4781 {
4782 	return pcie_wait_for_link_delay(pdev, active, 100);
4783 }
4784 
4785 /*
4786  * Find maximum D3cold delay required by all the devices on the bus.  The
4787  * spec says 100 ms, but firmware can lower it and we allow drivers to
4788  * increase it as well.
4789  *
4790  * Called with @pci_bus_sem locked for reading.
4791  */
4792 static int pci_bus_max_d3cold_delay(const struct pci_bus *bus)
4793 {
4794 	const struct pci_dev *pdev;
4795 	int min_delay = 100;
4796 	int max_delay = 0;
4797 
4798 	list_for_each_entry(pdev, &bus->devices, bus_list) {
4799 		if (pdev->d3cold_delay < min_delay)
4800 			min_delay = pdev->d3cold_delay;
4801 		if (pdev->d3cold_delay > max_delay)
4802 			max_delay = pdev->d3cold_delay;
4803 	}
4804 
4805 	return max(min_delay, max_delay);
4806 }
4807 
4808 /**
4809  * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible
4810  * @dev: PCI bridge
4811  * @reset_type: reset type in human-readable form
4812  *
4813  * Handle necessary delays before access to the devices on the secondary
4814  * side of the bridge are permitted after D3cold to D0 transition
4815  * or Conventional Reset.
4816  *
4817  * For PCIe this means the delays in PCIe 5.0 section 6.6.1. For
4818  * conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section
4819  * 4.3.2.
4820  *
4821  * Return 0 on success or -ENOTTY if the first device on the secondary bus
4822  * failed to become accessible.
4823  */
4824 int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type)
4825 {
4826 	struct pci_dev *child __free(pci_dev_put) = NULL;
4827 	int delay;
4828 
4829 	if (pci_dev_is_disconnected(dev))
4830 		return 0;
4831 
4832 	if (!pci_is_bridge(dev))
4833 		return 0;
4834 
4835 	down_read(&pci_bus_sem);
4836 
4837 	/*
4838 	 * We only deal with devices that are present currently on the bus.
4839 	 * For any hot-added devices the access delay is handled in pciehp
4840 	 * board_added(). In case of ACPI hotplug the firmware is expected
4841 	 * to configure the devices before OS is notified.
4842 	 */
4843 	if (!dev->subordinate || list_empty(&dev->subordinate->devices)) {
4844 		up_read(&pci_bus_sem);
4845 		return 0;
4846 	}
4847 
4848 	/* Take d3cold_delay requirements into account */
4849 	delay = pci_bus_max_d3cold_delay(dev->subordinate);
4850 	if (!delay) {
4851 		up_read(&pci_bus_sem);
4852 		return 0;
4853 	}
4854 
4855 	child = pci_dev_get(list_first_entry(&dev->subordinate->devices,
4856 					     struct pci_dev, bus_list));
4857 	up_read(&pci_bus_sem);
4858 
4859 	/*
4860 	 * Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before
4861 	 * accessing the device after reset (that is 1000 ms + 100 ms).
4862 	 */
4863 	if (!pci_is_pcie(dev)) {
4864 		pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay);
4865 		msleep(1000 + delay);
4866 		return 0;
4867 	}
4868 
4869 	/*
4870 	 * For PCIe downstream and root ports that do not support speeds
4871 	 * greater than 5 GT/s need to wait minimum 100 ms. For higher
4872 	 * speeds (gen3) we need to wait first for the data link layer to
4873 	 * become active.
4874 	 *
4875 	 * However, 100 ms is the minimum and the PCIe spec says the
4876 	 * software must allow at least 1s before it can determine that the
4877 	 * device that did not respond is a broken device. Also device can
4878 	 * take longer than that to respond if it indicates so through Request
4879 	 * Retry Status completions.
4880 	 *
4881 	 * Therefore we wait for 100 ms and check for the device presence
4882 	 * until the timeout expires.
4883 	 */
4884 	if (!pcie_downstream_port(dev))
4885 		return 0;
4886 
4887 	if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) {
4888 		u16 status;
4889 
4890 		pci_dbg(dev, "waiting %d ms for downstream link\n", delay);
4891 		msleep(delay);
4892 
4893 		if (!pci_dev_wait(child, reset_type, PCI_RESET_WAIT - delay))
4894 			return 0;
4895 
4896 		/*
4897 		 * If the port supports active link reporting we now check
4898 		 * whether the link is active and if not bail out early with
4899 		 * the assumption that the device is not present anymore.
4900 		 */
4901 		if (!dev->link_active_reporting)
4902 			return -ENOTTY;
4903 
4904 		pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &status);
4905 		if (!(status & PCI_EXP_LNKSTA_DLLLA))
4906 			return -ENOTTY;
4907 
4908 		return pci_dev_wait(child, reset_type,
4909 				    PCIE_RESET_READY_POLL_MS - PCI_RESET_WAIT);
4910 	}
4911 
4912 	pci_dbg(dev, "waiting %d ms for downstream link, after activation\n",
4913 		delay);
4914 	if (!pcie_wait_for_link_delay(dev, true, delay)) {
4915 		/* Did not train, no need to wait any further */
4916 		pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n");
4917 		return -ENOTTY;
4918 	}
4919 
4920 	return pci_dev_wait(child, reset_type,
4921 			    PCIE_RESET_READY_POLL_MS - delay);
4922 }
4923 
4924 void pci_reset_secondary_bus(struct pci_dev *dev)
4925 {
4926 	u16 ctrl;
4927 
4928 	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
4929 	ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
4930 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
4931 
4932 	/*
4933 	 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms.  Double
4934 	 * this to 2ms to ensure that we meet the minimum requirement.
4935 	 */
4936 	msleep(2);
4937 
4938 	ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
4939 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
4940 }
4941 
4942 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
4943 {
4944 	pci_reset_secondary_bus(dev);
4945 }
4946 
4947 /**
4948  * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge.
4949  * @dev: Bridge device
4950  *
4951  * Use the bridge control register to assert reset on the secondary bus.
4952  * Devices on the secondary bus are left in power-on state.
4953  */
4954 int pci_bridge_secondary_bus_reset(struct pci_dev *dev)
4955 {
4956 	if (!dev->block_cfg_access)
4957 		pci_warn_once(dev, "unlocked secondary bus reset via: %pS\n",
4958 			      __builtin_return_address(0));
4959 	pcibios_reset_secondary_bus(dev);
4960 
4961 	return pci_bridge_wait_for_secondary_bus(dev, "bus reset");
4962 }
4963 EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset);
4964 
4965 static int pci_parent_bus_reset(struct pci_dev *dev, bool probe)
4966 {
4967 	struct pci_dev *pdev;
4968 
4969 	if (pci_is_root_bus(dev->bus) || dev->subordinate ||
4970 	    !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4971 		return -ENOTTY;
4972 
4973 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
4974 		if (pdev != dev)
4975 			return -ENOTTY;
4976 
4977 	if (probe)
4978 		return 0;
4979 
4980 	return pci_bridge_secondary_bus_reset(dev->bus->self);
4981 }
4982 
4983 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe)
4984 {
4985 	int rc = -ENOTTY;
4986 
4987 	if (!hotplug || !try_module_get(hotplug->owner))
4988 		return rc;
4989 
4990 	if (hotplug->ops->reset_slot)
4991 		rc = hotplug->ops->reset_slot(hotplug, probe);
4992 
4993 	module_put(hotplug->owner);
4994 
4995 	return rc;
4996 }
4997 
4998 static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe)
4999 {
5000 	if (dev->multifunction || dev->subordinate || !dev->slot ||
5001 	    dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
5002 		return -ENOTTY;
5003 
5004 	return pci_reset_hotplug_slot(dev->slot->hotplug, probe);
5005 }
5006 
5007 static u16 cxl_port_dvsec(struct pci_dev *dev)
5008 {
5009 	return pci_find_dvsec_capability(dev, PCI_VENDOR_ID_CXL,
5010 					 PCI_DVSEC_CXL_PORT);
5011 }
5012 
5013 static bool cxl_sbr_masked(struct pci_dev *dev)
5014 {
5015 	u16 dvsec, reg;
5016 	int rc;
5017 
5018 	dvsec = cxl_port_dvsec(dev);
5019 	if (!dvsec)
5020 		return false;
5021 
5022 	rc = pci_read_config_word(dev, dvsec + PCI_DVSEC_CXL_PORT_CTL, &reg);
5023 	if (rc || PCI_POSSIBLE_ERROR(reg))
5024 		return false;
5025 
5026 	/*
5027 	 * Per CXL spec r3.1, sec 8.1.5.2, when "Unmask SBR" is 0, the SBR
5028 	 * bit in Bridge Control has no effect.  When 1, the Port generates
5029 	 * hot reset when the SBR bit is set to 1.
5030 	 */
5031 	if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR)
5032 		return false;
5033 
5034 	return true;
5035 }
5036 
5037 static int pci_reset_bus_function(struct pci_dev *dev, bool probe)
5038 {
5039 	struct pci_dev *bridge = pci_upstream_bridge(dev);
5040 	int rc;
5041 
5042 	/*
5043 	 * If "dev" is below a CXL port that has SBR control masked, SBR
5044 	 * won't do anything, so return error.
5045 	 */
5046 	if (bridge && cxl_sbr_masked(bridge)) {
5047 		if (probe)
5048 			return 0;
5049 
5050 		return -ENOTTY;
5051 	}
5052 
5053 	rc = pci_dev_reset_slot_function(dev, probe);
5054 	if (rc != -ENOTTY)
5055 		return rc;
5056 	return pci_parent_bus_reset(dev, probe);
5057 }
5058 
5059 static int cxl_reset_bus_function(struct pci_dev *dev, bool probe)
5060 {
5061 	struct pci_dev *bridge;
5062 	u16 dvsec, reg, val;
5063 	int rc;
5064 
5065 	bridge = pci_upstream_bridge(dev);
5066 	if (!bridge)
5067 		return -ENOTTY;
5068 
5069 	dvsec = cxl_port_dvsec(bridge);
5070 	if (!dvsec)
5071 		return -ENOTTY;
5072 
5073 	if (probe)
5074 		return 0;
5075 
5076 	rc = pci_read_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, &reg);
5077 	if (rc)
5078 		return -ENOTTY;
5079 
5080 	if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) {
5081 		val = reg;
5082 	} else {
5083 		val = reg | PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR;
5084 		pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL,
5085 				      val);
5086 	}
5087 
5088 	rc = pci_reset_bus_function(dev, probe);
5089 
5090 	if (reg != val)
5091 		pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL,
5092 				      reg);
5093 
5094 	return rc;
5095 }
5096 
5097 void pci_dev_lock(struct pci_dev *dev)
5098 {
5099 	/* block PM suspend, driver probe, etc. */
5100 	device_lock(&dev->dev);
5101 	pci_cfg_access_lock(dev);
5102 }
5103 EXPORT_SYMBOL_GPL(pci_dev_lock);
5104 
5105 /* Return 1 on successful lock, 0 on contention */
5106 int pci_dev_trylock(struct pci_dev *dev)
5107 {
5108 	if (device_trylock(&dev->dev)) {
5109 		if (pci_cfg_access_trylock(dev))
5110 			return 1;
5111 		device_unlock(&dev->dev);
5112 	}
5113 
5114 	return 0;
5115 }
5116 EXPORT_SYMBOL_GPL(pci_dev_trylock);
5117 
5118 void pci_dev_unlock(struct pci_dev *dev)
5119 {
5120 	pci_cfg_access_unlock(dev);
5121 	device_unlock(&dev->dev);
5122 }
5123 EXPORT_SYMBOL_GPL(pci_dev_unlock);
5124 
5125 static void pci_dev_save_and_disable(struct pci_dev *dev)
5126 {
5127 	const struct pci_error_handlers *err_handler =
5128 			dev->driver ? dev->driver->err_handler : NULL;
5129 
5130 	/*
5131 	 * dev->driver->err_handler->reset_prepare() is protected against
5132 	 * races with ->remove() by the device lock, which must be held by
5133 	 * the caller.
5134 	 */
5135 	if (err_handler && err_handler->reset_prepare)
5136 		err_handler->reset_prepare(dev);
5137 
5138 	/*
5139 	 * Wake-up device prior to save.  PM registers default to D0 after
5140 	 * reset and a simple register restore doesn't reliably return
5141 	 * to a non-D0 state anyway.
5142 	 */
5143 	pci_set_power_state(dev, PCI_D0);
5144 
5145 	pci_save_state(dev);
5146 	/*
5147 	 * Disable the device by clearing the Command register, except for
5148 	 * INTx-disable which is set.  This not only disables MMIO and I/O port
5149 	 * BARs, but also prevents the device from being Bus Master, preventing
5150 	 * DMA from the device including MSI/MSI-X interrupts.  For PCI 2.3
5151 	 * compliant devices, INTx-disable prevents legacy interrupts.
5152 	 */
5153 	pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
5154 }
5155 
5156 static void pci_dev_restore(struct pci_dev *dev)
5157 {
5158 	const struct pci_error_handlers *err_handler =
5159 			dev->driver ? dev->driver->err_handler : NULL;
5160 
5161 	pci_restore_state(dev);
5162 
5163 	/*
5164 	 * dev->driver->err_handler->reset_done() is protected against
5165 	 * races with ->remove() by the device lock, which must be held by
5166 	 * the caller.
5167 	 */
5168 	if (err_handler && err_handler->reset_done)
5169 		err_handler->reset_done(dev);
5170 }
5171 
5172 /* dev->reset_methods[] is a 0-terminated list of indices into this array */
5173 static const struct pci_reset_fn_method pci_reset_fn_methods[] = {
5174 	{ },
5175 	{ pci_dev_specific_reset, .name = "device_specific" },
5176 	{ pci_dev_acpi_reset, .name = "acpi" },
5177 	{ pcie_reset_flr, .name = "flr" },
5178 	{ pci_af_flr, .name = "af_flr" },
5179 	{ pci_pm_reset, .name = "pm" },
5180 	{ pci_reset_bus_function, .name = "bus" },
5181 	{ cxl_reset_bus_function, .name = "cxl_bus" },
5182 };
5183 
5184 static ssize_t reset_method_show(struct device *dev,
5185 				 struct device_attribute *attr, char *buf)
5186 {
5187 	struct pci_dev *pdev = to_pci_dev(dev);
5188 	ssize_t len = 0;
5189 	int i, m;
5190 
5191 	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5192 		m = pdev->reset_methods[i];
5193 		if (!m)
5194 			break;
5195 
5196 		len += sysfs_emit_at(buf, len, "%s%s", len ? " " : "",
5197 				     pci_reset_fn_methods[m].name);
5198 	}
5199 
5200 	if (len)
5201 		len += sysfs_emit_at(buf, len, "\n");
5202 
5203 	return len;
5204 }
5205 
5206 static int reset_method_lookup(const char *name)
5207 {
5208 	int m;
5209 
5210 	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5211 		if (sysfs_streq(name, pci_reset_fn_methods[m].name))
5212 			return m;
5213 	}
5214 
5215 	return 0;	/* not found */
5216 }
5217 
5218 static ssize_t reset_method_store(struct device *dev,
5219 				  struct device_attribute *attr,
5220 				  const char *buf, size_t count)
5221 {
5222 	struct pci_dev *pdev = to_pci_dev(dev);
5223 	char *options, *name;
5224 	int m, n;
5225 	u8 reset_methods[PCI_NUM_RESET_METHODS] = { 0 };
5226 
5227 	if (sysfs_streq(buf, "")) {
5228 		pdev->reset_methods[0] = 0;
5229 		pci_warn(pdev, "All device reset methods disabled by user");
5230 		return count;
5231 	}
5232 
5233 	if (sysfs_streq(buf, "default")) {
5234 		pci_init_reset_methods(pdev);
5235 		return count;
5236 	}
5237 
5238 	options = kstrndup(buf, count, GFP_KERNEL);
5239 	if (!options)
5240 		return -ENOMEM;
5241 
5242 	n = 0;
5243 	while ((name = strsep(&options, " ")) != NULL) {
5244 		if (sysfs_streq(name, ""))
5245 			continue;
5246 
5247 		name = strim(name);
5248 
5249 		m = reset_method_lookup(name);
5250 		if (!m) {
5251 			pci_err(pdev, "Invalid reset method '%s'", name);
5252 			goto error;
5253 		}
5254 
5255 		if (pci_reset_fn_methods[m].reset_fn(pdev, PCI_RESET_PROBE)) {
5256 			pci_err(pdev, "Unsupported reset method '%s'", name);
5257 			goto error;
5258 		}
5259 
5260 		if (n == PCI_NUM_RESET_METHODS - 1) {
5261 			pci_err(pdev, "Too many reset methods\n");
5262 			goto error;
5263 		}
5264 
5265 		reset_methods[n++] = m;
5266 	}
5267 
5268 	reset_methods[n] = 0;
5269 
5270 	/* Warn if dev-specific supported but not highest priority */
5271 	if (pci_reset_fn_methods[1].reset_fn(pdev, PCI_RESET_PROBE) == 0 &&
5272 	    reset_methods[0] != 1)
5273 		pci_warn(pdev, "Device-specific reset disabled/de-prioritized by user");
5274 	memcpy(pdev->reset_methods, reset_methods, sizeof(pdev->reset_methods));
5275 	kfree(options);
5276 	return count;
5277 
5278 error:
5279 	/* Leave previous methods unchanged */
5280 	kfree(options);
5281 	return -EINVAL;
5282 }
5283 static DEVICE_ATTR_RW(reset_method);
5284 
5285 static struct attribute *pci_dev_reset_method_attrs[] = {
5286 	&dev_attr_reset_method.attr,
5287 	NULL,
5288 };
5289 
5290 static umode_t pci_dev_reset_method_attr_is_visible(struct kobject *kobj,
5291 						    struct attribute *a, int n)
5292 {
5293 	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
5294 
5295 	if (!pci_reset_supported(pdev))
5296 		return 0;
5297 
5298 	return a->mode;
5299 }
5300 
5301 const struct attribute_group pci_dev_reset_method_attr_group = {
5302 	.attrs = pci_dev_reset_method_attrs,
5303 	.is_visible = pci_dev_reset_method_attr_is_visible,
5304 };
5305 
5306 /**
5307  * __pci_reset_function_locked - reset a PCI device function while holding
5308  * the @dev mutex lock.
5309  * @dev: PCI device to reset
5310  *
5311  * Some devices allow an individual function to be reset without affecting
5312  * other functions in the same device.  The PCI device must be responsive
5313  * to PCI config space in order to use this function.
5314  *
5315  * The device function is presumed to be unused and the caller is holding
5316  * the device mutex lock when this function is called.
5317  *
5318  * Resetting the device will make the contents of PCI configuration space
5319  * random, so any caller of this must be prepared to reinitialise the
5320  * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
5321  * etc.
5322  *
5323  * Returns 0 if the device function was successfully reset or negative if the
5324  * device doesn't support resetting a single function.
5325  */
5326 int __pci_reset_function_locked(struct pci_dev *dev)
5327 {
5328 	int i, m, rc;
5329 
5330 	might_sleep();
5331 
5332 	/*
5333 	 * A reset method returns -ENOTTY if it doesn't support this device and
5334 	 * we should try the next method.
5335 	 *
5336 	 * If it returns 0 (success), we're finished.  If it returns any other
5337 	 * error, we're also finished: this indicates that further reset
5338 	 * mechanisms might be broken on the device.
5339 	 */
5340 	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5341 		m = dev->reset_methods[i];
5342 		if (!m)
5343 			return -ENOTTY;
5344 
5345 		rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_DO_RESET);
5346 		if (!rc)
5347 			return 0;
5348 		if (rc != -ENOTTY)
5349 			return rc;
5350 	}
5351 
5352 	return -ENOTTY;
5353 }
5354 EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
5355 
5356 /**
5357  * pci_init_reset_methods - check whether device can be safely reset
5358  * and store supported reset mechanisms.
5359  * @dev: PCI device to check for reset mechanisms
5360  *
5361  * Some devices allow an individual function to be reset without affecting
5362  * other functions in the same device.  The PCI device must be in D0-D3hot
5363  * state.
5364  *
5365  * Stores reset mechanisms supported by device in reset_methods byte array
5366  * which is a member of struct pci_dev.
5367  */
5368 void pci_init_reset_methods(struct pci_dev *dev)
5369 {
5370 	int m, i, rc;
5371 
5372 	BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS);
5373 
5374 	might_sleep();
5375 
5376 	i = 0;
5377 	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5378 		rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE);
5379 		if (!rc)
5380 			dev->reset_methods[i++] = m;
5381 		else if (rc != -ENOTTY)
5382 			break;
5383 	}
5384 
5385 	dev->reset_methods[i] = 0;
5386 }
5387 
5388 /**
5389  * pci_reset_function - quiesce and reset a PCI device function
5390  * @dev: PCI device to reset
5391  *
5392  * Some devices allow an individual function to be reset without affecting
5393  * other functions in the same device.  The PCI device must be responsive
5394  * to PCI config space in order to use this function.
5395  *
5396  * This function does not just reset the PCI portion of a device, but
5397  * clears all the state associated with the device.  This function differs
5398  * from __pci_reset_function_locked() in that it saves and restores device state
5399  * over the reset and takes the PCI device lock.
5400  *
5401  * Returns 0 if the device function was successfully reset or negative if the
5402  * device doesn't support resetting a single function.
5403  */
5404 int pci_reset_function(struct pci_dev *dev)
5405 {
5406 	struct pci_dev *bridge;
5407 	int rc;
5408 
5409 	if (!pci_reset_supported(dev))
5410 		return -ENOTTY;
5411 
5412 	/*
5413 	 * If there's no upstream bridge, no locking is needed since there is
5414 	 * no upstream bridge configuration to hold consistent.
5415 	 */
5416 	bridge = pci_upstream_bridge(dev);
5417 	if (bridge)
5418 		pci_dev_lock(bridge);
5419 
5420 	pci_dev_lock(dev);
5421 	pci_dev_save_and_disable(dev);
5422 
5423 	rc = __pci_reset_function_locked(dev);
5424 
5425 	pci_dev_restore(dev);
5426 	pci_dev_unlock(dev);
5427 
5428 	if (bridge)
5429 		pci_dev_unlock(bridge);
5430 
5431 	return rc;
5432 }
5433 EXPORT_SYMBOL_GPL(pci_reset_function);
5434 
5435 /**
5436  * pci_reset_function_locked - quiesce and reset a PCI device function
5437  * @dev: PCI device to reset
5438  *
5439  * Some devices allow an individual function to be reset without affecting
5440  * other functions in the same device.  The PCI device must be responsive
5441  * to PCI config space in order to use this function.
5442  *
5443  * This function does not just reset the PCI portion of a device, but
5444  * clears all the state associated with the device.  This function differs
5445  * from __pci_reset_function_locked() in that it saves and restores device state
5446  * over the reset.  It also differs from pci_reset_function() in that it
5447  * requires the PCI device lock to be held.
5448  *
5449  * Returns 0 if the device function was successfully reset or negative if the
5450  * device doesn't support resetting a single function.
5451  */
5452 int pci_reset_function_locked(struct pci_dev *dev)
5453 {
5454 	int rc;
5455 
5456 	if (!pci_reset_supported(dev))
5457 		return -ENOTTY;
5458 
5459 	pci_dev_save_and_disable(dev);
5460 
5461 	rc = __pci_reset_function_locked(dev);
5462 
5463 	pci_dev_restore(dev);
5464 
5465 	return rc;
5466 }
5467 EXPORT_SYMBOL_GPL(pci_reset_function_locked);
5468 
5469 /**
5470  * pci_try_reset_function - quiesce and reset a PCI device function
5471  * @dev: PCI device to reset
5472  *
5473  * Same as above, except return -EAGAIN if unable to lock device.
5474  */
5475 int pci_try_reset_function(struct pci_dev *dev)
5476 {
5477 	int rc;
5478 
5479 	if (!pci_reset_supported(dev))
5480 		return -ENOTTY;
5481 
5482 	if (!pci_dev_trylock(dev))
5483 		return -EAGAIN;
5484 
5485 	pci_dev_save_and_disable(dev);
5486 	rc = __pci_reset_function_locked(dev);
5487 	pci_dev_restore(dev);
5488 	pci_dev_unlock(dev);
5489 
5490 	return rc;
5491 }
5492 EXPORT_SYMBOL_GPL(pci_try_reset_function);
5493 
5494 /* Do any devices on or below this bus prevent a bus reset? */
5495 static bool pci_bus_resettable(struct pci_bus *bus)
5496 {
5497 	struct pci_dev *dev;
5498 
5499 
5500 	if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5501 		return false;
5502 
5503 	list_for_each_entry(dev, &bus->devices, bus_list) {
5504 		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5505 		    (dev->subordinate && !pci_bus_resettable(dev->subordinate)))
5506 			return false;
5507 	}
5508 
5509 	return true;
5510 }
5511 
5512 /* Lock devices from the top of the tree down */
5513 static void pci_bus_lock(struct pci_bus *bus)
5514 {
5515 	struct pci_dev *dev;
5516 
5517 	pci_dev_lock(bus->self);
5518 	list_for_each_entry(dev, &bus->devices, bus_list) {
5519 		if (dev->subordinate)
5520 			pci_bus_lock(dev->subordinate);
5521 		else
5522 			pci_dev_lock(dev);
5523 	}
5524 }
5525 
5526 /* Unlock devices from the bottom of the tree up */
5527 static void pci_bus_unlock(struct pci_bus *bus)
5528 {
5529 	struct pci_dev *dev;
5530 
5531 	list_for_each_entry(dev, &bus->devices, bus_list) {
5532 		if (dev->subordinate)
5533 			pci_bus_unlock(dev->subordinate);
5534 		else
5535 			pci_dev_unlock(dev);
5536 	}
5537 	pci_dev_unlock(bus->self);
5538 }
5539 
5540 /* Return 1 on successful lock, 0 on contention */
5541 static int pci_bus_trylock(struct pci_bus *bus)
5542 {
5543 	struct pci_dev *dev;
5544 
5545 	if (!pci_dev_trylock(bus->self))
5546 		return 0;
5547 
5548 	list_for_each_entry(dev, &bus->devices, bus_list) {
5549 		if (dev->subordinate) {
5550 			if (!pci_bus_trylock(dev->subordinate))
5551 				goto unlock;
5552 		} else if (!pci_dev_trylock(dev))
5553 			goto unlock;
5554 	}
5555 	return 1;
5556 
5557 unlock:
5558 	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
5559 		if (dev->subordinate)
5560 			pci_bus_unlock(dev->subordinate);
5561 		else
5562 			pci_dev_unlock(dev);
5563 	}
5564 	pci_dev_unlock(bus->self);
5565 	return 0;
5566 }
5567 
5568 /* Do any devices on or below this slot prevent a bus reset? */
5569 static bool pci_slot_resettable(struct pci_slot *slot)
5570 {
5571 	struct pci_dev *dev;
5572 
5573 	if (slot->bus->self &&
5574 	    (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5575 		return false;
5576 
5577 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5578 		if (!dev->slot || dev->slot != slot)
5579 			continue;
5580 		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5581 		    (dev->subordinate && !pci_bus_resettable(dev->subordinate)))
5582 			return false;
5583 	}
5584 
5585 	return true;
5586 }
5587 
5588 /* Lock devices from the top of the tree down */
5589 static void pci_slot_lock(struct pci_slot *slot)
5590 {
5591 	struct pci_dev *dev;
5592 
5593 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5594 		if (!dev->slot || dev->slot != slot)
5595 			continue;
5596 		if (dev->subordinate)
5597 			pci_bus_lock(dev->subordinate);
5598 		else
5599 			pci_dev_lock(dev);
5600 	}
5601 }
5602 
5603 /* Unlock devices from the bottom of the tree up */
5604 static void pci_slot_unlock(struct pci_slot *slot)
5605 {
5606 	struct pci_dev *dev;
5607 
5608 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5609 		if (!dev->slot || dev->slot != slot)
5610 			continue;
5611 		if (dev->subordinate)
5612 			pci_bus_unlock(dev->subordinate);
5613 		pci_dev_unlock(dev);
5614 	}
5615 }
5616 
5617 /* Return 1 on successful lock, 0 on contention */
5618 static int pci_slot_trylock(struct pci_slot *slot)
5619 {
5620 	struct pci_dev *dev;
5621 
5622 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5623 		if (!dev->slot || dev->slot != slot)
5624 			continue;
5625 		if (dev->subordinate) {
5626 			if (!pci_bus_trylock(dev->subordinate)) {
5627 				pci_dev_unlock(dev);
5628 				goto unlock;
5629 			}
5630 		} else if (!pci_dev_trylock(dev))
5631 			goto unlock;
5632 	}
5633 	return 1;
5634 
5635 unlock:
5636 	list_for_each_entry_continue_reverse(dev,
5637 					     &slot->bus->devices, bus_list) {
5638 		if (!dev->slot || dev->slot != slot)
5639 			continue;
5640 		if (dev->subordinate)
5641 			pci_bus_unlock(dev->subordinate);
5642 		else
5643 			pci_dev_unlock(dev);
5644 	}
5645 	return 0;
5646 }
5647 
5648 /*
5649  * Save and disable devices from the top of the tree down while holding
5650  * the @dev mutex lock for the entire tree.
5651  */
5652 static void pci_bus_save_and_disable_locked(struct pci_bus *bus)
5653 {
5654 	struct pci_dev *dev;
5655 
5656 	list_for_each_entry(dev, &bus->devices, bus_list) {
5657 		pci_dev_save_and_disable(dev);
5658 		if (dev->subordinate)
5659 			pci_bus_save_and_disable_locked(dev->subordinate);
5660 	}
5661 }
5662 
5663 /*
5664  * Restore devices from top of the tree down while holding @dev mutex lock
5665  * for the entire tree.  Parent bridges need to be restored before we can
5666  * get to subordinate devices.
5667  */
5668 static void pci_bus_restore_locked(struct pci_bus *bus)
5669 {
5670 	struct pci_dev *dev;
5671 
5672 	list_for_each_entry(dev, &bus->devices, bus_list) {
5673 		pci_dev_restore(dev);
5674 		if (dev->subordinate)
5675 			pci_bus_restore_locked(dev->subordinate);
5676 	}
5677 }
5678 
5679 /*
5680  * Save and disable devices from the top of the tree down while holding
5681  * the @dev mutex lock for the entire tree.
5682  */
5683 static void pci_slot_save_and_disable_locked(struct pci_slot *slot)
5684 {
5685 	struct pci_dev *dev;
5686 
5687 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5688 		if (!dev->slot || dev->slot != slot)
5689 			continue;
5690 		pci_dev_save_and_disable(dev);
5691 		if (dev->subordinate)
5692 			pci_bus_save_and_disable_locked(dev->subordinate);
5693 	}
5694 }
5695 
5696 /*
5697  * Restore devices from top of the tree down while holding @dev mutex lock
5698  * for the entire tree.  Parent bridges need to be restored before we can
5699  * get to subordinate devices.
5700  */
5701 static void pci_slot_restore_locked(struct pci_slot *slot)
5702 {
5703 	struct pci_dev *dev;
5704 
5705 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5706 		if (!dev->slot || dev->slot != slot)
5707 			continue;
5708 		pci_dev_restore(dev);
5709 		if (dev->subordinate)
5710 			pci_bus_restore_locked(dev->subordinate);
5711 	}
5712 }
5713 
5714 static int pci_slot_reset(struct pci_slot *slot, bool probe)
5715 {
5716 	int rc;
5717 
5718 	if (!slot || !pci_slot_resettable(slot))
5719 		return -ENOTTY;
5720 
5721 	if (!probe)
5722 		pci_slot_lock(slot);
5723 
5724 	might_sleep();
5725 
5726 	rc = pci_reset_hotplug_slot(slot->hotplug, probe);
5727 
5728 	if (!probe)
5729 		pci_slot_unlock(slot);
5730 
5731 	return rc;
5732 }
5733 
5734 /**
5735  * pci_probe_reset_slot - probe whether a PCI slot can be reset
5736  * @slot: PCI slot to probe
5737  *
5738  * Return 0 if slot can be reset, negative if a slot reset is not supported.
5739  */
5740 int pci_probe_reset_slot(struct pci_slot *slot)
5741 {
5742 	return pci_slot_reset(slot, PCI_RESET_PROBE);
5743 }
5744 EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
5745 
5746 /**
5747  * __pci_reset_slot - Try to reset a PCI slot
5748  * @slot: PCI slot to reset
5749  *
5750  * A PCI bus may host multiple slots, each slot may support a reset mechanism
5751  * independent of other slots.  For instance, some slots may support slot power
5752  * control.  In the case of a 1:1 bus to slot architecture, this function may
5753  * wrap the bus reset to avoid spurious slot related events such as hotplug.
5754  * Generally a slot reset should be attempted before a bus reset.  All of the
5755  * function of the slot and any subordinate buses behind the slot are reset
5756  * through this function.  PCI config space of all devices in the slot and
5757  * behind the slot is saved before and restored after reset.
5758  *
5759  * Same as above except return -EAGAIN if the slot cannot be locked
5760  */
5761 static int __pci_reset_slot(struct pci_slot *slot)
5762 {
5763 	int rc;
5764 
5765 	rc = pci_slot_reset(slot, PCI_RESET_PROBE);
5766 	if (rc)
5767 		return rc;
5768 
5769 	if (pci_slot_trylock(slot)) {
5770 		pci_slot_save_and_disable_locked(slot);
5771 		might_sleep();
5772 		rc = pci_reset_hotplug_slot(slot->hotplug, PCI_RESET_DO_RESET);
5773 		pci_slot_restore_locked(slot);
5774 		pci_slot_unlock(slot);
5775 	} else
5776 		rc = -EAGAIN;
5777 
5778 	return rc;
5779 }
5780 
5781 static int pci_bus_reset(struct pci_bus *bus, bool probe)
5782 {
5783 	int ret;
5784 
5785 	if (!bus->self || !pci_bus_resettable(bus))
5786 		return -ENOTTY;
5787 
5788 	if (probe)
5789 		return 0;
5790 
5791 	pci_bus_lock(bus);
5792 
5793 	might_sleep();
5794 
5795 	ret = pci_bridge_secondary_bus_reset(bus->self);
5796 
5797 	pci_bus_unlock(bus);
5798 
5799 	return ret;
5800 }
5801 
5802 /**
5803  * pci_bus_error_reset - reset the bridge's subordinate bus
5804  * @bridge: The parent device that connects to the bus to reset
5805  *
5806  * This function will first try to reset the slots on this bus if the method is
5807  * available. If slot reset fails or is not available, this will fall back to a
5808  * secondary bus reset.
5809  */
5810 int pci_bus_error_reset(struct pci_dev *bridge)
5811 {
5812 	struct pci_bus *bus = bridge->subordinate;
5813 	struct pci_slot *slot;
5814 
5815 	if (!bus)
5816 		return -ENOTTY;
5817 
5818 	mutex_lock(&pci_slot_mutex);
5819 	if (list_empty(&bus->slots))
5820 		goto bus_reset;
5821 
5822 	list_for_each_entry(slot, &bus->slots, list)
5823 		if (pci_probe_reset_slot(slot))
5824 			goto bus_reset;
5825 
5826 	list_for_each_entry(slot, &bus->slots, list)
5827 		if (pci_slot_reset(slot, PCI_RESET_DO_RESET))
5828 			goto bus_reset;
5829 
5830 	mutex_unlock(&pci_slot_mutex);
5831 	return 0;
5832 bus_reset:
5833 	mutex_unlock(&pci_slot_mutex);
5834 	return pci_bus_reset(bridge->subordinate, PCI_RESET_DO_RESET);
5835 }
5836 
5837 /**
5838  * pci_probe_reset_bus - probe whether a PCI bus can be reset
5839  * @bus: PCI bus to probe
5840  *
5841  * Return 0 if bus can be reset, negative if a bus reset is not supported.
5842  */
5843 int pci_probe_reset_bus(struct pci_bus *bus)
5844 {
5845 	return pci_bus_reset(bus, PCI_RESET_PROBE);
5846 }
5847 EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
5848 
5849 /**
5850  * __pci_reset_bus - Try to reset a PCI bus
5851  * @bus: top level PCI bus to reset
5852  *
5853  * Same as above except return -EAGAIN if the bus cannot be locked
5854  */
5855 static int __pci_reset_bus(struct pci_bus *bus)
5856 {
5857 	int rc;
5858 
5859 	rc = pci_bus_reset(bus, PCI_RESET_PROBE);
5860 	if (rc)
5861 		return rc;
5862 
5863 	if (pci_bus_trylock(bus)) {
5864 		pci_bus_save_and_disable_locked(bus);
5865 		might_sleep();
5866 		rc = pci_bridge_secondary_bus_reset(bus->self);
5867 		pci_bus_restore_locked(bus);
5868 		pci_bus_unlock(bus);
5869 	} else
5870 		rc = -EAGAIN;
5871 
5872 	return rc;
5873 }
5874 
5875 /**
5876  * pci_reset_bus - Try to reset a PCI bus
5877  * @pdev: top level PCI device to reset via slot/bus
5878  *
5879  * Same as above except return -EAGAIN if the bus cannot be locked
5880  */
5881 int pci_reset_bus(struct pci_dev *pdev)
5882 {
5883 	return (!pci_probe_reset_slot(pdev->slot)) ?
5884 	    __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus);
5885 }
5886 EXPORT_SYMBOL_GPL(pci_reset_bus);
5887 
5888 /**
5889  * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
5890  * @dev: PCI device to query
5891  *
5892  * Returns mmrbc: maximum designed memory read count in bytes or
5893  * appropriate error value.
5894  */
5895 int pcix_get_max_mmrbc(struct pci_dev *dev)
5896 {
5897 	int cap;
5898 	u32 stat;
5899 
5900 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5901 	if (!cap)
5902 		return -EINVAL;
5903 
5904 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
5905 		return -EINVAL;
5906 
5907 	return 512 << FIELD_GET(PCI_X_STATUS_MAX_READ, stat);
5908 }
5909 EXPORT_SYMBOL(pcix_get_max_mmrbc);
5910 
5911 /**
5912  * pcix_get_mmrbc - get PCI-X maximum memory read byte count
5913  * @dev: PCI device to query
5914  *
5915  * Returns mmrbc: maximum memory read count in bytes or appropriate error
5916  * value.
5917  */
5918 int pcix_get_mmrbc(struct pci_dev *dev)
5919 {
5920 	int cap;
5921 	u16 cmd;
5922 
5923 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5924 	if (!cap)
5925 		return -EINVAL;
5926 
5927 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
5928 		return -EINVAL;
5929 
5930 	return 512 << FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
5931 }
5932 EXPORT_SYMBOL(pcix_get_mmrbc);
5933 
5934 /**
5935  * pcix_set_mmrbc - set PCI-X maximum memory read byte count
5936  * @dev: PCI device to query
5937  * @mmrbc: maximum memory read count in bytes
5938  *    valid values are 512, 1024, 2048, 4096
5939  *
5940  * If possible sets maximum memory read byte count, some bridges have errata
5941  * that prevent this.
5942  */
5943 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
5944 {
5945 	int cap;
5946 	u32 stat, v, o;
5947 	u16 cmd;
5948 
5949 	if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc))
5950 		return -EINVAL;
5951 
5952 	v = ffs(mmrbc) - 10;
5953 
5954 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5955 	if (!cap)
5956 		return -EINVAL;
5957 
5958 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
5959 		return -EINVAL;
5960 
5961 	if (v > FIELD_GET(PCI_X_STATUS_MAX_READ, stat))
5962 		return -E2BIG;
5963 
5964 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
5965 		return -EINVAL;
5966 
5967 	o = FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
5968 	if (o != v) {
5969 		if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
5970 			return -EIO;
5971 
5972 		cmd &= ~PCI_X_CMD_MAX_READ;
5973 		cmd |= FIELD_PREP(PCI_X_CMD_MAX_READ, v);
5974 		if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd))
5975 			return -EIO;
5976 	}
5977 	return 0;
5978 }
5979 EXPORT_SYMBOL(pcix_set_mmrbc);
5980 
5981 /**
5982  * pcie_get_readrq - get PCI Express read request size
5983  * @dev: PCI device to query
5984  *
5985  * Returns maximum memory read request in bytes or appropriate error value.
5986  */
5987 int pcie_get_readrq(struct pci_dev *dev)
5988 {
5989 	u16 ctl;
5990 
5991 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
5992 
5993 	return 128 << FIELD_GET(PCI_EXP_DEVCTL_READRQ, ctl);
5994 }
5995 EXPORT_SYMBOL(pcie_get_readrq);
5996 
5997 /**
5998  * pcie_set_readrq - set PCI Express maximum memory read request
5999  * @dev: PCI device to query
6000  * @rq: maximum memory read count in bytes
6001  *    valid values are 128, 256, 512, 1024, 2048, 4096
6002  *
6003  * If possible sets maximum memory read request in bytes
6004  */
6005 int pcie_set_readrq(struct pci_dev *dev, int rq)
6006 {
6007 	u16 v;
6008 	int ret;
6009 	struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus);
6010 
6011 	if (rq < 128 || rq > 4096 || !is_power_of_2(rq))
6012 		return -EINVAL;
6013 
6014 	/*
6015 	 * If using the "performance" PCIe config, we clamp the read rq
6016 	 * size to the max packet size to keep the host bridge from
6017 	 * generating requests larger than we can cope with.
6018 	 */
6019 	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
6020 		int mps = pcie_get_mps(dev);
6021 
6022 		if (mps < rq)
6023 			rq = mps;
6024 	}
6025 
6026 	v = FIELD_PREP(PCI_EXP_DEVCTL_READRQ, ffs(rq) - 8);
6027 
6028 	if (bridge->no_inc_mrrs) {
6029 		int max_mrrs = pcie_get_readrq(dev);
6030 
6031 		if (rq > max_mrrs) {
6032 			pci_info(dev, "can't set Max_Read_Request_Size to %d; max is %d\n", rq, max_mrrs);
6033 			return -EINVAL;
6034 		}
6035 	}
6036 
6037 	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
6038 						  PCI_EXP_DEVCTL_READRQ, v);
6039 
6040 	return pcibios_err_to_errno(ret);
6041 }
6042 EXPORT_SYMBOL(pcie_set_readrq);
6043 
6044 /**
6045  * pcie_get_mps - get PCI Express maximum payload size
6046  * @dev: PCI device to query
6047  *
6048  * Returns maximum payload size in bytes
6049  */
6050 int pcie_get_mps(struct pci_dev *dev)
6051 {
6052 	u16 ctl;
6053 
6054 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
6055 
6056 	return 128 << FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, ctl);
6057 }
6058 EXPORT_SYMBOL(pcie_get_mps);
6059 
6060 /**
6061  * pcie_set_mps - set PCI Express maximum payload size
6062  * @dev: PCI device to query
6063  * @mps: maximum payload size in bytes
6064  *    valid values are 128, 256, 512, 1024, 2048, 4096
6065  *
6066  * If possible sets maximum payload size
6067  */
6068 int pcie_set_mps(struct pci_dev *dev, int mps)
6069 {
6070 	u16 v;
6071 	int ret;
6072 
6073 	if (mps < 128 || mps > 4096 || !is_power_of_2(mps))
6074 		return -EINVAL;
6075 
6076 	v = ffs(mps) - 8;
6077 	if (v > dev->pcie_mpss)
6078 		return -EINVAL;
6079 	v = FIELD_PREP(PCI_EXP_DEVCTL_PAYLOAD, v);
6080 
6081 	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
6082 						  PCI_EXP_DEVCTL_PAYLOAD, v);
6083 
6084 	return pcibios_err_to_errno(ret);
6085 }
6086 EXPORT_SYMBOL(pcie_set_mps);
6087 
6088 static enum pci_bus_speed to_pcie_link_speed(u16 lnksta)
6089 {
6090 	return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)];
6091 }
6092 
6093 int pcie_link_speed_mbps(struct pci_dev *pdev)
6094 {
6095 	u16 lnksta;
6096 	int err;
6097 
6098 	err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
6099 	if (err)
6100 		return err;
6101 
6102 	return pcie_dev_speed_mbps(to_pcie_link_speed(lnksta));
6103 }
6104 EXPORT_SYMBOL(pcie_link_speed_mbps);
6105 
6106 /**
6107  * pcie_bandwidth_available - determine minimum link settings of a PCIe
6108  *			      device and its bandwidth limitation
6109  * @dev: PCI device to query
6110  * @limiting_dev: storage for device causing the bandwidth limitation
6111  * @speed: storage for speed of limiting device
6112  * @width: storage for width of limiting device
6113  *
6114  * Walk up the PCI device chain and find the point where the minimum
6115  * bandwidth is available.  Return the bandwidth available there and (if
6116  * limiting_dev, speed, and width pointers are supplied) information about
6117  * that point.  The bandwidth returned is in Mb/s, i.e., megabits/second of
6118  * raw bandwidth.
6119  */
6120 u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
6121 			     enum pci_bus_speed *speed,
6122 			     enum pcie_link_width *width)
6123 {
6124 	u16 lnksta;
6125 	enum pci_bus_speed next_speed;
6126 	enum pcie_link_width next_width;
6127 	u32 bw, next_bw;
6128 
6129 	if (speed)
6130 		*speed = PCI_SPEED_UNKNOWN;
6131 	if (width)
6132 		*width = PCIE_LNK_WIDTH_UNKNOWN;
6133 
6134 	bw = 0;
6135 
6136 	while (dev) {
6137 		pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
6138 
6139 		next_speed = to_pcie_link_speed(lnksta);
6140 		next_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
6141 
6142 		next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed);
6143 
6144 		/* Check if current device limits the total bandwidth */
6145 		if (!bw || next_bw <= bw) {
6146 			bw = next_bw;
6147 
6148 			if (limiting_dev)
6149 				*limiting_dev = dev;
6150 			if (speed)
6151 				*speed = next_speed;
6152 			if (width)
6153 				*width = next_width;
6154 		}
6155 
6156 		dev = pci_upstream_bridge(dev);
6157 	}
6158 
6159 	return bw;
6160 }
6161 EXPORT_SYMBOL(pcie_bandwidth_available);
6162 
6163 /**
6164  * pcie_get_speed_cap - query for the PCI device's link speed capability
6165  * @dev: PCI device to query
6166  *
6167  * Query the PCI device speed capability.  Return the maximum link speed
6168  * supported by the device.
6169  */
6170 enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev)
6171 {
6172 	u32 lnkcap2, lnkcap;
6173 
6174 	/*
6175 	 * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18.  The
6176 	 * implementation note there recommends using the Supported Link
6177 	 * Speeds Vector in Link Capabilities 2 when supported.
6178 	 *
6179 	 * Without Link Capabilities 2, i.e., prior to PCIe r3.0, software
6180 	 * should use the Supported Link Speeds field in Link Capabilities,
6181 	 * where only 2.5 GT/s and 5.0 GT/s speeds were defined.
6182 	 */
6183 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2);
6184 
6185 	/* PCIe r3.0-compliant */
6186 	if (lnkcap2)
6187 		return PCIE_LNKCAP2_SLS2SPEED(lnkcap2);
6188 
6189 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
6190 	if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB)
6191 		return PCIE_SPEED_5_0GT;
6192 	else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB)
6193 		return PCIE_SPEED_2_5GT;
6194 
6195 	return PCI_SPEED_UNKNOWN;
6196 }
6197 EXPORT_SYMBOL(pcie_get_speed_cap);
6198 
6199 /**
6200  * pcie_get_width_cap - query for the PCI device's link width capability
6201  * @dev: PCI device to query
6202  *
6203  * Query the PCI device width capability.  Return the maximum link width
6204  * supported by the device.
6205  */
6206 enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev)
6207 {
6208 	u32 lnkcap;
6209 
6210 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
6211 	if (lnkcap)
6212 		return FIELD_GET(PCI_EXP_LNKCAP_MLW, lnkcap);
6213 
6214 	return PCIE_LNK_WIDTH_UNKNOWN;
6215 }
6216 EXPORT_SYMBOL(pcie_get_width_cap);
6217 
6218 /**
6219  * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability
6220  * @dev: PCI device
6221  * @speed: storage for link speed
6222  * @width: storage for link width
6223  *
6224  * Calculate a PCI device's link bandwidth by querying for its link speed
6225  * and width, multiplying them, and applying encoding overhead.  The result
6226  * is in Mb/s, i.e., megabits/second of raw bandwidth.
6227  */
6228 static u32 pcie_bandwidth_capable(struct pci_dev *dev,
6229 				  enum pci_bus_speed *speed,
6230 				  enum pcie_link_width *width)
6231 {
6232 	*speed = pcie_get_speed_cap(dev);
6233 	*width = pcie_get_width_cap(dev);
6234 
6235 	if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
6236 		return 0;
6237 
6238 	return *width * PCIE_SPEED2MBS_ENC(*speed);
6239 }
6240 
6241 /**
6242  * __pcie_print_link_status - Report the PCI device's link speed and width
6243  * @dev: PCI device to query
6244  * @verbose: Print info even when enough bandwidth is available
6245  *
6246  * If the available bandwidth at the device is less than the device is
6247  * capable of, report the device's maximum possible bandwidth and the
6248  * upstream link that limits its performance.  If @verbose, always print
6249  * the available bandwidth, even if the device isn't constrained.
6250  */
6251 void __pcie_print_link_status(struct pci_dev *dev, bool verbose)
6252 {
6253 	enum pcie_link_width width, width_cap;
6254 	enum pci_bus_speed speed, speed_cap;
6255 	struct pci_dev *limiting_dev = NULL;
6256 	u32 bw_avail, bw_cap;
6257 
6258 	bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap);
6259 	bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width);
6260 
6261 	if (bw_avail >= bw_cap && verbose)
6262 		pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n",
6263 			 bw_cap / 1000, bw_cap % 1000,
6264 			 pci_speed_string(speed_cap), width_cap);
6265 	else if (bw_avail < bw_cap)
6266 		pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n",
6267 			 bw_avail / 1000, bw_avail % 1000,
6268 			 pci_speed_string(speed), width,
6269 			 limiting_dev ? pci_name(limiting_dev) : "<unknown>",
6270 			 bw_cap / 1000, bw_cap % 1000,
6271 			 pci_speed_string(speed_cap), width_cap);
6272 }
6273 
6274 /**
6275  * pcie_print_link_status - Report the PCI device's link speed and width
6276  * @dev: PCI device to query
6277  *
6278  * Report the available bandwidth at the device.
6279  */
6280 void pcie_print_link_status(struct pci_dev *dev)
6281 {
6282 	__pcie_print_link_status(dev, true);
6283 }
6284 EXPORT_SYMBOL(pcie_print_link_status);
6285 
6286 /**
6287  * pci_select_bars - Make BAR mask from the type of resource
6288  * @dev: the PCI device for which BAR mask is made
6289  * @flags: resource type mask to be selected
6290  *
6291  * This helper routine makes bar mask from the type of resource.
6292  */
6293 int pci_select_bars(struct pci_dev *dev, unsigned long flags)
6294 {
6295 	int i, bars = 0;
6296 	for (i = 0; i < PCI_NUM_RESOURCES; i++)
6297 		if (pci_resource_flags(dev, i) & flags)
6298 			bars |= (1 << i);
6299 	return bars;
6300 }
6301 EXPORT_SYMBOL(pci_select_bars);
6302 
6303 /* Some architectures require additional programming to enable VGA */
6304 static arch_set_vga_state_t arch_set_vga_state;
6305 
6306 void __init pci_register_set_vga_state(arch_set_vga_state_t func)
6307 {
6308 	arch_set_vga_state = func;	/* NULL disables */
6309 }
6310 
6311 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
6312 				  unsigned int command_bits, u32 flags)
6313 {
6314 	if (arch_set_vga_state)
6315 		return arch_set_vga_state(dev, decode, command_bits,
6316 						flags);
6317 	return 0;
6318 }
6319 
6320 /**
6321  * pci_set_vga_state - set VGA decode state on device and parents if requested
6322  * @dev: the PCI device
6323  * @decode: true = enable decoding, false = disable decoding
6324  * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
6325  * @flags: traverse ancestors and change bridges
6326  * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
6327  */
6328 int pci_set_vga_state(struct pci_dev *dev, bool decode,
6329 		      unsigned int command_bits, u32 flags)
6330 {
6331 	struct pci_bus *bus;
6332 	struct pci_dev *bridge;
6333 	u16 cmd;
6334 	int rc;
6335 
6336 	WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
6337 
6338 	/* ARCH specific VGA enables */
6339 	rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
6340 	if (rc)
6341 		return rc;
6342 
6343 	if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
6344 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
6345 		if (decode)
6346 			cmd |= command_bits;
6347 		else
6348 			cmd &= ~command_bits;
6349 		pci_write_config_word(dev, PCI_COMMAND, cmd);
6350 	}
6351 
6352 	if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
6353 		return 0;
6354 
6355 	bus = dev->bus;
6356 	while (bus) {
6357 		bridge = bus->self;
6358 		if (bridge) {
6359 			pci_read_config_word(bridge, PCI_BRIDGE_CONTROL,
6360 					     &cmd);
6361 			if (decode)
6362 				cmd |= PCI_BRIDGE_CTL_VGA;
6363 			else
6364 				cmd &= ~PCI_BRIDGE_CTL_VGA;
6365 			pci_write_config_word(bridge, PCI_BRIDGE_CONTROL,
6366 					      cmd);
6367 		}
6368 		bus = bus->parent;
6369 	}
6370 	return 0;
6371 }
6372 
6373 #ifdef CONFIG_ACPI
6374 bool pci_pr3_present(struct pci_dev *pdev)
6375 {
6376 	struct acpi_device *adev;
6377 
6378 	if (acpi_disabled)
6379 		return false;
6380 
6381 	adev = ACPI_COMPANION(&pdev->dev);
6382 	if (!adev)
6383 		return false;
6384 
6385 	return adev->power.flags.power_resources &&
6386 		acpi_has_method(adev->handle, "_PR3");
6387 }
6388 EXPORT_SYMBOL_GPL(pci_pr3_present);
6389 #endif
6390 
6391 /**
6392  * pci_add_dma_alias - Add a DMA devfn alias for a device
6393  * @dev: the PCI device for which alias is added
6394  * @devfn_from: alias slot and function
6395  * @nr_devfns: number of subsequent devfns to alias
6396  *
6397  * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask
6398  * which is used to program permissible bus-devfn source addresses for DMA
6399  * requests in an IOMMU.  These aliases factor into IOMMU group creation
6400  * and are useful for devices generating DMA requests beyond or different
6401  * from their logical bus-devfn.  Examples include device quirks where the
6402  * device simply uses the wrong devfn, as well as non-transparent bridges
6403  * where the alias may be a proxy for devices in another domain.
6404  *
6405  * IOMMU group creation is performed during device discovery or addition,
6406  * prior to any potential DMA mapping and therefore prior to driver probing
6407  * (especially for userspace assigned devices where IOMMU group definition
6408  * cannot be left as a userspace activity).  DMA aliases should therefore
6409  * be configured via quirks, such as the PCI fixup header quirk.
6410  */
6411 void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from,
6412 		       unsigned int nr_devfns)
6413 {
6414 	int devfn_to;
6415 
6416 	nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from);
6417 	devfn_to = devfn_from + nr_devfns - 1;
6418 
6419 	if (!dev->dma_alias_mask)
6420 		dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL);
6421 	if (!dev->dma_alias_mask) {
6422 		pci_warn(dev, "Unable to allocate DMA alias mask\n");
6423 		return;
6424 	}
6425 
6426 	bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns);
6427 
6428 	if (nr_devfns == 1)
6429 		pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n",
6430 				PCI_SLOT(devfn_from), PCI_FUNC(devfn_from));
6431 	else if (nr_devfns > 1)
6432 		pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n",
6433 				PCI_SLOT(devfn_from), PCI_FUNC(devfn_from),
6434 				PCI_SLOT(devfn_to), PCI_FUNC(devfn_to));
6435 }
6436 
6437 bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2)
6438 {
6439 	return (dev1->dma_alias_mask &&
6440 		test_bit(dev2->devfn, dev1->dma_alias_mask)) ||
6441 	       (dev2->dma_alias_mask &&
6442 		test_bit(dev1->devfn, dev2->dma_alias_mask)) ||
6443 	       pci_real_dma_dev(dev1) == dev2 ||
6444 	       pci_real_dma_dev(dev2) == dev1;
6445 }
6446 
6447 bool pci_device_is_present(struct pci_dev *pdev)
6448 {
6449 	u32 v;
6450 
6451 	/* Check PF if pdev is a VF, since VF Vendor/Device IDs are 0xffff */
6452 	pdev = pci_physfn(pdev);
6453 	if (pci_dev_is_disconnected(pdev))
6454 		return false;
6455 	return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0);
6456 }
6457 EXPORT_SYMBOL_GPL(pci_device_is_present);
6458 
6459 void pci_ignore_hotplug(struct pci_dev *dev)
6460 {
6461 	struct pci_dev *bridge = dev->bus->self;
6462 
6463 	dev->ignore_hotplug = 1;
6464 	/* Propagate the "ignore hotplug" setting to the parent bridge. */
6465 	if (bridge)
6466 		bridge->ignore_hotplug = 1;
6467 }
6468 EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
6469 
6470 /**
6471  * pci_real_dma_dev - Get PCI DMA device for PCI device
6472  * @dev: the PCI device that may have a PCI DMA alias
6473  *
6474  * Permits the platform to provide architecture-specific functionality to
6475  * devices needing to alias DMA to another PCI device on another PCI bus. If
6476  * the PCI device is on the same bus, it is recommended to use
6477  * pci_add_dma_alias(). This is the default implementation. Architecture
6478  * implementations can override this.
6479  */
6480 struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev)
6481 {
6482 	return dev;
6483 }
6484 
6485 resource_size_t __weak pcibios_default_alignment(void)
6486 {
6487 	return 0;
6488 }
6489 
6490 /*
6491  * Arches that don't want to expose struct resource to userland as-is in
6492  * sysfs and /proc can implement their own pci_resource_to_user().
6493  */
6494 void __weak pci_resource_to_user(const struct pci_dev *dev, int bar,
6495 				 const struct resource *rsrc,
6496 				 resource_size_t *start, resource_size_t *end)
6497 {
6498 	*start = rsrc->start;
6499 	*end = rsrc->end;
6500 }
6501 
6502 static char *resource_alignment_param;
6503 static DEFINE_SPINLOCK(resource_alignment_lock);
6504 
6505 /**
6506  * pci_specified_resource_alignment - get resource alignment specified by user.
6507  * @dev: the PCI device to get
6508  * @resize: whether or not to change resources' size when reassigning alignment
6509  *
6510  * RETURNS: Resource alignment if it is specified.
6511  *          Zero if it is not specified.
6512  */
6513 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev,
6514 							bool *resize)
6515 {
6516 	int align_order, count;
6517 	resource_size_t align = pcibios_default_alignment();
6518 	const char *p;
6519 	int ret;
6520 
6521 	spin_lock(&resource_alignment_lock);
6522 	p = resource_alignment_param;
6523 	if (!p || !*p)
6524 		goto out;
6525 	if (pci_has_flag(PCI_PROBE_ONLY)) {
6526 		align = 0;
6527 		pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
6528 		goto out;
6529 	}
6530 
6531 	while (*p) {
6532 		count = 0;
6533 		if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
6534 		    p[count] == '@') {
6535 			p += count + 1;
6536 			if (align_order > 63) {
6537 				pr_err("PCI: Invalid requested alignment (order %d)\n",
6538 				       align_order);
6539 				align_order = PAGE_SHIFT;
6540 			}
6541 		} else {
6542 			align_order = PAGE_SHIFT;
6543 		}
6544 
6545 		ret = pci_dev_str_match(dev, p, &p);
6546 		if (ret == 1) {
6547 			*resize = true;
6548 			align = 1ULL << align_order;
6549 			break;
6550 		} else if (ret < 0) {
6551 			pr_err("PCI: Can't parse resource_alignment parameter: %s\n",
6552 			       p);
6553 			break;
6554 		}
6555 
6556 		if (*p != ';' && *p != ',') {
6557 			/* End of param or invalid format */
6558 			break;
6559 		}
6560 		p++;
6561 	}
6562 out:
6563 	spin_unlock(&resource_alignment_lock);
6564 	return align;
6565 }
6566 
6567 static void pci_request_resource_alignment(struct pci_dev *dev, int bar,
6568 					   resource_size_t align, bool resize)
6569 {
6570 	struct resource *r = &dev->resource[bar];
6571 	const char *r_name = pci_resource_name(dev, bar);
6572 	resource_size_t size;
6573 
6574 	if (!(r->flags & IORESOURCE_MEM))
6575 		return;
6576 
6577 	if (r->flags & IORESOURCE_PCI_FIXED) {
6578 		pci_info(dev, "%s %pR: ignoring requested alignment %#llx\n",
6579 			 r_name, r, (unsigned long long)align);
6580 		return;
6581 	}
6582 
6583 	size = resource_size(r);
6584 	if (size >= align)
6585 		return;
6586 
6587 	/*
6588 	 * Increase the alignment of the resource.  There are two ways we
6589 	 * can do this:
6590 	 *
6591 	 * 1) Increase the size of the resource.  BARs are aligned on their
6592 	 *    size, so when we reallocate space for this resource, we'll
6593 	 *    allocate it with the larger alignment.  This also prevents
6594 	 *    assignment of any other BARs inside the alignment region, so
6595 	 *    if we're requesting page alignment, this means no other BARs
6596 	 *    will share the page.
6597 	 *
6598 	 *    The disadvantage is that this makes the resource larger than
6599 	 *    the hardware BAR, which may break drivers that compute things
6600 	 *    based on the resource size, e.g., to find registers at a
6601 	 *    fixed offset before the end of the BAR.
6602 	 *
6603 	 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and
6604 	 *    set r->start to the desired alignment.  By itself this
6605 	 *    doesn't prevent other BARs being put inside the alignment
6606 	 *    region, but if we realign *every* resource of every device in
6607 	 *    the system, none of them will share an alignment region.
6608 	 *
6609 	 * When the user has requested alignment for only some devices via
6610 	 * the "pci=resource_alignment" argument, "resize" is true and we
6611 	 * use the first method.  Otherwise we assume we're aligning all
6612 	 * devices and we use the second.
6613 	 */
6614 
6615 	pci_info(dev, "%s %pR: requesting alignment to %#llx\n",
6616 		 r_name, r, (unsigned long long)align);
6617 
6618 	if (resize) {
6619 		r->start = 0;
6620 		r->end = align - 1;
6621 	} else {
6622 		r->flags &= ~IORESOURCE_SIZEALIGN;
6623 		r->flags |= IORESOURCE_STARTALIGN;
6624 		r->start = align;
6625 		r->end = r->start + size - 1;
6626 	}
6627 	r->flags |= IORESOURCE_UNSET;
6628 }
6629 
6630 /*
6631  * This function disables memory decoding and releases memory resources
6632  * of the device specified by kernel's boot parameter 'pci=resource_alignment='.
6633  * It also rounds up size to specified alignment.
6634  * Later on, the kernel will assign page-aligned memory resource back
6635  * to the device.
6636  */
6637 void pci_reassigndev_resource_alignment(struct pci_dev *dev)
6638 {
6639 	int i;
6640 	struct resource *r;
6641 	resource_size_t align;
6642 	u16 command;
6643 	bool resize = false;
6644 
6645 	/*
6646 	 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec
6647 	 * 3.4.1.11.  Their resources are allocated from the space
6648 	 * described by the VF BARx register in the PF's SR-IOV capability.
6649 	 * We can't influence their alignment here.
6650 	 */
6651 	if (dev->is_virtfn)
6652 		return;
6653 
6654 	/* check if specified PCI is target device to reassign */
6655 	align = pci_specified_resource_alignment(dev, &resize);
6656 	if (!align)
6657 		return;
6658 
6659 	if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
6660 	    (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
6661 		pci_warn(dev, "Can't reassign resources to host bridge\n");
6662 		return;
6663 	}
6664 
6665 	pci_read_config_word(dev, PCI_COMMAND, &command);
6666 	command &= ~PCI_COMMAND_MEMORY;
6667 	pci_write_config_word(dev, PCI_COMMAND, command);
6668 
6669 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
6670 		pci_request_resource_alignment(dev, i, align, resize);
6671 
6672 	/*
6673 	 * Need to disable bridge's resource window,
6674 	 * to enable the kernel to reassign new resource
6675 	 * window later on.
6676 	 */
6677 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
6678 		for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
6679 			r = &dev->resource[i];
6680 			if (!(r->flags & IORESOURCE_MEM))
6681 				continue;
6682 			r->flags |= IORESOURCE_UNSET;
6683 			r->end = resource_size(r) - 1;
6684 			r->start = 0;
6685 		}
6686 		pci_disable_bridge_window(dev);
6687 	}
6688 }
6689 
6690 static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf)
6691 {
6692 	size_t count = 0;
6693 
6694 	spin_lock(&resource_alignment_lock);
6695 	if (resource_alignment_param)
6696 		count = sysfs_emit(buf, "%s\n", resource_alignment_param);
6697 	spin_unlock(&resource_alignment_lock);
6698 
6699 	return count;
6700 }
6701 
6702 static ssize_t resource_alignment_store(const struct bus_type *bus,
6703 					const char *buf, size_t count)
6704 {
6705 	char *param, *old, *end;
6706 
6707 	if (count >= (PAGE_SIZE - 1))
6708 		return -EINVAL;
6709 
6710 	param = kstrndup(buf, count, GFP_KERNEL);
6711 	if (!param)
6712 		return -ENOMEM;
6713 
6714 	end = strchr(param, '\n');
6715 	if (end)
6716 		*end = '\0';
6717 
6718 	spin_lock(&resource_alignment_lock);
6719 	old = resource_alignment_param;
6720 	if (strlen(param)) {
6721 		resource_alignment_param = param;
6722 	} else {
6723 		kfree(param);
6724 		resource_alignment_param = NULL;
6725 	}
6726 	spin_unlock(&resource_alignment_lock);
6727 
6728 	kfree(old);
6729 
6730 	return count;
6731 }
6732 
6733 static BUS_ATTR_RW(resource_alignment);
6734 
6735 static int __init pci_resource_alignment_sysfs_init(void)
6736 {
6737 	return bus_create_file(&pci_bus_type,
6738 					&bus_attr_resource_alignment);
6739 }
6740 late_initcall(pci_resource_alignment_sysfs_init);
6741 
6742 static void pci_no_domains(void)
6743 {
6744 #ifdef CONFIG_PCI_DOMAINS
6745 	pci_domains_supported = 0;
6746 #endif
6747 }
6748 
6749 #ifdef CONFIG_PCI_DOMAINS_GENERIC
6750 static DEFINE_IDA(pci_domain_nr_static_ida);
6751 static DEFINE_IDA(pci_domain_nr_dynamic_ida);
6752 
6753 static void of_pci_reserve_static_domain_nr(void)
6754 {
6755 	struct device_node *np;
6756 	int domain_nr;
6757 
6758 	for_each_node_by_type(np, "pci") {
6759 		domain_nr = of_get_pci_domain_nr(np);
6760 		if (domain_nr < 0)
6761 			continue;
6762 		/*
6763 		 * Permanently allocate domain_nr in dynamic_ida
6764 		 * to prevent it from dynamic allocation.
6765 		 */
6766 		ida_alloc_range(&pci_domain_nr_dynamic_ida,
6767 				domain_nr, domain_nr, GFP_KERNEL);
6768 	}
6769 }
6770 
6771 static int of_pci_bus_find_domain_nr(struct device *parent)
6772 {
6773 	static bool static_domains_reserved = false;
6774 	int domain_nr;
6775 
6776 	/* On the first call scan device tree for static allocations. */
6777 	if (!static_domains_reserved) {
6778 		of_pci_reserve_static_domain_nr();
6779 		static_domains_reserved = true;
6780 	}
6781 
6782 	if (parent) {
6783 		/*
6784 		 * If domain is in DT, allocate it in static IDA.  This
6785 		 * prevents duplicate static allocations in case of errors
6786 		 * in DT.
6787 		 */
6788 		domain_nr = of_get_pci_domain_nr(parent->of_node);
6789 		if (domain_nr >= 0)
6790 			return ida_alloc_range(&pci_domain_nr_static_ida,
6791 					       domain_nr, domain_nr,
6792 					       GFP_KERNEL);
6793 	}
6794 
6795 	/*
6796 	 * If domain was not specified in DT, choose a free ID from dynamic
6797 	 * allocations. All domain numbers from DT are permanently in
6798 	 * dynamic allocations to prevent assigning them to other DT nodes
6799 	 * without static domain.
6800 	 */
6801 	return ida_alloc(&pci_domain_nr_dynamic_ida, GFP_KERNEL);
6802 }
6803 
6804 static void of_pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent)
6805 {
6806 	if (bus->domain_nr < 0)
6807 		return;
6808 
6809 	/* Release domain from IDA where it was allocated. */
6810 	if (of_get_pci_domain_nr(parent->of_node) == bus->domain_nr)
6811 		ida_free(&pci_domain_nr_static_ida, bus->domain_nr);
6812 	else
6813 		ida_free(&pci_domain_nr_dynamic_ida, bus->domain_nr);
6814 }
6815 
6816 int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
6817 {
6818 	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
6819 			       acpi_pci_bus_find_domain_nr(bus);
6820 }
6821 
6822 void pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent)
6823 {
6824 	if (!acpi_disabled)
6825 		return;
6826 	of_pci_bus_release_domain_nr(bus, parent);
6827 }
6828 #endif
6829 
6830 /**
6831  * pci_ext_cfg_avail - can we access extended PCI config space?
6832  *
6833  * Returns 1 if we can access PCI extended config space (offsets
6834  * greater than 0xff). This is the default implementation. Architecture
6835  * implementations can override this.
6836  */
6837 int __weak pci_ext_cfg_avail(void)
6838 {
6839 	return 1;
6840 }
6841 
6842 void __weak pci_fixup_cardbus(struct pci_bus *bus)
6843 {
6844 }
6845 EXPORT_SYMBOL(pci_fixup_cardbus);
6846 
6847 static int __init pci_setup(char *str)
6848 {
6849 	while (str) {
6850 		char *k = strchr(str, ',');
6851 		if (k)
6852 			*k++ = 0;
6853 		if (*str && (str = pcibios_setup(str)) && *str) {
6854 			if (!strcmp(str, "nomsi")) {
6855 				pci_no_msi();
6856 			} else if (!strncmp(str, "noats", 5)) {
6857 				pr_info("PCIe: ATS is disabled\n");
6858 				pcie_ats_disabled = true;
6859 			} else if (!strcmp(str, "noaer")) {
6860 				pci_no_aer();
6861 			} else if (!strcmp(str, "earlydump")) {
6862 				pci_early_dump = true;
6863 			} else if (!strncmp(str, "realloc=", 8)) {
6864 				pci_realloc_get_opt(str + 8);
6865 			} else if (!strncmp(str, "realloc", 7)) {
6866 				pci_realloc_get_opt("on");
6867 			} else if (!strcmp(str, "nodomains")) {
6868 				pci_no_domains();
6869 			} else if (!strncmp(str, "noari", 5)) {
6870 				pcie_ari_disabled = true;
6871 			} else if (!strncmp(str, "cbiosize=", 9)) {
6872 				pci_cardbus_io_size = memparse(str + 9, &str);
6873 			} else if (!strncmp(str, "cbmemsize=", 10)) {
6874 				pci_cardbus_mem_size = memparse(str + 10, &str);
6875 			} else if (!strncmp(str, "resource_alignment=", 19)) {
6876 				resource_alignment_param = str + 19;
6877 			} else if (!strncmp(str, "ecrc=", 5)) {
6878 				pcie_ecrc_get_policy(str + 5);
6879 			} else if (!strncmp(str, "hpiosize=", 9)) {
6880 				pci_hotplug_io_size = memparse(str + 9, &str);
6881 			} else if (!strncmp(str, "hpmmiosize=", 11)) {
6882 				pci_hotplug_mmio_size = memparse(str + 11, &str);
6883 			} else if (!strncmp(str, "hpmmioprefsize=", 15)) {
6884 				pci_hotplug_mmio_pref_size = memparse(str + 15, &str);
6885 			} else if (!strncmp(str, "hpmemsize=", 10)) {
6886 				pci_hotplug_mmio_size = memparse(str + 10, &str);
6887 				pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size;
6888 			} else if (!strncmp(str, "hpbussize=", 10)) {
6889 				pci_hotplug_bus_size =
6890 					simple_strtoul(str + 10, &str, 0);
6891 				if (pci_hotplug_bus_size > 0xff)
6892 					pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
6893 			} else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
6894 				pcie_bus_config = PCIE_BUS_TUNE_OFF;
6895 			} else if (!strncmp(str, "pcie_bus_safe", 13)) {
6896 				pcie_bus_config = PCIE_BUS_SAFE;
6897 			} else if (!strncmp(str, "pcie_bus_perf", 13)) {
6898 				pcie_bus_config = PCIE_BUS_PERFORMANCE;
6899 			} else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
6900 				pcie_bus_config = PCIE_BUS_PEER2PEER;
6901 			} else if (!strncmp(str, "pcie_scan_all", 13)) {
6902 				pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS);
6903 			} else if (!strncmp(str, "disable_acs_redir=", 18)) {
6904 				disable_acs_redir_param = str + 18;
6905 			} else if (!strncmp(str, "config_acs=", 11)) {
6906 				config_acs_param = str + 11;
6907 			} else {
6908 				pr_err("PCI: Unknown option `%s'\n", str);
6909 			}
6910 		}
6911 		str = k;
6912 	}
6913 	return 0;
6914 }
6915 early_param("pci", pci_setup);
6916 
6917 /*
6918  * 'resource_alignment_param' and 'disable_acs_redir_param' are initialized
6919  * in pci_setup(), above, to point to data in the __initdata section which
6920  * will be freed after the init sequence is complete. We can't allocate memory
6921  * in pci_setup() because some architectures do not have any memory allocation
6922  * service available during an early_param() call. So we allocate memory and
6923  * copy the variable here before the init section is freed.
6924  *
6925  */
6926 static int __init pci_realloc_setup_params(void)
6927 {
6928 	resource_alignment_param = kstrdup(resource_alignment_param,
6929 					   GFP_KERNEL);
6930 	disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL);
6931 	config_acs_param = kstrdup(config_acs_param, GFP_KERNEL);
6932 
6933 	return 0;
6934 }
6935 pure_initcall(pci_realloc_setup_params);
6936