xref: /linux/drivers/pci/probe.c (revision e67bb0da332c6058b29a9c46cc4035647d049a0c)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI detection and setup code
4  */
5 
6 #include <linux/array_size.h>
7 #include <linux/kernel.h>
8 #include <linux/delay.h>
9 #include <linux/init.h>
10 #include <linux/pci.h>
11 #include <linux/msi.h>
12 #include <linux/of_pci.h>
13 #include <linux/of_platform.h>
14 #include <linux/platform_device.h>
15 #include <linux/pci_hotplug.h>
16 #include <linux/slab.h>
17 #include <linux/module.h>
18 #include <linux/cpumask.h>
19 #include <linux/aer.h>
20 #include <linux/acpi.h>
21 #include <linux/hypervisor.h>
22 #include <linux/irqdomain.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/bitfield.h>
25 #include "pci.h"
26 
27 #define CARDBUS_LATENCY_TIMER	176	/* secondary latency timer */
28 #define CARDBUS_RESERVE_BUSNR	3
29 
30 static struct resource busn_resource = {
31 	.name	= "PCI busn",
32 	.start	= 0,
33 	.end	= 255,
34 	.flags	= IORESOURCE_BUS,
35 };
36 
37 /* Ugh.  Need to stop exporting this to modules. */
38 LIST_HEAD(pci_root_buses);
39 EXPORT_SYMBOL(pci_root_buses);
40 
41 static LIST_HEAD(pci_domain_busn_res_list);
42 
43 struct pci_domain_busn_res {
44 	struct list_head list;
45 	struct resource res;
46 	int domain_nr;
47 };
48 
get_pci_domain_busn_res(int domain_nr)49 static struct resource *get_pci_domain_busn_res(int domain_nr)
50 {
51 	struct pci_domain_busn_res *r;
52 
53 	list_for_each_entry(r, &pci_domain_busn_res_list, list)
54 		if (r->domain_nr == domain_nr)
55 			return &r->res;
56 
57 	r = kzalloc(sizeof(*r), GFP_KERNEL);
58 	if (!r)
59 		return NULL;
60 
61 	r->domain_nr = domain_nr;
62 	r->res.start = 0;
63 	r->res.end = 0xff;
64 	r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
65 
66 	list_add_tail(&r->list, &pci_domain_busn_res_list);
67 
68 	return &r->res;
69 }
70 
71 /*
72  * Some device drivers need know if PCI is initiated.
73  * Basically, we think PCI is not initiated when there
74  * is no device to be found on the pci_bus_type.
75  */
no_pci_devices(void)76 int no_pci_devices(void)
77 {
78 	struct device *dev;
79 	int no_devices;
80 
81 	dev = bus_find_next_device(&pci_bus_type, NULL);
82 	no_devices = (dev == NULL);
83 	put_device(dev);
84 	return no_devices;
85 }
86 EXPORT_SYMBOL(no_pci_devices);
87 
88 /*
89  * PCI Bus Class
90  */
release_pcibus_dev(struct device * dev)91 static void release_pcibus_dev(struct device *dev)
92 {
93 	struct pci_bus *pci_bus = to_pci_bus(dev);
94 
95 	put_device(pci_bus->bridge);
96 	pci_bus_remove_resources(pci_bus);
97 	pci_release_bus_of_node(pci_bus);
98 	kfree(pci_bus);
99 }
100 
101 static const struct class pcibus_class = {
102 	.name		= "pci_bus",
103 	.dev_release	= &release_pcibus_dev,
104 	.dev_groups	= pcibus_groups,
105 };
106 
pcibus_class_init(void)107 static int __init pcibus_class_init(void)
108 {
109 	return class_register(&pcibus_class);
110 }
111 postcore_initcall(pcibus_class_init);
112 
pci_size(u64 base,u64 maxbase,u64 mask)113 static u64 pci_size(u64 base, u64 maxbase, u64 mask)
114 {
115 	u64 size = mask & maxbase;	/* Find the significant bits */
116 	if (!size)
117 		return 0;
118 
119 	/*
120 	 * Get the lowest of them to find the decode size, and from that
121 	 * the extent.
122 	 */
123 	size = size & ~(size-1);
124 
125 	/*
126 	 * base == maxbase can be valid only if the BAR has already been
127 	 * programmed with all 1s.
128 	 */
129 	if (base == maxbase && ((base | (size - 1)) & mask) != mask)
130 		return 0;
131 
132 	return size;
133 }
134 
decode_bar(struct pci_dev * dev,u32 bar)135 static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar)
136 {
137 	u32 mem_type;
138 	unsigned long flags;
139 
140 	if ((bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
141 		flags = bar & ~PCI_BASE_ADDRESS_IO_MASK;
142 		flags |= IORESOURCE_IO;
143 		return flags;
144 	}
145 
146 	flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK;
147 	flags |= IORESOURCE_MEM;
148 	if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH)
149 		flags |= IORESOURCE_PREFETCH;
150 
151 	mem_type = bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
152 	switch (mem_type) {
153 	case PCI_BASE_ADDRESS_MEM_TYPE_32:
154 		break;
155 	case PCI_BASE_ADDRESS_MEM_TYPE_1M:
156 		/* 1M mem BAR treated as 32-bit BAR */
157 		break;
158 	case PCI_BASE_ADDRESS_MEM_TYPE_64:
159 		flags |= IORESOURCE_MEM_64;
160 		break;
161 	default:
162 		/* mem unknown type treated as 32-bit BAR */
163 		break;
164 	}
165 	return flags;
166 }
167 
168 #define PCI_COMMAND_DECODE_ENABLE	(PCI_COMMAND_MEMORY | PCI_COMMAND_IO)
169 
170 /**
171  * __pci_size_bars - Read the raw BAR mask for a range of PCI BARs
172  * @dev: the PCI device
173  * @count: number of BARs to size
174  * @pos: starting config space position
175  * @sizes: array to store mask values
176  * @rom: indicate whether to use ROM mask, which avoids enabling ROM BARs
177  *
178  * Provided @sizes array must be sufficiently sized to store results for
179  * @count u32 BARs.  Caller is responsible for disabling decode to specified
180  * BAR range around calling this function.  This function is intended to avoid
181  * disabling decode around sizing each BAR individually, which can result in
182  * non-trivial overhead in virtualized environments with very large PCI BARs.
183  */
__pci_size_bars(struct pci_dev * dev,int count,unsigned int pos,u32 * sizes,bool rom)184 static void __pci_size_bars(struct pci_dev *dev, int count,
185 			    unsigned int pos, u32 *sizes, bool rom)
186 {
187 	u32 orig, mask = rom ? PCI_ROM_ADDRESS_MASK : ~0;
188 	int i;
189 
190 	for (i = 0; i < count; i++, pos += 4, sizes++) {
191 		pci_read_config_dword(dev, pos, &orig);
192 		pci_write_config_dword(dev, pos, mask);
193 		pci_read_config_dword(dev, pos, sizes);
194 		pci_write_config_dword(dev, pos, orig);
195 	}
196 }
197 
__pci_size_stdbars(struct pci_dev * dev,int count,unsigned int pos,u32 * sizes)198 void __pci_size_stdbars(struct pci_dev *dev, int count,
199 			unsigned int pos, u32 *sizes)
200 {
201 	__pci_size_bars(dev, count, pos, sizes, false);
202 }
203 
__pci_size_rom(struct pci_dev * dev,unsigned int pos,u32 * sizes)204 static void __pci_size_rom(struct pci_dev *dev, unsigned int pos, u32 *sizes)
205 {
206 	__pci_size_bars(dev, 1, pos, sizes, true);
207 }
208 
209 /**
210  * __pci_read_base - Read a PCI BAR
211  * @dev: the PCI device
212  * @type: type of the BAR
213  * @res: resource buffer to be filled in
214  * @pos: BAR position in the config space
215  * @sizes: array of one or more pre-read BAR masks
216  *
217  * Returns 1 if the BAR is 64-bit, or 0 if 32-bit.
218  */
__pci_read_base(struct pci_dev * dev,enum pci_bar_type type,struct resource * res,unsigned int pos,u32 * sizes)219 int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
220 		    struct resource *res, unsigned int pos, u32 *sizes)
221 {
222 	u32 l = 0, sz;
223 	u64 l64, sz64, mask64;
224 	struct pci_bus_region region, inverted_region;
225 	const char *res_name = pci_resource_name(dev, res - dev->resource);
226 
227 	res->name = pci_name(dev);
228 
229 	pci_read_config_dword(dev, pos, &l);
230 	sz = sizes[0];
231 
232 	/*
233 	 * All bits set in sz means the device isn't working properly.
234 	 * If the BAR isn't implemented, all bits must be 0.  If it's a
235 	 * memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit
236 	 * 1 must be clear.
237 	 */
238 	if (PCI_POSSIBLE_ERROR(sz))
239 		sz = 0;
240 
241 	/*
242 	 * I don't know how l can have all bits set.  Copied from old code.
243 	 * Maybe it fixes a bug on some ancient platform.
244 	 */
245 	if (PCI_POSSIBLE_ERROR(l))
246 		l = 0;
247 
248 	if (type == pci_bar_unknown) {
249 		res->flags = decode_bar(dev, l);
250 		res->flags |= IORESOURCE_SIZEALIGN;
251 		if (res->flags & IORESOURCE_IO) {
252 			l64 = l & PCI_BASE_ADDRESS_IO_MASK;
253 			sz64 = sz & PCI_BASE_ADDRESS_IO_MASK;
254 			mask64 = PCI_BASE_ADDRESS_IO_MASK & (u32)IO_SPACE_LIMIT;
255 		} else {
256 			l64 = l & PCI_BASE_ADDRESS_MEM_MASK;
257 			sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK;
258 			mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK;
259 		}
260 	} else {
261 		if (l & PCI_ROM_ADDRESS_ENABLE)
262 			res->flags |= IORESOURCE_ROM_ENABLE;
263 		l64 = l & PCI_ROM_ADDRESS_MASK;
264 		sz64 = sz & PCI_ROM_ADDRESS_MASK;
265 		mask64 = PCI_ROM_ADDRESS_MASK;
266 	}
267 
268 	if (res->flags & IORESOURCE_MEM_64) {
269 		pci_read_config_dword(dev, pos + 4, &l);
270 		sz = sizes[1];
271 
272 		l64 |= ((u64)l << 32);
273 		sz64 |= ((u64)sz << 32);
274 		mask64 |= ((u64)~0 << 32);
275 	}
276 
277 	if (!sz64)
278 		goto fail;
279 
280 	sz64 = pci_size(l64, sz64, mask64);
281 	if (!sz64) {
282 		pci_info(dev, FW_BUG "%s: invalid; can't size\n", res_name);
283 		goto fail;
284 	}
285 
286 	if (res->flags & IORESOURCE_MEM_64) {
287 		if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8)
288 		    && sz64 > 0x100000000ULL) {
289 			res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
290 			res->start = 0;
291 			res->end = 0;
292 			pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n",
293 				res_name, (unsigned long long)sz64);
294 			goto out;
295 		}
296 
297 		if ((sizeof(pci_bus_addr_t) < 8) && l) {
298 			/* Above 32-bit boundary; try to reallocate */
299 			res->flags |= IORESOURCE_UNSET;
300 			res->start = 0;
301 			res->end = sz64 - 1;
302 			pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n",
303 				 res_name, (unsigned long long)l64);
304 			goto out;
305 		}
306 	}
307 
308 	region.start = l64;
309 	region.end = l64 + sz64 - 1;
310 
311 	pcibios_bus_to_resource(dev->bus, res, &region);
312 	pcibios_resource_to_bus(dev->bus, &inverted_region, res);
313 
314 	/*
315 	 * If "A" is a BAR value (a bus address), "bus_to_resource(A)" is
316 	 * the corresponding resource address (the physical address used by
317 	 * the CPU.  Converting that resource address back to a bus address
318 	 * should yield the original BAR value:
319 	 *
320 	 *     resource_to_bus(bus_to_resource(A)) == A
321 	 *
322 	 * If it doesn't, CPU accesses to "bus_to_resource(A)" will not
323 	 * be claimed by the device.
324 	 */
325 	if (inverted_region.start != region.start) {
326 		res->flags |= IORESOURCE_UNSET;
327 		res->start = 0;
328 		res->end = region.end - region.start;
329 		pci_info(dev, "%s: initial BAR value %#010llx invalid\n",
330 			 res_name, (unsigned long long)region.start);
331 	}
332 
333 	goto out;
334 
335 
336 fail:
337 	res->flags = 0;
338 out:
339 	if (res->flags)
340 		pci_info(dev, "%s %pR\n", res_name, res);
341 
342 	return (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
343 }
344 
pci_read_bases(struct pci_dev * dev,unsigned int howmany,int rom)345 static __always_inline void pci_read_bases(struct pci_dev *dev,
346 					   unsigned int howmany, int rom)
347 {
348 	u32 rombar, stdbars[PCI_STD_NUM_BARS];
349 	unsigned int pos, reg;
350 	u16 orig_cmd;
351 
352 	BUILD_BUG_ON(statically_true(howmany > PCI_STD_NUM_BARS));
353 
354 	if (dev->non_compliant_bars)
355 		return;
356 
357 	/* Per PCIe r4.0, sec 9.3.4.1.11, the VF BARs are all RO Zero */
358 	if (dev->is_virtfn)
359 		return;
360 
361 	/* No printks while decoding is disabled! */
362 	if (!dev->mmio_always_on) {
363 		pci_read_config_word(dev, PCI_COMMAND, &orig_cmd);
364 		if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) {
365 			pci_write_config_word(dev, PCI_COMMAND,
366 				orig_cmd & ~PCI_COMMAND_DECODE_ENABLE);
367 		}
368 	}
369 
370 	__pci_size_stdbars(dev, howmany, PCI_BASE_ADDRESS_0, stdbars);
371 	if (rom)
372 		__pci_size_rom(dev, rom, &rombar);
373 
374 	if (!dev->mmio_always_on &&
375 	    (orig_cmd & PCI_COMMAND_DECODE_ENABLE))
376 		pci_write_config_word(dev, PCI_COMMAND, orig_cmd);
377 
378 	for (pos = 0; pos < howmany; pos++) {
379 		struct resource *res = &dev->resource[pos];
380 		reg = PCI_BASE_ADDRESS_0 + (pos << 2);
381 		pos += __pci_read_base(dev, pci_bar_unknown,
382 				       res, reg, &stdbars[pos]);
383 	}
384 
385 	if (rom) {
386 		struct resource *res = &dev->resource[PCI_ROM_RESOURCE];
387 		dev->rom_base_reg = rom;
388 		res->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH |
389 				IORESOURCE_READONLY | IORESOURCE_SIZEALIGN;
390 		__pci_read_base(dev, pci_bar_mem32, res, rom, &rombar);
391 	}
392 }
393 
pci_read_bridge_io(struct pci_dev * dev,struct resource * res,bool log)394 static void pci_read_bridge_io(struct pci_dev *dev, struct resource *res,
395 			       bool log)
396 {
397 	u8 io_base_lo, io_limit_lo;
398 	unsigned long io_mask, io_granularity, base, limit;
399 	struct pci_bus_region region;
400 
401 	io_mask = PCI_IO_RANGE_MASK;
402 	io_granularity = 0x1000;
403 	if (dev->io_window_1k) {
404 		/* Support 1K I/O space granularity */
405 		io_mask = PCI_IO_1K_RANGE_MASK;
406 		io_granularity = 0x400;
407 	}
408 
409 	pci_read_config_byte(dev, PCI_IO_BASE, &io_base_lo);
410 	pci_read_config_byte(dev, PCI_IO_LIMIT, &io_limit_lo);
411 	base = (io_base_lo & io_mask) << 8;
412 	limit = (io_limit_lo & io_mask) << 8;
413 
414 	if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
415 		u16 io_base_hi, io_limit_hi;
416 
417 		pci_read_config_word(dev, PCI_IO_BASE_UPPER16, &io_base_hi);
418 		pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, &io_limit_hi);
419 		base |= ((unsigned long) io_base_hi << 16);
420 		limit |= ((unsigned long) io_limit_hi << 16);
421 	}
422 
423 	res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO;
424 
425 	if (base <= limit) {
426 		region.start = base;
427 		region.end = limit + io_granularity - 1;
428 		pcibios_bus_to_resource(dev->bus, res, &region);
429 		if (log)
430 			pci_info(dev, "  bridge window %pR\n", res);
431 	} else {
432 		resource_set_range(res, 0, 0);
433 		res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
434 	}
435 }
436 
pci_read_bridge_mmio(struct pci_dev * dev,struct resource * res,bool log)437 static void pci_read_bridge_mmio(struct pci_dev *dev, struct resource *res,
438 				 bool log)
439 {
440 	u16 mem_base_lo, mem_limit_lo;
441 	unsigned long base, limit;
442 	struct pci_bus_region region;
443 
444 	pci_read_config_word(dev, PCI_MEMORY_BASE, &mem_base_lo);
445 	pci_read_config_word(dev, PCI_MEMORY_LIMIT, &mem_limit_lo);
446 	base = ((unsigned long) mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16;
447 	limit = ((unsigned long) mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16;
448 
449 	res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM;
450 
451 	if (base <= limit) {
452 		region.start = base;
453 		region.end = limit + 0xfffff;
454 		pcibios_bus_to_resource(dev->bus, res, &region);
455 		if (log)
456 			pci_info(dev, "  bridge window %pR\n", res);
457 	} else {
458 		resource_set_range(res, 0, 0);
459 		res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
460 	}
461 }
462 
pci_read_bridge_mmio_pref(struct pci_dev * dev,struct resource * res,bool log)463 static void pci_read_bridge_mmio_pref(struct pci_dev *dev, struct resource *res,
464 				      bool log)
465 {
466 	u16 mem_base_lo, mem_limit_lo;
467 	u64 base64, limit64;
468 	pci_bus_addr_t base, limit;
469 	struct pci_bus_region region;
470 
471 	pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, &mem_base_lo);
472 	pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, &mem_limit_lo);
473 	base64 = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16;
474 	limit64 = (mem_limit_lo & PCI_PREF_RANGE_MASK) << 16;
475 
476 	if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
477 		u32 mem_base_hi, mem_limit_hi;
478 
479 		pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, &mem_base_hi);
480 		pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, &mem_limit_hi);
481 
482 		/*
483 		 * Some bridges set the base > limit by default, and some
484 		 * (broken) BIOSes do not initialize them.  If we find
485 		 * this, just assume they are not being used.
486 		 */
487 		if (mem_base_hi <= mem_limit_hi) {
488 			base64 |= (u64) mem_base_hi << 32;
489 			limit64 |= (u64) mem_limit_hi << 32;
490 		}
491 	}
492 
493 	base = (pci_bus_addr_t) base64;
494 	limit = (pci_bus_addr_t) limit64;
495 
496 	if (base != base64) {
497 		pci_err(dev, "can't handle bridge window above 4GB (bus address %#010llx)\n",
498 			(unsigned long long) base64);
499 		return;
500 	}
501 
502 	res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) | IORESOURCE_MEM |
503 		     IORESOURCE_PREFETCH;
504 	if (res->flags & PCI_PREF_RANGE_TYPE_64)
505 		res->flags |= IORESOURCE_MEM_64;
506 
507 	if (base <= limit) {
508 		region.start = base;
509 		region.end = limit + 0xfffff;
510 		pcibios_bus_to_resource(dev->bus, res, &region);
511 		if (log)
512 			pci_info(dev, "  bridge window %pR\n", res);
513 	} else {
514 		resource_set_range(res, 0, 0);
515 		res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
516 	}
517 }
518 
pci_read_bridge_windows(struct pci_dev * bridge)519 static void pci_read_bridge_windows(struct pci_dev *bridge)
520 {
521 	u32 buses;
522 	u16 io;
523 	u32 pmem, tmp;
524 	struct resource res;
525 
526 	pci_read_config_dword(bridge, PCI_PRIMARY_BUS, &buses);
527 	res.flags = IORESOURCE_BUS;
528 	res.start = (buses >> 8) & 0xff;
529 	res.end = (buses >> 16) & 0xff;
530 	pci_info(bridge, "PCI bridge to %pR%s\n", &res,
531 		 bridge->transparent ? " (subtractive decode)" : "");
532 
533 	pci_read_config_word(bridge, PCI_IO_BASE, &io);
534 	if (!io) {
535 		pci_write_config_word(bridge, PCI_IO_BASE, 0xe0f0);
536 		pci_read_config_word(bridge, PCI_IO_BASE, &io);
537 		pci_write_config_word(bridge, PCI_IO_BASE, 0x0);
538 	}
539 	if (io) {
540 		bridge->io_window = 1;
541 		pci_read_bridge_io(bridge, &res, true);
542 	}
543 
544 	pci_read_bridge_mmio(bridge, &res, true);
545 
546 	/*
547 	 * DECchip 21050 pass 2 errata: the bridge may miss an address
548 	 * disconnect boundary by one PCI data phase.  Workaround: do not
549 	 * use prefetching on this device.
550 	 */
551 	if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == 0x0001)
552 		return;
553 
554 	pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem);
555 	if (!pmem) {
556 		pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE,
557 					       0xffe0fff0);
558 		pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem);
559 		pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE, 0x0);
560 	}
561 	if (!pmem)
562 		return;
563 
564 	bridge->pref_window = 1;
565 
566 	if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
567 
568 		/*
569 		 * Bridge claims to have a 64-bit prefetchable memory
570 		 * window; verify that the upper bits are actually
571 		 * writable.
572 		 */
573 		pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &pmem);
574 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32,
575 				       0xffffffff);
576 		pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &tmp);
577 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, pmem);
578 		if (tmp)
579 			bridge->pref_64_window = 1;
580 	}
581 
582 	pci_read_bridge_mmio_pref(bridge, &res, true);
583 }
584 
pci_read_bridge_bases(struct pci_bus * child)585 void pci_read_bridge_bases(struct pci_bus *child)
586 {
587 	struct pci_dev *dev = child->self;
588 	struct resource *res;
589 	int i;
590 
591 	if (pci_is_root_bus(child))	/* It's a host bus, nothing to read */
592 		return;
593 
594 	pci_info(dev, "PCI bridge to %pR%s\n",
595 		 &child->busn_res,
596 		 dev->transparent ? " (subtractive decode)" : "");
597 
598 	pci_bus_remove_resources(child);
599 	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
600 		child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i];
601 
602 	pci_read_bridge_io(child->self,
603 			   child->resource[PCI_BUS_BRIDGE_IO_WINDOW], false);
604 	pci_read_bridge_mmio(child->self,
605 			     child->resource[PCI_BUS_BRIDGE_MEM_WINDOW], false);
606 	pci_read_bridge_mmio_pref(child->self,
607 				  child->resource[PCI_BUS_BRIDGE_PREF_MEM_WINDOW],
608 				  false);
609 
610 	if (!dev->transparent)
611 		return;
612 
613 	pci_bus_for_each_resource(child->parent, res) {
614 		if (!res || !res->flags)
615 			continue;
616 
617 		pci_bus_add_resource(child, res);
618 		pci_info(dev, "  bridge window %pR (subtractive decode)\n", res);
619 	}
620 }
621 
pci_alloc_bus(struct pci_bus * parent)622 static struct pci_bus *pci_alloc_bus(struct pci_bus *parent)
623 {
624 	struct pci_bus *b;
625 
626 	b = kzalloc(sizeof(*b), GFP_KERNEL);
627 	if (!b)
628 		return NULL;
629 
630 	INIT_LIST_HEAD(&b->node);
631 	INIT_LIST_HEAD(&b->children);
632 	INIT_LIST_HEAD(&b->devices);
633 	INIT_LIST_HEAD(&b->slots);
634 	INIT_LIST_HEAD(&b->resources);
635 	b->max_bus_speed = PCI_SPEED_UNKNOWN;
636 	b->cur_bus_speed = PCI_SPEED_UNKNOWN;
637 #ifdef CONFIG_PCI_DOMAINS_GENERIC
638 	if (parent)
639 		b->domain_nr = parent->domain_nr;
640 #endif
641 	return b;
642 }
643 
pci_release_host_bridge_dev(struct device * dev)644 static void pci_release_host_bridge_dev(struct device *dev)
645 {
646 	struct pci_host_bridge *bridge = to_pci_host_bridge(dev);
647 
648 	if (bridge->release_fn)
649 		bridge->release_fn(bridge);
650 
651 	pci_free_resource_list(&bridge->windows);
652 	pci_free_resource_list(&bridge->dma_ranges);
653 	kfree(bridge);
654 }
655 
pci_init_host_bridge(struct pci_host_bridge * bridge)656 static void pci_init_host_bridge(struct pci_host_bridge *bridge)
657 {
658 	INIT_LIST_HEAD(&bridge->windows);
659 	INIT_LIST_HEAD(&bridge->dma_ranges);
660 
661 	/*
662 	 * We assume we can manage these PCIe features.  Some systems may
663 	 * reserve these for use by the platform itself, e.g., an ACPI BIOS
664 	 * may implement its own AER handling and use _OSC to prevent the
665 	 * OS from interfering.
666 	 */
667 	bridge->native_aer = 1;
668 	bridge->native_pcie_hotplug = 1;
669 	bridge->native_shpc_hotplug = 1;
670 	bridge->native_pme = 1;
671 	bridge->native_ltr = 1;
672 	bridge->native_dpc = 1;
673 	bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET;
674 	bridge->native_cxl_error = 1;
675 
676 	device_initialize(&bridge->dev);
677 }
678 
pci_alloc_host_bridge(size_t priv)679 struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
680 {
681 	struct pci_host_bridge *bridge;
682 
683 	bridge = kzalloc(sizeof(*bridge) + priv, GFP_KERNEL);
684 	if (!bridge)
685 		return NULL;
686 
687 	pci_init_host_bridge(bridge);
688 	bridge->dev.release = pci_release_host_bridge_dev;
689 
690 	return bridge;
691 }
692 EXPORT_SYMBOL(pci_alloc_host_bridge);
693 
devm_pci_alloc_host_bridge_release(void * data)694 static void devm_pci_alloc_host_bridge_release(void *data)
695 {
696 	pci_free_host_bridge(data);
697 }
698 
devm_pci_alloc_host_bridge(struct device * dev,size_t priv)699 struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev,
700 						   size_t priv)
701 {
702 	int ret;
703 	struct pci_host_bridge *bridge;
704 
705 	bridge = pci_alloc_host_bridge(priv);
706 	if (!bridge)
707 		return NULL;
708 
709 	bridge->dev.parent = dev;
710 
711 	ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release,
712 				       bridge);
713 	if (ret)
714 		return NULL;
715 
716 	ret = devm_of_pci_bridge_init(dev, bridge);
717 	if (ret)
718 		return NULL;
719 
720 	return bridge;
721 }
722 EXPORT_SYMBOL(devm_pci_alloc_host_bridge);
723 
pci_free_host_bridge(struct pci_host_bridge * bridge)724 void pci_free_host_bridge(struct pci_host_bridge *bridge)
725 {
726 	put_device(&bridge->dev);
727 }
728 EXPORT_SYMBOL(pci_free_host_bridge);
729 
730 /* Indexed by PCI_X_SSTATUS_FREQ (secondary bus mode and frequency) */
731 static const unsigned char pcix_bus_speed[] = {
732 	PCI_SPEED_UNKNOWN,		/* 0 */
733 	PCI_SPEED_66MHz_PCIX,		/* 1 */
734 	PCI_SPEED_100MHz_PCIX,		/* 2 */
735 	PCI_SPEED_133MHz_PCIX,		/* 3 */
736 	PCI_SPEED_UNKNOWN,		/* 4 */
737 	PCI_SPEED_66MHz_PCIX_ECC,	/* 5 */
738 	PCI_SPEED_100MHz_PCIX_ECC,	/* 6 */
739 	PCI_SPEED_133MHz_PCIX_ECC,	/* 7 */
740 	PCI_SPEED_UNKNOWN,		/* 8 */
741 	PCI_SPEED_66MHz_PCIX_266,	/* 9 */
742 	PCI_SPEED_100MHz_PCIX_266,	/* A */
743 	PCI_SPEED_133MHz_PCIX_266,	/* B */
744 	PCI_SPEED_UNKNOWN,		/* C */
745 	PCI_SPEED_66MHz_PCIX_533,	/* D */
746 	PCI_SPEED_100MHz_PCIX_533,	/* E */
747 	PCI_SPEED_133MHz_PCIX_533	/* F */
748 };
749 
750 /* Indexed by PCI_EXP_LNKCAP_SLS, PCI_EXP_LNKSTA_CLS */
751 const unsigned char pcie_link_speed[] = {
752 	PCI_SPEED_UNKNOWN,		/* 0 */
753 	PCIE_SPEED_2_5GT,		/* 1 */
754 	PCIE_SPEED_5_0GT,		/* 2 */
755 	PCIE_SPEED_8_0GT,		/* 3 */
756 	PCIE_SPEED_16_0GT,		/* 4 */
757 	PCIE_SPEED_32_0GT,		/* 5 */
758 	PCIE_SPEED_64_0GT,		/* 6 */
759 	PCI_SPEED_UNKNOWN,		/* 7 */
760 	PCI_SPEED_UNKNOWN,		/* 8 */
761 	PCI_SPEED_UNKNOWN,		/* 9 */
762 	PCI_SPEED_UNKNOWN,		/* A */
763 	PCI_SPEED_UNKNOWN,		/* B */
764 	PCI_SPEED_UNKNOWN,		/* C */
765 	PCI_SPEED_UNKNOWN,		/* D */
766 	PCI_SPEED_UNKNOWN,		/* E */
767 	PCI_SPEED_UNKNOWN		/* F */
768 };
769 EXPORT_SYMBOL_GPL(pcie_link_speed);
770 
pci_speed_string(enum pci_bus_speed speed)771 const char *pci_speed_string(enum pci_bus_speed speed)
772 {
773 	/* Indexed by the pci_bus_speed enum */
774 	static const char *speed_strings[] = {
775 	    "33 MHz PCI",		/* 0x00 */
776 	    "66 MHz PCI",		/* 0x01 */
777 	    "66 MHz PCI-X",		/* 0x02 */
778 	    "100 MHz PCI-X",		/* 0x03 */
779 	    "133 MHz PCI-X",		/* 0x04 */
780 	    NULL,			/* 0x05 */
781 	    NULL,			/* 0x06 */
782 	    NULL,			/* 0x07 */
783 	    NULL,			/* 0x08 */
784 	    "66 MHz PCI-X 266",		/* 0x09 */
785 	    "100 MHz PCI-X 266",	/* 0x0a */
786 	    "133 MHz PCI-X 266",	/* 0x0b */
787 	    "Unknown AGP",		/* 0x0c */
788 	    "1x AGP",			/* 0x0d */
789 	    "2x AGP",			/* 0x0e */
790 	    "4x AGP",			/* 0x0f */
791 	    "8x AGP",			/* 0x10 */
792 	    "66 MHz PCI-X 533",		/* 0x11 */
793 	    "100 MHz PCI-X 533",	/* 0x12 */
794 	    "133 MHz PCI-X 533",	/* 0x13 */
795 	    "2.5 GT/s PCIe",		/* 0x14 */
796 	    "5.0 GT/s PCIe",		/* 0x15 */
797 	    "8.0 GT/s PCIe",		/* 0x16 */
798 	    "16.0 GT/s PCIe",		/* 0x17 */
799 	    "32.0 GT/s PCIe",		/* 0x18 */
800 	    "64.0 GT/s PCIe",		/* 0x19 */
801 	};
802 
803 	if (speed < ARRAY_SIZE(speed_strings))
804 		return speed_strings[speed];
805 	return "Unknown";
806 }
807 EXPORT_SYMBOL_GPL(pci_speed_string);
808 
pcie_update_link_speed(struct pci_bus * bus)809 void pcie_update_link_speed(struct pci_bus *bus)
810 {
811 	struct pci_dev *bridge = bus->self;
812 	u16 linksta, linksta2;
813 
814 	pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &linksta);
815 	pcie_capability_read_word(bridge, PCI_EXP_LNKSTA2, &linksta2);
816 	__pcie_update_link_speed(bus, linksta, linksta2);
817 }
818 EXPORT_SYMBOL_GPL(pcie_update_link_speed);
819 
820 static unsigned char agp_speeds[] = {
821 	AGP_UNKNOWN,
822 	AGP_1X,
823 	AGP_2X,
824 	AGP_4X,
825 	AGP_8X
826 };
827 
agp_speed(int agp3,int agpstat)828 static enum pci_bus_speed agp_speed(int agp3, int agpstat)
829 {
830 	int index = 0;
831 
832 	if (agpstat & 4)
833 		index = 3;
834 	else if (agpstat & 2)
835 		index = 2;
836 	else if (agpstat & 1)
837 		index = 1;
838 	else
839 		goto out;
840 
841 	if (agp3) {
842 		index += 2;
843 		if (index == 5)
844 			index = 0;
845 	}
846 
847  out:
848 	return agp_speeds[index];
849 }
850 
pci_set_bus_speed(struct pci_bus * bus)851 static void pci_set_bus_speed(struct pci_bus *bus)
852 {
853 	struct pci_dev *bridge = bus->self;
854 	int pos;
855 
856 	pos = pci_find_capability(bridge, PCI_CAP_ID_AGP);
857 	if (!pos)
858 		pos = pci_find_capability(bridge, PCI_CAP_ID_AGP3);
859 	if (pos) {
860 		u32 agpstat, agpcmd;
861 
862 		pci_read_config_dword(bridge, pos + PCI_AGP_STATUS, &agpstat);
863 		bus->max_bus_speed = agp_speed(agpstat & 8, agpstat & 7);
864 
865 		pci_read_config_dword(bridge, pos + PCI_AGP_COMMAND, &agpcmd);
866 		bus->cur_bus_speed = agp_speed(agpstat & 8, agpcmd & 7);
867 	}
868 
869 	pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
870 	if (pos) {
871 		u16 status;
872 		enum pci_bus_speed max;
873 
874 		pci_read_config_word(bridge, pos + PCI_X_BRIDGE_SSTATUS,
875 				     &status);
876 
877 		if (status & PCI_X_SSTATUS_533MHZ) {
878 			max = PCI_SPEED_133MHz_PCIX_533;
879 		} else if (status & PCI_X_SSTATUS_266MHZ) {
880 			max = PCI_SPEED_133MHz_PCIX_266;
881 		} else if (status & PCI_X_SSTATUS_133MHZ) {
882 			if ((status & PCI_X_SSTATUS_VERS) == PCI_X_SSTATUS_V2)
883 				max = PCI_SPEED_133MHz_PCIX_ECC;
884 			else
885 				max = PCI_SPEED_133MHz_PCIX;
886 		} else {
887 			max = PCI_SPEED_66MHz_PCIX;
888 		}
889 
890 		bus->max_bus_speed = max;
891 		bus->cur_bus_speed =
892 			pcix_bus_speed[FIELD_GET(PCI_X_SSTATUS_FREQ, status)];
893 
894 		return;
895 	}
896 
897 	if (pci_is_pcie(bridge)) {
898 		u32 linkcap;
899 
900 		pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &linkcap);
901 		bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS];
902 
903 		pcie_update_link_speed(bus);
904 	}
905 }
906 
pci_host_bridge_msi_domain(struct pci_bus * bus)907 static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus)
908 {
909 	struct irq_domain *d;
910 
911 	/* If the host bridge driver sets a MSI domain of the bridge, use it */
912 	d = dev_get_msi_domain(bus->bridge);
913 
914 	/*
915 	 * Any firmware interface that can resolve the msi_domain
916 	 * should be called from here.
917 	 */
918 	if (!d)
919 		d = pci_host_bridge_of_msi_domain(bus);
920 	if (!d)
921 		d = pci_host_bridge_acpi_msi_domain(bus);
922 
923 	/*
924 	 * If no IRQ domain was found via the OF tree, try looking it up
925 	 * directly through the fwnode_handle.
926 	 */
927 	if (!d) {
928 		struct fwnode_handle *fwnode = pci_root_bus_fwnode(bus);
929 
930 		if (fwnode)
931 			d = irq_find_matching_fwnode(fwnode,
932 						     DOMAIN_BUS_PCI_MSI);
933 	}
934 
935 	return d;
936 }
937 
pci_set_bus_msi_domain(struct pci_bus * bus)938 static void pci_set_bus_msi_domain(struct pci_bus *bus)
939 {
940 	struct irq_domain *d;
941 	struct pci_bus *b;
942 
943 	/*
944 	 * The bus can be a root bus, a subordinate bus, or a virtual bus
945 	 * created by an SR-IOV device.  Walk up to the first bridge device
946 	 * found or derive the domain from the host bridge.
947 	 */
948 	for (b = bus, d = NULL; !d && !pci_is_root_bus(b); b = b->parent) {
949 		if (b->self)
950 			d = dev_get_msi_domain(&b->self->dev);
951 	}
952 
953 	if (!d)
954 		d = pci_host_bridge_msi_domain(b);
955 
956 	dev_set_msi_domain(&bus->dev, d);
957 }
958 
pci_preserve_config(struct pci_host_bridge * host_bridge)959 static bool pci_preserve_config(struct pci_host_bridge *host_bridge)
960 {
961 	if (pci_acpi_preserve_config(host_bridge))
962 		return true;
963 
964 	if (host_bridge->dev.parent && host_bridge->dev.parent->of_node)
965 		return of_pci_preserve_config(host_bridge->dev.parent->of_node);
966 
967 	return false;
968 }
969 
pci_register_host_bridge(struct pci_host_bridge * bridge)970 static int pci_register_host_bridge(struct pci_host_bridge *bridge)
971 {
972 	struct device *parent = bridge->dev.parent;
973 	struct resource_entry *window, *next, *n;
974 	struct pci_bus *bus, *b;
975 	resource_size_t offset, next_offset;
976 	LIST_HEAD(resources);
977 	struct resource *res, *next_res;
978 	bool bus_registered = false;
979 	char addr[64], *fmt;
980 	const char *name;
981 	int err;
982 
983 	bus = pci_alloc_bus(NULL);
984 	if (!bus)
985 		return -ENOMEM;
986 
987 	bridge->bus = bus;
988 
989 	bus->sysdata = bridge->sysdata;
990 	bus->ops = bridge->ops;
991 	bus->number = bus->busn_res.start = bridge->busnr;
992 #ifdef CONFIG_PCI_DOMAINS_GENERIC
993 	if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
994 		bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
995 	else
996 		bus->domain_nr = bridge->domain_nr;
997 	if (bus->domain_nr < 0) {
998 		err = bus->domain_nr;
999 		goto free;
1000 	}
1001 #endif
1002 
1003 	b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
1004 	if (b) {
1005 		/* Ignore it if we already got here via a different bridge */
1006 		dev_dbg(&b->dev, "bus already known\n");
1007 		err = -EEXIST;
1008 		goto free;
1009 	}
1010 
1011 	dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus),
1012 		     bridge->busnr);
1013 
1014 	err = pcibios_root_bridge_prepare(bridge);
1015 	if (err)
1016 		goto free;
1017 
1018 	/* Temporarily move resources off the list */
1019 	list_splice_init(&bridge->windows, &resources);
1020 	err = device_add(&bridge->dev);
1021 	if (err)
1022 		goto free;
1023 
1024 	bus->bridge = get_device(&bridge->dev);
1025 	device_enable_async_suspend(bus->bridge);
1026 	pci_set_bus_of_node(bus);
1027 	pci_set_bus_msi_domain(bus);
1028 	if (bridge->msi_domain && !dev_get_msi_domain(&bus->dev) &&
1029 	    !pci_host_of_has_msi_map(parent))
1030 		bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI;
1031 
1032 	if (!parent)
1033 		set_dev_node(bus->bridge, pcibus_to_node(bus));
1034 
1035 	bus->dev.class = &pcibus_class;
1036 	bus->dev.parent = bus->bridge;
1037 
1038 	dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);
1039 	name = dev_name(&bus->dev);
1040 
1041 	err = device_register(&bus->dev);
1042 	bus_registered = true;
1043 	if (err)
1044 		goto unregister;
1045 
1046 	pcibios_add_bus(bus);
1047 
1048 	if (bus->ops->add_bus) {
1049 		err = bus->ops->add_bus(bus);
1050 		if (WARN_ON(err < 0))
1051 			dev_err(&bus->dev, "failed to add bus: %d\n", err);
1052 	}
1053 
1054 	/* Create legacy_io and legacy_mem files for this bus */
1055 	pci_create_legacy_files(bus);
1056 
1057 	if (parent)
1058 		dev_info(parent, "PCI host bridge to bus %s\n", name);
1059 	else
1060 		pr_info("PCI host bridge to bus %s\n", name);
1061 
1062 	if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE)
1063 		dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n");
1064 
1065 	/* Check if the boot configuration by FW needs to be preserved */
1066 	bridge->preserve_config = pci_preserve_config(bridge);
1067 
1068 	/* Coalesce contiguous windows */
1069 	resource_list_for_each_entry_safe(window, n, &resources) {
1070 		if (list_is_last(&window->node, &resources))
1071 			break;
1072 
1073 		next = list_next_entry(window, node);
1074 		offset = window->offset;
1075 		res = window->res;
1076 		next_offset = next->offset;
1077 		next_res = next->res;
1078 
1079 		if (res->flags != next_res->flags || offset != next_offset)
1080 			continue;
1081 
1082 		if (res->end + 1 == next_res->start) {
1083 			next_res->start = res->start;
1084 			res->flags = res->start = res->end = 0;
1085 		}
1086 	}
1087 
1088 	/* Add initial resources to the bus */
1089 	resource_list_for_each_entry_safe(window, n, &resources) {
1090 		offset = window->offset;
1091 		res = window->res;
1092 		if (!res->flags && !res->start && !res->end) {
1093 			release_resource(res);
1094 			resource_list_destroy_entry(window);
1095 			continue;
1096 		}
1097 
1098 		list_move_tail(&window->node, &bridge->windows);
1099 
1100 		if (res->flags & IORESOURCE_BUS)
1101 			pci_bus_insert_busn_res(bus, bus->number, res->end);
1102 		else
1103 			pci_bus_add_resource(bus, res);
1104 
1105 		if (offset) {
1106 			if (resource_type(res) == IORESOURCE_IO)
1107 				fmt = " (bus address [%#06llx-%#06llx])";
1108 			else
1109 				fmt = " (bus address [%#010llx-%#010llx])";
1110 
1111 			snprintf(addr, sizeof(addr), fmt,
1112 				 (unsigned long long)(res->start - offset),
1113 				 (unsigned long long)(res->end - offset));
1114 		} else
1115 			addr[0] = '\0';
1116 
1117 		dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
1118 	}
1119 
1120 	of_pci_make_host_bridge_node(bridge);
1121 
1122 	down_write(&pci_bus_sem);
1123 	list_add_tail(&bus->node, &pci_root_buses);
1124 	up_write(&pci_bus_sem);
1125 
1126 	return 0;
1127 
1128 unregister:
1129 	put_device(&bridge->dev);
1130 	device_del(&bridge->dev);
1131 free:
1132 #ifdef CONFIG_PCI_DOMAINS_GENERIC
1133 	pci_bus_release_domain_nr(parent, bus->domain_nr);
1134 #endif
1135 	if (bus_registered)
1136 		put_device(&bus->dev);
1137 	else
1138 		kfree(bus);
1139 
1140 	return err;
1141 }
1142 
pci_bridge_child_ext_cfg_accessible(struct pci_dev * bridge)1143 static bool pci_bridge_child_ext_cfg_accessible(struct pci_dev *bridge)
1144 {
1145 	int pos;
1146 	u32 status;
1147 
1148 	/*
1149 	 * If extended config space isn't accessible on a bridge's primary
1150 	 * bus, we certainly can't access it on the secondary bus.
1151 	 */
1152 	if (bridge->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1153 		return false;
1154 
1155 	/*
1156 	 * PCIe Root Ports and switch ports are PCIe on both sides, so if
1157 	 * extended config space is accessible on the primary, it's also
1158 	 * accessible on the secondary.
1159 	 */
1160 	if (pci_is_pcie(bridge) &&
1161 	    (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT ||
1162 	     pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM ||
1163 	     pci_pcie_type(bridge) == PCI_EXP_TYPE_DOWNSTREAM))
1164 		return true;
1165 
1166 	/*
1167 	 * For the other bridge types:
1168 	 *   - PCI-to-PCI bridges
1169 	 *   - PCIe-to-PCI/PCI-X forward bridges
1170 	 *   - PCI/PCI-X-to-PCIe reverse bridges
1171 	 * extended config space on the secondary side is only accessible
1172 	 * if the bridge supports PCI-X Mode 2.
1173 	 */
1174 	pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
1175 	if (!pos)
1176 		return false;
1177 
1178 	pci_read_config_dword(bridge, pos + PCI_X_STATUS, &status);
1179 	return status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ);
1180 }
1181 
pci_alloc_child_bus(struct pci_bus * parent,struct pci_dev * bridge,int busnr)1182 static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
1183 					   struct pci_dev *bridge, int busnr)
1184 {
1185 	struct pci_bus *child;
1186 	struct pci_host_bridge *host;
1187 	int i;
1188 	int ret;
1189 
1190 	/* Allocate a new bus and inherit stuff from the parent */
1191 	child = pci_alloc_bus(parent);
1192 	if (!child)
1193 		return NULL;
1194 
1195 	child->parent = parent;
1196 	child->sysdata = parent->sysdata;
1197 	child->bus_flags = parent->bus_flags;
1198 
1199 	host = pci_find_host_bridge(parent);
1200 	if (host->child_ops)
1201 		child->ops = host->child_ops;
1202 	else
1203 		child->ops = parent->ops;
1204 
1205 	/*
1206 	 * Initialize some portions of the bus device, but don't register
1207 	 * it now as the parent is not properly set up yet.
1208 	 */
1209 	child->dev.class = &pcibus_class;
1210 	dev_set_name(&child->dev, "%04x:%02x", pci_domain_nr(child), busnr);
1211 
1212 	/* Set up the primary, secondary and subordinate bus numbers */
1213 	child->number = child->busn_res.start = busnr;
1214 	child->primary = parent->busn_res.start;
1215 	child->busn_res.end = 0xff;
1216 
1217 	if (!bridge) {
1218 		child->dev.parent = parent->bridge;
1219 		goto add_dev;
1220 	}
1221 
1222 	child->self = bridge;
1223 	child->bridge = get_device(&bridge->dev);
1224 	child->dev.parent = child->bridge;
1225 	pci_set_bus_of_node(child);
1226 	pci_set_bus_speed(child);
1227 
1228 	/*
1229 	 * Check whether extended config space is accessible on the child
1230 	 * bus.  Note that we currently assume it is always accessible on
1231 	 * the root bus.
1232 	 */
1233 	if (!pci_bridge_child_ext_cfg_accessible(bridge)) {
1234 		child->bus_flags |= PCI_BUS_FLAGS_NO_EXTCFG;
1235 		pci_info(child, "extended config space not accessible\n");
1236 	}
1237 
1238 	/* Set up default resource pointers and names */
1239 	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
1240 		child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
1241 		child->resource[i]->name = child->name;
1242 	}
1243 	bridge->subordinate = child;
1244 
1245 add_dev:
1246 	pci_set_bus_msi_domain(child);
1247 	ret = device_register(&child->dev);
1248 	if (WARN_ON(ret < 0)) {
1249 		put_device(&child->dev);
1250 		return NULL;
1251 	}
1252 
1253 	pcibios_add_bus(child);
1254 
1255 	if (child->ops->add_bus) {
1256 		ret = child->ops->add_bus(child);
1257 		if (WARN_ON(ret < 0))
1258 			dev_err(&child->dev, "failed to add bus: %d\n", ret);
1259 	}
1260 
1261 	/* Create legacy_io and legacy_mem files for this bus */
1262 	pci_create_legacy_files(child);
1263 
1264 	return child;
1265 }
1266 
pci_add_new_bus(struct pci_bus * parent,struct pci_dev * dev,int busnr)1267 struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev,
1268 				int busnr)
1269 {
1270 	struct pci_bus *child;
1271 
1272 	child = pci_alloc_child_bus(parent, dev, busnr);
1273 	if (child) {
1274 		down_write(&pci_bus_sem);
1275 		list_add_tail(&child->node, &parent->children);
1276 		up_write(&pci_bus_sem);
1277 	}
1278 	return child;
1279 }
1280 EXPORT_SYMBOL(pci_add_new_bus);
1281 
pci_enable_rrs_sv(struct pci_dev * pdev)1282 static void pci_enable_rrs_sv(struct pci_dev *pdev)
1283 {
1284 	u16 root_cap = 0;
1285 
1286 	/* Enable Configuration RRS Software Visibility if supported */
1287 	pcie_capability_read_word(pdev, PCI_EXP_RTCAP, &root_cap);
1288 	if (root_cap & PCI_EXP_RTCAP_RRS_SV) {
1289 		pcie_capability_set_word(pdev, PCI_EXP_RTCTL,
1290 					 PCI_EXP_RTCTL_RRS_SVE);
1291 		pdev->config_rrs_sv = 1;
1292 	}
1293 }
1294 
1295 static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
1296 					      unsigned int available_buses);
1297 /**
1298  * pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus
1299  * numbers from EA capability.
1300  * @dev: Bridge
1301  * @sec: updated with secondary bus number from EA
1302  * @sub: updated with subordinate bus number from EA
1303  *
1304  * If @dev is a bridge with EA capability that specifies valid secondary
1305  * and subordinate bus numbers, return true with the bus numbers in @sec
1306  * and @sub.  Otherwise return false.
1307  */
pci_ea_fixed_busnrs(struct pci_dev * dev,u8 * sec,u8 * sub)1308 static bool pci_ea_fixed_busnrs(struct pci_dev *dev, u8 *sec, u8 *sub)
1309 {
1310 	int ea, offset;
1311 	u32 dw;
1312 	u8 ea_sec, ea_sub;
1313 
1314 	if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
1315 		return false;
1316 
1317 	/* find PCI EA capability in list */
1318 	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
1319 	if (!ea)
1320 		return false;
1321 
1322 	offset = ea + PCI_EA_FIRST_ENT;
1323 	pci_read_config_dword(dev, offset, &dw);
1324 	ea_sec = FIELD_GET(PCI_EA_SEC_BUS_MASK, dw);
1325 	ea_sub = FIELD_GET(PCI_EA_SUB_BUS_MASK, dw);
1326 	if (ea_sec  == 0 || ea_sub < ea_sec)
1327 		return false;
1328 
1329 	*sec = ea_sec;
1330 	*sub = ea_sub;
1331 	return true;
1332 }
1333 
1334 /*
1335  * pci_scan_bridge_extend() - Scan buses behind a bridge
1336  * @bus: Parent bus the bridge is on
1337  * @dev: Bridge itself
1338  * @max: Starting subordinate number of buses behind this bridge
1339  * @available_buses: Total number of buses available for this bridge and
1340  *		     the devices below. After the minimal bus space has
1341  *		     been allocated the remaining buses will be
1342  *		     distributed equally between hotplug-capable bridges.
1343  * @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1344  *        that need to be reconfigured.
1345  *
1346  * If it's a bridge, configure it and scan the bus behind it.
1347  * For CardBus bridges, we don't scan behind as the devices will
1348  * be handled by the bridge driver itself.
1349  *
1350  * We need to process bridges in two passes -- first we scan those
1351  * already configured by the BIOS and after we are done with all of
1352  * them, we proceed to assigning numbers to the remaining buses in
1353  * order to avoid overlaps between old and new bus numbers.
1354  *
1355  * Return: New subordinate number covering all buses behind this bridge.
1356  */
pci_scan_bridge_extend(struct pci_bus * bus,struct pci_dev * dev,int max,unsigned int available_buses,int pass)1357 static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
1358 				  int max, unsigned int available_buses,
1359 				  int pass)
1360 {
1361 	struct pci_bus *child;
1362 	int is_cardbus = (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS);
1363 	u32 buses, i, j = 0;
1364 	u16 bctl;
1365 	u8 primary, secondary, subordinate;
1366 	int broken = 0;
1367 	bool fixed_buses;
1368 	u8 fixed_sec, fixed_sub;
1369 	int next_busnr;
1370 
1371 	/*
1372 	 * Make sure the bridge is powered on to be able to access config
1373 	 * space of devices below it.
1374 	 */
1375 	pm_runtime_get_sync(&dev->dev);
1376 
1377 	pci_read_config_dword(dev, PCI_PRIMARY_BUS, &buses);
1378 	primary = buses & 0xFF;
1379 	secondary = (buses >> 8) & 0xFF;
1380 	subordinate = (buses >> 16) & 0xFF;
1381 
1382 	pci_dbg(dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
1383 		secondary, subordinate, pass);
1384 
1385 	if (!primary && (primary != bus->number) && secondary && subordinate) {
1386 		pci_warn(dev, "Primary bus is hard wired to 0\n");
1387 		primary = bus->number;
1388 	}
1389 
1390 	/* Check if setup is sensible at all */
1391 	if (!pass &&
1392 	    (primary != bus->number || secondary <= bus->number ||
1393 	     secondary > subordinate)) {
1394 		pci_info(dev, "bridge configuration invalid ([bus %02x-%02x]), reconfiguring\n",
1395 			 secondary, subordinate);
1396 		broken = 1;
1397 	}
1398 
1399 	/*
1400 	 * Disable Master-Abort Mode during probing to avoid reporting of
1401 	 * bus errors in some architectures.
1402 	 */
1403 	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &bctl);
1404 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
1405 			      bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
1406 
1407 	if ((secondary || subordinate) && !pcibios_assign_all_busses() &&
1408 	    !is_cardbus && !broken) {
1409 		unsigned int cmax, buses;
1410 
1411 		/*
1412 		 * Bus already configured by firmware, process it in the
1413 		 * first pass and just note the configuration.
1414 		 */
1415 		if (pass)
1416 			goto out;
1417 
1418 		/*
1419 		 * The bus might already exist for two reasons: Either we
1420 		 * are rescanning the bus or the bus is reachable through
1421 		 * more than one bridge. The second case can happen with
1422 		 * the i450NX chipset.
1423 		 */
1424 		child = pci_find_bus(pci_domain_nr(bus), secondary);
1425 		if (!child) {
1426 			child = pci_add_new_bus(bus, dev, secondary);
1427 			if (!child)
1428 				goto out;
1429 			child->primary = primary;
1430 			pci_bus_insert_busn_res(child, secondary, subordinate);
1431 			child->bridge_ctl = bctl;
1432 		}
1433 
1434 		buses = subordinate - secondary;
1435 		cmax = pci_scan_child_bus_extend(child, buses);
1436 		if (cmax > subordinate)
1437 			pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n",
1438 				 subordinate, cmax);
1439 
1440 		/* Subordinate should equal child->busn_res.end */
1441 		if (subordinate > max)
1442 			max = subordinate;
1443 	} else {
1444 
1445 		/*
1446 		 * We need to assign a number to this bus which we always
1447 		 * do in the second pass.
1448 		 */
1449 		if (!pass) {
1450 			if (pcibios_assign_all_busses() || broken || is_cardbus)
1451 
1452 				/*
1453 				 * Temporarily disable forwarding of the
1454 				 * configuration cycles on all bridges in
1455 				 * this bus segment to avoid possible
1456 				 * conflicts in the second pass between two
1457 				 * bridges programmed with overlapping bus
1458 				 * ranges.
1459 				 */
1460 				pci_write_config_dword(dev, PCI_PRIMARY_BUS,
1461 						       buses & ~0xffffff);
1462 			goto out;
1463 		}
1464 
1465 		/* Clear errors */
1466 		pci_write_config_word(dev, PCI_STATUS, 0xffff);
1467 
1468 		/* Read bus numbers from EA Capability (if present) */
1469 		fixed_buses = pci_ea_fixed_busnrs(dev, &fixed_sec, &fixed_sub);
1470 		if (fixed_buses)
1471 			next_busnr = fixed_sec;
1472 		else
1473 			next_busnr = max + 1;
1474 
1475 		/*
1476 		 * Prevent assigning a bus number that already exists.
1477 		 * This can happen when a bridge is hot-plugged, so in this
1478 		 * case we only re-scan this bus.
1479 		 */
1480 		child = pci_find_bus(pci_domain_nr(bus), next_busnr);
1481 		if (!child) {
1482 			child = pci_add_new_bus(bus, dev, next_busnr);
1483 			if (!child)
1484 				goto out;
1485 			pci_bus_insert_busn_res(child, next_busnr,
1486 						bus->busn_res.end);
1487 		}
1488 		max++;
1489 		if (available_buses)
1490 			available_buses--;
1491 
1492 		buses = (buses & 0xff000000)
1493 		      | ((unsigned int)(child->primary)     <<  0)
1494 		      | ((unsigned int)(child->busn_res.start)   <<  8)
1495 		      | ((unsigned int)(child->busn_res.end) << 16);
1496 
1497 		/*
1498 		 * yenta.c forces a secondary latency timer of 176.
1499 		 * Copy that behaviour here.
1500 		 */
1501 		if (is_cardbus) {
1502 			buses &= ~0xff000000;
1503 			buses |= CARDBUS_LATENCY_TIMER << 24;
1504 		}
1505 
1506 		/* We need to blast all three values with a single write */
1507 		pci_write_config_dword(dev, PCI_PRIMARY_BUS, buses);
1508 
1509 		if (!is_cardbus) {
1510 			child->bridge_ctl = bctl;
1511 			max = pci_scan_child_bus_extend(child, available_buses);
1512 		} else {
1513 
1514 			/*
1515 			 * For CardBus bridges, we leave 4 bus numbers as
1516 			 * cards with a PCI-to-PCI bridge can be inserted
1517 			 * later.
1518 			 */
1519 			for (i = 0; i < CARDBUS_RESERVE_BUSNR; i++) {
1520 				struct pci_bus *parent = bus;
1521 				if (pci_find_bus(pci_domain_nr(bus),
1522 							max+i+1))
1523 					break;
1524 				while (parent->parent) {
1525 					if ((!pcibios_assign_all_busses()) &&
1526 					    (parent->busn_res.end > max) &&
1527 					    (parent->busn_res.end <= max+i)) {
1528 						j = 1;
1529 					}
1530 					parent = parent->parent;
1531 				}
1532 				if (j) {
1533 
1534 					/*
1535 					 * Often, there are two CardBus
1536 					 * bridges -- try to leave one
1537 					 * valid bus number for each one.
1538 					 */
1539 					i /= 2;
1540 					break;
1541 				}
1542 			}
1543 			max += i;
1544 		}
1545 
1546 		/*
1547 		 * Set subordinate bus number to its real value.
1548 		 * If fixed subordinate bus number exists from EA
1549 		 * capability then use it.
1550 		 */
1551 		if (fixed_buses)
1552 			max = fixed_sub;
1553 		pci_bus_update_busn_res_end(child, max);
1554 		pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, max);
1555 	}
1556 
1557 	sprintf(child->name,
1558 		(is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
1559 		pci_domain_nr(bus), child->number);
1560 
1561 	/* Check that all devices are accessible */
1562 	while (bus->parent) {
1563 		if ((child->busn_res.end > bus->busn_res.end) ||
1564 		    (child->number > bus->busn_res.end) ||
1565 		    (child->number < bus->number) ||
1566 		    (child->busn_res.end < bus->number)) {
1567 			dev_info(&dev->dev, "devices behind bridge are unusable because %pR cannot be assigned for them\n",
1568 				 &child->busn_res);
1569 			break;
1570 		}
1571 		bus = bus->parent;
1572 	}
1573 
1574 out:
1575 	/* Clear errors in the Secondary Status Register */
1576 	pci_write_config_word(dev, PCI_SEC_STATUS, 0xffff);
1577 
1578 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl);
1579 
1580 	pm_runtime_put(&dev->dev);
1581 
1582 	return max;
1583 }
1584 
1585 /*
1586  * pci_scan_bridge() - Scan buses behind a bridge
1587  * @bus: Parent bus the bridge is on
1588  * @dev: Bridge itself
1589  * @max: Starting subordinate number of buses behind this bridge
1590  * @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1591  *        that need to be reconfigured.
1592  *
1593  * If it's a bridge, configure it and scan the bus behind it.
1594  * For CardBus bridges, we don't scan behind as the devices will
1595  * be handled by the bridge driver itself.
1596  *
1597  * We need to process bridges in two passes -- first we scan those
1598  * already configured by the BIOS and after we are done with all of
1599  * them, we proceed to assigning numbers to the remaining buses in
1600  * order to avoid overlaps between old and new bus numbers.
1601  *
1602  * Return: New subordinate number covering all buses behind this bridge.
1603  */
pci_scan_bridge(struct pci_bus * bus,struct pci_dev * dev,int max,int pass)1604 int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, int pass)
1605 {
1606 	return pci_scan_bridge_extend(bus, dev, max, 0, pass);
1607 }
1608 EXPORT_SYMBOL(pci_scan_bridge);
1609 
1610 /*
1611  * Read interrupt line and base address registers.
1612  * The architecture-dependent code can tweak these, of course.
1613  */
pci_read_irq(struct pci_dev * dev)1614 static void pci_read_irq(struct pci_dev *dev)
1615 {
1616 	unsigned char irq;
1617 
1618 	/* VFs are not allowed to use INTx, so skip the config reads */
1619 	if (dev->is_virtfn) {
1620 		dev->pin = 0;
1621 		dev->irq = 0;
1622 		return;
1623 	}
1624 
1625 	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &irq);
1626 	dev->pin = irq;
1627 	if (irq)
1628 		pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &irq);
1629 	dev->irq = irq;
1630 }
1631 
set_pcie_port_type(struct pci_dev * pdev)1632 void set_pcie_port_type(struct pci_dev *pdev)
1633 {
1634 	int pos;
1635 	u16 reg16;
1636 	u32 reg32;
1637 	int type;
1638 	struct pci_dev *parent;
1639 
1640 	pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
1641 	if (!pos)
1642 		return;
1643 
1644 	pdev->pcie_cap = pos;
1645 	pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, &reg16);
1646 	pdev->pcie_flags_reg = reg16;
1647 
1648 	type = pci_pcie_type(pdev);
1649 	if (type == PCI_EXP_TYPE_ROOT_PORT)
1650 		pci_enable_rrs_sv(pdev);
1651 
1652 	pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, &pdev->devcap);
1653 	pdev->pcie_mpss = FIELD_GET(PCI_EXP_DEVCAP_PAYLOAD, pdev->devcap);
1654 
1655 	pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &reg32);
1656 	if (reg32 & PCI_EXP_LNKCAP_DLLLARC)
1657 		pdev->link_active_reporting = 1;
1658 
1659 	parent = pci_upstream_bridge(pdev);
1660 	if (!parent)
1661 		return;
1662 
1663 	/*
1664 	 * Some systems do not identify their upstream/downstream ports
1665 	 * correctly so detect impossible configurations here and correct
1666 	 * the port type accordingly.
1667 	 */
1668 	if (type == PCI_EXP_TYPE_DOWNSTREAM) {
1669 		/*
1670 		 * If pdev claims to be downstream port but the parent
1671 		 * device is also downstream port assume pdev is actually
1672 		 * upstream port.
1673 		 */
1674 		if (pcie_downstream_port(parent)) {
1675 			pci_info(pdev, "claims to be downstream port but is acting as upstream port, correcting type\n");
1676 			pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1677 			pdev->pcie_flags_reg |= PCI_EXP_TYPE_UPSTREAM;
1678 		}
1679 	} else if (type == PCI_EXP_TYPE_UPSTREAM) {
1680 		/*
1681 		 * If pdev claims to be upstream port but the parent
1682 		 * device is also upstream port assume pdev is actually
1683 		 * downstream port.
1684 		 */
1685 		if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM) {
1686 			pci_info(pdev, "claims to be upstream port but is acting as downstream port, correcting type\n");
1687 			pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1688 			pdev->pcie_flags_reg |= PCI_EXP_TYPE_DOWNSTREAM;
1689 		}
1690 	}
1691 }
1692 
set_pcie_hotplug_bridge(struct pci_dev * pdev)1693 void set_pcie_hotplug_bridge(struct pci_dev *pdev)
1694 {
1695 	u32 reg32;
1696 
1697 	pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &reg32);
1698 	if (reg32 & PCI_EXP_SLTCAP_HPC)
1699 		pdev->is_hotplug_bridge = pdev->is_pciehp = 1;
1700 }
1701 
set_pcie_thunderbolt(struct pci_dev * dev)1702 static void set_pcie_thunderbolt(struct pci_dev *dev)
1703 {
1704 	u16 vsec;
1705 
1706 	/* Is the device part of a Thunderbolt controller? */
1707 	vsec = pci_find_vsec_capability(dev, PCI_VENDOR_ID_INTEL, PCI_VSEC_ID_INTEL_TBT);
1708 	if (vsec)
1709 		dev->is_thunderbolt = 1;
1710 }
1711 
set_pcie_untrusted(struct pci_dev * dev)1712 static void set_pcie_untrusted(struct pci_dev *dev)
1713 {
1714 	struct pci_dev *parent = pci_upstream_bridge(dev);
1715 
1716 	if (!parent)
1717 		return;
1718 	/*
1719 	 * If the upstream bridge is untrusted we treat this device as
1720 	 * untrusted as well.
1721 	 */
1722 	if (parent->untrusted) {
1723 		dev->untrusted = true;
1724 		return;
1725 	}
1726 
1727 	if (arch_pci_dev_is_removable(dev)) {
1728 		pci_dbg(dev, "marking as untrusted\n");
1729 		dev->untrusted = true;
1730 	}
1731 }
1732 
pci_set_removable(struct pci_dev * dev)1733 static void pci_set_removable(struct pci_dev *dev)
1734 {
1735 	struct pci_dev *parent = pci_upstream_bridge(dev);
1736 
1737 	if (!parent)
1738 		return;
1739 	/*
1740 	 * We (only) consider everything tunneled below an external_facing
1741 	 * device to be removable by the user. We're mainly concerned with
1742 	 * consumer platforms with user accessible thunderbolt ports that are
1743 	 * vulnerable to DMA attacks, and we expect those ports to be marked by
1744 	 * the firmware as external_facing. Devices in traditional hotplug
1745 	 * slots can technically be removed, but the expectation is that unless
1746 	 * the port is marked with external_facing, such devices are less
1747 	 * accessible to user / may not be removed by end user, and thus not
1748 	 * exposed as "removable" to userspace.
1749 	 */
1750 	if (dev_is_removable(&parent->dev)) {
1751 		dev_set_removable(&dev->dev, DEVICE_REMOVABLE);
1752 		return;
1753 	}
1754 
1755 	if (arch_pci_dev_is_removable(dev)) {
1756 		pci_dbg(dev, "marking as removable\n");
1757 		dev_set_removable(&dev->dev, DEVICE_REMOVABLE);
1758 	}
1759 }
1760 
1761 /**
1762  * pci_ext_cfg_is_aliased - Is ext config space just an alias of std config?
1763  * @dev: PCI device
1764  *
1765  * PCI Express to PCI/PCI-X Bridge Specification, rev 1.0, 4.1.4 says that
1766  * when forwarding a type1 configuration request the bridge must check that
1767  * the extended register address field is zero.  The bridge is not permitted
1768  * to forward the transactions and must handle it as an Unsupported Request.
1769  * Some bridges do not follow this rule and simply drop the extended register
1770  * bits, resulting in the standard config space being aliased, every 256
1771  * bytes across the entire configuration space.  Test for this condition by
1772  * comparing the first dword of each potential alias to the vendor/device ID.
1773  * Known offenders:
1774  *   ASM1083/1085 PCIe-to-PCI Reversible Bridge (1b21:1080, rev 01 & 03)
1775  *   AMD/ATI SBx00 PCI to PCI Bridge (1002:4384, rev 40)
1776  */
pci_ext_cfg_is_aliased(struct pci_dev * dev)1777 static bool pci_ext_cfg_is_aliased(struct pci_dev *dev)
1778 {
1779 #ifdef CONFIG_PCI_QUIRKS
1780 	int pos, ret;
1781 	u32 header, tmp;
1782 
1783 	pci_read_config_dword(dev, PCI_VENDOR_ID, &header);
1784 
1785 	for (pos = PCI_CFG_SPACE_SIZE;
1786 	     pos < PCI_CFG_SPACE_EXP_SIZE; pos += PCI_CFG_SPACE_SIZE) {
1787 		ret = pci_read_config_dword(dev, pos, &tmp);
1788 		if ((ret != PCIBIOS_SUCCESSFUL) || (header != tmp))
1789 			return false;
1790 	}
1791 
1792 	return true;
1793 #else
1794 	return false;
1795 #endif
1796 }
1797 
1798 /**
1799  * pci_cfg_space_size_ext - Get the configuration space size of the PCI device
1800  * @dev: PCI device
1801  *
1802  * Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices
1803  * have 4096 bytes.  Even if the device is capable, that doesn't mean we can
1804  * access it.  Maybe we don't have a way to generate extended config space
1805  * accesses, or the device is behind a reverse Express bridge.  So we try
1806  * reading the dword at 0x100 which must either be 0 or a valid extended
1807  * capability header.
1808  */
pci_cfg_space_size_ext(struct pci_dev * dev)1809 static int pci_cfg_space_size_ext(struct pci_dev *dev)
1810 {
1811 	u32 status;
1812 	int pos = PCI_CFG_SPACE_SIZE;
1813 
1814 	if (pci_read_config_dword(dev, pos, &status) != PCIBIOS_SUCCESSFUL)
1815 		return PCI_CFG_SPACE_SIZE;
1816 	if (PCI_POSSIBLE_ERROR(status) || pci_ext_cfg_is_aliased(dev))
1817 		return PCI_CFG_SPACE_SIZE;
1818 
1819 	return PCI_CFG_SPACE_EXP_SIZE;
1820 }
1821 
pci_cfg_space_size(struct pci_dev * dev)1822 int pci_cfg_space_size(struct pci_dev *dev)
1823 {
1824 	int pos;
1825 	u32 status;
1826 	u16 class;
1827 
1828 #ifdef CONFIG_PCI_IOV
1829 	/*
1830 	 * Per the SR-IOV specification (rev 1.1, sec 3.5), VFs are required to
1831 	 * implement a PCIe capability and therefore must implement extended
1832 	 * config space.  We can skip the NO_EXTCFG test below and the
1833 	 * reachability/aliasing test in pci_cfg_space_size_ext() by virtue of
1834 	 * the fact that the SR-IOV capability on the PF resides in extended
1835 	 * config space and must be accessible and non-aliased to have enabled
1836 	 * support for this VF.  This is a micro performance optimization for
1837 	 * systems supporting many VFs.
1838 	 */
1839 	if (dev->is_virtfn)
1840 		return PCI_CFG_SPACE_EXP_SIZE;
1841 #endif
1842 
1843 	if (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1844 		return PCI_CFG_SPACE_SIZE;
1845 
1846 	class = dev->class >> 8;
1847 	if (class == PCI_CLASS_BRIDGE_HOST)
1848 		return pci_cfg_space_size_ext(dev);
1849 
1850 	if (pci_is_pcie(dev))
1851 		return pci_cfg_space_size_ext(dev);
1852 
1853 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1854 	if (!pos)
1855 		return PCI_CFG_SPACE_SIZE;
1856 
1857 	pci_read_config_dword(dev, pos + PCI_X_STATUS, &status);
1858 	if (status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ))
1859 		return pci_cfg_space_size_ext(dev);
1860 
1861 	return PCI_CFG_SPACE_SIZE;
1862 }
1863 
pci_class(struct pci_dev * dev)1864 static u32 pci_class(struct pci_dev *dev)
1865 {
1866 	u32 class;
1867 
1868 #ifdef CONFIG_PCI_IOV
1869 	if (dev->is_virtfn)
1870 		return dev->physfn->sriov->class;
1871 #endif
1872 	pci_read_config_dword(dev, PCI_CLASS_REVISION, &class);
1873 	return class;
1874 }
1875 
pci_subsystem_ids(struct pci_dev * dev,u16 * vendor,u16 * device)1876 static void pci_subsystem_ids(struct pci_dev *dev, u16 *vendor, u16 *device)
1877 {
1878 #ifdef CONFIG_PCI_IOV
1879 	if (dev->is_virtfn) {
1880 		*vendor = dev->physfn->sriov->subsystem_vendor;
1881 		*device = dev->physfn->sriov->subsystem_device;
1882 		return;
1883 	}
1884 #endif
1885 	pci_read_config_word(dev, PCI_SUBSYSTEM_VENDOR_ID, vendor);
1886 	pci_read_config_word(dev, PCI_SUBSYSTEM_ID, device);
1887 }
1888 
pci_hdr_type(struct pci_dev * dev)1889 static u8 pci_hdr_type(struct pci_dev *dev)
1890 {
1891 	u8 hdr_type;
1892 
1893 #ifdef CONFIG_PCI_IOV
1894 	if (dev->is_virtfn)
1895 		return dev->physfn->sriov->hdr_type;
1896 #endif
1897 	pci_read_config_byte(dev, PCI_HEADER_TYPE, &hdr_type);
1898 	return hdr_type;
1899 }
1900 
1901 #define LEGACY_IO_RESOURCE	(IORESOURCE_IO | IORESOURCE_PCI_FIXED)
1902 
1903 /**
1904  * pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability
1905  * @dev: PCI device
1906  *
1907  * Test whether PCI_COMMAND_INTX_DISABLE is writable for @dev.  Check this
1908  * at enumeration-time to avoid modifying PCI_COMMAND at run-time.
1909  */
pci_intx_mask_broken(struct pci_dev * dev)1910 static int pci_intx_mask_broken(struct pci_dev *dev)
1911 {
1912 	u16 orig, toggle, new;
1913 
1914 	pci_read_config_word(dev, PCI_COMMAND, &orig);
1915 	toggle = orig ^ PCI_COMMAND_INTX_DISABLE;
1916 	pci_write_config_word(dev, PCI_COMMAND, toggle);
1917 	pci_read_config_word(dev, PCI_COMMAND, &new);
1918 
1919 	pci_write_config_word(dev, PCI_COMMAND, orig);
1920 
1921 	/*
1922 	 * PCI_COMMAND_INTX_DISABLE was reserved and read-only prior to PCI
1923 	 * r2.3, so strictly speaking, a device is not *broken* if it's not
1924 	 * writable.  But we'll live with the misnomer for now.
1925 	 */
1926 	if (new != toggle)
1927 		return 1;
1928 	return 0;
1929 }
1930 
early_dump_pci_device(struct pci_dev * pdev)1931 static void early_dump_pci_device(struct pci_dev *pdev)
1932 {
1933 	u32 value[PCI_CFG_SPACE_SIZE / sizeof(u32)];
1934 	int i;
1935 
1936 	pci_info(pdev, "config space:\n");
1937 
1938 	for (i = 0; i < ARRAY_SIZE(value); i++)
1939 		pci_read_config_dword(pdev, i * sizeof(u32), &value[i]);
1940 
1941 	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
1942 		       value, ARRAY_SIZE(value) * sizeof(u32), false);
1943 }
1944 
pci_type_str(struct pci_dev * dev)1945 static const char *pci_type_str(struct pci_dev *dev)
1946 {
1947 	static const char * const str[] = {
1948 		"PCIe Endpoint",
1949 		"PCIe Legacy Endpoint",
1950 		"PCIe unknown",
1951 		"PCIe unknown",
1952 		"PCIe Root Port",
1953 		"PCIe Switch Upstream Port",
1954 		"PCIe Switch Downstream Port",
1955 		"PCIe to PCI/PCI-X bridge",
1956 		"PCI/PCI-X to PCIe bridge",
1957 		"PCIe Root Complex Integrated Endpoint",
1958 		"PCIe Root Complex Event Collector",
1959 	};
1960 	int type;
1961 
1962 	if (pci_is_pcie(dev)) {
1963 		type = pci_pcie_type(dev);
1964 		if (type < ARRAY_SIZE(str))
1965 			return str[type];
1966 
1967 		return "PCIe unknown";
1968 	}
1969 
1970 	switch (dev->hdr_type) {
1971 	case PCI_HEADER_TYPE_NORMAL:
1972 		return "conventional PCI endpoint";
1973 	case PCI_HEADER_TYPE_BRIDGE:
1974 		return "conventional PCI bridge";
1975 	case PCI_HEADER_TYPE_CARDBUS:
1976 		return "CardBus bridge";
1977 	default:
1978 		return "conventional PCI";
1979 	}
1980 }
1981 
1982 /**
1983  * pci_setup_device - Fill in class and map information of a device
1984  * @dev: the device structure to fill
1985  *
1986  * Initialize the device structure with information about the device's
1987  * vendor,class,memory and IO-space addresses, IRQ lines etc.
1988  * Called at initialisation of the PCI subsystem and by CardBus services.
1989  * Returns 0 on success and negative if unknown type of device (not normal,
1990  * bridge or CardBus).
1991  */
pci_setup_device(struct pci_dev * dev)1992 int pci_setup_device(struct pci_dev *dev)
1993 {
1994 	u32 class;
1995 	u16 cmd;
1996 	u8 hdr_type;
1997 	int err, pos = 0;
1998 	struct pci_bus_region region;
1999 	struct resource *res;
2000 
2001 	hdr_type = pci_hdr_type(dev);
2002 
2003 	dev->sysdata = dev->bus->sysdata;
2004 	dev->dev.parent = dev->bus->bridge;
2005 	dev->dev.bus = &pci_bus_type;
2006 	dev->hdr_type = FIELD_GET(PCI_HEADER_TYPE_MASK, hdr_type);
2007 	dev->multifunction = FIELD_GET(PCI_HEADER_TYPE_MFD, hdr_type);
2008 	dev->error_state = pci_channel_io_normal;
2009 	set_pcie_port_type(dev);
2010 
2011 	err = pci_set_of_node(dev);
2012 	if (err)
2013 		return err;
2014 	pci_set_acpi_fwnode(dev);
2015 
2016 	pci_dev_assign_slot(dev);
2017 
2018 	/*
2019 	 * Assume 32-bit PCI; let 64-bit PCI cards (which are far rarer)
2020 	 * set this higher, assuming the system even supports it.
2021 	 */
2022 	dev->dma_mask = 0xffffffff;
2023 
2024 	dev_set_name(&dev->dev, "%04x:%02x:%02x.%d", pci_domain_nr(dev->bus),
2025 		     dev->bus->number, PCI_SLOT(dev->devfn),
2026 		     PCI_FUNC(dev->devfn));
2027 
2028 	class = pci_class(dev);
2029 
2030 	dev->revision = class & 0xff;
2031 	dev->class = class >> 8;		    /* upper 3 bytes */
2032 
2033 	if (pci_early_dump)
2034 		early_dump_pci_device(dev);
2035 
2036 	/* Need to have dev->class ready */
2037 	dev->cfg_size = pci_cfg_space_size(dev);
2038 
2039 	/* Need to have dev->cfg_size ready */
2040 	set_pcie_thunderbolt(dev);
2041 
2042 	set_pcie_untrusted(dev);
2043 
2044 	if (pci_is_pcie(dev))
2045 		dev->supported_speeds = pcie_get_supported_speeds(dev);
2046 
2047 	/* "Unknown power state" */
2048 	dev->current_state = PCI_UNKNOWN;
2049 
2050 	/* Early fixups, before probing the BARs */
2051 	pci_fixup_device(pci_fixup_early, dev);
2052 
2053 	pci_set_removable(dev);
2054 
2055 	pci_info(dev, "[%04x:%04x] type %02x class %#08x %s\n",
2056 		 dev->vendor, dev->device, dev->hdr_type, dev->class,
2057 		 pci_type_str(dev));
2058 
2059 	/* Device class may be changed after fixup */
2060 	class = dev->class >> 8;
2061 
2062 	if (dev->non_compliant_bars && !dev->mmio_always_on) {
2063 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
2064 		if (cmd & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY)) {
2065 			pci_info(dev, "device has non-compliant BARs; disabling IO/MEM decoding\n");
2066 			cmd &= ~PCI_COMMAND_IO;
2067 			cmd &= ~PCI_COMMAND_MEMORY;
2068 			pci_write_config_word(dev, PCI_COMMAND, cmd);
2069 		}
2070 	}
2071 
2072 	dev->broken_intx_masking = pci_intx_mask_broken(dev);
2073 
2074 	switch (dev->hdr_type) {		    /* header type */
2075 	case PCI_HEADER_TYPE_NORMAL:		    /* standard header */
2076 		if (class == PCI_CLASS_BRIDGE_PCI)
2077 			goto bad;
2078 		pci_read_irq(dev);
2079 		pci_read_bases(dev, PCI_STD_NUM_BARS, PCI_ROM_ADDRESS);
2080 
2081 		pci_subsystem_ids(dev, &dev->subsystem_vendor, &dev->subsystem_device);
2082 
2083 		/*
2084 		 * Do the ugly legacy mode stuff here rather than broken chip
2085 		 * quirk code. Legacy mode ATA controllers have fixed
2086 		 * addresses. These are not always echoed in BAR0-3, and
2087 		 * BAR0-3 in a few cases contain junk!
2088 		 */
2089 		if (class == PCI_CLASS_STORAGE_IDE) {
2090 			u8 progif;
2091 			pci_read_config_byte(dev, PCI_CLASS_PROG, &progif);
2092 			if ((progif & 1) == 0) {
2093 				region.start = 0x1F0;
2094 				region.end = 0x1F7;
2095 				res = &dev->resource[0];
2096 				res->flags = LEGACY_IO_RESOURCE;
2097 				pcibios_bus_to_resource(dev->bus, res, &region);
2098 				pci_info(dev, "BAR 0 %pR: legacy IDE quirk\n",
2099 					 res);
2100 				region.start = 0x3F6;
2101 				region.end = 0x3F6;
2102 				res = &dev->resource[1];
2103 				res->flags = LEGACY_IO_RESOURCE;
2104 				pcibios_bus_to_resource(dev->bus, res, &region);
2105 				pci_info(dev, "BAR 1 %pR: legacy IDE quirk\n",
2106 					 res);
2107 			}
2108 			if ((progif & 4) == 0) {
2109 				region.start = 0x170;
2110 				region.end = 0x177;
2111 				res = &dev->resource[2];
2112 				res->flags = LEGACY_IO_RESOURCE;
2113 				pcibios_bus_to_resource(dev->bus, res, &region);
2114 				pci_info(dev, "BAR 2 %pR: legacy IDE quirk\n",
2115 					 res);
2116 				region.start = 0x376;
2117 				region.end = 0x376;
2118 				res = &dev->resource[3];
2119 				res->flags = LEGACY_IO_RESOURCE;
2120 				pcibios_bus_to_resource(dev->bus, res, &region);
2121 				pci_info(dev, "BAR 3 %pR: legacy IDE quirk\n",
2122 					 res);
2123 			}
2124 		}
2125 		break;
2126 
2127 	case PCI_HEADER_TYPE_BRIDGE:		    /* bridge header */
2128 		/*
2129 		 * The PCI-to-PCI bridge spec requires that subtractive
2130 		 * decoding (i.e. transparent) bridge must have programming
2131 		 * interface code of 0x01.
2132 		 */
2133 		pci_read_irq(dev);
2134 		dev->transparent = ((dev->class & 0xff) == 1);
2135 		pci_read_bases(dev, 2, PCI_ROM_ADDRESS1);
2136 		pci_read_bridge_windows(dev);
2137 		set_pcie_hotplug_bridge(dev);
2138 		pos = pci_find_capability(dev, PCI_CAP_ID_SSVID);
2139 		if (pos) {
2140 			pci_read_config_word(dev, pos + PCI_SSVID_VENDOR_ID, &dev->subsystem_vendor);
2141 			pci_read_config_word(dev, pos + PCI_SSVID_DEVICE_ID, &dev->subsystem_device);
2142 		}
2143 		break;
2144 
2145 	case PCI_HEADER_TYPE_CARDBUS:		    /* CardBus bridge header */
2146 		if (class != PCI_CLASS_BRIDGE_CARDBUS)
2147 			goto bad;
2148 		pci_read_irq(dev);
2149 		pci_read_bases(dev, 1, 0);
2150 		pci_read_config_word(dev, PCI_CB_SUBSYSTEM_VENDOR_ID, &dev->subsystem_vendor);
2151 		pci_read_config_word(dev, PCI_CB_SUBSYSTEM_ID, &dev->subsystem_device);
2152 		break;
2153 
2154 	default:				    /* unknown header */
2155 		pci_err(dev, "unknown header type %02x, ignoring device\n",
2156 			dev->hdr_type);
2157 		pci_release_of_node(dev);
2158 		return -EIO;
2159 
2160 	bad:
2161 		pci_err(dev, "ignoring class %#08x (doesn't match header type %02x)\n",
2162 			dev->class, dev->hdr_type);
2163 		dev->class = PCI_CLASS_NOT_DEFINED << 8;
2164 	}
2165 
2166 	/* We found a fine healthy device, go go go... */
2167 	return 0;
2168 }
2169 
pci_configure_mps(struct pci_dev * dev)2170 static void pci_configure_mps(struct pci_dev *dev)
2171 {
2172 	struct pci_dev *bridge = pci_upstream_bridge(dev);
2173 	int mps, mpss, p_mps, rc;
2174 
2175 	if (!pci_is_pcie(dev))
2176 		return;
2177 
2178 	/* MPS and MRRS fields are of type 'RsvdP' for VFs, short-circuit out */
2179 	if (dev->is_virtfn)
2180 		return;
2181 
2182 	/*
2183 	 * For Root Complex Integrated Endpoints, program the maximum
2184 	 * supported value unless limited by the PCIE_BUS_PEER2PEER case.
2185 	 */
2186 	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
2187 		if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2188 			mps = 128;
2189 		else
2190 			mps = 128 << dev->pcie_mpss;
2191 		rc = pcie_set_mps(dev, mps);
2192 		if (rc) {
2193 			pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2194 				 mps);
2195 		}
2196 		return;
2197 	}
2198 
2199 	if (!bridge || !pci_is_pcie(bridge))
2200 		return;
2201 
2202 	mps = pcie_get_mps(dev);
2203 	p_mps = pcie_get_mps(bridge);
2204 
2205 	if (mps == p_mps)
2206 		return;
2207 
2208 	if (pcie_bus_config == PCIE_BUS_TUNE_OFF) {
2209 		pci_warn(dev, "Max Payload Size %d, but upstream %s set to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2210 			 mps, pci_name(bridge), p_mps);
2211 		return;
2212 	}
2213 
2214 	/*
2215 	 * Fancier MPS configuration is done later by
2216 	 * pcie_bus_configure_settings()
2217 	 */
2218 	if (pcie_bus_config != PCIE_BUS_DEFAULT)
2219 		return;
2220 
2221 	mpss = 128 << dev->pcie_mpss;
2222 	if (mpss < p_mps && pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
2223 		pcie_set_mps(bridge, mpss);
2224 		pci_info(dev, "Upstream bridge's Max Payload Size set to %d (was %d, max %d)\n",
2225 			 mpss, p_mps, 128 << bridge->pcie_mpss);
2226 		p_mps = pcie_get_mps(bridge);
2227 	}
2228 
2229 	rc = pcie_set_mps(dev, p_mps);
2230 	if (rc) {
2231 		pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2232 			 p_mps);
2233 		return;
2234 	}
2235 
2236 	pci_info(dev, "Max Payload Size set to %d (was %d, max %d)\n",
2237 		 p_mps, mps, mpss);
2238 }
2239 
pci_configure_extended_tags(struct pci_dev * dev,void * ign)2240 int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
2241 {
2242 	struct pci_host_bridge *host;
2243 	u32 cap;
2244 	u16 ctl;
2245 	int ret;
2246 
2247 	if (!pci_is_pcie(dev))
2248 		return 0;
2249 
2250 	ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
2251 	if (ret)
2252 		return 0;
2253 
2254 	if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
2255 		return 0;
2256 
2257 	ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
2258 	if (ret)
2259 		return 0;
2260 
2261 	host = pci_find_host_bridge(dev->bus);
2262 	if (!host)
2263 		return 0;
2264 
2265 	/*
2266 	 * If some device in the hierarchy doesn't handle Extended Tags
2267 	 * correctly, make sure they're disabled.
2268 	 */
2269 	if (host->no_ext_tags) {
2270 		if (ctl & PCI_EXP_DEVCTL_EXT_TAG) {
2271 			pci_info(dev, "disabling Extended Tags\n");
2272 			pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2273 						   PCI_EXP_DEVCTL_EXT_TAG);
2274 		}
2275 		return 0;
2276 	}
2277 
2278 	if (!(ctl & PCI_EXP_DEVCTL_EXT_TAG)) {
2279 		pci_info(dev, "enabling Extended Tags\n");
2280 		pcie_capability_set_word(dev, PCI_EXP_DEVCTL,
2281 					 PCI_EXP_DEVCTL_EXT_TAG);
2282 	}
2283 	return 0;
2284 }
2285 
2286 /**
2287  * pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
2288  * @dev: PCI device to query
2289  *
2290  * Returns true if the device has enabled relaxed ordering attribute.
2291  */
pcie_relaxed_ordering_enabled(struct pci_dev * dev)2292 bool pcie_relaxed_ordering_enabled(struct pci_dev *dev)
2293 {
2294 	u16 v;
2295 
2296 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &v);
2297 
2298 	return !!(v & PCI_EXP_DEVCTL_RELAX_EN);
2299 }
2300 EXPORT_SYMBOL(pcie_relaxed_ordering_enabled);
2301 
pci_configure_relaxed_ordering(struct pci_dev * dev)2302 static void pci_configure_relaxed_ordering(struct pci_dev *dev)
2303 {
2304 	struct pci_dev *root;
2305 
2306 	/* PCI_EXP_DEVCTL_RELAX_EN is RsvdP in VFs */
2307 	if (dev->is_virtfn)
2308 		return;
2309 
2310 	if (!pcie_relaxed_ordering_enabled(dev))
2311 		return;
2312 
2313 	/*
2314 	 * For now, we only deal with Relaxed Ordering issues with Root
2315 	 * Ports. Peer-to-Peer DMA is another can of worms.
2316 	 */
2317 	root = pcie_find_root_port(dev);
2318 	if (!root)
2319 		return;
2320 
2321 	if (root->dev_flags & PCI_DEV_FLAGS_NO_RELAXED_ORDERING) {
2322 		pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2323 					   PCI_EXP_DEVCTL_RELAX_EN);
2324 		pci_info(dev, "Relaxed Ordering disabled because the Root Port didn't support it\n");
2325 	}
2326 }
2327 
pci_configure_eetlp_prefix(struct pci_dev * dev)2328 static void pci_configure_eetlp_prefix(struct pci_dev *dev)
2329 {
2330 	struct pci_dev *bridge;
2331 	unsigned int eetlp_max;
2332 	int pcie_type;
2333 	u32 cap;
2334 
2335 	if (!pci_is_pcie(dev))
2336 		return;
2337 
2338 	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap);
2339 	if (!(cap & PCI_EXP_DEVCAP2_EE_PREFIX))
2340 		return;
2341 
2342 	pcie_type = pci_pcie_type(dev);
2343 
2344 	eetlp_max = FIELD_GET(PCI_EXP_DEVCAP2_EE_PREFIX_MAX, cap);
2345 	/* 00b means 4 */
2346 	eetlp_max = eetlp_max ?: 4;
2347 
2348 	if (pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
2349 	    pcie_type == PCI_EXP_TYPE_RC_END)
2350 		dev->eetlp_prefix_max = eetlp_max;
2351 	else {
2352 		bridge = pci_upstream_bridge(dev);
2353 		if (bridge && bridge->eetlp_prefix_max)
2354 			dev->eetlp_prefix_max = eetlp_max;
2355 	}
2356 }
2357 
pci_configure_serr(struct pci_dev * dev)2358 static void pci_configure_serr(struct pci_dev *dev)
2359 {
2360 	u16 control;
2361 
2362 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
2363 
2364 		/*
2365 		 * A bridge will not forward ERR_ messages coming from an
2366 		 * endpoint unless SERR# forwarding is enabled.
2367 		 */
2368 		pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &control);
2369 		if (!(control & PCI_BRIDGE_CTL_SERR)) {
2370 			control |= PCI_BRIDGE_CTL_SERR;
2371 			pci_write_config_word(dev, PCI_BRIDGE_CONTROL, control);
2372 		}
2373 	}
2374 }
2375 
pci_configure_device(struct pci_dev * dev)2376 static void pci_configure_device(struct pci_dev *dev)
2377 {
2378 	pci_configure_mps(dev);
2379 	pci_configure_extended_tags(dev, NULL);
2380 	pci_configure_relaxed_ordering(dev);
2381 	pci_configure_ltr(dev);
2382 	pci_configure_aspm_l1ss(dev);
2383 	pci_configure_eetlp_prefix(dev);
2384 	pci_configure_serr(dev);
2385 
2386 	pci_acpi_program_hp_params(dev);
2387 }
2388 
pci_release_capabilities(struct pci_dev * dev)2389 static void pci_release_capabilities(struct pci_dev *dev)
2390 {
2391 	pci_aer_exit(dev);
2392 	pci_rcec_exit(dev);
2393 	pci_iov_release(dev);
2394 	pci_free_cap_save_buffers(dev);
2395 }
2396 
2397 /**
2398  * pci_release_dev - Free a PCI device structure when all users of it are
2399  *		     finished
2400  * @dev: device that's been disconnected
2401  *
2402  * Will be called only by the device core when all users of this PCI device are
2403  * done.
2404  */
pci_release_dev(struct device * dev)2405 static void pci_release_dev(struct device *dev)
2406 {
2407 	struct pci_dev *pci_dev;
2408 
2409 	pci_dev = to_pci_dev(dev);
2410 	pci_release_capabilities(pci_dev);
2411 	pci_release_of_node(pci_dev);
2412 	pcibios_release_device(pci_dev);
2413 	pci_bus_put(pci_dev->bus);
2414 	kfree(pci_dev->driver_override);
2415 	bitmap_free(pci_dev->dma_alias_mask);
2416 	dev_dbg(dev, "device released\n");
2417 	kfree(pci_dev);
2418 }
2419 
2420 static const struct device_type pci_dev_type = {
2421 	.groups = pci_dev_attr_groups,
2422 };
2423 
pci_alloc_dev(struct pci_bus * bus)2424 struct pci_dev *pci_alloc_dev(struct pci_bus *bus)
2425 {
2426 	struct pci_dev *dev;
2427 
2428 	dev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL);
2429 	if (!dev)
2430 		return NULL;
2431 
2432 	INIT_LIST_HEAD(&dev->bus_list);
2433 	dev->dev.type = &pci_dev_type;
2434 	dev->bus = pci_bus_get(bus);
2435 	dev->driver_exclusive_resource = (struct resource) {
2436 		.name = "PCI Exclusive",
2437 		.start = 0,
2438 		.end = -1,
2439 	};
2440 
2441 	spin_lock_init(&dev->pcie_cap_lock);
2442 #ifdef CONFIG_PCI_MSI
2443 	raw_spin_lock_init(&dev->msi_lock);
2444 #endif
2445 	return dev;
2446 }
2447 EXPORT_SYMBOL(pci_alloc_dev);
2448 
pci_bus_wait_rrs(struct pci_bus * bus,int devfn,u32 * l,int timeout)2449 static bool pci_bus_wait_rrs(struct pci_bus *bus, int devfn, u32 *l,
2450 			     int timeout)
2451 {
2452 	int delay = 1;
2453 
2454 	if (!pci_bus_rrs_vendor_id(*l))
2455 		return true;	/* not a Configuration RRS completion */
2456 
2457 	if (!timeout)
2458 		return false;	/* RRS, but caller doesn't want to wait */
2459 
2460 	/*
2461 	 * We got the reserved Vendor ID that indicates a completion with
2462 	 * Configuration Request Retry Status (RRS).  Retry until we get a
2463 	 * valid Vendor ID or we time out.
2464 	 */
2465 	while (pci_bus_rrs_vendor_id(*l)) {
2466 		if (delay > timeout) {
2467 			pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n",
2468 				pci_domain_nr(bus), bus->number,
2469 				PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2470 
2471 			return false;
2472 		}
2473 		if (delay >= 1000)
2474 			pr_info("pci %04x:%02x:%02x.%d: not ready after %dms; waiting\n",
2475 				pci_domain_nr(bus), bus->number,
2476 				PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2477 
2478 		msleep(delay);
2479 		delay *= 2;
2480 
2481 		if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
2482 			return false;
2483 	}
2484 
2485 	if (delay >= 1000)
2486 		pr_info("pci %04x:%02x:%02x.%d: ready after %dms\n",
2487 			pci_domain_nr(bus), bus->number,
2488 			PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2489 
2490 	return true;
2491 }
2492 
pci_bus_generic_read_dev_vendor_id(struct pci_bus * bus,int devfn,u32 * l,int timeout)2493 bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2494 					int timeout)
2495 {
2496 	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
2497 		return false;
2498 
2499 	/* Some broken boards return 0 or ~0 (PCI_ERROR_RESPONSE) if a slot is empty: */
2500 	if (PCI_POSSIBLE_ERROR(*l) || *l == 0x00000000 ||
2501 	    *l == 0x0000ffff || *l == 0xffff0000)
2502 		return false;
2503 
2504 	if (pci_bus_rrs_vendor_id(*l))
2505 		return pci_bus_wait_rrs(bus, devfn, l, timeout);
2506 
2507 	return true;
2508 }
2509 
pci_bus_read_dev_vendor_id(struct pci_bus * bus,int devfn,u32 * l,int timeout)2510 bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2511 				int timeout)
2512 {
2513 #ifdef CONFIG_PCI_QUIRKS
2514 	struct pci_dev *bridge = bus->self;
2515 
2516 	/*
2517 	 * Certain IDT switches have an issue where they improperly trigger
2518 	 * ACS Source Validation errors on completions for config reads.
2519 	 */
2520 	if (bridge && bridge->vendor == PCI_VENDOR_ID_IDT &&
2521 	    bridge->device == 0x80b5)
2522 		return pci_idt_bus_quirk(bus, devfn, l, timeout);
2523 #endif
2524 
2525 	return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
2526 }
2527 EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
2528 
2529 #if IS_ENABLED(CONFIG_PCI_PWRCTRL)
pci_pwrctrl_create_device(struct pci_bus * bus,int devfn)2530 static struct platform_device *pci_pwrctrl_create_device(struct pci_bus *bus, int devfn)
2531 {
2532 	struct pci_host_bridge *host = pci_find_host_bridge(bus);
2533 	struct platform_device *pdev;
2534 	struct device_node *np;
2535 
2536 	np = of_pci_find_child_device(dev_of_node(&bus->dev), devfn);
2537 	if (!np)
2538 		return NULL;
2539 
2540 	pdev = of_find_device_by_node(np);
2541 	if (pdev) {
2542 		put_device(&pdev->dev);
2543 		goto err_put_of_node;
2544 	}
2545 
2546 	/*
2547 	 * First check whether the pwrctrl device really needs to be created or
2548 	 * not. This is decided based on at least one of the power supplies
2549 	 * being defined in the devicetree node of the device.
2550 	 */
2551 	if (!of_pci_supply_present(np)) {
2552 		pr_debug("PCI/pwrctrl: Skipping OF node: %s\n", np->name);
2553 		goto err_put_of_node;
2554 	}
2555 
2556 	/* Now create the pwrctrl device */
2557 	pdev = of_platform_device_create(np, NULL, &host->dev);
2558 	if (!pdev) {
2559 		pr_err("PCI/pwrctrl: Failed to create pwrctrl device for node: %s\n", np->name);
2560 		goto err_put_of_node;
2561 	}
2562 
2563 	of_node_put(np);
2564 
2565 	return pdev;
2566 
2567 err_put_of_node:
2568 	of_node_put(np);
2569 
2570 	return NULL;
2571 }
2572 #else
pci_pwrctrl_create_device(struct pci_bus * bus,int devfn)2573 static struct platform_device *pci_pwrctrl_create_device(struct pci_bus *bus, int devfn)
2574 {
2575 	return NULL;
2576 }
2577 #endif
2578 
2579 /*
2580  * Read the config data for a PCI device, sanity-check it,
2581  * and fill in the dev structure.
2582  */
pci_scan_device(struct pci_bus * bus,int devfn)2583 static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
2584 {
2585 	struct pci_dev *dev;
2586 	u32 l;
2587 
2588 	/*
2589 	 * Create pwrctrl device (if required) for the PCI device to handle the
2590 	 * power state. If the pwrctrl device is created, then skip scanning
2591 	 * further as the pwrctrl core will rescan the bus after powering on
2592 	 * the device.
2593 	 */
2594 	if (pci_pwrctrl_create_device(bus, devfn))
2595 		return NULL;
2596 
2597 	if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000))
2598 		return NULL;
2599 
2600 	dev = pci_alloc_dev(bus);
2601 	if (!dev)
2602 		return NULL;
2603 
2604 	dev->devfn = devfn;
2605 	dev->vendor = l & 0xffff;
2606 	dev->device = (l >> 16) & 0xffff;
2607 
2608 	if (pci_setup_device(dev)) {
2609 		pci_bus_put(dev->bus);
2610 		kfree(dev);
2611 		return NULL;
2612 	}
2613 
2614 	return dev;
2615 }
2616 
pcie_report_downtraining(struct pci_dev * dev)2617 void pcie_report_downtraining(struct pci_dev *dev)
2618 {
2619 	if (!pci_is_pcie(dev))
2620 		return;
2621 
2622 	/* Look from the device up to avoid downstream ports with no devices */
2623 	if ((pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT) &&
2624 	    (pci_pcie_type(dev) != PCI_EXP_TYPE_LEG_END) &&
2625 	    (pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM))
2626 		return;
2627 
2628 	/* Multi-function PCIe devices share the same link/status */
2629 	if (PCI_FUNC(dev->devfn) != 0 || dev->is_virtfn)
2630 		return;
2631 
2632 	/* Print link status only if the device is constrained by the fabric */
2633 	__pcie_print_link_status(dev, false);
2634 }
2635 
pci_imm_ready_init(struct pci_dev * dev)2636 static void pci_imm_ready_init(struct pci_dev *dev)
2637 {
2638 	u16 status;
2639 
2640 	pci_read_config_word(dev, PCI_STATUS, &status);
2641 	if (status & PCI_STATUS_IMM_READY)
2642 		dev->imm_ready = 1;
2643 }
2644 
pci_init_capabilities(struct pci_dev * dev)2645 static void pci_init_capabilities(struct pci_dev *dev)
2646 {
2647 	pci_ea_init(dev);		/* Enhanced Allocation */
2648 	pci_msi_init(dev);		/* Disable MSI */
2649 	pci_msix_init(dev);		/* Disable MSI-X */
2650 
2651 	/* Buffers for saving PCIe and PCI-X capabilities */
2652 	pci_allocate_cap_save_buffers(dev);
2653 
2654 	pci_imm_ready_init(dev);	/* Immediate Readiness */
2655 	pci_pm_init(dev);		/* Power Management */
2656 	pci_vpd_init(dev);		/* Vital Product Data */
2657 	pci_configure_ari(dev);		/* Alternative Routing-ID Forwarding */
2658 	pci_iov_init(dev);		/* Single Root I/O Virtualization */
2659 	pci_ats_init(dev);		/* Address Translation Services */
2660 	pci_pri_init(dev);		/* Page Request Interface */
2661 	pci_pasid_init(dev);		/* Process Address Space ID */
2662 	pci_acs_init(dev);		/* Access Control Services */
2663 	pci_ptm_init(dev);		/* Precision Time Measurement */
2664 	pci_aer_init(dev);		/* Advanced Error Reporting */
2665 	pci_dpc_init(dev);		/* Downstream Port Containment */
2666 	pci_rcec_init(dev);		/* Root Complex Event Collector */
2667 	pci_doe_init(dev);		/* Data Object Exchange */
2668 	pci_tph_init(dev);		/* TLP Processing Hints */
2669 	pci_rebar_init(dev);		/* Resizable BAR */
2670 
2671 	pcie_report_downtraining(dev);
2672 	pci_init_reset_methods(dev);
2673 }
2674 
2675 /*
2676  * This is the equivalent of pci_host_bridge_msi_domain() that acts on
2677  * devices. Firmware interfaces that can select the MSI domain on a
2678  * per-device basis should be called from here.
2679  */
pci_dev_msi_domain(struct pci_dev * dev)2680 static struct irq_domain *pci_dev_msi_domain(struct pci_dev *dev)
2681 {
2682 	struct irq_domain *d;
2683 
2684 	/*
2685 	 * If a domain has been set through the pcibios_device_add()
2686 	 * callback, then this is the one (platform code knows best).
2687 	 */
2688 	d = dev_get_msi_domain(&dev->dev);
2689 	if (d)
2690 		return d;
2691 
2692 	/*
2693 	 * Let's see if we have a firmware interface able to provide
2694 	 * the domain.
2695 	 */
2696 	d = pci_msi_get_device_domain(dev);
2697 	if (d)
2698 		return d;
2699 
2700 	return NULL;
2701 }
2702 
pci_set_msi_domain(struct pci_dev * dev)2703 static void pci_set_msi_domain(struct pci_dev *dev)
2704 {
2705 	struct irq_domain *d;
2706 
2707 	/*
2708 	 * If the platform or firmware interfaces cannot supply a
2709 	 * device-specific MSI domain, then inherit the default domain
2710 	 * from the host bridge itself.
2711 	 */
2712 	d = pci_dev_msi_domain(dev);
2713 	if (!d)
2714 		d = dev_get_msi_domain(&dev->bus->dev);
2715 
2716 	dev_set_msi_domain(&dev->dev, d);
2717 }
2718 
pci_device_add(struct pci_dev * dev,struct pci_bus * bus)2719 void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
2720 {
2721 	int ret;
2722 
2723 	pci_configure_device(dev);
2724 
2725 	device_initialize(&dev->dev);
2726 	dev->dev.release = pci_release_dev;
2727 
2728 	set_dev_node(&dev->dev, pcibus_to_node(bus));
2729 	dev->dev.dma_mask = &dev->dma_mask;
2730 	dev->dev.dma_parms = &dev->dma_parms;
2731 	dev->dev.coherent_dma_mask = 0xffffffffull;
2732 
2733 	dma_set_max_seg_size(&dev->dev, 65536);
2734 	dma_set_seg_boundary(&dev->dev, 0xffffffff);
2735 
2736 	pcie_failed_link_retrain(dev);
2737 
2738 	/* Fix up broken headers */
2739 	pci_fixup_device(pci_fixup_header, dev);
2740 
2741 	pci_reassigndev_resource_alignment(dev);
2742 
2743 	dev->state_saved = false;
2744 
2745 	pci_init_capabilities(dev);
2746 
2747 	/*
2748 	 * Add the device to our list of discovered devices
2749 	 * and the bus list for fixup functions, etc.
2750 	 */
2751 	down_write(&pci_bus_sem);
2752 	list_add_tail(&dev->bus_list, &bus->devices);
2753 	up_write(&pci_bus_sem);
2754 
2755 	ret = pcibios_device_add(dev);
2756 	WARN_ON(ret < 0);
2757 
2758 	/* Set up MSI IRQ domain */
2759 	pci_set_msi_domain(dev);
2760 
2761 	/* Notifier could use PCI capabilities */
2762 	ret = device_add(&dev->dev);
2763 	WARN_ON(ret < 0);
2764 
2765 	pci_npem_create(dev);
2766 
2767 	pci_doe_sysfs_init(dev);
2768 }
2769 
pci_scan_single_device(struct pci_bus * bus,int devfn)2770 struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn)
2771 {
2772 	struct pci_dev *dev;
2773 
2774 	dev = pci_get_slot(bus, devfn);
2775 	if (dev) {
2776 		pci_dev_put(dev);
2777 		return dev;
2778 	}
2779 
2780 	dev = pci_scan_device(bus, devfn);
2781 	if (!dev)
2782 		return NULL;
2783 
2784 	pci_device_add(dev, bus);
2785 
2786 	return dev;
2787 }
2788 EXPORT_SYMBOL(pci_scan_single_device);
2789 
next_ari_fn(struct pci_bus * bus,struct pci_dev * dev,int fn)2790 static int next_ari_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2791 {
2792 	int pos;
2793 	u16 cap = 0;
2794 	unsigned int next_fn;
2795 
2796 	if (!dev)
2797 		return -ENODEV;
2798 
2799 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI);
2800 	if (!pos)
2801 		return -ENODEV;
2802 
2803 	pci_read_config_word(dev, pos + PCI_ARI_CAP, &cap);
2804 	next_fn = PCI_ARI_CAP_NFN(cap);
2805 	if (next_fn <= fn)
2806 		return -ENODEV;	/* protect against malformed list */
2807 
2808 	return next_fn;
2809 }
2810 
next_fn(struct pci_bus * bus,struct pci_dev * dev,int fn)2811 static int next_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2812 {
2813 	if (pci_ari_enabled(bus))
2814 		return next_ari_fn(bus, dev, fn);
2815 
2816 	if (fn >= 7)
2817 		return -ENODEV;
2818 	/* only multifunction devices may have more functions */
2819 	if (dev && !dev->multifunction)
2820 		return -ENODEV;
2821 
2822 	return fn + 1;
2823 }
2824 
only_one_child(struct pci_bus * bus)2825 static int only_one_child(struct pci_bus *bus)
2826 {
2827 	struct pci_dev *bridge = bus->self;
2828 
2829 	/*
2830 	 * Systems with unusual topologies set PCI_SCAN_ALL_PCIE_DEVS so
2831 	 * we scan for all possible devices, not just Device 0.
2832 	 */
2833 	if (pci_has_flag(PCI_SCAN_ALL_PCIE_DEVS))
2834 		return 0;
2835 
2836 	/*
2837 	 * A PCIe Downstream Port normally leads to a Link with only Device
2838 	 * 0 on it (PCIe spec r3.1, sec 7.3.1).  As an optimization, scan
2839 	 * only for Device 0 in that situation.
2840 	 */
2841 	if (bridge && pci_is_pcie(bridge) && pcie_downstream_port(bridge))
2842 		return 1;
2843 
2844 	return 0;
2845 }
2846 
2847 /**
2848  * pci_scan_slot - Scan a PCI slot on a bus for devices
2849  * @bus: PCI bus to scan
2850  * @devfn: slot number to scan (must have zero function)
2851  *
2852  * Scan a PCI slot on the specified PCI bus for devices, adding
2853  * discovered devices to the @bus->devices list.  New devices
2854  * will not have is_added set.
2855  *
2856  * Returns the number of new devices found.
2857  */
pci_scan_slot(struct pci_bus * bus,int devfn)2858 int pci_scan_slot(struct pci_bus *bus, int devfn)
2859 {
2860 	struct pci_dev *dev;
2861 	int fn = 0, nr = 0;
2862 
2863 	if (only_one_child(bus) && (devfn > 0))
2864 		return 0; /* Already scanned the entire slot */
2865 
2866 	do {
2867 		dev = pci_scan_single_device(bus, devfn + fn);
2868 		if (dev) {
2869 			if (!pci_dev_is_added(dev))
2870 				nr++;
2871 			if (fn > 0)
2872 				dev->multifunction = 1;
2873 		} else if (fn == 0) {
2874 			/*
2875 			 * Function 0 is required unless we are running on
2876 			 * a hypervisor that passes through individual PCI
2877 			 * functions.
2878 			 */
2879 			if (!hypervisor_isolated_pci_functions())
2880 				break;
2881 		}
2882 		fn = next_fn(bus, dev, fn);
2883 	} while (fn >= 0);
2884 
2885 	/* Only one slot has PCIe device */
2886 	if (bus->self && nr)
2887 		pcie_aspm_init_link_state(bus->self);
2888 
2889 	return nr;
2890 }
2891 EXPORT_SYMBOL(pci_scan_slot);
2892 
pcie_find_smpss(struct pci_dev * dev,void * data)2893 static int pcie_find_smpss(struct pci_dev *dev, void *data)
2894 {
2895 	u8 *smpss = data;
2896 
2897 	if (!pci_is_pcie(dev))
2898 		return 0;
2899 
2900 	/*
2901 	 * We don't have a way to change MPS settings on devices that have
2902 	 * drivers attached.  A hot-added device might support only the minimum
2903 	 * MPS setting (MPS=128).  Therefore, if the fabric contains a bridge
2904 	 * where devices may be hot-added, we limit the fabric MPS to 128 so
2905 	 * hot-added devices will work correctly.
2906 	 *
2907 	 * However, if we hot-add a device to a slot directly below a Root
2908 	 * Port, it's impossible for there to be other existing devices below
2909 	 * the port.  We don't limit the MPS in this case because we can
2910 	 * reconfigure MPS on both the Root Port and the hot-added device,
2911 	 * and there are no other devices involved.
2912 	 *
2913 	 * Note that this PCIE_BUS_SAFE path assumes no peer-to-peer DMA.
2914 	 */
2915 	if (dev->is_hotplug_bridge &&
2916 	    pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT)
2917 		*smpss = 0;
2918 
2919 	if (*smpss > dev->pcie_mpss)
2920 		*smpss = dev->pcie_mpss;
2921 
2922 	return 0;
2923 }
2924 
pcie_write_mps(struct pci_dev * dev,int mps)2925 static void pcie_write_mps(struct pci_dev *dev, int mps)
2926 {
2927 	int rc;
2928 
2929 	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
2930 		mps = 128 << dev->pcie_mpss;
2931 
2932 		if (pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT &&
2933 		    dev->bus->self)
2934 
2935 			/*
2936 			 * For "Performance", the assumption is made that
2937 			 * downstream communication will never be larger than
2938 			 * the MRRS.  So, the MPS only needs to be configured
2939 			 * for the upstream communication.  This being the case,
2940 			 * walk from the top down and set the MPS of the child
2941 			 * to that of the parent bus.
2942 			 *
2943 			 * Configure the device MPS with the smaller of the
2944 			 * device MPSS or the bridge MPS (which is assumed to be
2945 			 * properly configured at this point to the largest
2946 			 * allowable MPS based on its parent bus).
2947 			 */
2948 			mps = min(mps, pcie_get_mps(dev->bus->self));
2949 	}
2950 
2951 	rc = pcie_set_mps(dev, mps);
2952 	if (rc)
2953 		pci_err(dev, "Failed attempting to set the MPS\n");
2954 }
2955 
pcie_write_mrrs(struct pci_dev * dev)2956 static void pcie_write_mrrs(struct pci_dev *dev)
2957 {
2958 	int rc, mrrs;
2959 
2960 	/*
2961 	 * In the "safe" case, do not configure the MRRS.  There appear to be
2962 	 * issues with setting MRRS to 0 on a number of devices.
2963 	 */
2964 	if (pcie_bus_config != PCIE_BUS_PERFORMANCE)
2965 		return;
2966 
2967 	/*
2968 	 * For max performance, the MRRS must be set to the largest supported
2969 	 * value.  However, it cannot be configured larger than the MPS the
2970 	 * device or the bus can support.  This should already be properly
2971 	 * configured by a prior call to pcie_write_mps().
2972 	 */
2973 	mrrs = pcie_get_mps(dev);
2974 
2975 	/*
2976 	 * MRRS is a R/W register.  Invalid values can be written, but a
2977 	 * subsequent read will verify if the value is acceptable or not.
2978 	 * If the MRRS value provided is not acceptable (e.g., too large),
2979 	 * shrink the value until it is acceptable to the HW.
2980 	 */
2981 	while (mrrs != pcie_get_readrq(dev) && mrrs >= 128) {
2982 		rc = pcie_set_readrq(dev, mrrs);
2983 		if (!rc)
2984 			break;
2985 
2986 		pci_warn(dev, "Failed attempting to set the MRRS\n");
2987 		mrrs /= 2;
2988 	}
2989 
2990 	if (mrrs < 128)
2991 		pci_err(dev, "MRRS was unable to be configured with a safe value.  If problems are experienced, try running with pci=pcie_bus_safe\n");
2992 }
2993 
pcie_bus_configure_set(struct pci_dev * dev,void * data)2994 static int pcie_bus_configure_set(struct pci_dev *dev, void *data)
2995 {
2996 	int mps, orig_mps;
2997 
2998 	if (!pci_is_pcie(dev))
2999 		return 0;
3000 
3001 	if (pcie_bus_config == PCIE_BUS_TUNE_OFF ||
3002 	    pcie_bus_config == PCIE_BUS_DEFAULT)
3003 		return 0;
3004 
3005 	mps = 128 << *(u8 *)data;
3006 	orig_mps = pcie_get_mps(dev);
3007 
3008 	pcie_write_mps(dev, mps);
3009 	pcie_write_mrrs(dev);
3010 
3011 	pci_info(dev, "Max Payload Size set to %4d/%4d (was %4d), Max Read Rq %4d\n",
3012 		 pcie_get_mps(dev), 128 << dev->pcie_mpss,
3013 		 orig_mps, pcie_get_readrq(dev));
3014 
3015 	return 0;
3016 }
3017 
3018 /*
3019  * pcie_bus_configure_settings() requires that pci_walk_bus work in a top-down,
3020  * parents then children fashion.  If this changes, then this code will not
3021  * work as designed.
3022  */
pcie_bus_configure_settings(struct pci_bus * bus)3023 void pcie_bus_configure_settings(struct pci_bus *bus)
3024 {
3025 	u8 smpss = 0;
3026 
3027 	if (!bus->self)
3028 		return;
3029 
3030 	if (!pci_is_pcie(bus->self))
3031 		return;
3032 
3033 	/*
3034 	 * FIXME - Peer to peer DMA is possible, though the endpoint would need
3035 	 * to be aware of the MPS of the destination.  To work around this,
3036 	 * simply force the MPS of the entire system to the smallest possible.
3037 	 */
3038 	if (pcie_bus_config == PCIE_BUS_PEER2PEER)
3039 		smpss = 0;
3040 
3041 	if (pcie_bus_config == PCIE_BUS_SAFE) {
3042 		smpss = bus->self->pcie_mpss;
3043 
3044 		pcie_find_smpss(bus->self, &smpss);
3045 		pci_walk_bus(bus, pcie_find_smpss, &smpss);
3046 	}
3047 
3048 	pcie_bus_configure_set(bus->self, &smpss);
3049 	pci_walk_bus(bus, pcie_bus_configure_set, &smpss);
3050 }
3051 EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
3052 
3053 /*
3054  * Called after each bus is probed, but before its children are examined.  This
3055  * is marked as __weak because multiple architectures define it.
3056  */
pcibios_fixup_bus(struct pci_bus * bus)3057 void __weak pcibios_fixup_bus(struct pci_bus *bus)
3058 {
3059        /* nothing to do, expected to be removed in the future */
3060 }
3061 
3062 /**
3063  * pci_scan_child_bus_extend() - Scan devices below a bus
3064  * @bus: Bus to scan for devices
3065  * @available_buses: Total number of buses available (%0 does not try to
3066  *		     extend beyond the minimal)
3067  *
3068  * Scans devices below @bus including subordinate buses. Returns new
3069  * subordinate number including all the found devices. Passing
3070  * @available_buses causes the remaining bus space to be distributed
3071  * equally between hotplug-capable bridges to allow future extension of the
3072  * hierarchy.
3073  */
pci_scan_child_bus_extend(struct pci_bus * bus,unsigned int available_buses)3074 static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
3075 					      unsigned int available_buses)
3076 {
3077 	unsigned int used_buses, normal_bridges = 0, hotplug_bridges = 0;
3078 	unsigned int start = bus->busn_res.start;
3079 	unsigned int devnr, cmax, max = start;
3080 	struct pci_dev *dev;
3081 
3082 	dev_dbg(&bus->dev, "scanning bus\n");
3083 
3084 	/* Go find them, Rover! */
3085 	for (devnr = 0; devnr < PCI_MAX_NR_DEVS; devnr++)
3086 		pci_scan_slot(bus, PCI_DEVFN(devnr, 0));
3087 
3088 	/* Reserve buses for SR-IOV capability */
3089 	used_buses = pci_iov_bus_range(bus);
3090 	max += used_buses;
3091 
3092 	/*
3093 	 * After performing arch-dependent fixup of the bus, look behind
3094 	 * all PCI-to-PCI bridges on this bus.
3095 	 */
3096 	if (!bus->is_added) {
3097 		dev_dbg(&bus->dev, "fixups for bus\n");
3098 		pcibios_fixup_bus(bus);
3099 		bus->is_added = 1;
3100 	}
3101 
3102 	/*
3103 	 * Calculate how many hotplug bridges and normal bridges there
3104 	 * are on this bus. We will distribute the additional available
3105 	 * buses between hotplug bridges.
3106 	 */
3107 	for_each_pci_bridge(dev, bus) {
3108 		if (dev->is_hotplug_bridge)
3109 			hotplug_bridges++;
3110 		else
3111 			normal_bridges++;
3112 	}
3113 
3114 	/*
3115 	 * Scan bridges that are already configured. We don't touch them
3116 	 * unless they are misconfigured (which will be done in the second
3117 	 * scan below).
3118 	 */
3119 	for_each_pci_bridge(dev, bus) {
3120 		cmax = max;
3121 		max = pci_scan_bridge_extend(bus, dev, max, 0, 0);
3122 
3123 		/*
3124 		 * Reserve one bus for each bridge now to avoid extending
3125 		 * hotplug bridges too much during the second scan below.
3126 		 */
3127 		used_buses++;
3128 		if (max - cmax > 1)
3129 			used_buses += max - cmax - 1;
3130 	}
3131 
3132 	/* Scan bridges that need to be reconfigured */
3133 	for_each_pci_bridge(dev, bus) {
3134 		unsigned int buses = 0;
3135 
3136 		if (!hotplug_bridges && normal_bridges == 1) {
3137 			/*
3138 			 * There is only one bridge on the bus (upstream
3139 			 * port) so it gets all available buses which it
3140 			 * can then distribute to the possible hotplug
3141 			 * bridges below.
3142 			 */
3143 			buses = available_buses;
3144 		} else if (dev->is_hotplug_bridge) {
3145 			/*
3146 			 * Distribute the extra buses between hotplug
3147 			 * bridges if any.
3148 			 */
3149 			buses = available_buses / hotplug_bridges;
3150 			buses = min(buses, available_buses - used_buses + 1);
3151 		}
3152 
3153 		cmax = max;
3154 		max = pci_scan_bridge_extend(bus, dev, cmax, buses, 1);
3155 		/* One bus is already accounted so don't add it again */
3156 		if (max - cmax > 1)
3157 			used_buses += max - cmax - 1;
3158 	}
3159 
3160 	/*
3161 	 * Make sure a hotplug bridge has at least the minimum requested
3162 	 * number of buses but allow it to grow up to the maximum available
3163 	 * bus number if there is room.
3164 	 */
3165 	if (bus->self && bus->self->is_hotplug_bridge) {
3166 		used_buses = max_t(unsigned int, available_buses,
3167 				   pci_hotplug_bus_size - 1);
3168 		if (max - start < used_buses) {
3169 			max = start + used_buses;
3170 
3171 			/* Do not allocate more buses than we have room left */
3172 			if (max > bus->busn_res.end)
3173 				max = bus->busn_res.end;
3174 
3175 			dev_dbg(&bus->dev, "%pR extended by %#02x\n",
3176 				&bus->busn_res, max - start);
3177 		}
3178 	}
3179 
3180 	/*
3181 	 * We've scanned the bus and so we know all about what's on
3182 	 * the other side of any bridges that may be on this bus plus
3183 	 * any devices.
3184 	 *
3185 	 * Return how far we've got finding sub-buses.
3186 	 */
3187 	dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
3188 	return max;
3189 }
3190 
3191 /**
3192  * pci_scan_child_bus() - Scan devices below a bus
3193  * @bus: Bus to scan for devices
3194  *
3195  * Scans devices below @bus including subordinate buses. Returns new
3196  * subordinate number including all the found devices.
3197  */
pci_scan_child_bus(struct pci_bus * bus)3198 unsigned int pci_scan_child_bus(struct pci_bus *bus)
3199 {
3200 	return pci_scan_child_bus_extend(bus, 0);
3201 }
3202 EXPORT_SYMBOL_GPL(pci_scan_child_bus);
3203 
3204 /**
3205  * pcibios_root_bridge_prepare - Platform-specific host bridge setup
3206  * @bridge: Host bridge to set up
3207  *
3208  * Default empty implementation.  Replace with an architecture-specific setup
3209  * routine, if necessary.
3210  */
pcibios_root_bridge_prepare(struct pci_host_bridge * bridge)3211 int __weak pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
3212 {
3213 	return 0;
3214 }
3215 
pcibios_add_bus(struct pci_bus * bus)3216 void __weak pcibios_add_bus(struct pci_bus *bus)
3217 {
3218 }
3219 
pcibios_remove_bus(struct pci_bus * bus)3220 void __weak pcibios_remove_bus(struct pci_bus *bus)
3221 {
3222 }
3223 
pci_create_root_bus(struct device * parent,int bus,struct pci_ops * ops,void * sysdata,struct list_head * resources)3224 struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
3225 		struct pci_ops *ops, void *sysdata, struct list_head *resources)
3226 {
3227 	int error;
3228 	struct pci_host_bridge *bridge;
3229 
3230 	bridge = pci_alloc_host_bridge(0);
3231 	if (!bridge)
3232 		return NULL;
3233 
3234 	bridge->dev.parent = parent;
3235 
3236 	list_splice_init(resources, &bridge->windows);
3237 	bridge->sysdata = sysdata;
3238 	bridge->busnr = bus;
3239 	bridge->ops = ops;
3240 
3241 	error = pci_register_host_bridge(bridge);
3242 	if (error < 0)
3243 		goto err_out;
3244 
3245 	return bridge->bus;
3246 
3247 err_out:
3248 	put_device(&bridge->dev);
3249 	return NULL;
3250 }
3251 EXPORT_SYMBOL_GPL(pci_create_root_bus);
3252 
pci_host_probe(struct pci_host_bridge * bridge)3253 int pci_host_probe(struct pci_host_bridge *bridge)
3254 {
3255 	struct pci_bus *bus, *child;
3256 	int ret;
3257 
3258 	pci_lock_rescan_remove();
3259 	ret = pci_scan_root_bus_bridge(bridge);
3260 	pci_unlock_rescan_remove();
3261 	if (ret < 0) {
3262 		dev_err(bridge->dev.parent, "Scanning root bridge failed");
3263 		return ret;
3264 	}
3265 
3266 	bus = bridge->bus;
3267 
3268 	/* If we must preserve the resource configuration, claim now */
3269 	if (bridge->preserve_config)
3270 		pci_bus_claim_resources(bus);
3271 
3272 	/*
3273 	 * Assign whatever was left unassigned. If we didn't claim above,
3274 	 * this will reassign everything.
3275 	 */
3276 	pci_assign_unassigned_root_bus_resources(bus);
3277 
3278 	list_for_each_entry(child, &bus->children, node)
3279 		pcie_bus_configure_settings(child);
3280 
3281 	pci_lock_rescan_remove();
3282 	pci_bus_add_devices(bus);
3283 	pci_unlock_rescan_remove();
3284 
3285 	/*
3286 	 * Ensure pm_runtime_enable() is called for the controller drivers
3287 	 * before calling pci_host_probe(). The PM framework expects that
3288 	 * if the parent device supports runtime PM, it will be enabled
3289 	 * before child runtime PM is enabled.
3290 	 */
3291 	pm_runtime_set_active(&bridge->dev);
3292 	pm_runtime_no_callbacks(&bridge->dev);
3293 	devm_pm_runtime_enable(&bridge->dev);
3294 
3295 	return 0;
3296 }
3297 EXPORT_SYMBOL_GPL(pci_host_probe);
3298 
pci_bus_insert_busn_res(struct pci_bus * b,int bus,int bus_max)3299 int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
3300 {
3301 	struct resource *res = &b->busn_res;
3302 	struct resource *parent_res, *conflict;
3303 
3304 	res->start = bus;
3305 	res->end = bus_max;
3306 	res->flags = IORESOURCE_BUS;
3307 
3308 	if (!pci_is_root_bus(b))
3309 		parent_res = &b->parent->busn_res;
3310 	else {
3311 		parent_res = get_pci_domain_busn_res(pci_domain_nr(b));
3312 		res->flags |= IORESOURCE_PCI_FIXED;
3313 	}
3314 
3315 	conflict = request_resource_conflict(parent_res, res);
3316 
3317 	if (conflict)
3318 		dev_info(&b->dev,
3319 			   "busn_res: can not insert %pR under %s%pR (conflicts with %s %pR)\n",
3320 			    res, pci_is_root_bus(b) ? "domain " : "",
3321 			    parent_res, conflict->name, conflict);
3322 
3323 	return conflict == NULL;
3324 }
3325 
pci_bus_update_busn_res_end(struct pci_bus * b,int bus_max)3326 int pci_bus_update_busn_res_end(struct pci_bus *b, int bus_max)
3327 {
3328 	struct resource *res = &b->busn_res;
3329 	struct resource old_res = *res;
3330 	resource_size_t size;
3331 	int ret;
3332 
3333 	if (res->start > bus_max)
3334 		return -EINVAL;
3335 
3336 	size = bus_max - res->start + 1;
3337 	ret = adjust_resource(res, res->start, size);
3338 	dev_info(&b->dev, "busn_res: %pR end %s updated to %02x\n",
3339 			&old_res, ret ? "can not be" : "is", bus_max);
3340 
3341 	if (!ret && !res->parent)
3342 		pci_bus_insert_busn_res(b, res->start, res->end);
3343 
3344 	return ret;
3345 }
3346 
pci_bus_release_busn_res(struct pci_bus * b)3347 void pci_bus_release_busn_res(struct pci_bus *b)
3348 {
3349 	struct resource *res = &b->busn_res;
3350 	int ret;
3351 
3352 	if (!res->flags || !res->parent)
3353 		return;
3354 
3355 	ret = release_resource(res);
3356 	dev_info(&b->dev, "busn_res: %pR %s released\n",
3357 			res, ret ? "can not be" : "is");
3358 }
3359 
pci_scan_root_bus_bridge(struct pci_host_bridge * bridge)3360 int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge)
3361 {
3362 	struct resource_entry *window;
3363 	bool found = false;
3364 	struct pci_bus *b;
3365 	int max, bus, ret;
3366 
3367 	if (!bridge)
3368 		return -EINVAL;
3369 
3370 	resource_list_for_each_entry(window, &bridge->windows)
3371 		if (window->res->flags & IORESOURCE_BUS) {
3372 			bridge->busnr = window->res->start;
3373 			found = true;
3374 			break;
3375 		}
3376 
3377 	ret = pci_register_host_bridge(bridge);
3378 	if (ret < 0)
3379 		return ret;
3380 
3381 	b = bridge->bus;
3382 	bus = bridge->busnr;
3383 
3384 	if (!found) {
3385 		dev_info(&b->dev,
3386 		 "No busn resource found for root bus, will use [bus %02x-ff]\n",
3387 			bus);
3388 		pci_bus_insert_busn_res(b, bus, 255);
3389 	}
3390 
3391 	max = pci_scan_child_bus(b);
3392 
3393 	if (!found)
3394 		pci_bus_update_busn_res_end(b, max);
3395 
3396 	return 0;
3397 }
3398 EXPORT_SYMBOL(pci_scan_root_bus_bridge);
3399 
pci_scan_root_bus(struct device * parent,int bus,struct pci_ops * ops,void * sysdata,struct list_head * resources)3400 struct pci_bus *pci_scan_root_bus(struct device *parent, int bus,
3401 		struct pci_ops *ops, void *sysdata, struct list_head *resources)
3402 {
3403 	struct resource_entry *window;
3404 	bool found = false;
3405 	struct pci_bus *b;
3406 	int max;
3407 
3408 	resource_list_for_each_entry(window, resources)
3409 		if (window->res->flags & IORESOURCE_BUS) {
3410 			found = true;
3411 			break;
3412 		}
3413 
3414 	b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
3415 	if (!b)
3416 		return NULL;
3417 
3418 	if (!found) {
3419 		dev_info(&b->dev,
3420 		 "No busn resource found for root bus, will use [bus %02x-ff]\n",
3421 			bus);
3422 		pci_bus_insert_busn_res(b, bus, 255);
3423 	}
3424 
3425 	max = pci_scan_child_bus(b);
3426 
3427 	if (!found)
3428 		pci_bus_update_busn_res_end(b, max);
3429 
3430 	return b;
3431 }
3432 EXPORT_SYMBOL(pci_scan_root_bus);
3433 
pci_scan_bus(int bus,struct pci_ops * ops,void * sysdata)3434 struct pci_bus *pci_scan_bus(int bus, struct pci_ops *ops,
3435 					void *sysdata)
3436 {
3437 	LIST_HEAD(resources);
3438 	struct pci_bus *b;
3439 
3440 	pci_add_resource(&resources, &ioport_resource);
3441 	pci_add_resource(&resources, &iomem_resource);
3442 	pci_add_resource(&resources, &busn_resource);
3443 	b = pci_create_root_bus(NULL, bus, ops, sysdata, &resources);
3444 	if (b) {
3445 		pci_scan_child_bus(b);
3446 	} else {
3447 		pci_free_resource_list(&resources);
3448 	}
3449 	return b;
3450 }
3451 EXPORT_SYMBOL(pci_scan_bus);
3452 
3453 /**
3454  * pci_rescan_bus_bridge_resize - Scan a PCI bus for devices
3455  * @bridge: PCI bridge for the bus to scan
3456  *
3457  * Scan a PCI bus and child buses for new devices, add them,
3458  * and enable them, resizing bridge mmio/io resource if necessary
3459  * and possible.  The caller must ensure the child devices are already
3460  * removed for resizing to occur.
3461  *
3462  * Returns the max number of subordinate bus discovered.
3463  */
pci_rescan_bus_bridge_resize(struct pci_dev * bridge)3464 unsigned int pci_rescan_bus_bridge_resize(struct pci_dev *bridge)
3465 {
3466 	unsigned int max;
3467 	struct pci_bus *bus = bridge->subordinate;
3468 
3469 	max = pci_scan_child_bus(bus);
3470 
3471 	pci_assign_unassigned_bridge_resources(bridge);
3472 
3473 	pci_bus_add_devices(bus);
3474 
3475 	return max;
3476 }
3477 
3478 /**
3479  * pci_rescan_bus - Scan a PCI bus for devices
3480  * @bus: PCI bus to scan
3481  *
3482  * Scan a PCI bus and child buses for new devices, add them,
3483  * and enable them.
3484  *
3485  * Returns the max number of subordinate bus discovered.
3486  */
pci_rescan_bus(struct pci_bus * bus)3487 unsigned int pci_rescan_bus(struct pci_bus *bus)
3488 {
3489 	unsigned int max;
3490 
3491 	max = pci_scan_child_bus(bus);
3492 	pci_assign_unassigned_bus_resources(bus);
3493 	pci_bus_add_devices(bus);
3494 
3495 	return max;
3496 }
3497 EXPORT_SYMBOL_GPL(pci_rescan_bus);
3498 
3499 /*
3500  * pci_rescan_bus(), pci_rescan_bus_bridge_resize() and PCI device removal
3501  * routines should always be executed under this mutex.
3502  */
3503 DEFINE_MUTEX(pci_rescan_remove_lock);
3504 
pci_lock_rescan_remove(void)3505 void pci_lock_rescan_remove(void)
3506 {
3507 	mutex_lock(&pci_rescan_remove_lock);
3508 }
3509 EXPORT_SYMBOL_GPL(pci_lock_rescan_remove);
3510 
pci_unlock_rescan_remove(void)3511 void pci_unlock_rescan_remove(void)
3512 {
3513 	mutex_unlock(&pci_rescan_remove_lock);
3514 }
3515 EXPORT_SYMBOL_GPL(pci_unlock_rescan_remove);
3516 
pci_sort_bf_cmp(const struct device * d_a,const struct device * d_b)3517 static int __init pci_sort_bf_cmp(const struct device *d_a,
3518 				  const struct device *d_b)
3519 {
3520 	const struct pci_dev *a = to_pci_dev(d_a);
3521 	const struct pci_dev *b = to_pci_dev(d_b);
3522 
3523 	if      (pci_domain_nr(a->bus) < pci_domain_nr(b->bus)) return -1;
3524 	else if (pci_domain_nr(a->bus) > pci_domain_nr(b->bus)) return  1;
3525 
3526 	if      (a->bus->number < b->bus->number) return -1;
3527 	else if (a->bus->number > b->bus->number) return  1;
3528 
3529 	if      (a->devfn < b->devfn) return -1;
3530 	else if (a->devfn > b->devfn) return  1;
3531 
3532 	return 0;
3533 }
3534 
pci_sort_breadthfirst(void)3535 void __init pci_sort_breadthfirst(void)
3536 {
3537 	bus_sort_breadthfirst(&pci_bus_type, &pci_sort_bf_cmp);
3538 }
3539 
pci_hp_add_bridge(struct pci_dev * dev)3540 int pci_hp_add_bridge(struct pci_dev *dev)
3541 {
3542 	struct pci_bus *parent = dev->bus;
3543 	int busnr, start = parent->busn_res.start;
3544 	unsigned int available_buses = 0;
3545 	int end = parent->busn_res.end;
3546 
3547 	for (busnr = start; busnr <= end; busnr++) {
3548 		if (!pci_find_bus(pci_domain_nr(parent), busnr))
3549 			break;
3550 	}
3551 	if (busnr-- > end) {
3552 		pci_err(dev, "No bus number available for hot-added bridge\n");
3553 		return -1;
3554 	}
3555 
3556 	/* Scan bridges that are already configured */
3557 	busnr = pci_scan_bridge(parent, dev, busnr, 0);
3558 
3559 	/*
3560 	 * Distribute the available bus numbers between hotplug-capable
3561 	 * bridges to make extending the chain later possible.
3562 	 */
3563 	available_buses = end - busnr;
3564 
3565 	/* Scan bridges that need to be reconfigured */
3566 	pci_scan_bridge_extend(parent, dev, busnr, available_buses, 1);
3567 
3568 	if (!dev->subordinate)
3569 		return -1;
3570 
3571 	return 0;
3572 }
3573 EXPORT_SYMBOL_GPL(pci_hp_add_bridge);
3574