1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
6 * Copyright (c) 2000, BSDi
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice unmodified, this list of conditions, and the following
14 * disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include "opt_acpi.h"
33 #include "opt_iommu.h"
34 #include "opt_bus.h"
35
36 #include <sys/param.h>
37 #include <sys/conf.h>
38 #include <sys/endian.h>
39 #include <sys/eventhandler.h>
40 #include <sys/fcntl.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/linker.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/queue.h>
47 #include <sys/sbuf.h>
48 #include <sys/stdarg.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 #include <sys/taskqueue.h>
52 #include <sys/tree.h>
53
54 #include <vm/vm.h>
55 #include <vm/pmap.h>
56 #include <vm/vm_extern.h>
57
58 #include <sys/bus.h>
59 #include <machine/bus.h>
60 #include <sys/rman.h>
61 #include <machine/resource.h>
62
63 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
64 #include <machine/intr_machdep.h>
65 #endif
66
67 #include <sys/pciio.h>
68 #include <dev/pci/pcireg.h>
69 #include <dev/pci/pcivar.h>
70 #include <dev/pci/pci_private.h>
71
72 #ifdef PCI_IOV
73 #include <sys/nv.h>
74 #include <dev/pci/pci_iov_private.h>
75 #endif
76
77 #include <dev/usb/controller/xhcireg.h>
78 #include <dev/usb/controller/ehcireg.h>
79 #include <dev/usb/controller/ohcireg.h>
80 #include <dev/usb/controller/uhcireg.h>
81
82 #include <dev/iommu/iommu.h>
83
84 #include "pcib_if.h"
85 #include "pci_if.h"
86
87 #define PCIR_IS_BIOS(cfg, reg) \
88 (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \
89 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))
90
91 static device_probe_t pci_probe;
92
93 static bus_reset_post_t pci_reset_post;
94 static bus_reset_prepare_t pci_reset_prepare;
95 static bus_reset_child_t pci_reset_child;
96 static bus_hint_device_unit_t pci_hint_device_unit;
97 static bus_remap_intr_t pci_remap_intr_method;
98
99 static pci_get_id_t pci_get_id_method;
100
101 static int pci_has_quirk(uint32_t devid, int quirk);
102 static pci_addr_t pci_mapbase(uint64_t mapreg);
103 static const char *pci_maptype(uint64_t mapreg);
104 static int pci_maprange(uint64_t mapreg);
105 static pci_addr_t pci_rombase(uint64_t mapreg);
106 static int pci_romsize(uint64_t testval);
107 static void pci_fixancient(pcicfgregs *cfg);
108 static int pci_printf(pcicfgregs *cfg, const char *fmt, ...);
109
110 static int pci_porten(device_t dev);
111 static int pci_memen(device_t dev);
112 static void pci_assign_interrupt(device_t bus, device_t dev,
113 int force_route);
114 static int pci_add_map(device_t bus, device_t dev, int reg,
115 struct resource_list *rl, int force, int prefetch);
116 static void pci_load_vendor_data(void);
117 static int pci_describe_parse_line(char **ptr, int *vendor,
118 int *device, char **desc);
119 static char *pci_describe_device(device_t dev);
120 static int pci_modevent(module_t mod, int what, void *arg);
121 static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
122 pcicfgregs *cfg);
123 static void pci_read_cap(device_t pcib, pcicfgregs *cfg);
124 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg,
125 int reg, uint32_t *data);
126 #if 0
127 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
128 int reg, uint32_t data);
129 #endif
130 static void pci_read_vpd(device_t pcib, pcicfgregs *cfg);
131 static void pci_mask_msix(device_t dev, u_int index);
132 static void pci_unmask_msix(device_t dev, u_int index);
133 static int pci_msi_blacklisted(void);
134 static int pci_msix_blacklisted(void);
135 static void pci_resume_msi(device_t dev);
136 static void pci_resume_msix(device_t dev);
137 static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d,
138 int b, int s, int f, uint16_t vid, uint16_t did);
139
140 static device_method_t pci_methods[] = {
141 /* Device interface */
142 DEVMETHOD(device_probe, pci_probe),
143 DEVMETHOD(device_attach, pci_attach),
144 DEVMETHOD(device_detach, pci_detach),
145 DEVMETHOD(device_shutdown, bus_generic_shutdown),
146 DEVMETHOD(device_suspend, bus_generic_suspend),
147 DEVMETHOD(device_resume, pci_resume),
148
149 /* Bus interface */
150 DEVMETHOD(bus_print_child, pci_print_child),
151 DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch),
152 DEVMETHOD(bus_read_ivar, pci_read_ivar),
153 DEVMETHOD(bus_write_ivar, pci_write_ivar),
154 DEVMETHOD(bus_driver_added, pci_driver_added),
155 DEVMETHOD(bus_setup_intr, pci_setup_intr),
156 DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
157 DEVMETHOD(bus_reset_prepare, pci_reset_prepare),
158 DEVMETHOD(bus_reset_post, pci_reset_post),
159 DEVMETHOD(bus_reset_child, pci_reset_child),
160
161 DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag),
162 DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
163 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
164 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
165 DEVMETHOD(bus_delete_resource, pci_delete_resource),
166 DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
167 DEVMETHOD(bus_adjust_resource, pci_adjust_resource),
168 DEVMETHOD(bus_release_resource, pci_release_resource),
169 DEVMETHOD(bus_activate_resource, pci_activate_resource),
170 DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
171 DEVMETHOD(bus_map_resource, pci_map_resource),
172 DEVMETHOD(bus_unmap_resource, pci_unmap_resource),
173 DEVMETHOD(bus_child_deleted, pci_child_deleted),
174 DEVMETHOD(bus_child_detached, pci_child_detached),
175 DEVMETHOD(bus_child_pnpinfo, pci_child_pnpinfo_method),
176 DEVMETHOD(bus_child_location, pci_child_location_method),
177 DEVMETHOD(bus_get_device_path, pci_get_device_path_method),
178 DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit),
179 DEVMETHOD(bus_remap_intr, pci_remap_intr_method),
180 DEVMETHOD(bus_suspend_child, pci_suspend_child),
181 DEVMETHOD(bus_resume_child, pci_resume_child),
182 DEVMETHOD(bus_rescan, pci_rescan_method),
183
184 /* PCI interface */
185 DEVMETHOD(pci_read_config, pci_read_config_method),
186 DEVMETHOD(pci_write_config, pci_write_config_method),
187 DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method),
188 DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method),
189 DEVMETHOD(pci_enable_io, pci_enable_io_method),
190 DEVMETHOD(pci_disable_io, pci_disable_io_method),
191 DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method),
192 DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method),
193 DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method),
194 DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method),
195 DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method),
196 DEVMETHOD(pci_find_cap, pci_find_cap_method),
197 DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method),
198 DEVMETHOD(pci_find_extcap, pci_find_extcap_method),
199 DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method),
200 DEVMETHOD(pci_find_htcap, pci_find_htcap_method),
201 DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method),
202 DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method),
203 DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method),
204 DEVMETHOD(pci_enable_msi, pci_enable_msi_method),
205 DEVMETHOD(pci_enable_msix, pci_enable_msix_method),
206 DEVMETHOD(pci_disable_msi, pci_disable_msi_method),
207 DEVMETHOD(pci_remap_msix, pci_remap_msix_method),
208 DEVMETHOD(pci_release_msi, pci_release_msi_method),
209 DEVMETHOD(pci_msi_count, pci_msi_count_method),
210 DEVMETHOD(pci_msix_count, pci_msix_count_method),
211 DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method),
212 DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method),
213 DEVMETHOD(pci_get_id, pci_get_id_method),
214 DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method),
215 DEVMETHOD(pci_child_added, pci_child_added_method),
216 #ifdef PCI_IOV
217 DEVMETHOD(pci_iov_attach, pci_iov_attach_method),
218 DEVMETHOD(pci_iov_detach, pci_iov_detach_method),
219 DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method),
220 #endif
221
222 DEVMETHOD_END
223 };
224
225 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));
226
227 EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_modevent, NULL, BUS_PASS_BUS);
228 MODULE_VERSION(pci, 1);
229
230 static char *pci_vendordata;
231 static size_t pci_vendordata_size;
232
233 struct pci_quirk {
234 uint32_t devid; /* Vendor/device of the card */
235 int type;
236 #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */
237 #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */
238 #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */
239 #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */
240 #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */
241 #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */
242 #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */
243 #define PCI_QUIRK_DISABLE_FLR 8 /* Function-Level Reset (FLR) not working. */
244 #define PCI_QUIRK_ENABLE_FLR 9 /* FLR works but is not advertised. */
245 int arg1;
246 int arg2;
247 };
248
249 static const struct pci_quirk pci_quirks[] = {
250 /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
251 { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 },
252 { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 },
253 /* As does the Serverworks OSB4 (the SMBus mapping register) */
254 { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 },
255
256 /*
257 * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge
258 * or the CMIC-SL (AKA ServerWorks GC_LE).
259 */
260 { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
261 { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
262
263 /*
264 * MSI doesn't work on earlier Intel chipsets including
265 * E7500, E7501, E7505, 845, 865, 875/E7210, and 855.
266 */
267 { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
268 { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
269 { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
270 { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
271 { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
272 { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
273 { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
274
275 /*
276 * MSI doesn't work with devices behind the AMD 8131 HT-PCIX
277 * bridge.
278 */
279 { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 },
280
281 /*
282 * Some virtualization environments emulate an older chipset
283 * but support MSI just fine. QEMU uses the Intel 82440.
284 */
285 { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 },
286
287 /*
288 * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus
289 * controller depending on SoftPciRst register (PM_IO 0x55 [7]).
290 * It prevents us from attaching hpet(4) when the bit is unset.
291 * Note this quirk only affects SB600 revision A13 and earlier.
292 * For SB600 A21 and later, firmware must set the bit to hide it.
293 * For SB700 and later, it is unused and hardcoded to zero.
294 */
295 { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 },
296
297 /*
298 * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have
299 * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit
300 * of the command register is set.
301 */
302 { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
303 { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
304 { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
305 { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
306 { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
307
308 /*
309 * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't
310 * issue MSI interrupts with PCIM_CMD_INTxDIS set either.
311 */
312 { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */
313 { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */
314 { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */
315 { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */
316 { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */
317 { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */
318
319 /*
320 * HPE Gen 10 VGA has a memory range that can't be allocated in the
321 * expected place.
322 */
323 { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 },
324
325 /*
326 * The Intel 82599 VF implements FLR without advertising it; see
327 * 82599 Specification Update, erratum 35.
328 */
329 { 0x10ed8086, PCI_QUIRK_ENABLE_FLR, 0, 0 },
330
331 /*
332 * With some MediaTek mt76 WiFi FLR does not work despite advertised.
333 */
334 { 0x061614c3, PCI_QUIRK_DISABLE_FLR, 0, 0 }, /* mt76 7922 */
335
336 /* end of table */
337 { 0 }
338 };
339
340 /* map register information */
341 #define PCI_MAPMEM 0x01 /* memory map */
342 #define PCI_MAPMEMP 0x02 /* prefetchable memory map */
343 #define PCI_MAPPORT 0x04 /* port map */
344
345 struct devlist pci_devq;
346 uint32_t pci_generation;
347 uint32_t pci_numdevs = 0;
348 static int pcie_chipset, pcix_chipset;
349
350 /* sysctl vars */
351 SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
352 "PCI bus tuning parameters");
353
354 static int pci_enable_io_modes = 1;
355 SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN,
356 &pci_enable_io_modes, 1,
357 "Enable I/O and memory bits in the config register. Some BIOSes do not"
358 " enable these bits correctly. We'd like to do this all the time, but"
359 " there are some peripherals that this causes problems with.");
360
361 static int pci_do_realloc_bars = 1;
362 SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN,
363 &pci_do_realloc_bars, 0,
364 "Attempt to allocate a new range for any BARs whose original "
365 "firmware-assigned ranges fail to allocate during the initial device scan.");
366
367 static int pci_do_power_nodriver = 0;
368 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN,
369 &pci_do_power_nodriver, 0,
370 "Place a function into D3 state when no driver attaches to it. 0 means"
371 " disable. 1 means conservatively place function into D3 state. 2 means"
372 " aggressively place function into D3 state. 3 means put absolutely"
373 " everything in D3 state.");
374
375 int pci_do_power_resume = 1;
376 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN,
377 &pci_do_power_resume, 1,
378 "Transition from D3 -> D0 on resume.");
379
380 int pci_do_power_suspend = 1;
381 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN,
382 &pci_do_power_suspend, 1,
383 "Transition from D0 -> D3 on suspend.");
384
385 static int pci_do_msi = 1;
386 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1,
387 "Enable support for MSI interrupts");
388
389 static int pci_do_msix = 1;
390 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1,
391 "Enable support for MSI-X interrupts");
392
393 static int pci_msix_rewrite_table = 0;
394 SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
395 &pci_msix_rewrite_table, 0,
396 "Rewrite entire MSI-X table when updating MSI-X entries");
397
398 static int pci_honor_msi_blacklist = 1;
399 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN,
400 &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");
401
402 #if defined(__i386__) || defined(__amd64__)
403 static int pci_usb_takeover = 1;
404 #else
405 static int pci_usb_takeover = 0;
406 #endif
407 SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN,
408 &pci_usb_takeover, 1,
409 "Enable early takeover of USB controllers. Disable this if you depend on"
410 " BIOS emulation of USB devices, that is you use USB devices (like"
411 " keyboard or mouse) but do not load USB drivers");
412
413 static int pci_clear_bars;
414 SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0,
415 "Ignore firmware-assigned resources for BARs.");
416
417 static int pci_clear_buses;
418 SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
419 "Ignore firmware-assigned bus numbers.");
420
421 static int pci_enable_ari = 1;
422 SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
423 0, "Enable support for PCIe Alternative RID Interpretation");
424
425 /*
426 * Some x86 firmware only enables PCIe hotplug if we claim to support aspm,
427 * however enabling it breaks some arm64 firmware as it powers off devices.
428 */
429 #if defined(__i386__) || defined(__amd64__)
430 int pci_enable_aspm = 1;
431 #else
432 int pci_enable_aspm = 0;
433 #endif
434 SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm,
435 0, "Enable support for PCIe Active State Power Management");
436
437 static int pci_clear_aer_on_attach = 0;
438 SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN,
439 &pci_clear_aer_on_attach, 0,
440 "Clear port and device AER state on driver attach");
441
442 static bool pci_enable_mps_tune = true;
443 SYSCTL_BOOL(_hw_pci, OID_AUTO, enable_mps_tune, CTLFLAG_RWTUN,
444 &pci_enable_mps_tune, 1,
445 "Enable tuning of MPS(maximum payload size)." );
446
447 static int pci_mps_limit;
448 SYSCTL_INT(_hw_pci, OID_AUTO, mps_limit, CTLFLAG_RDTUN, &pci_mps_limit, 0,
449 "Limit PCIe MPS to this many bytes (power of two from 128 to 4096)");
450 static bool pci_mps_limit_warned;
451
452 static bool pci_mps_enforce;
453 SYSCTL_BOOL(_hw_pci, OID_AUTO, mps_enforce, CTLFLAG_RDTUN,
454 &pci_mps_enforce, 0,
455 "Disable PCIe endpoints with an MPS incompatible with their shared path");
456
457 static bool pci_intx_reroute = true;
458 SYSCTL_BOOL(_hw_pci, OID_AUTO, intx_reroute, CTLFLAG_RWTUN,
459 &pci_intx_reroute, 0, "Re-route INTx interrupts when scanning devices");
460
461 static int
pci_has_quirk(uint32_t devid,int quirk)462 pci_has_quirk(uint32_t devid, int quirk)
463 {
464 const struct pci_quirk *q;
465
466 for (q = &pci_quirks[0]; q->devid; q++) {
467 if (q->devid == devid && q->type == quirk)
468 return (1);
469 }
470 return (0);
471 }
472
473 /* Find a device_t by bus/slot/function in domain 0 */
474
475 device_t
pci_find_bsf(uint8_t bus,uint8_t slot,uint8_t func)476 pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func)
477 {
478
479 return (pci_find_dbsf(0, bus, slot, func));
480 }
481
482 /* Find a device_t by domain/bus/slot/function */
483
484 device_t
pci_find_dbsf(uint32_t domain,uint8_t bus,uint8_t slot,uint8_t func)485 pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func)
486 {
487 struct pci_devinfo *dinfo = NULL;
488
489 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
490 if ((dinfo->cfg.domain == domain) &&
491 (dinfo->cfg.bus == bus) &&
492 (dinfo->cfg.slot == slot) &&
493 (dinfo->cfg.func == func)) {
494 break;
495 }
496 }
497
498 return (dinfo != NULL ? dinfo->cfg.dev : NULL);
499 }
500
501 /* Find a device_t by vendor/device ID */
502
503 device_t
pci_find_device(uint16_t vendor,uint16_t device)504 pci_find_device(uint16_t vendor, uint16_t device)
505 {
506 struct pci_devinfo *dinfo;
507
508 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
509 if ((dinfo->cfg.vendor == vendor) &&
510 (dinfo->cfg.device == device)) {
511 return (dinfo->cfg.dev);
512 }
513 }
514
515 return (NULL);
516 }
517
518 device_t
pci_find_class(uint8_t class,uint8_t subclass)519 pci_find_class(uint8_t class, uint8_t subclass)
520 {
521 struct pci_devinfo *dinfo;
522
523 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
524 if (dinfo->cfg.baseclass == class &&
525 dinfo->cfg.subclass == subclass) {
526 return (dinfo->cfg.dev);
527 }
528 }
529
530 return (NULL);
531 }
532
533 device_t
pci_find_class_from(uint8_t class,uint8_t subclass,device_t from)534 pci_find_class_from(uint8_t class, uint8_t subclass, device_t from)
535 {
536 struct pci_devinfo *dinfo;
537 bool found = false;
538
539 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
540 if (from != NULL && found == false) {
541 if (from != dinfo->cfg.dev)
542 continue;
543 found = true;
544 continue;
545 }
546 if (dinfo->cfg.baseclass == class &&
547 dinfo->cfg.subclass == subclass) {
548 return (dinfo->cfg.dev);
549 }
550 }
551
552 return (NULL);
553 }
554
555 device_t
pci_find_base_class_from(uint8_t class,device_t from)556 pci_find_base_class_from(uint8_t class, device_t from)
557 {
558 struct pci_devinfo *dinfo;
559 bool found = false;
560
561 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
562 if (from != NULL && found == false) {
563 if (from != dinfo->cfg.dev)
564 continue;
565 found = true;
566 continue;
567 }
568 if (dinfo->cfg.baseclass == class) {
569 return (dinfo->cfg.dev);
570 }
571 }
572
573 return (NULL);
574 }
575
576 static int
pci_printf(pcicfgregs * cfg,const char * fmt,...)577 pci_printf(pcicfgregs *cfg, const char *fmt, ...)
578 {
579 va_list ap;
580 int retval;
581
582 retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot,
583 cfg->func);
584 va_start(ap, fmt);
585 retval += vprintf(fmt, ap);
586 va_end(ap);
587 return (retval);
588 }
589
590 /* return base address of memory or port map */
591
592 static pci_addr_t
pci_mapbase(uint64_t mapreg)593 pci_mapbase(uint64_t mapreg)
594 {
595
596 if (PCI_BAR_MEM(mapreg))
597 return (mapreg & PCIM_BAR_MEM_BASE);
598 else
599 return (mapreg & PCIM_BAR_IO_BASE);
600 }
601
602 /* return map type of memory or port map */
603
604 static const char *
pci_maptype(uint64_t mapreg)605 pci_maptype(uint64_t mapreg)
606 {
607
608 if (PCI_BAR_IO(mapreg))
609 return ("I/O Port");
610 if (mapreg & PCIM_BAR_MEM_PREFETCH)
611 return ("Prefetchable Memory");
612 return ("Memory");
613 }
614
615 /* return log2 of map size decoded for memory or port map */
616
617 int
pci_mapsize(uint64_t testval)618 pci_mapsize(uint64_t testval)
619 {
620 int ln2size;
621
622 testval = pci_mapbase(testval);
623 ln2size = 0;
624 if (testval != 0) {
625 while ((testval & 1) == 0)
626 {
627 ln2size++;
628 testval >>= 1;
629 }
630 }
631 return (ln2size);
632 }
633
634 /* return base address of device ROM */
635
636 static pci_addr_t
pci_rombase(uint64_t mapreg)637 pci_rombase(uint64_t mapreg)
638 {
639
640 return (mapreg & PCIM_BIOS_ADDR_MASK);
641 }
642
643 /* return log2 of map size decided for device ROM */
644
645 static int
pci_romsize(uint64_t testval)646 pci_romsize(uint64_t testval)
647 {
648 int ln2size;
649
650 testval = pci_rombase(testval);
651 ln2size = 0;
652 if (testval != 0) {
653 while ((testval & 1) == 0)
654 {
655 ln2size++;
656 testval >>= 1;
657 }
658 }
659 return (ln2size);
660 }
661
662 /* return log2 of address range supported by map register */
663
664 static int
pci_maprange(uint64_t mapreg)665 pci_maprange(uint64_t mapreg)
666 {
667 int ln2range = 0;
668
669 if (PCI_BAR_IO(mapreg))
670 ln2range = 32;
671 else
672 switch (mapreg & PCIM_BAR_MEM_TYPE) {
673 case PCIM_BAR_MEM_32:
674 ln2range = 32;
675 break;
676 case PCIM_BAR_MEM_1MB:
677 ln2range = 20;
678 break;
679 case PCIM_BAR_MEM_64:
680 ln2range = 64;
681 break;
682 }
683 return (ln2range);
684 }
685
686 /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */
687
688 static void
pci_fixancient(pcicfgregs * cfg)689 pci_fixancient(pcicfgregs *cfg)
690 {
691 if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
692 return;
693
694 /* PCI to PCI bridges use header type 1 */
695 if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI)
696 cfg->hdrtype = PCIM_HDRTYPE_BRIDGE;
697 }
698
699 /* extract header type specific config data */
700
701 static void
pci_hdrtypedata(device_t pcib,int b,int s,int f,pcicfgregs * cfg)702 pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg)
703 {
704 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
705 switch (cfg->hdrtype & PCIM_HDRTYPE) {
706 case PCIM_HDRTYPE_NORMAL:
707 cfg->subvendor = REG(PCIR_SUBVEND_0, 2);
708 cfg->subdevice = REG(PCIR_SUBDEV_0, 2);
709 cfg->mingnt = REG(PCIR_MINGNT, 1);
710 cfg->maxlat = REG(PCIR_MAXLAT, 1);
711 cfg->nummaps = PCI_MAXMAPS_0;
712 break;
713 case PCIM_HDRTYPE_BRIDGE:
714 cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1);
715 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1);
716 cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1);
717 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1);
718 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2);
719 cfg->nummaps = PCI_MAXMAPS_1;
720 break;
721 case PCIM_HDRTYPE_CARDBUS:
722 cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1);
723 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1);
724 cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1);
725 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1);
726 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2);
727 cfg->subvendor = REG(PCIR_SUBVEND_2, 2);
728 cfg->subdevice = REG(PCIR_SUBDEV_2, 2);
729 cfg->nummaps = PCI_MAXMAPS_2;
730 break;
731 }
732 #undef REG
733 }
734
735 /* read configuration header into pcicfgregs structure */
736 struct pci_devinfo *
pci_read_device(device_t pcib,device_t bus,int d,int b,int s,int f)737 pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f)
738 {
739 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
740 uint16_t vid, did;
741
742 vid = REG(PCIR_VENDOR, 2);
743 if (vid == PCIV_INVALID)
744 return (NULL);
745
746 did = REG(PCIR_DEVICE, 2);
747
748 return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did));
749 }
750
751 struct pci_devinfo *
pci_alloc_devinfo_method(device_t dev)752 pci_alloc_devinfo_method(device_t dev)
753 {
754
755 return (malloc(sizeof(struct pci_devinfo), M_DEVBUF,
756 M_WAITOK | M_ZERO));
757 }
758
759 static struct pci_devinfo *
pci_fill_devinfo(device_t pcib,device_t bus,int d,int b,int s,int f,uint16_t vid,uint16_t did)760 pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f,
761 uint16_t vid, uint16_t did)
762 {
763 struct pci_devinfo *devlist_entry;
764 pcicfgregs *cfg;
765
766 devlist_entry = PCI_ALLOC_DEVINFO(bus);
767
768 cfg = &devlist_entry->cfg;
769
770 cfg->domain = d;
771 cfg->bus = b;
772 cfg->slot = s;
773 cfg->func = f;
774 cfg->vendor = vid;
775 cfg->device = did;
776 cfg->cmdreg = REG(PCIR_COMMAND, 2);
777 cfg->statreg = REG(PCIR_STATUS, 2);
778 cfg->baseclass = REG(PCIR_CLASS, 1);
779 cfg->subclass = REG(PCIR_SUBCLASS, 1);
780 cfg->progif = REG(PCIR_PROGIF, 1);
781 cfg->revid = REG(PCIR_REVID, 1);
782 cfg->hdrtype = REG(PCIR_HDRTYPE, 1);
783 cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1);
784 cfg->lattimer = REG(PCIR_LATTIMER, 1);
785 cfg->intpin = REG(PCIR_INTPIN, 1);
786 cfg->intline = REG(PCIR_INTLINE, 1);
787
788 cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0;
789 cfg->hdrtype &= ~PCIM_MFDEV;
790 STAILQ_INIT(&cfg->maps);
791
792 cfg->iov = NULL;
793
794 pci_fixancient(cfg);
795 pci_hdrtypedata(pcib, b, s, f, cfg);
796
797 if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)
798 pci_read_cap(pcib, cfg);
799
800 STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links);
801
802 devlist_entry->conf.pc_sel.pc_domain = cfg->domain;
803 devlist_entry->conf.pc_sel.pc_bus = cfg->bus;
804 devlist_entry->conf.pc_sel.pc_dev = cfg->slot;
805 devlist_entry->conf.pc_sel.pc_func = cfg->func;
806 devlist_entry->conf.pc_hdr = cfg->hdrtype;
807
808 devlist_entry->conf.pc_subvendor = cfg->subvendor;
809 devlist_entry->conf.pc_subdevice = cfg->subdevice;
810 devlist_entry->conf.pc_vendor = cfg->vendor;
811 devlist_entry->conf.pc_device = cfg->device;
812
813 devlist_entry->conf.pc_class = cfg->baseclass;
814 devlist_entry->conf.pc_subclass = cfg->subclass;
815 devlist_entry->conf.pc_progif = cfg->progif;
816 devlist_entry->conf.pc_revid = cfg->revid;
817
818 devlist_entry->conf.pc_secbus = cfg->bridge.br_secbus;
819 devlist_entry->conf.pc_subbus = cfg->bridge.br_subbus;
820
821 pci_numdevs++;
822 pci_generation++;
823
824 return (devlist_entry);
825 }
826 #undef REG
827
828 static void
pci_ea_fill_info(device_t pcib,pcicfgregs * cfg)829 pci_ea_fill_info(device_t pcib, pcicfgregs *cfg)
830 {
831 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \
832 cfg->ea.ea_location + (n), w)
833 int num_ent;
834 int ptr;
835 int a, b;
836 uint32_t val;
837 int ent_size;
838 uint32_t dw[4];
839 uint64_t base, max_offset;
840 struct pci_ea_entry *eae;
841
842 if (cfg->ea.ea_location == 0)
843 return;
844
845 STAILQ_INIT(&cfg->ea.ea_entries);
846
847 /* Determine the number of entries */
848 num_ent = REG(PCIR_EA_NUM_ENT, 2);
849 num_ent &= PCIM_EA_NUM_ENT_MASK;
850
851 /* Find the first entry to care of */
852 ptr = PCIR_EA_FIRST_ENT;
853
854 /* Skip DWORD 2 for type 1 functions */
855 if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE)
856 ptr += 4;
857
858 for (a = 0; a < num_ent; a++) {
859 eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO);
860 eae->eae_cfg_offset = cfg->ea.ea_location + ptr;
861
862 /* Read a number of dwords in the entry */
863 val = REG(ptr, 4);
864 ptr += 4;
865 ent_size = (val & PCIM_EA_ES);
866
867 for (b = 0; b < ent_size; b++) {
868 dw[b] = REG(ptr, 4);
869 ptr += 4;
870 }
871
872 eae->eae_flags = val;
873 eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET;
874
875 base = dw[0] & PCIM_EA_FIELD_MASK;
876 max_offset = dw[1] | ~PCIM_EA_FIELD_MASK;
877 b = 2;
878 if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) {
879 base |= (uint64_t)dw[b] << 32UL;
880 b++;
881 }
882 if (((dw[1] & PCIM_EA_IS_64) != 0)
883 && (b < ent_size)) {
884 max_offset |= (uint64_t)dw[b] << 32UL;
885 b++;
886 }
887
888 eae->eae_base = base;
889 eae->eae_max_offset = max_offset;
890
891 STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link);
892
893 if (bootverbose) {
894 printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n",
895 cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags,
896 (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset);
897 }
898 }
899 }
900 #undef REG
901
902 static void
pci_read_cap(device_t pcib,pcicfgregs * cfg)903 pci_read_cap(device_t pcib, pcicfgregs *cfg)
904 {
905 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w)
906 #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w)
907 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
908 uint64_t addr;
909 #endif
910 uint32_t val;
911 int ptr, nextptr, ptrptr;
912
913 switch (cfg->hdrtype & PCIM_HDRTYPE) {
914 case PCIM_HDRTYPE_NORMAL:
915 case PCIM_HDRTYPE_BRIDGE:
916 ptrptr = PCIR_CAP_PTR;
917 break;
918 case PCIM_HDRTYPE_CARDBUS:
919 ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */
920 break;
921 default:
922 return; /* no extended capabilities support */
923 }
924 nextptr = REG(ptrptr, 1); /* sanity check? */
925
926 /*
927 * Read capability entries.
928 */
929 while (nextptr != 0) {
930 /* Sanity check */
931 if (nextptr > 255) {
932 printf("illegal PCI extended capability offset %d\n",
933 nextptr);
934 return;
935 }
936 /* Find the next entry */
937 ptr = nextptr;
938 nextptr = REG(ptr + PCICAP_NEXTPTR, 1);
939
940 /* Process this entry */
941 switch (REG(ptr + PCICAP_ID, 1)) {
942 case PCIY_PMG: /* PCI power management */
943 cfg->pp.pp_location = ptr;
944 cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2);
945 break;
946 case PCIY_HT: /* HyperTransport */
947 /* Determine HT-specific capability type. */
948 val = REG(ptr + PCIR_HT_COMMAND, 2);
949
950 if ((val & 0xe000) == PCIM_HTCAP_SLAVE)
951 cfg->ht.ht_slave = ptr;
952
953 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
954 switch (val & PCIM_HTCMD_CAP_MASK) {
955 case PCIM_HTCAP_MSI_MAPPING:
956 if (!(val & PCIM_HTCMD_MSI_FIXED)) {
957 /* Sanity check the mapping window. */
958 addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI,
959 4);
960 addr <<= 32;
961 addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO,
962 4);
963 if (addr != MSI_INTEL_ADDR_BASE)
964 device_printf(pcib,
965 "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n",
966 cfg->domain, cfg->bus,
967 cfg->slot, cfg->func,
968 (long long)addr);
969 } else
970 addr = MSI_INTEL_ADDR_BASE;
971
972 cfg->ht.ht_msimap = ptr;
973 cfg->ht.ht_msictrl = val;
974 cfg->ht.ht_msiaddr = addr;
975 break;
976 }
977 #endif
978 break;
979 case PCIY_MSI: /* PCI MSI */
980 cfg->msi.msi_location = ptr;
981 cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2);
982 break;
983 case PCIY_MSIX: /* PCI MSI-X */
984 cfg->msix.msix_location = ptr;
985 cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2);
986 val = REG(ptr + PCIR_MSIX_TABLE, 4);
987 cfg->msix.msix_table_bar = PCIR_BAR(val &
988 PCIM_MSIX_BIR_MASK);
989 cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK;
990 val = REG(ptr + PCIR_MSIX_PBA, 4);
991 cfg->msix.msix_pba_bar = PCIR_BAR(val &
992 PCIM_MSIX_BIR_MASK);
993 cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
994 break;
995 case PCIY_VPD: /* PCI Vital Product Data */
996 cfg->vpd.vpd_reg = ptr;
997 break;
998 case PCIY_SUBVENDOR:
999 /* Should always be true. */
1000 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
1001 PCIM_HDRTYPE_BRIDGE) {
1002 val = REG(ptr + PCIR_SUBVENDCAP_ID, 4);
1003 cfg->subvendor = val & 0xffff;
1004 cfg->subdevice = val >> 16;
1005 }
1006 break;
1007 case PCIY_PCIX: /* PCI-X */
1008 /*
1009 * Assume we have a PCI-X chipset if we have
1010 * at least one PCI-PCI bridge with a PCI-X
1011 * capability. Note that some systems with
1012 * PCI-express or HT chipsets might match on
1013 * this check as well.
1014 */
1015 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
1016 PCIM_HDRTYPE_BRIDGE)
1017 pcix_chipset = 1;
1018 cfg->pcix.pcix_location = ptr;
1019 break;
1020 case PCIY_EXPRESS: /* PCI-express */
1021 /*
1022 * Assume we have a PCI-express chipset if we have
1023 * at least one PCI-express device.
1024 */
1025 pcie_chipset = 1;
1026 cfg->pcie.pcie_location = ptr;
1027 val = REG(ptr + PCIER_FLAGS, 2);
1028 cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE;
1029 break;
1030 case PCIY_EA: /* Enhanced Allocation */
1031 cfg->ea.ea_location = ptr;
1032 pci_ea_fill_info(pcib, cfg);
1033 break;
1034 default:
1035 break;
1036 }
1037 }
1038
1039 #if defined(__powerpc__)
1040 /*
1041 * Enable the MSI mapping window for all HyperTransport
1042 * slaves. PCI-PCI bridges have their windows enabled via
1043 * PCIB_MAP_MSI().
1044 */
1045 if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 &&
1046 !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
1047 device_printf(pcib,
1048 "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n",
1049 cfg->domain, cfg->bus, cfg->slot, cfg->func);
1050 cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
1051 WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl,
1052 2);
1053 }
1054 #endif
1055 /* REG and WREG use carry through to next functions */
1056 }
1057
1058 /*
1059 * PCI Vital Product Data
1060 */
1061
1062 #define PCI_VPD_TIMEOUT 1000000
1063
1064 static int
pci_read_vpd_reg(device_t pcib,pcicfgregs * cfg,int reg,uint32_t * data)1065 pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data)
1066 {
1067 int count = PCI_VPD_TIMEOUT;
1068
1069 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1070
1071 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2);
1072
1073 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) {
1074 if (--count < 0)
1075 return (ENXIO);
1076 DELAY(1); /* limit looping */
1077 }
1078 *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4));
1079
1080 return (0);
1081 }
1082
1083 #if 0
1084 static int
1085 pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data)
1086 {
1087 int count = PCI_VPD_TIMEOUT;
1088
1089 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1090
1091 WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4);
1092 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2);
1093 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) {
1094 if (--count < 0)
1095 return (ENXIO);
1096 DELAY(1); /* limit looping */
1097 }
1098
1099 return (0);
1100 }
1101 #endif
1102
1103 #undef PCI_VPD_TIMEOUT
1104
1105 struct vpd_readstate {
1106 device_t pcib;
1107 pcicfgregs *cfg;
1108 uint32_t val;
1109 int bytesinval;
1110 int off;
1111 uint8_t cksum;
1112 };
1113
1114 /* return 0 and one byte in *data if no read error, -1 else */
1115 static int
vpd_nextbyte(struct vpd_readstate * vrs,uint8_t * data)1116 vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data)
1117 {
1118 uint32_t reg;
1119 uint8_t byte;
1120
1121 if (vrs->bytesinval == 0) {
1122 if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®))
1123 return (-1);
1124 vrs->val = le32toh(reg);
1125 vrs->off += 4;
1126 byte = vrs->val & 0xff;
1127 vrs->bytesinval = 3;
1128 } else {
1129 vrs->val = vrs->val >> 8;
1130 byte = vrs->val & 0xff;
1131 vrs->bytesinval--;
1132 }
1133
1134 vrs->cksum += byte;
1135 *data = byte;
1136 return (0);
1137 }
1138
1139 /* return 0 on match, -1 and "unget" byte on no match */
1140 static int
vpd_expectbyte(struct vpd_readstate * vrs,uint8_t expected)1141 vpd_expectbyte(struct vpd_readstate *vrs, uint8_t expected)
1142 {
1143 uint8_t data;
1144
1145 if (vpd_nextbyte(vrs, &data) != 0)
1146 return (-1);
1147
1148 if (data == expected)
1149 return (0);
1150
1151 vrs->cksum -= data;
1152 vrs->val = (vrs->val << 8) + data;
1153 vrs->bytesinval++;
1154 return (-1);
1155 }
1156
1157 /* return size if tag matches, -1 on no match, -2 on read error */
1158 static int
vpd_read_tag_size(struct vpd_readstate * vrs,uint8_t vpd_tag)1159 vpd_read_tag_size(struct vpd_readstate *vrs, uint8_t vpd_tag)
1160 {
1161 uint8_t byte1, byte2;
1162
1163 if (vpd_expectbyte(vrs, vpd_tag) != 0)
1164 return (-1);
1165
1166 if ((vpd_tag & 0x80) == 0)
1167 return (vpd_tag & 0x07);
1168
1169 if (vpd_nextbyte(vrs, &byte1) != 0)
1170 return (-2);
1171 if (vpd_nextbyte(vrs, &byte2) != 0)
1172 return (-2);
1173
1174 return ((byte2 << 8) + byte1);
1175 }
1176
1177 /* (re)allocate buffer in multiples of 8 elements */
1178 static void*
alloc_buffer(void * buffer,size_t element_size,int needed)1179 alloc_buffer(void* buffer, size_t element_size, int needed)
1180 {
1181 int alloc, new_alloc;
1182
1183 alloc = roundup2(needed, 8);
1184 new_alloc = roundup2(needed + 1, 8);
1185 if (alloc != new_alloc) {
1186 buffer = reallocf(buffer,
1187 new_alloc * element_size, M_DEVBUF, M_WAITOK | M_ZERO);
1188 }
1189
1190 return (buffer);
1191 }
1192
1193 /* read VPD keyword and return element size, return -1 on read error */
1194 static int
vpd_read_elem_head(struct vpd_readstate * vrs,char keyword[2])1195 vpd_read_elem_head(struct vpd_readstate *vrs, char keyword[2])
1196 {
1197 uint8_t data;
1198
1199 if (vpd_nextbyte(vrs, &keyword[0]) != 0)
1200 return (-1);
1201 if (vpd_nextbyte(vrs, &keyword[1]) != 0)
1202 return (-1);
1203 if (vpd_nextbyte(vrs, &data) != 0)
1204 return (-1);
1205
1206 return (data);
1207 }
1208
1209 /* read VPD data element of given size into allocated buffer */
1210 static char *
vpd_read_value(struct vpd_readstate * vrs,int size)1211 vpd_read_value(struct vpd_readstate *vrs, int size)
1212 {
1213 int i;
1214 char char1;
1215 char *value;
1216
1217 value = malloc(size + 1, M_DEVBUF, M_WAITOK);
1218 for (i = 0; i < size; i++) {
1219 if (vpd_nextbyte(vrs, &char1) != 0) {
1220 free(value, M_DEVBUF);
1221 return (NULL);
1222 }
1223 value[i] = char1;
1224 }
1225 value[size] = '\0';
1226
1227 return (value);
1228 }
1229
1230 /* read VPD into *keyword and *value, return length of data element */
1231 static int
vpd_read_elem_data(struct vpd_readstate * vrs,char keyword[2],char ** value,int maxlen)1232 vpd_read_elem_data(struct vpd_readstate *vrs, char keyword[2], char **value, int maxlen)
1233 {
1234 int len;
1235
1236 len = vpd_read_elem_head(vrs, keyword);
1237 if (len < 0 || len > maxlen)
1238 return (-1);
1239 *value = vpd_read_value(vrs, len);
1240
1241 return (len);
1242 }
1243
1244 /* subtract all data following first byte from checksum of RV element */
1245 static void
vpd_fixup_cksum(struct vpd_readstate * vrs,char * rvstring,int len)1246 vpd_fixup_cksum(struct vpd_readstate *vrs, char *rvstring, int len)
1247 {
1248 int i;
1249 uint8_t fixup;
1250
1251 fixup = 0;
1252 for (i = 1; i < len; i++)
1253 fixup += rvstring[i];
1254 vrs->cksum -= fixup;
1255 }
1256
1257 /* fetch one read-only element and return size of heading + data */
1258 static int
next_vpd_ro_elem(struct vpd_readstate * vrs,int maxsize)1259 next_vpd_ro_elem(struct vpd_readstate *vrs, int maxsize)
1260 {
1261 struct pcicfg_vpd *vpd;
1262 pcicfgregs *cfg;
1263 struct vpd_readonly *vpd_ros;
1264 int len;
1265
1266 cfg = vrs->cfg;
1267 vpd = &cfg->vpd;
1268
1269 if (maxsize < 3)
1270 return (-1);
1271 vpd->vpd_ros = alloc_buffer(vpd->vpd_ros, sizeof(*vpd->vpd_ros), vpd->vpd_rocnt);
1272 vpd_ros = &vpd->vpd_ros[vpd->vpd_rocnt];
1273 maxsize -= 3;
1274 len = vpd_read_elem_data(vrs, vpd_ros->keyword, &vpd_ros->value, maxsize);
1275 if (vpd_ros->value == NULL)
1276 return (-1);
1277 vpd_ros->len = len;
1278 if (vpd_ros->keyword[0] == 'R' && vpd_ros->keyword[1] == 'V') {
1279 vpd_fixup_cksum(vrs, vpd_ros->value, len);
1280 if (vrs->cksum != 0) {
1281 pci_printf(cfg,
1282 "invalid VPD checksum %#hhx\n", vrs->cksum);
1283 return (-1);
1284 }
1285 }
1286 vpd->vpd_rocnt++;
1287
1288 return (len + 3);
1289 }
1290
1291 /* fetch one writable element and return size of heading + data */
1292 static int
next_vpd_rw_elem(struct vpd_readstate * vrs,int maxsize)1293 next_vpd_rw_elem(struct vpd_readstate *vrs, int maxsize)
1294 {
1295 struct pcicfg_vpd *vpd;
1296 pcicfgregs *cfg;
1297 struct vpd_write *vpd_w;
1298 int len;
1299
1300 cfg = vrs->cfg;
1301 vpd = &cfg->vpd;
1302
1303 if (maxsize < 3)
1304 return (-1);
1305 vpd->vpd_w = alloc_buffer(vpd->vpd_w, sizeof(*vpd->vpd_w), vpd->vpd_wcnt);
1306 if (vpd->vpd_w == NULL) {
1307 pci_printf(cfg, "out of memory");
1308 return (-1);
1309 }
1310 vpd_w = &vpd->vpd_w[vpd->vpd_wcnt];
1311 maxsize -= 3;
1312 vpd_w->start = vrs->off + 3 - vrs->bytesinval;
1313 len = vpd_read_elem_data(vrs, vpd_w->keyword, &vpd_w->value, maxsize);
1314 if (vpd_w->value == NULL)
1315 return (-1);
1316 vpd_w->len = len;
1317 vpd->vpd_wcnt++;
1318
1319 return (len + 3);
1320 }
1321
1322 /* free all memory allocated for VPD data */
1323 static void
vpd_free(struct pcicfg_vpd * vpd)1324 vpd_free(struct pcicfg_vpd *vpd)
1325 {
1326 int i;
1327
1328 free(vpd->vpd_ident, M_DEVBUF);
1329 for (i = 0; i < vpd->vpd_rocnt; i++)
1330 free(vpd->vpd_ros[i].value, M_DEVBUF);
1331 free(vpd->vpd_ros, M_DEVBUF);
1332 vpd->vpd_rocnt = 0;
1333 for (i = 0; i < vpd->vpd_wcnt; i++)
1334 free(vpd->vpd_w[i].value, M_DEVBUF);
1335 free(vpd->vpd_w, M_DEVBUF);
1336 vpd->vpd_wcnt = 0;
1337 }
1338
1339 #define VPD_TAG_END ((0x0f << 3) | 0) /* small tag, len == 0 */
1340 #define VPD_TAG_IDENT (0x02 | 0x80) /* large tag */
1341 #define VPD_TAG_RO (0x10 | 0x80) /* large tag */
1342 #define VPD_TAG_RW (0x11 | 0x80) /* large tag */
1343
1344 static int
pci_parse_vpd(device_t pcib,pcicfgregs * cfg)1345 pci_parse_vpd(device_t pcib, pcicfgregs *cfg)
1346 {
1347 struct vpd_readstate vrs;
1348 int cksumvalid;
1349 int size, elem_size;
1350
1351 /* init vpd reader */
1352 vrs.bytesinval = 0;
1353 vrs.off = 0;
1354 vrs.pcib = pcib;
1355 vrs.cfg = cfg;
1356 vrs.cksum = 0;
1357
1358 /* read VPD ident element - mandatory */
1359 size = vpd_read_tag_size(&vrs, VPD_TAG_IDENT);
1360 if (size <= 0) {
1361 pci_printf(cfg, "no VPD ident found\n");
1362 return (0);
1363 }
1364 cfg->vpd.vpd_ident = vpd_read_value(&vrs, size);
1365 if (cfg->vpd.vpd_ident == NULL) {
1366 pci_printf(cfg, "error accessing VPD ident data\n");
1367 return (0);
1368 }
1369
1370 /* read VPD RO elements - mandatory */
1371 size = vpd_read_tag_size(&vrs, VPD_TAG_RO);
1372 if (size <= 0) {
1373 pci_printf(cfg, "no read-only VPD data found\n");
1374 return (0);
1375 }
1376 while (size > 0) {
1377 elem_size = next_vpd_ro_elem(&vrs, size);
1378 if (elem_size < 0) {
1379 pci_printf(cfg, "error accessing read-only VPD data\n");
1380 return (-1);
1381 }
1382 size -= elem_size;
1383 }
1384 cksumvalid = (vrs.cksum == 0);
1385 if (!cksumvalid)
1386 return (-1);
1387
1388 /* read VPD RW elements - optional */
1389 size = vpd_read_tag_size(&vrs, VPD_TAG_RW);
1390 if (size == -2)
1391 return (-1);
1392 while (size > 0) {
1393 elem_size = next_vpd_rw_elem(&vrs, size);
1394 if (elem_size < 0) {
1395 pci_printf(cfg, "error accessing writeable VPD data\n");
1396 return (-1);
1397 }
1398 size -= elem_size;
1399 }
1400
1401 /* read empty END tag - mandatory */
1402 size = vpd_read_tag_size(&vrs, VPD_TAG_END);
1403 if (size != 0) {
1404 pci_printf(cfg, "No valid VPD end tag found\n");
1405 }
1406 return (0);
1407 }
1408
1409 static void
pci_read_vpd(device_t pcib,pcicfgregs * cfg)1410 pci_read_vpd(device_t pcib, pcicfgregs *cfg)
1411 {
1412 int status;
1413
1414 status = pci_parse_vpd(pcib, cfg);
1415 if (status < 0)
1416 vpd_free(&cfg->vpd);
1417 cfg->vpd.vpd_cached = 1;
1418 #undef REG
1419 #undef WREG
1420 }
1421
1422 int
pci_get_vpd_ident_method(device_t dev,device_t child,const char ** identptr)1423 pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr)
1424 {
1425 struct pci_devinfo *dinfo = device_get_ivars(child);
1426 pcicfgregs *cfg = &dinfo->cfg;
1427
1428 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1429 pci_read_vpd(device_get_parent(dev), cfg);
1430
1431 *identptr = cfg->vpd.vpd_ident;
1432
1433 if (*identptr == NULL)
1434 return (ENXIO);
1435
1436 return (0);
1437 }
1438
1439 int
pci_get_vpd_readonly_method(device_t dev,device_t child,const char * kw,const char ** vptr)1440 pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
1441 const char **vptr)
1442 {
1443 struct pci_devinfo *dinfo = device_get_ivars(child);
1444 pcicfgregs *cfg = &dinfo->cfg;
1445 int i;
1446
1447 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1448 pci_read_vpd(device_get_parent(dev), cfg);
1449
1450 for (i = 0; i < cfg->vpd.vpd_rocnt; i++)
1451 if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword,
1452 sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) {
1453 *vptr = cfg->vpd.vpd_ros[i].value;
1454 return (0);
1455 }
1456
1457 *vptr = NULL;
1458 return (ENXIO);
1459 }
1460
1461 struct pcicfg_vpd *
pci_fetch_vpd_list(device_t dev)1462 pci_fetch_vpd_list(device_t dev)
1463 {
1464 struct pci_devinfo *dinfo = device_get_ivars(dev);
1465 pcicfgregs *cfg = &dinfo->cfg;
1466
1467 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1468 pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg);
1469 return (&cfg->vpd);
1470 }
1471
1472 /*
1473 * Find the requested HyperTransport capability and return the offset
1474 * in configuration space via the pointer provided. The function
1475 * returns 0 on success and an error code otherwise.
1476 */
1477 int
pci_find_htcap_method(device_t dev,device_t child,int capability,int * capreg)1478 pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg)
1479 {
1480 int ptr, error;
1481 uint16_t val;
1482
1483 error = pci_find_cap(child, PCIY_HT, &ptr);
1484 if (error)
1485 return (error);
1486
1487 /*
1488 * Traverse the capabilities list checking each HT capability
1489 * to see if it matches the requested HT capability.
1490 */
1491 for (;;) {
1492 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1493 if (capability == PCIM_HTCAP_SLAVE ||
1494 capability == PCIM_HTCAP_HOST)
1495 val &= 0xe000;
1496 else
1497 val &= PCIM_HTCMD_CAP_MASK;
1498 if (val == capability) {
1499 if (capreg != NULL)
1500 *capreg = ptr;
1501 return (0);
1502 }
1503
1504 /* Skip to the next HT capability. */
1505 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1506 break;
1507 }
1508
1509 return (ENOENT);
1510 }
1511
1512 /*
1513 * Find the next requested HyperTransport capability after start and return
1514 * the offset in configuration space via the pointer provided. The function
1515 * returns 0 on success and an error code otherwise.
1516 */
1517 int
pci_find_next_htcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1518 pci_find_next_htcap_method(device_t dev, device_t child, int capability,
1519 int start, int *capreg)
1520 {
1521 int ptr;
1522 uint16_t val;
1523
1524 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT,
1525 ("start capability is not HyperTransport capability"));
1526 ptr = start;
1527
1528 /*
1529 * Traverse the capabilities list checking each HT capability
1530 * to see if it matches the requested HT capability.
1531 */
1532 for (;;) {
1533 /* Skip to the next HT capability. */
1534 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1535 break;
1536
1537 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1538 if (capability == PCIM_HTCAP_SLAVE ||
1539 capability == PCIM_HTCAP_HOST)
1540 val &= 0xe000;
1541 else
1542 val &= PCIM_HTCMD_CAP_MASK;
1543 if (val == capability) {
1544 if (capreg != NULL)
1545 *capreg = ptr;
1546 return (0);
1547 }
1548 }
1549
1550 return (ENOENT);
1551 }
1552
1553 /*
1554 * Find the requested capability and return the offset in
1555 * configuration space via the pointer provided. The function returns
1556 * 0 on success and an error code otherwise.
1557 */
1558 int
pci_find_cap_method(device_t dev,device_t child,int capability,int * capreg)1559 pci_find_cap_method(device_t dev, device_t child, int capability,
1560 int *capreg)
1561 {
1562 struct pci_devinfo *dinfo = device_get_ivars(child);
1563 pcicfgregs *cfg = &dinfo->cfg;
1564 uint32_t status;
1565 uint8_t ptr;
1566 int cnt;
1567
1568 /*
1569 * Check the CAP_LIST bit of the PCI status register first.
1570 */
1571 status = pci_read_config(child, PCIR_STATUS, 2);
1572 if (!(status & PCIM_STATUS_CAPPRESENT))
1573 return (ENXIO);
1574
1575 /*
1576 * Determine the start pointer of the capabilities list.
1577 */
1578 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1579 case PCIM_HDRTYPE_NORMAL:
1580 case PCIM_HDRTYPE_BRIDGE:
1581 ptr = PCIR_CAP_PTR;
1582 break;
1583 case PCIM_HDRTYPE_CARDBUS:
1584 ptr = PCIR_CAP_PTR_2;
1585 break;
1586 default:
1587 /* XXX: panic? */
1588 return (ENXIO); /* no extended capabilities support */
1589 }
1590 ptr = pci_read_config(child, ptr, 1);
1591
1592 /*
1593 * Traverse the capabilities list. Limit by total theoretical
1594 * maximum number of caps: capability needs at least id and
1595 * next registers, and any type X header cannot contain caps.
1596 */
1597 for (cnt = 0; ptr != 0 && cnt < (PCIE_REGMAX - 0x40) / 2; cnt++) {
1598 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1599 if (capreg != NULL)
1600 *capreg = ptr;
1601 return (0);
1602 }
1603 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1604 }
1605
1606 return (ENOENT);
1607 }
1608
1609 /*
1610 * Find the next requested capability after start and return the offset in
1611 * configuration space via the pointer provided. The function returns
1612 * 0 on success and an error code otherwise.
1613 */
1614 int
pci_find_next_cap_method(device_t dev,device_t child,int capability,int start,int * capreg)1615 pci_find_next_cap_method(device_t dev, device_t child, int capability,
1616 int start, int *capreg)
1617 {
1618 uint8_t ptr;
1619
1620 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability,
1621 ("start capability is not expected capability"));
1622
1623 ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1);
1624 while (ptr != 0) {
1625 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1626 if (capreg != NULL)
1627 *capreg = ptr;
1628 return (0);
1629 }
1630 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1631 }
1632
1633 return (ENOENT);
1634 }
1635
1636 /*
1637 * Find the requested extended capability and return the offset in
1638 * configuration space via the pointer provided. The function returns
1639 * 0 on success and an error code otherwise.
1640 */
1641 int
pci_find_extcap_method(device_t dev,device_t child,int capability,int * capreg)1642 pci_find_extcap_method(device_t dev, device_t child, int capability,
1643 int *capreg)
1644 {
1645 struct pci_devinfo *dinfo = device_get_ivars(child);
1646 pcicfgregs *cfg = &dinfo->cfg;
1647 uint32_t ecap;
1648 uint16_t ptr;
1649
1650 /* Only supported for PCI-express devices. */
1651 if (cfg->pcie.pcie_location == 0)
1652 return (ENXIO);
1653
1654 ptr = PCIR_EXTCAP;
1655 ecap = pci_read_config(child, ptr, 4);
1656 if (ecap == 0xffffffff || ecap == 0)
1657 return (ENOENT);
1658 for (;;) {
1659 if (PCI_EXTCAP_ID(ecap) == capability) {
1660 if (capreg != NULL)
1661 *capreg = ptr;
1662 return (0);
1663 }
1664 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1665 if (ptr == 0)
1666 break;
1667 ecap = pci_read_config(child, ptr, 4);
1668 }
1669
1670 return (ENOENT);
1671 }
1672
1673 /*
1674 * Find the next requested extended capability after start and return the
1675 * offset in configuration space via the pointer provided. The function
1676 * returns 0 on success and an error code otherwise.
1677 */
1678 int
pci_find_next_extcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1679 pci_find_next_extcap_method(device_t dev, device_t child, int capability,
1680 int start, int *capreg)
1681 {
1682 struct pci_devinfo *dinfo = device_get_ivars(child);
1683 pcicfgregs *cfg = &dinfo->cfg;
1684 uint32_t ecap;
1685 uint16_t ptr;
1686
1687 /* Only supported for PCI-express devices. */
1688 if (cfg->pcie.pcie_location == 0)
1689 return (ENXIO);
1690
1691 ecap = pci_read_config(child, start, 4);
1692 KASSERT(PCI_EXTCAP_ID(ecap) == capability,
1693 ("start extended capability is not expected capability"));
1694 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1695 while (ptr != 0) {
1696 ecap = pci_read_config(child, ptr, 4);
1697 if (PCI_EXTCAP_ID(ecap) == capability) {
1698 if (capreg != NULL)
1699 *capreg = ptr;
1700 return (0);
1701 }
1702 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1703 }
1704
1705 return (ENOENT);
1706 }
1707
1708 /*
1709 * Support for MSI-X message interrupts.
1710 */
1711 static void
pci_write_msix_entry(device_t dev,u_int index,uint64_t address,uint32_t data)1712 pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data)
1713 {
1714 struct pci_devinfo *dinfo = device_get_ivars(dev);
1715 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1716 uint32_t offset;
1717
1718 KASSERT(msix->msix_table_len > index, ("bogus index"));
1719 offset = msix->msix_table_offset + index * 16;
1720 bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
1721 bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
1722 bus_write_4(msix->msix_table_res, offset + 8, data);
1723 }
1724
1725 void
pci_enable_msix_method(device_t dev,device_t child,u_int index,uint64_t address,uint32_t data)1726 pci_enable_msix_method(device_t dev, device_t child, u_int index,
1727 uint64_t address, uint32_t data)
1728 {
1729
1730 if (pci_msix_rewrite_table) {
1731 struct pci_devinfo *dinfo = device_get_ivars(child);
1732 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1733
1734 /*
1735 * Some VM hosts require MSIX to be disabled in the
1736 * control register before updating the MSIX table
1737 * entries are allowed. It is not enough to only
1738 * disable MSIX while updating a single entry. MSIX
1739 * must be disabled while updating all entries in the
1740 * table.
1741 */
1742 pci_write_config(child,
1743 msix->msix_location + PCIR_MSIX_CTRL,
1744 msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
1745 pci_resume_msix(child);
1746 } else
1747 pci_write_msix_entry(child, index, address, data);
1748
1749 /* Enable MSI -> HT mapping. */
1750 pci_ht_map_msi(child, address);
1751 }
1752
1753 void
pci_mask_msix(device_t dev,u_int index)1754 pci_mask_msix(device_t dev, u_int index)
1755 {
1756 struct pci_devinfo *dinfo = device_get_ivars(dev);
1757 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1758 uint32_t offset, val;
1759
1760 KASSERT(PCI_MSIX_MSGNUM(msix->msix_ctrl) > index, ("bogus index"));
1761 offset = msix->msix_table_offset + index * 16 + 12;
1762 val = bus_read_4(msix->msix_table_res, offset);
1763 val |= PCIM_MSIX_VCTRL_MASK;
1764
1765 /*
1766 * Some devices (e.g. Samsung PM961) do not support reads of this
1767 * register, so always write the new value.
1768 */
1769 bus_write_4(msix->msix_table_res, offset, val);
1770 }
1771
1772 void
pci_unmask_msix(device_t dev,u_int index)1773 pci_unmask_msix(device_t dev, u_int index)
1774 {
1775 struct pci_devinfo *dinfo = device_get_ivars(dev);
1776 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1777 uint32_t offset, val;
1778
1779 KASSERT(PCI_MSIX_MSGNUM(msix->msix_ctrl) > index, ("bogus index"));
1780 offset = msix->msix_table_offset + index * 16 + 12;
1781 val = bus_read_4(msix->msix_table_res, offset);
1782 val &= ~PCIM_MSIX_VCTRL_MASK;
1783
1784 /*
1785 * Some devices (e.g. Samsung PM961) do not support reads of this
1786 * register, so always write the new value.
1787 */
1788 bus_write_4(msix->msix_table_res, offset, val);
1789 }
1790
1791 int
pci_pending_msix(device_t dev,u_int index)1792 pci_pending_msix(device_t dev, u_int index)
1793 {
1794 struct pci_devinfo *dinfo = device_get_ivars(dev);
1795 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1796 uint32_t offset, bit;
1797
1798 KASSERT(msix->msix_table_len > index, ("bogus index"));
1799 offset = msix->msix_pba_offset + (index / 32) * 4;
1800 bit = 1 << index % 32;
1801 return (bus_read_4(msix->msix_pba_res, offset) & bit);
1802 }
1803
1804 /*
1805 * Restore MSI-X registers and table during resume. If MSI-X is
1806 * enabled then walk the virtual table to restore the actual MSI-X
1807 * table.
1808 */
1809 static void
pci_resume_msix(device_t dev)1810 pci_resume_msix(device_t dev)
1811 {
1812 struct pci_devinfo *dinfo = device_get_ivars(dev);
1813 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1814 struct msix_table_entry *mte;
1815 struct msix_vector *mv;
1816 u_int i, msgnum;
1817
1818 if (msix->msix_alloc > 0) {
1819 msgnum = PCI_MSIX_MSGNUM(msix->msix_ctrl);
1820
1821 /* First, mask all vectors. */
1822 for (i = 0; i < msgnum; i++)
1823 pci_mask_msix(dev, i);
1824
1825 /* Second, program any messages with at least one handler. */
1826 for (i = 0; i < msix->msix_table_len; i++) {
1827 mte = &msix->msix_table[i];
1828 if (mte->mte_vector == 0 || mte->mte_handlers == 0)
1829 continue;
1830 mv = &msix->msix_vectors[mte->mte_vector - 1];
1831 pci_write_msix_entry(dev, i, mv->mv_address,
1832 mv->mv_data);
1833 pci_unmask_msix(dev, i);
1834 }
1835 }
1836 pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
1837 msix->msix_ctrl, 2);
1838 }
1839
1840 /*
1841 * Attempt to allocate *count MSI-X messages. The actual number allocated is
1842 * returned in *count. After this function returns, each message will be
1843 * available to the driver as SYS_RES_IRQ resources starting at rid 1.
1844 */
1845 int
pci_alloc_msix_method(device_t dev,device_t child,int * count)1846 pci_alloc_msix_method(device_t dev, device_t child, int *count)
1847 {
1848 struct pci_devinfo *dinfo = device_get_ivars(child);
1849 pcicfgregs *cfg = &dinfo->cfg;
1850 struct resource_list_entry *rle;
1851 u_int actual, i, max;
1852 int error, irq;
1853 uint16_t ctrl, msgnum;
1854
1855 /* Don't let count == 0 get us into trouble. */
1856 if (*count < 1)
1857 return (EINVAL);
1858
1859 /* If rid 0 is allocated, then fail. */
1860 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
1861 if (rle != NULL && rle->res != NULL)
1862 return (ENXIO);
1863
1864 /* Already have allocated messages? */
1865 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
1866 return (ENXIO);
1867
1868 /* If MSI-X is blacklisted for this system, fail. */
1869 if (pci_msix_blacklisted())
1870 return (ENXIO);
1871
1872 /* MSI-X capability present? */
1873 if (cfg->msix.msix_location == 0 || !pci_do_msix)
1874 return (ENODEV);
1875
1876 /* Make sure the appropriate BARs are mapped. */
1877 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1878 cfg->msix.msix_table_bar);
1879 if (rle == NULL || rle->res == NULL ||
1880 !(rman_get_flags(rle->res) & RF_ACTIVE))
1881 return (ENXIO);
1882 cfg->msix.msix_table_res = rle->res;
1883 if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
1884 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1885 cfg->msix.msix_pba_bar);
1886 if (rle == NULL || rle->res == NULL ||
1887 !(rman_get_flags(rle->res) & RF_ACTIVE))
1888 return (ENXIO);
1889 }
1890 cfg->msix.msix_pba_res = rle->res;
1891
1892 ctrl = pci_read_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1893 2);
1894 msgnum = PCI_MSIX_MSGNUM(ctrl);
1895 if (bootverbose)
1896 device_printf(child,
1897 "attempting to allocate %d MSI-X vectors (%d supported)\n",
1898 *count, msgnum);
1899 max = min(*count, msgnum);
1900 for (i = 0; i < max; i++) {
1901 /* Allocate a message. */
1902 error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
1903 if (error) {
1904 if (i == 0)
1905 return (error);
1906 break;
1907 }
1908 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
1909 irq, 1);
1910 }
1911 actual = i;
1912
1913 if (bootverbose) {
1914 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
1915 if (actual == 1)
1916 device_printf(child, "using IRQ %ju for MSI-X\n",
1917 rle->start);
1918 else {
1919 bool run;
1920
1921 /*
1922 * Be fancy and try to print contiguous runs of
1923 * IRQ values as ranges. 'irq' is the previous IRQ.
1924 * 'run' is true if we are in a range.
1925 */
1926 device_printf(child, "using IRQs %ju", rle->start);
1927 irq = rle->start;
1928 run = false;
1929 for (i = 1; i < actual; i++) {
1930 rle = resource_list_find(&dinfo->resources,
1931 SYS_RES_IRQ, i + 1);
1932
1933 /* Still in a run? */
1934 if (rle->start == irq + 1) {
1935 run = true;
1936 irq++;
1937 continue;
1938 }
1939
1940 /* Finish previous range. */
1941 if (run) {
1942 printf("-%d", irq);
1943 run = false;
1944 }
1945
1946 /* Start new range. */
1947 printf(",%ju", rle->start);
1948 irq = rle->start;
1949 }
1950
1951 /* Unfinished range? */
1952 if (run)
1953 printf("-%d", irq);
1954 printf(" for MSI-X\n");
1955 }
1956 }
1957
1958 /*
1959 * Mask all vectors. Note that the message index assertion in
1960 * pci_mask_msix requires msix_ctrl to be set.
1961 */
1962 cfg->msix.msix_ctrl = ctrl;
1963 for (i = 0; i < msgnum; i++)
1964 pci_mask_msix(child, i);
1965
1966 /* Allocate and initialize vector data and virtual table. */
1967 cfg->msix.msix_vectors = mallocarray(actual, sizeof(struct msix_vector),
1968 M_DEVBUF, M_WAITOK | M_ZERO);
1969 cfg->msix.msix_table = mallocarray(actual,
1970 sizeof(struct msix_table_entry), M_DEVBUF, M_WAITOK | M_ZERO);
1971 for (i = 0; i < actual; i++) {
1972 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
1973 cfg->msix.msix_vectors[i].mv_irq = rle->start;
1974 cfg->msix.msix_table[i].mte_vector = i + 1;
1975 }
1976
1977 /* Update control register to enable MSI-X. */
1978 ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1979 pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1980 ctrl, 2);
1981 cfg->msix.msix_ctrl = ctrl;
1982
1983 /* Update counts of alloc'd messages. */
1984 cfg->msix.msix_alloc = actual;
1985 cfg->msix.msix_table_len = actual;
1986 *count = actual;
1987 return (0);
1988 }
1989
1990 /*
1991 * By default, pci_alloc_msix() will assign the allocated IRQ
1992 * resources consecutively to the first N messages in the MSI-X table.
1993 * However, device drivers may want to use different layouts if they
1994 * either receive fewer messages than they asked for, or they wish to
1995 * populate the MSI-X table sparsely. This method allows the driver
1996 * to specify what layout it wants. It must be called after a
1997 * successful pci_alloc_msix() but before any of the associated
1998 * SYS_RES_IRQ resources are allocated via bus_alloc_resource().
1999 *
2000 * The 'vectors' array contains 'count' message vectors. The array
2001 * maps directly to the MSI-X table in that index 0 in the array
2002 * specifies the vector for the first message in the MSI-X table, etc.
2003 * The vector value in each array index can either be 0 to indicate
2004 * that no vector should be assigned to a message slot, or it can be a
2005 * number from 1 to N (where N is the count returned from a
2006 * succcessful call to pci_alloc_msix()) to indicate which message
2007 * vector (IRQ) to be used for the corresponding message.
2008 *
2009 * On successful return, each message with a non-zero vector will have
2010 * an associated SYS_RES_IRQ whose rid is equal to the array index +
2011 * 1. Additionally, if any of the IRQs allocated via the previous
2012 * call to pci_alloc_msix() are not used in the mapping, those IRQs
2013 * will be freed back to the system automatically.
2014 *
2015 * For example, suppose a driver has a MSI-X table with 6 messages and
2016 * asks for 6 messages, but pci_alloc_msix() only returns a count of
2017 * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and
2018 * C. After the call to pci_alloc_msix(), the device will be setup to
2019 * have an MSI-X table of ABC--- (where - means no vector assigned).
2020 * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 },
2021 * then the MSI-X table will look like A-AB-B, and the 'C' vector will
2022 * be freed back to the system. This device will also have valid
2023 * SYS_RES_IRQ rids of 1, 3, 4, and 6.
2024 *
2025 * In any case, the SYS_RES_IRQ rid X will always map to the message
2026 * at MSI-X table index X - 1 and will only be valid if a vector is
2027 * assigned to that table entry.
2028 */
2029 int
pci_remap_msix_method(device_t dev,device_t child,int count,const u_int * vectors)2030 pci_remap_msix_method(device_t dev, device_t child, int count,
2031 const u_int *vectors)
2032 {
2033 struct pci_devinfo *dinfo = device_get_ivars(child);
2034 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2035 struct resource_list_entry *rle;
2036 u_int i, irq, j;
2037 bool *used;
2038
2039 /*
2040 * Have to have at least one message in the table but the
2041 * table can't be bigger than the actual MSI-X table in the
2042 * device.
2043 */
2044 if (count < 1 || count > PCI_MSIX_MSGNUM(msix->msix_ctrl))
2045 return (EINVAL);
2046
2047 /* Sanity check the vectors. */
2048 for (i = 0; i < count; i++)
2049 if (vectors[i] > msix->msix_alloc)
2050 return (EINVAL);
2051
2052 /*
2053 * Make sure there aren't any holes in the vectors to be used.
2054 * It's a big pain to support it, and it doesn't really make
2055 * sense anyway. Also, at least one vector must be used.
2056 */
2057 used = mallocarray(msix->msix_alloc, sizeof(*used), M_DEVBUF, M_WAITOK |
2058 M_ZERO);
2059 for (i = 0; i < count; i++)
2060 if (vectors[i] != 0)
2061 used[vectors[i] - 1] = true;
2062 for (i = 0; i < msix->msix_alloc - 1; i++)
2063 if (!used[i] && used[i + 1]) {
2064 free(used, M_DEVBUF);
2065 return (EINVAL);
2066 }
2067 if (!used[0]) {
2068 free(used, M_DEVBUF);
2069 return (EINVAL);
2070 }
2071
2072 /* Make sure none of the resources are allocated. */
2073 for (i = 0; i < msix->msix_table_len; i++) {
2074 if (msix->msix_table[i].mte_vector == 0)
2075 continue;
2076 if (msix->msix_table[i].mte_handlers > 0) {
2077 free(used, M_DEVBUF);
2078 return (EBUSY);
2079 }
2080 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2081 KASSERT(rle != NULL, ("missing resource"));
2082 if (rle->res != NULL) {
2083 free(used, M_DEVBUF);
2084 return (EBUSY);
2085 }
2086 }
2087
2088 /* Free the existing resource list entries. */
2089 for (i = 0; i < msix->msix_table_len; i++) {
2090 if (msix->msix_table[i].mte_vector == 0)
2091 continue;
2092 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2093 }
2094
2095 /*
2096 * Build the new virtual table keeping track of which vectors are
2097 * used.
2098 */
2099 free(msix->msix_table, M_DEVBUF);
2100 msix->msix_table = mallocarray(count, sizeof(struct msix_table_entry),
2101 M_DEVBUF, M_WAITOK | M_ZERO);
2102 for (i = 0; i < count; i++)
2103 msix->msix_table[i].mte_vector = vectors[i];
2104 msix->msix_table_len = count;
2105
2106 /* Free any unused IRQs and resize the vectors array if necessary. */
2107 j = msix->msix_alloc - 1;
2108 if (!used[j]) {
2109 struct msix_vector *vec;
2110
2111 while (!used[j]) {
2112 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2113 msix->msix_vectors[j].mv_irq);
2114 j--;
2115 }
2116 vec = mallocarray(j + 1, sizeof(struct msix_vector), M_DEVBUF,
2117 M_WAITOK);
2118 bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) *
2119 (j + 1));
2120 free(msix->msix_vectors, M_DEVBUF);
2121 msix->msix_vectors = vec;
2122 msix->msix_alloc = j + 1;
2123 }
2124 free(used, M_DEVBUF);
2125
2126 /* Map the IRQs onto the rids. */
2127 for (i = 0; i < count; i++) {
2128 if (vectors[i] == 0)
2129 continue;
2130 irq = msix->msix_vectors[vectors[i] - 1].mv_irq;
2131 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
2132 irq, 1);
2133 }
2134
2135 if (bootverbose) {
2136 device_printf(child, "Remapped MSI-X IRQs as: ");
2137 for (i = 0; i < count; i++) {
2138 if (i != 0)
2139 printf(", ");
2140 if (vectors[i] == 0)
2141 printf("---");
2142 else
2143 printf("%d",
2144 msix->msix_vectors[vectors[i] - 1].mv_irq);
2145 }
2146 printf("\n");
2147 }
2148
2149 return (0);
2150 }
2151
2152 static int
pci_release_msix(device_t dev,device_t child)2153 pci_release_msix(device_t dev, device_t child)
2154 {
2155 struct pci_devinfo *dinfo = device_get_ivars(child);
2156 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2157 struct resource_list_entry *rle;
2158 u_int i;
2159
2160 /* Do we have any messages to release? */
2161 if (msix->msix_alloc == 0)
2162 return (ENODEV);
2163
2164 /* Make sure none of the resources are allocated. */
2165 for (i = 0; i < msix->msix_table_len; i++) {
2166 if (msix->msix_table[i].mte_vector == 0)
2167 continue;
2168 if (msix->msix_table[i].mte_handlers > 0)
2169 return (EBUSY);
2170 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2171 KASSERT(rle != NULL, ("missing resource"));
2172 if (rle->res != NULL)
2173 return (EBUSY);
2174 }
2175
2176 /* Update control register to disable MSI-X. */
2177 msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
2178 pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
2179 msix->msix_ctrl, 2);
2180
2181 /* Free the resource list entries. */
2182 for (i = 0; i < msix->msix_table_len; i++) {
2183 if (msix->msix_table[i].mte_vector == 0)
2184 continue;
2185 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2186 }
2187 free(msix->msix_table, M_DEVBUF);
2188 msix->msix_table_len = 0;
2189
2190 /* Release the IRQs. */
2191 for (i = 0; i < msix->msix_alloc; i++)
2192 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2193 msix->msix_vectors[i].mv_irq);
2194 free(msix->msix_vectors, M_DEVBUF);
2195 msix->msix_alloc = 0;
2196 return (0);
2197 }
2198
2199 /*
2200 * Return the max supported MSI-X messages this device supports.
2201 * Basically, assuming the MD code can alloc messages, this function
2202 * should return the maximum value that pci_alloc_msix() can return.
2203 * Thus, it is subject to the tunables, etc.
2204 */
2205 int
pci_msix_count_method(device_t dev,device_t child)2206 pci_msix_count_method(device_t dev, device_t child)
2207 {
2208 struct pci_devinfo *dinfo = device_get_ivars(child);
2209 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2210 uint16_t ctrl;
2211
2212 if (pci_do_msix && msix->msix_location != 0) {
2213 ctrl = pci_read_config(child, msix->msix_location +
2214 PCIR_MSI_CTRL, 2);
2215 return (PCI_MSIX_MSGNUM(ctrl));
2216 }
2217 return (0);
2218 }
2219
2220 int
pci_msix_pba_bar_method(device_t dev,device_t child)2221 pci_msix_pba_bar_method(device_t dev, device_t child)
2222 {
2223 struct pci_devinfo *dinfo = device_get_ivars(child);
2224 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2225
2226 if (pci_do_msix && msix->msix_location != 0)
2227 return (msix->msix_pba_bar);
2228 return (-1);
2229 }
2230
2231 int
pci_msix_table_bar_method(device_t dev,device_t child)2232 pci_msix_table_bar_method(device_t dev, device_t child)
2233 {
2234 struct pci_devinfo *dinfo = device_get_ivars(child);
2235 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2236
2237 if (pci_do_msix && msix->msix_location != 0)
2238 return (msix->msix_table_bar);
2239 return (-1);
2240 }
2241
2242 /*
2243 * HyperTransport MSI mapping control
2244 */
2245 void
pci_ht_map_msi(device_t dev,uint64_t addr)2246 pci_ht_map_msi(device_t dev, uint64_t addr)
2247 {
2248 struct pci_devinfo *dinfo = device_get_ivars(dev);
2249 struct pcicfg_ht *ht = &dinfo->cfg.ht;
2250
2251 if (!ht->ht_msimap)
2252 return;
2253
2254 if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) &&
2255 ht->ht_msiaddr >> 20 == addr >> 20) {
2256 /* Enable MSI -> HT mapping. */
2257 ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
2258 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2259 ht->ht_msictrl, 2);
2260 }
2261
2262 if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
2263 /* Disable MSI -> HT mapping. */
2264 ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
2265 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2266 ht->ht_msictrl, 2);
2267 }
2268 }
2269
2270 int
pci_get_relaxed_ordering_enabled(device_t dev)2271 pci_get_relaxed_ordering_enabled(device_t dev)
2272 {
2273 struct pci_devinfo *dinfo = device_get_ivars(dev);
2274 int cap;
2275 uint16_t val;
2276
2277 cap = dinfo->cfg.pcie.pcie_location;
2278 if (cap == 0)
2279 return (0);
2280 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2281 val &= PCIEM_CTL_RELAXED_ORD_ENABLE;
2282 return (val != 0);
2283 }
2284
2285 int
pci_get_max_payload(device_t dev)2286 pci_get_max_payload(device_t dev)
2287 {
2288 struct pci_devinfo *dinfo = device_get_ivars(dev);
2289 int cap;
2290 uint16_t val;
2291
2292 cap = dinfo->cfg.pcie.pcie_location;
2293 if (cap == 0)
2294 return (0);
2295 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2296 val &= PCIEM_CTL_MAX_PAYLOAD;
2297 val >>= 5;
2298 return (1 << (val + 7));
2299 }
2300
2301 int
pci_get_max_read_req(device_t dev)2302 pci_get_max_read_req(device_t dev)
2303 {
2304 struct pci_devinfo *dinfo = device_get_ivars(dev);
2305 int cap;
2306 uint16_t val;
2307
2308 cap = dinfo->cfg.pcie.pcie_location;
2309 if (cap == 0)
2310 return (0);
2311 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2312 val &= PCIEM_CTL_MAX_READ_REQUEST;
2313 val >>= 12;
2314 return (1 << (val + 7));
2315 }
2316
2317 int
pci_set_max_read_req(device_t dev,int size)2318 pci_set_max_read_req(device_t dev, int size)
2319 {
2320 struct pci_devinfo *dinfo = device_get_ivars(dev);
2321 int cap;
2322 uint16_t val;
2323
2324 cap = dinfo->cfg.pcie.pcie_location;
2325 if (cap == 0)
2326 return (0);
2327 if (size < 128)
2328 size = 128;
2329 if (size > 4096)
2330 size = 4096;
2331 size = (1 << (fls(size) - 1));
2332 val = (fls(size) - 8) << 12;
2333 pcie_adjust_config(dev, PCIER_DEVICE_CTL,
2334 PCIEM_CTL_MAX_READ_REQUEST, val, 2);
2335 return (size);
2336 }
2337
2338 uint32_t
pcie_read_config(device_t dev,int reg,int width)2339 pcie_read_config(device_t dev, int reg, int width)
2340 {
2341 struct pci_devinfo *dinfo = device_get_ivars(dev);
2342 int cap;
2343
2344 cap = dinfo->cfg.pcie.pcie_location;
2345 if (cap == 0) {
2346 if (width == 2)
2347 return (0xffff);
2348 return (0xffffffff);
2349 }
2350
2351 return (pci_read_config(dev, cap + reg, width));
2352 }
2353
2354 void
pcie_write_config(device_t dev,int reg,uint32_t value,int width)2355 pcie_write_config(device_t dev, int reg, uint32_t value, int width)
2356 {
2357 struct pci_devinfo *dinfo = device_get_ivars(dev);
2358 int cap;
2359
2360 cap = dinfo->cfg.pcie.pcie_location;
2361 if (cap == 0)
2362 return;
2363 pci_write_config(dev, cap + reg, value, width);
2364 }
2365
2366 /*
2367 * Adjusts a PCI-e capability register by clearing the bits in mask
2368 * and setting the bits in (value & mask). Bits not set in mask are
2369 * not adjusted.
2370 *
2371 * Returns the old value on success or all ones on failure.
2372 */
2373 uint32_t
pcie_adjust_config(device_t dev,int reg,uint32_t mask,uint32_t value,int width)2374 pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
2375 int width)
2376 {
2377 struct pci_devinfo *dinfo = device_get_ivars(dev);
2378 uint16_t *saved;
2379 uint32_t old, new;
2380 int cap;
2381
2382 cap = dinfo->cfg.pcie.pcie_location;
2383 if (cap == 0) {
2384 if (width == 2)
2385 return (0xffff);
2386 return (0xffffffff);
2387 }
2388
2389 old = pci_read_config(dev, cap + reg, width);
2390 new = old & ~mask;
2391 new |= (value & mask);
2392 pci_write_config(dev, cap + reg, new, width);
2393 /* Apply only the requested policy bits to the saved restore image. */
2394 if (width == 2) {
2395 saved = NULL;
2396 switch (reg) {
2397 case PCIER_DEVICE_CTL:
2398 saved = &dinfo->cfg.pcie.pcie_device_ctl;
2399 break;
2400 case PCIER_ROOT_CTL:
2401 saved = &dinfo->cfg.pcie.pcie_root_ctl;
2402 break;
2403 }
2404 if (saved != NULL) {
2405 *saved &= ~(uint16_t)mask;
2406 *saved |= (uint16_t)(value & mask);
2407 }
2408 }
2409 return (old);
2410 }
2411
2412 /*
2413 * Support for MSI message signalled interrupts.
2414 */
2415 void
pci_enable_msi_method(device_t dev,device_t child,uint64_t address,uint16_t data)2416 pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
2417 uint16_t data)
2418 {
2419 struct pci_devinfo *dinfo = device_get_ivars(child);
2420 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2421
2422 /* Write data and address values. */
2423 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
2424 address & 0xffffffff, 4);
2425 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2426 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
2427 address >> 32, 4);
2428 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
2429 data, 2);
2430 } else
2431 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
2432 2);
2433
2434 /* Enable MSI in the control register. */
2435 msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE;
2436 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2437 msi->msi_ctrl, 2);
2438
2439 /* Enable MSI -> HT mapping. */
2440 pci_ht_map_msi(child, address);
2441 }
2442
2443 void
pci_disable_msi_method(device_t dev,device_t child)2444 pci_disable_msi_method(device_t dev, device_t child)
2445 {
2446 struct pci_devinfo *dinfo = device_get_ivars(child);
2447 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2448
2449 /* Disable MSI -> HT mapping. */
2450 pci_ht_map_msi(child, 0);
2451
2452 /* Disable MSI in the control register. */
2453 msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
2454 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2455 msi->msi_ctrl, 2);
2456 }
2457
2458 /*
2459 * Restore MSI registers during resume. If MSI is enabled then
2460 * restore the data and address registers in addition to the control
2461 * register.
2462 */
2463 static void
pci_resume_msi(device_t dev)2464 pci_resume_msi(device_t dev)
2465 {
2466 struct pci_devinfo *dinfo = device_get_ivars(dev);
2467 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2468 uint64_t address;
2469 uint16_t data;
2470
2471 if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) {
2472 address = msi->msi_addr;
2473 data = msi->msi_data;
2474 pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
2475 address & 0xffffffff, 4);
2476 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2477 pci_write_config(dev, msi->msi_location +
2478 PCIR_MSI_ADDR_HIGH, address >> 32, 4);
2479 pci_write_config(dev, msi->msi_location +
2480 PCIR_MSI_DATA_64BIT, data, 2);
2481 } else
2482 pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA,
2483 data, 2);
2484 }
2485 pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
2486 2);
2487 }
2488
2489 static int
pci_remap_intr_method(device_t bus,device_t dev,u_int irq)2490 pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
2491 {
2492 struct pci_devinfo *dinfo = device_get_ivars(dev);
2493 pcicfgregs *cfg = &dinfo->cfg;
2494 struct resource_list_entry *rle;
2495 struct msix_table_entry *mte;
2496 struct msix_vector *mv;
2497 uint64_t addr;
2498 uint32_t data;
2499 u_int i, j;
2500 int error;
2501
2502 /*
2503 * Handle MSI first. We try to find this IRQ among our list
2504 * of MSI IRQs. If we find it, we request updated address and
2505 * data registers and apply the results.
2506 */
2507 if (cfg->msi.msi_alloc > 0) {
2508 /* If we don't have any active handlers, nothing to do. */
2509 if (cfg->msi.msi_handlers == 0)
2510 return (0);
2511 for (i = 0; i < cfg->msi.msi_alloc; i++) {
2512 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
2513 i + 1);
2514 if (rle->start == irq) {
2515 error = PCIB_MAP_MSI(device_get_parent(bus),
2516 dev, irq, &addr, &data);
2517 if (error)
2518 return (error);
2519 pci_disable_msi(dev);
2520 dinfo->cfg.msi.msi_addr = addr;
2521 dinfo->cfg.msi.msi_data = data;
2522 pci_enable_msi(dev, addr, data);
2523 return (0);
2524 }
2525 }
2526 return (ENOENT);
2527 }
2528
2529 /*
2530 * For MSI-X, we check to see if we have this IRQ. If we do,
2531 * we request the updated mapping info. If that works, we go
2532 * through all the slots that use this IRQ and update them.
2533 */
2534 if (cfg->msix.msix_alloc > 0) {
2535 bool found = false;
2536
2537 for (i = 0; i < cfg->msix.msix_alloc; i++) {
2538 mv = &cfg->msix.msix_vectors[i];
2539 if (mv->mv_irq == irq) {
2540 error = PCIB_MAP_MSI(device_get_parent(bus),
2541 dev, irq, &addr, &data);
2542 if (error)
2543 return (error);
2544 mv->mv_address = addr;
2545 mv->mv_data = data;
2546 for (j = 0; j < cfg->msix.msix_table_len; j++) {
2547 mte = &cfg->msix.msix_table[j];
2548 if (mte->mte_vector != i + 1)
2549 continue;
2550 if (mte->mte_handlers == 0)
2551 continue;
2552 pci_mask_msix(dev, j);
2553 pci_enable_msix(dev, j, addr, data);
2554 pci_unmask_msix(dev, j);
2555 }
2556 found = true;
2557 }
2558 }
2559 return (found ? 0 : ENOENT);
2560 }
2561
2562 return (ENOENT);
2563 }
2564
2565 /*
2566 * Returns true if the specified device is blacklisted because MSI
2567 * doesn't work.
2568 */
2569 int
pci_msi_device_blacklisted(device_t dev)2570 pci_msi_device_blacklisted(device_t dev)
2571 {
2572
2573 if (!pci_honor_msi_blacklist)
2574 return (0);
2575
2576 return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
2577 }
2578
2579 /*
2580 * Determine if MSI is blacklisted globally on this system. Currently,
2581 * we just check for blacklisted chipsets as represented by the
2582 * host-PCI bridge at device 0:0:0. In the future, it may become
2583 * necessary to check other system attributes, such as the kenv values
2584 * that give the motherboard manufacturer and model number.
2585 */
2586 static int
pci_msi_blacklisted(void)2587 pci_msi_blacklisted(void)
2588 {
2589 device_t dev;
2590
2591 if (!pci_honor_msi_blacklist)
2592 return (0);
2593
2594 /* Blacklist all non-PCI-express and non-PCI-X chipsets. */
2595 if (!(pcie_chipset || pcix_chipset)) {
2596 if (vm_guest != VM_GUEST_NO) {
2597 /*
2598 * Whitelist older chipsets in virtual
2599 * machines known to support MSI.
2600 */
2601 dev = pci_find_bsf(0, 0, 0);
2602 if (dev != NULL)
2603 return (!pci_has_quirk(pci_get_devid(dev),
2604 PCI_QUIRK_ENABLE_MSI_VM));
2605 }
2606 return (1);
2607 }
2608
2609 dev = pci_find_bsf(0, 0, 0);
2610 if (dev != NULL)
2611 return (pci_msi_device_blacklisted(dev));
2612 return (0);
2613 }
2614
2615 /*
2616 * Returns true if the specified device is blacklisted because MSI-X
2617 * doesn't work. Note that this assumes that if MSI doesn't work,
2618 * MSI-X doesn't either.
2619 */
2620 int
pci_msix_device_blacklisted(device_t dev)2621 pci_msix_device_blacklisted(device_t dev)
2622 {
2623
2624 if (!pci_honor_msi_blacklist)
2625 return (0);
2626
2627 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
2628 return (1);
2629
2630 return (pci_msi_device_blacklisted(dev));
2631 }
2632
2633 /*
2634 * Determine if MSI-X is blacklisted globally on this system. If MSI
2635 * is blacklisted, assume that MSI-X is as well. Check for additional
2636 * chipsets where MSI works but MSI-X does not.
2637 */
2638 static int
pci_msix_blacklisted(void)2639 pci_msix_blacklisted(void)
2640 {
2641 device_t dev;
2642
2643 if (!pci_honor_msi_blacklist)
2644 return (0);
2645
2646 dev = pci_find_bsf(0, 0, 0);
2647 if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
2648 PCI_QUIRK_DISABLE_MSIX))
2649 return (1);
2650
2651 return (pci_msi_blacklisted());
2652 }
2653
2654 /*
2655 * Attempt to allocate *count MSI messages. The actual number allocated is
2656 * returned in *count. After this function returns, each message will be
2657 * available to the driver as SYS_RES_IRQ resources starting at a rid 1.
2658 */
2659 int
pci_alloc_msi_method(device_t dev,device_t child,int * count)2660 pci_alloc_msi_method(device_t dev, device_t child, int *count)
2661 {
2662 struct pci_devinfo *dinfo = device_get_ivars(child);
2663 pcicfgregs *cfg = &dinfo->cfg;
2664 struct resource_list_entry *rle;
2665 u_int actual, i;
2666 int error, irqs[32];
2667 uint16_t ctrl, msgnum;
2668
2669 /* Don't let count == 0 get us into trouble. */
2670 if (*count < 1)
2671 return (EINVAL);
2672
2673 /* If rid 0 is allocated, then fail. */
2674 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
2675 if (rle != NULL && rle->res != NULL)
2676 return (ENXIO);
2677
2678 /* Already have allocated messages? */
2679 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
2680 return (ENXIO);
2681
2682 /* If MSI is blacklisted for this system, fail. */
2683 if (pci_msi_blacklisted())
2684 return (ENXIO);
2685
2686 /* MSI capability present? */
2687 if (cfg->msi.msi_location == 0 || !pci_do_msi)
2688 return (ENODEV);
2689
2690 ctrl = pci_read_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, 2);
2691 msgnum = PCI_MSI_MSGNUM(ctrl);
2692 if (bootverbose)
2693 device_printf(child,
2694 "attempting to allocate %d MSI vectors (%u supported)\n",
2695 *count, msgnum);
2696
2697 /* Don't ask for more than the device supports. */
2698 actual = min(*count, msgnum);
2699
2700 /* Don't ask for more than 32 messages. */
2701 actual = min(actual, 32);
2702
2703 /* MSI requires power of 2 number of messages. */
2704 if (!powerof2(actual))
2705 return (EINVAL);
2706
2707 for (;;) {
2708 /* Try to allocate N messages. */
2709 error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual,
2710 actual, irqs);
2711 if (error == 0)
2712 break;
2713 if (actual == 1)
2714 return (error);
2715
2716 /* Try N / 2. */
2717 actual >>= 1;
2718 }
2719
2720 /*
2721 * We now have N actual messages mapped onto SYS_RES_IRQ
2722 * resources in the irqs[] array, so add new resources
2723 * starting at rid 1.
2724 */
2725 for (i = 0; i < actual; i++)
2726 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1,
2727 irqs[i], irqs[i], 1);
2728
2729 if (bootverbose) {
2730 if (actual == 1)
2731 device_printf(child, "using IRQ %d for MSI\n", irqs[0]);
2732 else {
2733 bool run;
2734
2735 /*
2736 * Be fancy and try to print contiguous runs
2737 * of IRQ values as ranges. 'run' is true if
2738 * we are in a range.
2739 */
2740 device_printf(child, "using IRQs %d", irqs[0]);
2741 run = false;
2742 for (i = 1; i < actual; i++) {
2743 /* Still in a run? */
2744 if (irqs[i] == irqs[i - 1] + 1) {
2745 run = true;
2746 continue;
2747 }
2748
2749 /* Finish previous range. */
2750 if (run) {
2751 printf("-%d", irqs[i - 1]);
2752 run = false;
2753 }
2754
2755 /* Start new range. */
2756 printf(",%d", irqs[i]);
2757 }
2758
2759 /* Unfinished range? */
2760 if (run)
2761 printf("-%d", irqs[actual - 1]);
2762 printf(" for MSI\n");
2763 }
2764 }
2765
2766 /* Update control register with actual count. */
2767 ctrl &= ~PCIM_MSICTRL_MME_MASK;
2768 ctrl |= (ffs(actual) - 1) << 4;
2769 cfg->msi.msi_ctrl = ctrl;
2770 pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2);
2771
2772 /* Update counts of alloc'd messages. */
2773 cfg->msi.msi_alloc = actual;
2774 cfg->msi.msi_handlers = 0;
2775 *count = actual;
2776 return (0);
2777 }
2778
2779 /* Release the MSI messages associated with this device. */
2780 int
pci_release_msi_method(device_t dev,device_t child)2781 pci_release_msi_method(device_t dev, device_t child)
2782 {
2783 struct pci_devinfo *dinfo = device_get_ivars(child);
2784 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2785 struct resource_list_entry *rle;
2786 u_int i, irqs[32];
2787 int error;
2788
2789 /* Try MSI-X first. */
2790 error = pci_release_msix(dev, child);
2791 if (error != ENODEV)
2792 return (error);
2793
2794 /* Do we have any messages to release? */
2795 if (msi->msi_alloc == 0)
2796 return (ENODEV);
2797 KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));
2798
2799 /* Make sure none of the resources are allocated. */
2800 if (msi->msi_handlers > 0)
2801 return (EBUSY);
2802 for (i = 0; i < msi->msi_alloc; i++) {
2803 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2804 KASSERT(rle != NULL, ("missing MSI resource"));
2805 if (rle->res != NULL)
2806 return (EBUSY);
2807 irqs[i] = rle->start;
2808 }
2809
2810 /* Update control register with 0 count. */
2811 KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
2812 ("%s: MSI still enabled", __func__));
2813 msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
2814 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2815 msi->msi_ctrl, 2);
2816
2817 /* Release the messages. */
2818 PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs);
2819 for (i = 0; i < msi->msi_alloc; i++)
2820 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2821
2822 /* Update alloc count. */
2823 msi->msi_alloc = 0;
2824 msi->msi_addr = 0;
2825 msi->msi_data = 0;
2826 return (0);
2827 }
2828
2829 /*
2830 * Return the max supported MSI messages this device supports.
2831 * Basically, assuming the MD code can alloc messages, this function
2832 * should return the maximum value that pci_alloc_msi() can return.
2833 * Thus, it is subject to the tunables, etc.
2834 */
2835 int
pci_msi_count_method(device_t dev,device_t child)2836 pci_msi_count_method(device_t dev, device_t child)
2837 {
2838 struct pci_devinfo *dinfo = device_get_ivars(child);
2839 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2840 uint16_t ctrl;
2841
2842 if (pci_do_msi && msi->msi_location != 0) {
2843 ctrl = pci_read_config(child, msi->msi_location + PCIR_MSI_CTRL,
2844 2);
2845 return (PCI_MSI_MSGNUM(ctrl));
2846 }
2847 return (0);
2848 }
2849
2850 /* free pcicfgregs structure and all depending data structures */
2851
2852 int
pci_freecfg(struct pci_devinfo * dinfo)2853 pci_freecfg(struct pci_devinfo *dinfo)
2854 {
2855 struct devlist *devlist_head;
2856 struct pci_map *pm, *next;
2857
2858 devlist_head = &pci_devq;
2859
2860 if (dinfo->cfg.vpd.vpd_reg)
2861 vpd_free(&dinfo->cfg.vpd);
2862
2863 STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) {
2864 free(pm, M_DEVBUF);
2865 }
2866 STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links);
2867 free(dinfo, M_DEVBUF);
2868
2869 /* increment the generation count */
2870 pci_generation++;
2871
2872 /* we're losing one device */
2873 pci_numdevs--;
2874 return (0);
2875 }
2876
2877 /*
2878 * PCI power manangement
2879 */
2880 int
pci_set_powerstate_method(device_t dev,device_t child,int state)2881 pci_set_powerstate_method(device_t dev, device_t child, int state)
2882 {
2883 struct pci_devinfo *dinfo = device_get_ivars(child);
2884 pcicfgregs *cfg = &dinfo->cfg;
2885 uint16_t status;
2886 int oldstate, highest, delay;
2887
2888 if (cfg->pp.pp_location == 0)
2889 return (EOPNOTSUPP);
2890
2891 /*
2892 * Optimize a no state change request away. While it would be OK to
2893 * write to the hardware in theory, some devices have shown odd
2894 * behavior when going from D3 -> D3.
2895 */
2896 oldstate = pci_get_powerstate(child);
2897 if (oldstate == state)
2898 return (0);
2899
2900 /*
2901 * The PCI power management specification states that after a state
2902 * transition between PCI power states, system software must
2903 * guarantee a minimal delay before the function accesses the device.
2904 * Compute the worst case delay that we need to guarantee before we
2905 * access the device. Many devices will be responsive much more
2906 * quickly than this delay, but there are some that don't respond
2907 * instantly to state changes. Transitions to/from D3 state require
2908 * 10ms, while D2 requires 200us, and D0/1 require none. The delay
2909 * is done below with DELAY rather than a sleeper function because
2910 * this function can be called from contexts where we cannot sleep.
2911 */
2912 highest = (oldstate > state) ? oldstate : state;
2913 if (highest == PCI_POWERSTATE_D3)
2914 delay = 10000;
2915 else if (highest == PCI_POWERSTATE_D2)
2916 delay = 200;
2917 else
2918 delay = 0;
2919 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_location +
2920 PCIR_POWER_STATUS, 2) & ~PCIM_PSTAT_DMASK;
2921 switch (state) {
2922 case PCI_POWERSTATE_D0:
2923 status |= PCIM_PSTAT_D0;
2924 break;
2925 case PCI_POWERSTATE_D1:
2926 if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0)
2927 return (EOPNOTSUPP);
2928 status |= PCIM_PSTAT_D1;
2929 break;
2930 case PCI_POWERSTATE_D2:
2931 if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0)
2932 return (EOPNOTSUPP);
2933 status |= PCIM_PSTAT_D2;
2934 break;
2935 case PCI_POWERSTATE_D3:
2936 status |= PCIM_PSTAT_D3;
2937 break;
2938 default:
2939 return (EINVAL);
2940 }
2941
2942 if (bootverbose)
2943 pci_printf(cfg, "Transition from %s to %s\n",
2944 pci_powerstate_to_str(oldstate),
2945 pci_powerstate_to_str(state));
2946
2947 PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_location + PCIR_POWER_STATUS,
2948 status, 2);
2949 if (delay)
2950 DELAY(delay);
2951 return (0);
2952 }
2953
2954 int
pci_get_powerstate_method(device_t dev,device_t child)2955 pci_get_powerstate_method(device_t dev, device_t child)
2956 {
2957 struct pci_devinfo *dinfo = device_get_ivars(child);
2958 pcicfgregs *cfg = &dinfo->cfg;
2959 uint16_t status;
2960 int result;
2961
2962 if (cfg->pp.pp_location != 0) {
2963 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_location +
2964 PCIR_POWER_STATUS, 2);
2965 switch (status & PCIM_PSTAT_DMASK) {
2966 case PCIM_PSTAT_D0:
2967 result = PCI_POWERSTATE_D0;
2968 break;
2969 case PCIM_PSTAT_D1:
2970 result = PCI_POWERSTATE_D1;
2971 break;
2972 case PCIM_PSTAT_D2:
2973 result = PCI_POWERSTATE_D2;
2974 break;
2975 case PCIM_PSTAT_D3:
2976 result = PCI_POWERSTATE_D3;
2977 break;
2978 default:
2979 result = PCI_POWERSTATE_UNKNOWN;
2980 break;
2981 }
2982 } else {
2983 /* No support, device is always at D0 */
2984 result = PCI_POWERSTATE_D0;
2985 }
2986 return (result);
2987 }
2988
2989 /* Clear any active PME# and disable PME# generation. */
2990 void
pci_clear_pme(device_t dev)2991 pci_clear_pme(device_t dev)
2992 {
2993 struct pci_devinfo *dinfo = device_get_ivars(dev);
2994 pcicfgregs *cfg = &dinfo->cfg;
2995 uint16_t status;
2996
2997 if (cfg->pp.pp_location != 0) {
2998 status = pci_read_config(dev, dinfo->cfg.pp.pp_location +
2999 PCIR_POWER_STATUS, 2);
3000 status &= ~PCIM_PSTAT_PMEENABLE;
3001 status |= PCIM_PSTAT_PME;
3002 pci_write_config(dev, dinfo->cfg.pp.pp_location +
3003 PCIR_POWER_STATUS, status, 2);
3004 }
3005 }
3006
3007 /* Clear any active PME# and enable PME# generation. */
3008 void
pci_enable_pme(device_t dev)3009 pci_enable_pme(device_t dev)
3010 {
3011 struct pci_devinfo *dinfo = device_get_ivars(dev);
3012 pcicfgregs *cfg = &dinfo->cfg;
3013 uint16_t status;
3014
3015 if (cfg->pp.pp_location != 0) {
3016 status = pci_read_config(dev, dinfo->cfg.pp.pp_location +
3017 PCIR_POWER_STATUS, 2);
3018 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
3019 pci_write_config(dev, dinfo->cfg.pp.pp_location +
3020 PCIR_POWER_STATUS, status, 2);
3021 }
3022 }
3023
3024 bool
pci_has_pm(device_t dev)3025 pci_has_pm(device_t dev)
3026 {
3027 struct pci_devinfo *dinfo = device_get_ivars(dev);
3028 pcicfgregs *cfg = &dinfo->cfg;
3029
3030 return (cfg->pp.pp_location != 0);
3031 }
3032
3033 bool
pci_has_pme(device_t dev,int state)3034 pci_has_pme(device_t dev, int state)
3035 {
3036 static const uint16_t pme_mask[PCI_POWERSTATE_COUNT] = {
3037 [PCI_POWERSTATE_D0] = PCIM_PCAP_D0PME,
3038 [PCI_POWERSTATE_D1] = PCIM_PCAP_D1PME,
3039 [PCI_POWERSTATE_D2] = PCIM_PCAP_D2PME,
3040 [PCI_POWERSTATE_D3_HOT] = PCIM_PCAP_D3PME_HOT,
3041 [PCI_POWERSTATE_D3_COLD] = PCIM_PCAP_D3PME_COLD,
3042 };
3043 struct pci_devinfo *dinfo = device_get_ivars(dev);
3044 pcicfgregs *cfg = &dinfo->cfg;
3045
3046 if (state < PCI_POWERSTATE_D0 || state > PCI_POWERSTATE_MAX)
3047 return (false);
3048 return (cfg->pp.pp_location != 0 &&
3049 (cfg->pp.pp_cap & pme_mask[state]) != 0);
3050 }
3051
3052 /*
3053 * Some convenience functions for PCI device drivers.
3054 */
3055
3056 static __inline void
pci_set_command_bit(device_t dev,device_t child,uint16_t bit)3057 pci_set_command_bit(device_t dev, device_t child, uint16_t bit)
3058 {
3059 uint16_t command;
3060
3061 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
3062 command |= bit;
3063 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
3064 }
3065
3066 static __inline void
pci_clear_command_bit(device_t dev,device_t child,uint16_t bit)3067 pci_clear_command_bit(device_t dev, device_t child, uint16_t bit)
3068 {
3069 uint16_t command;
3070
3071 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
3072 command &= ~bit;
3073 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
3074 }
3075
3076 int
pci_enable_busmaster_method(device_t dev,device_t child)3077 pci_enable_busmaster_method(device_t dev, device_t child)
3078 {
3079 pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
3080 return (0);
3081 }
3082
3083 int
pci_disable_busmaster_method(device_t dev,device_t child)3084 pci_disable_busmaster_method(device_t dev, device_t child)
3085 {
3086 pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
3087 return (0);
3088 }
3089
3090 int
pci_enable_io_method(device_t dev,device_t child,int space)3091 pci_enable_io_method(device_t dev, device_t child, int space)
3092 {
3093 uint16_t bit;
3094
3095 switch(space) {
3096 case SYS_RES_IOPORT:
3097 bit = PCIM_CMD_PORTEN;
3098 break;
3099 case SYS_RES_MEMORY:
3100 bit = PCIM_CMD_MEMEN;
3101 break;
3102 default:
3103 return (EINVAL);
3104 }
3105 pci_set_command_bit(dev, child, bit);
3106 return (0);
3107 }
3108
3109 int
pci_disable_io_method(device_t dev,device_t child,int space)3110 pci_disable_io_method(device_t dev, device_t child, int space)
3111 {
3112 uint16_t bit;
3113
3114 switch(space) {
3115 case SYS_RES_IOPORT:
3116 bit = PCIM_CMD_PORTEN;
3117 break;
3118 case SYS_RES_MEMORY:
3119 bit = PCIM_CMD_MEMEN;
3120 break;
3121 default:
3122 return (EINVAL);
3123 }
3124 pci_clear_command_bit(dev, child, bit);
3125 return (0);
3126 }
3127
3128 /*
3129 * New style pci driver. Parent device is either a pci-host-bridge or a
3130 * pci-pci-bridge. Both kinds are represented by instances of pcib.
3131 */
3132
3133 void
pci_print_verbose(struct pci_devinfo * dinfo)3134 pci_print_verbose(struct pci_devinfo *dinfo)
3135 {
3136
3137 if (bootverbose) {
3138 pcicfgregs *cfg = &dinfo->cfg;
3139
3140 printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n",
3141 cfg->vendor, cfg->device, cfg->revid);
3142 printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n",
3143 cfg->domain, cfg->bus, cfg->slot, cfg->func);
3144 printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n",
3145 cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype,
3146 cfg->mfdev);
3147 printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n",
3148 cfg->cmdreg, cfg->statreg, cfg->cachelnsz);
3149 printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n",
3150 cfg->lattimer, cfg->lattimer * 30, cfg->mingnt,
3151 cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250);
3152 if (cfg->intpin > 0)
3153 printf("\tintpin=%c, irq=%d\n",
3154 cfg->intpin +'a' -1, cfg->intline);
3155 if (cfg->pp.pp_location) {
3156 uint16_t status;
3157
3158 status = pci_read_config(cfg->dev, cfg->pp.pp_location +
3159 PCIR_POWER_STATUS, 2);
3160 printf("\tpowerspec %d supports D0%s%s D3 current D%d\n",
3161 cfg->pp.pp_cap & PCIM_PCAP_SPEC,
3162 cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "",
3163 cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "",
3164 status & PCIM_PSTAT_DMASK);
3165 }
3166 if (cfg->msi.msi_location) {
3167 uint16_t ctrl, msgnum;
3168
3169 ctrl = cfg->msi.msi_ctrl;
3170 msgnum = PCI_MSI_MSGNUM(ctrl);
3171 printf("\tMSI supports %d message%s%s%s\n",
3172 msgnum, (msgnum == 1) ? "" : "s",
3173 (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "",
3174 (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":"");
3175 }
3176 if (cfg->msix.msix_location) {
3177 uint16_t msgnum;
3178
3179 msgnum = PCI_MSIX_MSGNUM(cfg->msix.msix_ctrl);
3180 printf("\tMSI-X supports %d message%s ",
3181 msgnum, (msgnum == 1) ? "" : "s");
3182 if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar)
3183 printf("in map 0x%x\n",
3184 cfg->msix.msix_table_bar);
3185 else
3186 printf("in maps 0x%x and 0x%x\n",
3187 cfg->msix.msix_table_bar,
3188 cfg->msix.msix_pba_bar);
3189 }
3190 }
3191 }
3192
3193 static int
pci_porten(device_t dev)3194 pci_porten(device_t dev)
3195 {
3196 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0;
3197 }
3198
3199 static int
pci_memen(device_t dev)3200 pci_memen(device_t dev)
3201 {
3202 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0;
3203 }
3204
3205 void
pci_read_bar(device_t dev,int reg,pci_addr_t * mapp,pci_addr_t * testvalp,int * bar64)3206 pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp,
3207 int *bar64)
3208 {
3209 struct pci_devinfo *dinfo;
3210 pci_addr_t map, testval;
3211 int ln2range;
3212 uint16_t cmd;
3213
3214 /*
3215 * The device ROM BAR is special. It is always a 32-bit
3216 * memory BAR. Bit 0 is special and should not be set when
3217 * sizing the BAR.
3218 */
3219 dinfo = device_get_ivars(dev);
3220 if (PCIR_IS_BIOS(&dinfo->cfg, reg)) {
3221 map = pci_read_config(dev, reg, 4);
3222 pci_write_config(dev, reg, 0xfffffffe, 4);
3223 testval = pci_read_config(dev, reg, 4);
3224 pci_write_config(dev, reg, map, 4);
3225 *mapp = map;
3226 *testvalp = testval;
3227 if (bar64 != NULL)
3228 *bar64 = 0;
3229 return;
3230 }
3231
3232 map = pci_read_config(dev, reg, 4);
3233 ln2range = pci_maprange(map);
3234 if (ln2range == 64)
3235 map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3236
3237 /*
3238 * Disable decoding via the command register before
3239 * determining the BAR's length since we will be placing it in
3240 * a weird state.
3241 */
3242 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3243 pci_write_config(dev, PCIR_COMMAND,
3244 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
3245
3246 /*
3247 * Determine the BAR's length by writing all 1's. The bottom
3248 * log_2(size) bits of the BAR will stick as 0 when we read
3249 * the value back.
3250 *
3251 * NB: according to the PCI Local Bus Specification, rev. 3.0:
3252 * "Software writes 0FFFFFFFFh to both registers, reads them back,
3253 * and combines the result into a 64-bit value." (section 6.2.5.1)
3254 *
3255 * Writes to both registers must be performed before attempting to
3256 * read back the size value.
3257 */
3258 testval = 0;
3259 pci_write_config(dev, reg, 0xffffffff, 4);
3260 if (ln2range == 64) {
3261 pci_write_config(dev, reg + 4, 0xffffffff, 4);
3262 testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3263 }
3264 testval |= pci_read_config(dev, reg, 4);
3265
3266 /*
3267 * Restore the original value of the BAR. We may have reprogrammed
3268 * the BAR of the low-level console device and when booting verbose,
3269 * we need the console device addressable.
3270 */
3271 pci_write_config(dev, reg, map, 4);
3272 if (ln2range == 64)
3273 pci_write_config(dev, reg + 4, map >> 32, 4);
3274 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3275
3276 *mapp = map;
3277 *testvalp = testval;
3278 if (bar64 != NULL)
3279 *bar64 = (ln2range == 64);
3280 }
3281
3282 static void
pci_write_bar(device_t dev,struct pci_map * pm,pci_addr_t base)3283 pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base)
3284 {
3285 struct pci_devinfo *dinfo;
3286 int ln2range;
3287
3288 /* The device ROM BAR is always a 32-bit memory BAR. */
3289 dinfo = device_get_ivars(dev);
3290 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3291 ln2range = 32;
3292 else
3293 ln2range = pci_maprange(pm->pm_value);
3294 pci_write_config(dev, pm->pm_reg, base, 4);
3295 if (ln2range == 64)
3296 pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4);
3297 pm->pm_value = pci_read_config(dev, pm->pm_reg, 4);
3298 if (ln2range == 64)
3299 pm->pm_value |= (pci_addr_t)pci_read_config(dev,
3300 pm->pm_reg + 4, 4) << 32;
3301 }
3302
3303 struct pci_map *
pci_find_bar(device_t dev,int reg)3304 pci_find_bar(device_t dev, int reg)
3305 {
3306 struct pci_devinfo *dinfo;
3307 struct pci_map *pm;
3308
3309 dinfo = device_get_ivars(dev);
3310 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3311 if (pm->pm_reg == reg)
3312 return (pm);
3313 }
3314 return (NULL);
3315 }
3316
3317 struct pci_map *
pci_first_bar(device_t dev)3318 pci_first_bar(device_t dev)
3319 {
3320 struct pci_devinfo *dinfo;
3321
3322 dinfo = device_get_ivars(dev);
3323 return (STAILQ_FIRST(&dinfo->cfg.maps));
3324 }
3325
3326 struct pci_map *
pci_next_bar(struct pci_map * pm)3327 pci_next_bar(struct pci_map *pm)
3328 {
3329 return (STAILQ_NEXT(pm, pm_link));
3330 }
3331
3332 int
pci_bar_enabled(device_t dev,struct pci_map * pm)3333 pci_bar_enabled(device_t dev, struct pci_map *pm)
3334 {
3335 struct pci_devinfo *dinfo;
3336 uint16_t cmd;
3337
3338 dinfo = device_get_ivars(dev);
3339 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) &&
3340 !(pm->pm_value & PCIM_BIOS_ENABLE))
3341 return (0);
3342 #ifdef PCI_IOV
3343 if ((dinfo->cfg.flags & PCICFG_VF) != 0) {
3344 struct pcicfg_iov *iov;
3345
3346 iov = dinfo->cfg.iov;
3347 cmd = pci_read_config(iov->iov_pf,
3348 iov->iov_pos + PCIR_SRIOV_CTL, 2);
3349 return ((cmd & PCIM_SRIOV_VF_MSE) != 0);
3350 }
3351 #endif
3352 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3353 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value))
3354 return ((cmd & PCIM_CMD_MEMEN) != 0);
3355 else
3356 return ((cmd & PCIM_CMD_PORTEN) != 0);
3357 }
3358
3359 struct pci_map *
pci_add_bar(device_t dev,int reg,pci_addr_t value,pci_addr_t size)3360 pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
3361 {
3362 struct pci_devinfo *dinfo;
3363 struct pci_map *pm, *prev;
3364
3365 dinfo = device_get_ivars(dev);
3366 pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO);
3367 pm->pm_reg = reg;
3368 pm->pm_value = value;
3369 pm->pm_size = size;
3370 STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) {
3371 KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x",
3372 reg));
3373 if (STAILQ_NEXT(prev, pm_link) == NULL ||
3374 STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg)
3375 break;
3376 }
3377 if (prev != NULL)
3378 STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link);
3379 else
3380 STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link);
3381 return (pm);
3382 }
3383
3384 static void
pci_restore_bars(device_t dev)3385 pci_restore_bars(device_t dev)
3386 {
3387 struct pci_devinfo *dinfo;
3388 struct pci_map *pm;
3389 int ln2range;
3390
3391 dinfo = device_get_ivars(dev);
3392 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3393 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3394 ln2range = 32;
3395 else
3396 ln2range = pci_maprange(pm->pm_value);
3397 pci_write_config(dev, pm->pm_reg, pm->pm_value, 4);
3398 if (ln2range == 64)
3399 pci_write_config(dev, pm->pm_reg + 4,
3400 pm->pm_value >> 32, 4);
3401 }
3402 }
3403
3404 /*
3405 * Add a resource based on a pci map register. Return 1 if the map
3406 * register is a 32bit map register or 2 if it is a 64bit register.
3407 */
3408 static int
pci_add_map(device_t bus,device_t dev,int reg,struct resource_list * rl,int force,int prefetch)3409 pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
3410 int force, int prefetch)
3411 {
3412 struct pci_map *pm;
3413 pci_addr_t base, map, testval;
3414 pci_addr_t start, end, count;
3415 int barlen, basezero, flags, maprange, mapsize, type;
3416 uint16_t cmd;
3417 struct resource *res;
3418
3419 /*
3420 * The BAR may already exist if the device is a CardBus card
3421 * whose CIS is stored in this BAR.
3422 */
3423 pm = pci_find_bar(dev, reg);
3424 if (pm != NULL) {
3425 maprange = pci_maprange(pm->pm_value);
3426 barlen = maprange == 64 ? 2 : 1;
3427 return (barlen);
3428 }
3429
3430 pci_read_bar(dev, reg, &map, &testval, NULL);
3431 if (PCI_BAR_MEM(map)) {
3432 type = SYS_RES_MEMORY;
3433 if (map & PCIM_BAR_MEM_PREFETCH)
3434 prefetch = 1;
3435 } else
3436 type = SYS_RES_IOPORT;
3437 mapsize = pci_mapsize(testval);
3438 base = pci_mapbase(map);
3439 #ifdef __PCI_BAR_ZERO_VALID
3440 basezero = 0;
3441 #else
3442 basezero = base == 0;
3443 #endif
3444 maprange = pci_maprange(map);
3445 barlen = maprange == 64 ? 2 : 1;
3446
3447 /*
3448 * For I/O registers, if bottom bit is set, and the next bit up
3449 * isn't clear, we know we have a BAR that doesn't conform to the
3450 * spec, so ignore it. Also, sanity check the size of the data
3451 * areas to the type of memory involved. Memory must be at least
3452 * 16 bytes in size, while I/O ranges must be at least 4.
3453 */
3454 if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0)
3455 return (barlen);
3456 if ((type == SYS_RES_MEMORY && mapsize < 4) ||
3457 (type == SYS_RES_IOPORT && mapsize < 2))
3458 return (barlen);
3459
3460 /* Save a record of this BAR. */
3461 pm = pci_add_bar(dev, reg, map, mapsize);
3462 if (bootverbose) {
3463 printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d",
3464 reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize);
3465 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3466 printf(", port disabled\n");
3467 else if (type == SYS_RES_MEMORY && !pci_memen(dev))
3468 printf(", memory disabled\n");
3469 else
3470 printf(", enabled\n");
3471 }
3472
3473 /*
3474 * If base is 0, then we have problems if this architecture does
3475 * not allow that. It is best to ignore such entries for the
3476 * moment. These will be allocated later if the driver specifically
3477 * requests them. However, some removable buses look better when
3478 * all resources are allocated, so allow '0' to be overridden.
3479 *
3480 * Similarly treat maps whose values is the same as the test value
3481 * read back. These maps have had all f's written to them by the
3482 * BIOS in an attempt to disable the resources.
3483 */
3484 if (!force && (basezero || map == testval))
3485 return (barlen);
3486 if ((u_long)base != base) {
3487 device_printf(bus,
3488 "pci%d:%d:%d:%d bar %#x too many address bits",
3489 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
3490 pci_get_function(dev), reg);
3491 return (barlen);
3492 }
3493
3494 /*
3495 * This code theoretically does the right thing, but has
3496 * undesirable side effects in some cases where peripherals
3497 * respond oddly to having these bits enabled. Let the user
3498 * be able to turn them off (since pci_enable_io_modes is 1 by
3499 * default).
3500 */
3501 if (pci_enable_io_modes) {
3502 /* Turn on resources that have been left off by a lazy BIOS */
3503 if (type == SYS_RES_IOPORT && !pci_porten(dev)) {
3504 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3505 cmd |= PCIM_CMD_PORTEN;
3506 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3507 }
3508 if (type == SYS_RES_MEMORY && !pci_memen(dev)) {
3509 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3510 cmd |= PCIM_CMD_MEMEN;
3511 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3512 }
3513 } else {
3514 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3515 return (barlen);
3516 if (type == SYS_RES_MEMORY && !pci_memen(dev))
3517 return (barlen);
3518 }
3519
3520 count = (pci_addr_t)1 << mapsize;
3521 flags = RF_ALIGNMENT_LOG2(mapsize);
3522 if (prefetch)
3523 flags |= RF_PREFETCHABLE;
3524 if (basezero || base == pci_mapbase(testval) || pci_clear_bars) {
3525 start = 0; /* Let the parent decide. */
3526 end = ~0;
3527 } else {
3528 start = base;
3529 end = base + count - 1;
3530 }
3531 resource_list_add(rl, type, reg, start, end, count);
3532
3533 /*
3534 * Try to allocate the resource for this BAR from our parent
3535 * so that this resource range is already reserved. The
3536 * driver for this device will later inherit this resource in
3537 * pci_alloc_resource().
3538 */
3539 res = resource_list_reserve(rl, bus, dev, type, reg, start, end, count,
3540 flags);
3541 if ((pci_do_realloc_bars
3542 || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR))
3543 && res == NULL && (start != 0 || end != ~0)) {
3544 /*
3545 * If the allocation fails, try to allocate a resource for
3546 * this BAR using any available range. The firmware felt
3547 * it was important enough to assign a resource, so don't
3548 * disable decoding if we can help it.
3549 */
3550 resource_list_delete(rl, type, reg);
3551 resource_list_add(rl, type, reg, 0, ~0, count);
3552 res = resource_list_reserve(rl, bus, dev, type, reg, 0, ~0,
3553 count, flags);
3554 }
3555 if (res == NULL) {
3556 /*
3557 * If the allocation fails, delete the resource list entry
3558 * and disable decoding for this device.
3559 *
3560 * If the driver requests this resource in the future,
3561 * pci_reserve_map() will try to allocate a fresh
3562 * resource range.
3563 */
3564 resource_list_delete(rl, type, reg);
3565 pci_disable_io(dev, type);
3566 if (bootverbose)
3567 device_printf(bus,
3568 "pci%d:%d:%d:%d bar %#x failed to allocate\n",
3569 pci_get_domain(dev), pci_get_bus(dev),
3570 pci_get_slot(dev), pci_get_function(dev), reg);
3571 } else {
3572 start = rman_get_start(res);
3573 pci_write_bar(dev, pm, start);
3574 }
3575 return (barlen);
3576 }
3577
3578 /*
3579 * For ATA devices we need to decide early what addressing mode to use.
3580 * Legacy demands that the primary and secondary ATA ports sits on the
3581 * same addresses that old ISA hardware did. This dictates that we use
3582 * those addresses and ignore the BAR's if we cannot set PCI native
3583 * addressing mode.
3584 */
3585 static void
pci_ata_maps(device_t bus,device_t dev,struct resource_list * rl,int force,uint32_t prefetchmask)3586 pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
3587 uint32_t prefetchmask)
3588 {
3589 int rid, type, progif;
3590 #if 0
3591 /* if this device supports PCI native addressing use it */
3592 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3593 if ((progif & 0x8a) == 0x8a) {
3594 if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) &&
3595 pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) {
3596 printf("Trying ATA native PCI addressing mode\n");
3597 pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1);
3598 }
3599 }
3600 #endif
3601 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3602 type = SYS_RES_IOPORT;
3603 if (progif & PCIP_STORAGE_IDE_MODEPRIM) {
3604 pci_add_map(bus, dev, PCIR_BAR(0), rl, force,
3605 prefetchmask & (1 << 0));
3606 pci_add_map(bus, dev, PCIR_BAR(1), rl, force,
3607 prefetchmask & (1 << 1));
3608 } else {
3609 rid = PCIR_BAR(0);
3610 resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
3611 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x1f0,
3612 0x1f7, 8, 0);
3613 rid = PCIR_BAR(1);
3614 resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
3615 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x3f6,
3616 0x3f6, 1, 0);
3617 }
3618 if (progif & PCIP_STORAGE_IDE_MODESEC) {
3619 pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
3620 prefetchmask & (1 << 2));
3621 pci_add_map(bus, dev, PCIR_BAR(3), rl, force,
3622 prefetchmask & (1 << 3));
3623 } else {
3624 rid = PCIR_BAR(2);
3625 resource_list_add(rl, type, rid, 0x170, 0x177, 8);
3626 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x170,
3627 0x177, 8, 0);
3628 rid = PCIR_BAR(3);
3629 resource_list_add(rl, type, rid, 0x376, 0x376, 1);
3630 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x376,
3631 0x376, 1, 0);
3632 }
3633 pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
3634 prefetchmask & (1 << 4));
3635 pci_add_map(bus, dev, PCIR_BAR(5), rl, force,
3636 prefetchmask & (1 << 5));
3637 }
3638
3639 static void
pci_assign_interrupt(device_t bus,device_t dev,int force_route)3640 pci_assign_interrupt(device_t bus, device_t dev, int force_route)
3641 {
3642 struct pci_devinfo *dinfo = device_get_ivars(dev);
3643 pcicfgregs *cfg = &dinfo->cfg;
3644 char tunable_name[64];
3645 int irq;
3646
3647 /* Has to have an intpin to have an interrupt. */
3648 if (cfg->intpin == 0)
3649 return;
3650
3651 /* Let the user override the IRQ with a tunable. */
3652 irq = PCI_INVALID_IRQ;
3653 snprintf(tunable_name, sizeof(tunable_name),
3654 "hw.pci%d.%d.%d.INT%c.irq",
3655 cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
3656 if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0))
3657 irq = PCI_INVALID_IRQ;
3658
3659 /*
3660 * If we didn't get an IRQ via the tunable, then we either use the
3661 * IRQ value in the intline register or we ask the bus to route an
3662 * interrupt for us. If force_route is true, then we only use the
3663 * value in the intline register if the bus was unable to assign an
3664 * IRQ.
3665 */
3666 if (!PCI_INTERRUPT_VALID(irq)) {
3667 if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route)
3668 irq = PCI_ASSIGN_INTERRUPT(bus, dev);
3669 if (!PCI_INTERRUPT_VALID(irq))
3670 irq = cfg->intline;
3671 }
3672
3673 /* If after all that we don't have an IRQ, just bail. */
3674 if (!PCI_INTERRUPT_VALID(irq))
3675 return;
3676
3677 /* Update the config register if it changed. */
3678 if (irq != cfg->intline) {
3679 cfg->intline = irq;
3680 pci_write_config(dev, PCIR_INTLINE, irq, 1);
3681 }
3682
3683 /* Add this IRQ as rid 0 interrupt resource. */
3684 resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1);
3685 }
3686
3687 /* Perform early OHCI takeover from SMM. */
3688 static void
ohci_early_takeover(device_t self)3689 ohci_early_takeover(device_t self)
3690 {
3691 struct resource *res;
3692 uint32_t ctl;
3693 int rid;
3694 int i;
3695
3696 rid = PCIR_BAR(0);
3697 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3698 if (res == NULL)
3699 return;
3700
3701 ctl = bus_read_4(res, OHCI_CONTROL);
3702 if (ctl & OHCI_IR) {
3703 if (bootverbose)
3704 printf("ohci early: "
3705 "SMM active, request owner change\n");
3706 bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR);
3707 for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) {
3708 DELAY(1000);
3709 ctl = bus_read_4(res, OHCI_CONTROL);
3710 }
3711 if (ctl & OHCI_IR) {
3712 if (bootverbose)
3713 printf("ohci early: "
3714 "SMM does not respond, resetting\n");
3715 bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET);
3716 }
3717 /* Disable interrupts */
3718 bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS);
3719 }
3720
3721 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3722 }
3723
3724 /* Perform early UHCI takeover from SMM. */
3725 static void
uhci_early_takeover(device_t self)3726 uhci_early_takeover(device_t self)
3727 {
3728 struct resource *res;
3729 int rid;
3730
3731 /*
3732 * Set the PIRQD enable bit and switch off all the others. We don't
3733 * want legacy support to interfere with us XXX Does this also mean
3734 * that the BIOS won't touch the keyboard anymore if it is connected
3735 * to the ports of the root hub?
3736 */
3737 pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);
3738
3739 /* Disable interrupts */
3740 rid = PCI_UHCI_BASE_REG;
3741 res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE);
3742 if (res != NULL) {
3743 bus_write_2(res, UHCI_INTR, 0);
3744 bus_release_resource(self, SYS_RES_IOPORT, rid, res);
3745 }
3746 }
3747
3748 /* Perform early EHCI takeover from SMM. */
3749 static void
ehci_early_takeover(device_t self)3750 ehci_early_takeover(device_t self)
3751 {
3752 struct resource *res;
3753 uint32_t cparams;
3754 uint32_t eec;
3755 uint8_t eecp;
3756 uint8_t bios_sem;
3757 uint8_t offs;
3758 int rid;
3759 int i;
3760
3761 rid = PCIR_BAR(0);
3762 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3763 if (res == NULL)
3764 return;
3765
3766 cparams = bus_read_4(res, EHCI_HCCPARAMS);
3767
3768 /* Synchronise with the BIOS if it owns the controller. */
3769 for (eecp = EHCI_HCC_EECP(cparams); eecp != 0;
3770 eecp = EHCI_EECP_NEXT(eec)) {
3771 eec = pci_read_config(self, eecp, 4);
3772 if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) {
3773 continue;
3774 }
3775 bios_sem = pci_read_config(self, eecp +
3776 EHCI_LEGSUP_BIOS_SEM, 1);
3777 if (bios_sem == 0) {
3778 continue;
3779 }
3780 if (bootverbose)
3781 printf("ehci early: "
3782 "SMM active, request owner change\n");
3783
3784 pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1);
3785
3786 for (i = 0; (i < 100) && (bios_sem != 0); i++) {
3787 DELAY(1000);
3788 bios_sem = pci_read_config(self, eecp +
3789 EHCI_LEGSUP_BIOS_SEM, 1);
3790 }
3791
3792 if (bios_sem != 0) {
3793 if (bootverbose)
3794 printf("ehci early: "
3795 "SMM does not respond\n");
3796 }
3797 /* Disable interrupts */
3798 offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION));
3799 bus_write_4(res, offs + EHCI_USBINTR, 0);
3800 }
3801 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3802 }
3803
3804 /* Perform early XHCI takeover from SMM. */
3805 static void
xhci_early_takeover(device_t self)3806 xhci_early_takeover(device_t self)
3807 {
3808 struct resource *res;
3809 uint32_t cparams;
3810 uint32_t eec;
3811 uint32_t eecp;
3812 uint8_t bios_sem;
3813 uint8_t offs;
3814 int rid;
3815 int i;
3816
3817 rid = PCIR_BAR(0);
3818 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3819 if (res == NULL)
3820 return;
3821
3822 cparams = bus_read_4(res, XHCI_HCCPARAMS1);
3823
3824 eec = -1;
3825
3826 /* Synchronise with the BIOS if it owns the controller. */
3827 for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec);
3828 eecp += XHCI_XECP_NEXT(eec) << 2) {
3829 eec = bus_read_4(res, eecp);
3830
3831 if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY)
3832 continue;
3833
3834 bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM);
3835 if (bios_sem == 0)
3836 continue;
3837
3838 if (bootverbose)
3839 printf("xhci early: "
3840 "SMM active, request owner change\n");
3841
3842 bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1);
3843
3844 /* wait a maximum of 5 second */
3845
3846 for (i = 0; (i < 5000) && (bios_sem != 0); i++) {
3847 DELAY(1000);
3848 bios_sem = bus_read_1(res, eecp +
3849 XHCI_XECP_BIOS_SEM);
3850 }
3851
3852 if (bios_sem != 0) {
3853 if (bootverbose)
3854 printf("xhci early: "
3855 "SMM does not respond\n");
3856 }
3857
3858 /* Disable interrupts */
3859 offs = bus_read_1(res, XHCI_CAPLENGTH);
3860 bus_write_4(res, offs + XHCI_USBCMD, 0);
3861 bus_read_4(res, offs + XHCI_USBSTS);
3862 }
3863 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3864 }
3865
3866 static void
pci_reserve_secbus(device_t bus,device_t dev,pcicfgregs * cfg,struct resource_list * rl)3867 pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg,
3868 struct resource_list *rl)
3869 {
3870 struct resource *res;
3871 char *cp;
3872 rman_res_t start, end, count;
3873 int sec_bus, sec_reg, sub_bus, sub_reg, sup_bus;
3874
3875 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3876 case PCIM_HDRTYPE_BRIDGE:
3877 sec_reg = PCIR_SECBUS_1;
3878 sub_reg = PCIR_SUBBUS_1;
3879 break;
3880 case PCIM_HDRTYPE_CARDBUS:
3881 sec_reg = PCIR_SECBUS_2;
3882 sub_reg = PCIR_SUBBUS_2;
3883 break;
3884 default:
3885 return;
3886 }
3887
3888 /*
3889 * If the existing bus range is valid, attempt to reserve it
3890 * from our parent. If this fails for any reason, clear the
3891 * secbus and subbus registers.
3892 *
3893 * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus?
3894 * This would at least preserve the existing sec_bus if it is
3895 * valid.
3896 */
3897 sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1);
3898 sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1);
3899
3900 /* Quirk handling. */
3901 switch (pci_get_devid(dev)) {
3902 case 0x12258086: /* Intel 82454KX/GX (Orion) */
3903 sup_bus = pci_read_config(dev, 0x41, 1);
3904 if (sup_bus != 0xff) {
3905 sec_bus = sup_bus + 1;
3906 sub_bus = sup_bus + 1;
3907 PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1);
3908 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3909 }
3910 break;
3911
3912 case 0x00dd10de:
3913 /* Compaq R3000 BIOS sets wrong subordinate bus number. */
3914 if ((cp = kern_getenv("smbios.planar.maker")) == NULL)
3915 break;
3916 if (strncmp(cp, "Compal", 6) != 0) {
3917 freeenv(cp);
3918 break;
3919 }
3920 freeenv(cp);
3921 if ((cp = kern_getenv("smbios.planar.product")) == NULL)
3922 break;
3923 if (strncmp(cp, "08A0", 4) != 0) {
3924 freeenv(cp);
3925 break;
3926 }
3927 freeenv(cp);
3928 if (sub_bus < 0xa) {
3929 sub_bus = 0xa;
3930 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3931 }
3932 break;
3933 }
3934
3935 if (bootverbose)
3936 printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus);
3937 if (sec_bus > 0 && sub_bus >= sec_bus) {
3938 start = sec_bus;
3939 end = sub_bus;
3940 count = end - start + 1;
3941
3942 resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count);
3943
3944 /*
3945 * If requested, clear secondary bus registers in
3946 * bridge devices to force a complete renumbering
3947 * rather than reserving the existing range. However,
3948 * preserve the existing size.
3949 */
3950 if (pci_clear_buses)
3951 goto clear;
3952
3953 res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, 0,
3954 start, end, count, 0);
3955 if (res != NULL)
3956 return;
3957
3958 if (bootverbose)
3959 device_printf(bus,
3960 "pci%d:%d:%d:%d secbus failed to allocate\n",
3961 pci_get_domain(dev), pci_get_bus(dev),
3962 pci_get_slot(dev), pci_get_function(dev));
3963 }
3964
3965 clear:
3966 PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1);
3967 PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1);
3968 }
3969
3970 static struct resource *
pci_alloc_secbus(device_t dev,device_t child,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)3971 pci_alloc_secbus(device_t dev, device_t child, int rid, rman_res_t start,
3972 rman_res_t end, rman_res_t count, u_int flags)
3973 {
3974 struct pci_devinfo *dinfo;
3975 pcicfgregs *cfg;
3976 struct resource_list *rl;
3977 struct resource *res;
3978 int sec_reg, sub_reg;
3979
3980 dinfo = device_get_ivars(child);
3981 cfg = &dinfo->cfg;
3982 rl = &dinfo->resources;
3983 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3984 case PCIM_HDRTYPE_BRIDGE:
3985 sec_reg = PCIR_SECBUS_1;
3986 sub_reg = PCIR_SUBBUS_1;
3987 break;
3988 case PCIM_HDRTYPE_CARDBUS:
3989 sec_reg = PCIR_SECBUS_2;
3990 sub_reg = PCIR_SUBBUS_2;
3991 break;
3992 default:
3993 return (NULL);
3994 }
3995
3996 if (rid != 0)
3997 return (NULL);
3998
3999 if (resource_list_find(rl, PCI_RES_BUS, rid) == NULL)
4000 resource_list_add(rl, PCI_RES_BUS, rid, start, end, count);
4001 if (!resource_list_reserved(rl, PCI_RES_BUS, rid)) {
4002 res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid,
4003 start, end, count, flags & ~RF_ACTIVE);
4004 if (res == NULL) {
4005 resource_list_delete(rl, PCI_RES_BUS, rid);
4006 device_printf(child, "allocating %ju bus%s failed\n",
4007 count, count == 1 ? "" : "es");
4008 return (NULL);
4009 }
4010 if (bootverbose)
4011 device_printf(child,
4012 "Lazy allocation of %ju bus%s at %ju\n", count,
4013 count == 1 ? "" : "es", rman_get_start(res));
4014 PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1);
4015 PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1);
4016 }
4017 return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start,
4018 end, count, flags));
4019 }
4020
4021 static int
pci_ea_bei_to_rid(device_t dev,int bei)4022 pci_ea_bei_to_rid(device_t dev, int bei)
4023 {
4024 #ifdef PCI_IOV
4025 struct pci_devinfo *dinfo;
4026 int iov_pos;
4027 struct pcicfg_iov *iov;
4028
4029 dinfo = device_get_ivars(dev);
4030 iov = dinfo->cfg.iov;
4031 if (iov != NULL)
4032 iov_pos = iov->iov_pos;
4033 else
4034 iov_pos = 0;
4035 #endif
4036
4037 /* Check if matches BAR */
4038 if ((bei >= PCIM_EA_BEI_BAR_0) &&
4039 (bei <= PCIM_EA_BEI_BAR_5))
4040 return (PCIR_BAR(bei));
4041
4042 /* Check ROM */
4043 if (bei == PCIM_EA_BEI_ROM)
4044 return (PCIR_BIOS);
4045
4046 #ifdef PCI_IOV
4047 /* Check if matches VF_BAR */
4048 if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) &&
4049 (bei <= PCIM_EA_BEI_VF_BAR_5))
4050 return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) +
4051 iov_pos);
4052 #endif
4053
4054 return (-1);
4055 }
4056
4057 int
pci_ea_is_enabled(device_t dev,int rid)4058 pci_ea_is_enabled(device_t dev, int rid)
4059 {
4060 struct pci_ea_entry *ea;
4061 struct pci_devinfo *dinfo;
4062
4063 dinfo = device_get_ivars(dev);
4064
4065 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
4066 if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid)
4067 return ((ea->eae_flags & PCIM_EA_ENABLE) > 0);
4068 }
4069
4070 return (0);
4071 }
4072
4073 void
pci_add_resources_ea(device_t bus,device_t dev,int alloc_iov)4074 pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
4075 {
4076 struct pci_ea_entry *ea;
4077 struct pci_devinfo *dinfo;
4078 pci_addr_t start, end, count;
4079 struct resource_list *rl;
4080 int type, flags, rid;
4081 struct resource *res;
4082 uint32_t tmp;
4083 #ifdef PCI_IOV
4084 struct pcicfg_iov *iov;
4085 #endif
4086
4087 dinfo = device_get_ivars(dev);
4088 rl = &dinfo->resources;
4089 flags = 0;
4090
4091 #ifdef PCI_IOV
4092 iov = dinfo->cfg.iov;
4093 #endif
4094
4095 if (dinfo->cfg.ea.ea_location == 0)
4096 return;
4097
4098 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
4099 /*
4100 * TODO: Ignore EA-BAR if is not enabled.
4101 * Currently the EA implementation supports
4102 * only situation, where EA structure contains
4103 * predefined entries. In case they are not enabled
4104 * leave them unallocated and proceed with
4105 * a legacy-BAR mechanism.
4106 */
4107 if ((ea->eae_flags & PCIM_EA_ENABLE) == 0)
4108 continue;
4109
4110 switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) {
4111 case PCIM_EA_P_MEM_PREFETCH:
4112 case PCIM_EA_P_VF_MEM_PREFETCH:
4113 flags = RF_PREFETCHABLE;
4114 /* FALLTHROUGH */
4115 case PCIM_EA_P_VF_MEM:
4116 case PCIM_EA_P_MEM:
4117 type = SYS_RES_MEMORY;
4118 break;
4119 case PCIM_EA_P_IO:
4120 type = SYS_RES_IOPORT;
4121 break;
4122 default:
4123 continue;
4124 }
4125
4126 if (alloc_iov != 0) {
4127 #ifdef PCI_IOV
4128 /* Allocating IOV, confirm BEI matches */
4129 if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) ||
4130 (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5))
4131 continue;
4132 #else
4133 continue;
4134 #endif
4135 } else {
4136 /* Allocating BAR, confirm BEI matches */
4137 if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) ||
4138 (ea->eae_bei > PCIM_EA_BEI_BAR_5)) &&
4139 (ea->eae_bei != PCIM_EA_BEI_ROM))
4140 continue;
4141 }
4142
4143 rid = pci_ea_bei_to_rid(dev, ea->eae_bei);
4144 if (rid < 0)
4145 continue;
4146
4147 /* Skip resources already allocated by EA */
4148 if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) ||
4149 (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL))
4150 continue;
4151
4152 start = ea->eae_base;
4153 count = ea->eae_max_offset + 1;
4154 #ifdef PCI_IOV
4155 if (iov != NULL)
4156 count = count * iov->iov_num_vfs;
4157 #endif
4158 end = start + count - 1;
4159 if (count == 0)
4160 continue;
4161
4162 resource_list_add(rl, type, rid, start, end, count);
4163 res = resource_list_reserve(rl, bus, dev, type, rid, start, end, count,
4164 flags);
4165 if (res == NULL) {
4166 resource_list_delete(rl, type, rid);
4167
4168 /*
4169 * Failed to allocate using EA, disable entry.
4170 * Another attempt to allocation will be performed
4171 * further, but this time using legacy BAR registers
4172 */
4173 tmp = pci_read_config(dev, ea->eae_cfg_offset, 4);
4174 tmp &= ~PCIM_EA_ENABLE;
4175 pci_write_config(dev, ea->eae_cfg_offset, tmp, 4);
4176
4177 /*
4178 * Disabling entry might fail in case it is hardwired.
4179 * Read flags again to match current status.
4180 */
4181 ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4);
4182
4183 continue;
4184 }
4185
4186 /* As per specification, fill BAR with zeros */
4187 pci_write_config(dev, rid, 0, 4);
4188 }
4189 }
4190
4191 void
pci_add_resources(device_t bus,device_t dev,int force,uint32_t prefetchmask)4192 pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
4193 {
4194 struct pci_devinfo *dinfo;
4195 pcicfgregs *cfg;
4196 struct resource_list *rl;
4197 const struct pci_quirk *q;
4198 uint32_t devid;
4199 int i;
4200
4201 dinfo = device_get_ivars(dev);
4202 cfg = &dinfo->cfg;
4203 rl = &dinfo->resources;
4204 devid = (cfg->device << 16) | cfg->vendor;
4205
4206 /* Allocate resources using Enhanced Allocation */
4207 pci_add_resources_ea(bus, dev, 0);
4208
4209 /* ATA devices needs special map treatment */
4210 if ((pci_get_class(dev) == PCIC_STORAGE) &&
4211 (pci_get_subclass(dev) == PCIS_STORAGE_IDE) &&
4212 ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) ||
4213 (!pci_read_config(dev, PCIR_BAR(0), 4) &&
4214 !pci_read_config(dev, PCIR_BAR(2), 4))) )
4215 pci_ata_maps(bus, dev, rl, force, prefetchmask);
4216 else
4217 for (i = 0; i < cfg->nummaps;) {
4218 /* Skip resources already managed by EA */
4219 if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) ||
4220 (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) ||
4221 pci_ea_is_enabled(dev, PCIR_BAR(i))) {
4222 i++;
4223 continue;
4224 }
4225
4226 /*
4227 * Skip quirked resources.
4228 */
4229 for (q = &pci_quirks[0]; q->devid != 0; q++)
4230 if (q->devid == devid &&
4231 q->type == PCI_QUIRK_UNMAP_REG &&
4232 q->arg1 == PCIR_BAR(i))
4233 break;
4234 if (q->devid != 0) {
4235 i++;
4236 continue;
4237 }
4238 i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force,
4239 prefetchmask & (1 << i));
4240 }
4241
4242 /*
4243 * Add additional, quirked resources.
4244 */
4245 for (q = &pci_quirks[0]; q->devid != 0; q++)
4246 if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG)
4247 pci_add_map(bus, dev, q->arg1, rl, force, 0);
4248
4249 if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline) &&
4250 pci_intx_reroute) {
4251 /*
4252 * Try to re-route interrupts. Sometimes the BIOS or
4253 * firmware may leave bogus values in these registers.
4254 * If the re-route fails, then just stick with what we
4255 * have.
4256 */
4257 pci_assign_interrupt(bus, dev, 1);
4258 }
4259
4260 if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS &&
4261 pci_get_subclass(dev) == PCIS_SERIALBUS_USB) {
4262 if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI)
4263 xhci_early_takeover(dev);
4264 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI)
4265 ehci_early_takeover(dev);
4266 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI)
4267 ohci_early_takeover(dev);
4268 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI)
4269 uhci_early_takeover(dev);
4270 }
4271
4272 /*
4273 * Reserve resources for secondary bus ranges behind bridge
4274 * devices.
4275 */
4276 pci_reserve_secbus(bus, dev, cfg, rl);
4277 }
4278
4279 static struct pci_devinfo *
pci_identify_function(device_t pcib,device_t dev,int domain,int busno,int slot,int func)4280 pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
4281 int slot, int func)
4282 {
4283 struct pci_devinfo *dinfo;
4284
4285 dinfo = pci_read_device(pcib, dev, domain, busno, slot, func);
4286 if (dinfo != NULL)
4287 pci_add_child(dev, dinfo);
4288
4289 return (dinfo);
4290 }
4291
4292 void
pci_add_children(device_t dev,int domain,int busno)4293 pci_add_children(device_t dev, int domain, int busno)
4294 {
4295 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4296 device_t pcib = device_get_parent(dev);
4297 struct pci_devinfo *dinfo;
4298 int maxslots;
4299 int s, f, pcifunchigh;
4300 uint8_t hdrtype;
4301 int first_func;
4302
4303 /*
4304 * Try to detect a device at slot 0, function 0. If it exists, try to
4305 * enable ARI. We must enable ARI before detecting the rest of the
4306 * functions on this bus as ARI changes the set of slots and functions
4307 * that are legal on this bus.
4308 */
4309 dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0);
4310 if (dinfo != NULL && pci_enable_ari)
4311 PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);
4312
4313 /*
4314 * Start looking for new devices on slot 0 at function 1 because we
4315 * just identified the device at slot 0, function 0.
4316 */
4317 first_func = 1;
4318
4319 maxslots = PCIB_MAXSLOTS(pcib);
4320 for (s = 0; s <= maxslots; s++, first_func = 0) {
4321 pcifunchigh = 0;
4322 f = 0;
4323 DELAY(1);
4324
4325 /* If function 0 is not present, skip to the next slot. */
4326 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4327 continue;
4328 hdrtype = REG(PCIR_HDRTYPE, 1);
4329 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4330 continue;
4331 if (hdrtype & PCIM_MFDEV)
4332 pcifunchigh = PCIB_MAXFUNCS(pcib);
4333 for (f = first_func; f <= pcifunchigh; f++)
4334 pci_identify_function(pcib, dev, domain, busno, s, f);
4335 }
4336 pcie_reconcile_link_mps(dev);
4337 #undef REG
4338 }
4339
4340 int
pci_rescan_method(device_t dev)4341 pci_rescan_method(device_t dev)
4342 {
4343 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4344 device_t pcib = device_get_parent(dev);
4345 device_t child, *devlist, *unchanged;
4346 int devcount, error, i, j, maxslots, oldcount;
4347 int busno, domain, s, f, pcifunchigh;
4348 uint8_t hdrtype;
4349
4350 /* No need to check for ARI on a rescan. */
4351 error = device_get_children(dev, &devlist, &devcount);
4352 if (error)
4353 return (error);
4354 if (devcount != 0) {
4355 unchanged = malloc(devcount * sizeof(device_t), M_TEMP,
4356 M_NOWAIT | M_ZERO);
4357 if (unchanged == NULL) {
4358 free(devlist, M_TEMP);
4359 return (ENOMEM);
4360 }
4361 } else
4362 unchanged = NULL;
4363
4364 domain = pcib_get_domain(dev);
4365 busno = pcib_get_bus(dev);
4366 maxslots = PCIB_MAXSLOTS(pcib);
4367 for (s = 0; s <= maxslots; s++) {
4368 /* If function 0 is not present, skip to the next slot. */
4369 f = 0;
4370 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4371 continue;
4372 pcifunchigh = 0;
4373 hdrtype = REG(PCIR_HDRTYPE, 1);
4374 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4375 continue;
4376 if (hdrtype & PCIM_MFDEV)
4377 pcifunchigh = PCIB_MAXFUNCS(pcib);
4378 for (f = 0; f <= pcifunchigh; f++) {
4379 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4380 continue;
4381
4382 /*
4383 * Found a valid function. Check if a
4384 * device_t for this device already exists.
4385 */
4386 for (i = 0; i < devcount; i++) {
4387 child = devlist[i];
4388 if (child == NULL)
4389 continue;
4390 if (pci_get_slot(child) == s &&
4391 pci_get_function(child) == f) {
4392 unchanged[i] = child;
4393 goto next_func;
4394 }
4395 }
4396
4397 pci_identify_function(pcib, dev, domain, busno, s, f);
4398 next_func:;
4399 }
4400 }
4401
4402 /* Remove devices that are no longer present. */
4403 for (i = 0; i < devcount; i++) {
4404 if (unchanged[i] != NULL)
4405 continue;
4406 device_delete_child(dev, devlist[i]);
4407 }
4408
4409 free(devlist, M_TEMP);
4410 oldcount = devcount;
4411
4412 /* Try to attach the devices just added. */
4413 error = device_get_children(dev, &devlist, &devcount);
4414 if (error) {
4415 free(unchanged, M_TEMP);
4416 return (error);
4417 }
4418
4419 for (i = 0; i < devcount; i++) {
4420 for (j = 0; j < oldcount; j++) {
4421 if (devlist[i] == unchanged[j])
4422 goto next_device;
4423 }
4424
4425 device_probe_and_attach(devlist[i]);
4426 next_device:;
4427 }
4428
4429 free(unchanged, M_TEMP);
4430 free(devlist, M_TEMP);
4431 return (0);
4432 #undef REG
4433 }
4434
4435 #ifdef PCI_IOV
4436 device_t
pci_iov_get_pf(device_t dev)4437 pci_iov_get_pf(device_t dev)
4438 {
4439 struct pci_devinfo *dinfo;
4440
4441 dinfo = device_get_ivars(dev);
4442 if (dinfo == NULL || (dinfo->cfg.flags & PCICFG_VF) == 0 ||
4443 dinfo->cfg.iov == NULL)
4444 return (NULL);
4445 return (dinfo->cfg.iov->iov_pf);
4446 }
4447
4448 device_t
pci_add_iov_child(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4449 pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid,
4450 uint16_t did)
4451 {
4452 struct pci_devinfo *pf_dinfo;
4453 struct pci_devinfo *vf_dinfo;
4454 device_t pcib;
4455 int busno, slot, func;
4456
4457 pcib = device_get_parent(bus);
4458
4459 PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func);
4460
4461 vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno,
4462 slot, func, vid, did);
4463
4464 /* Make the VF-to-PF relationship available to child-added callbacks. */
4465 pf_dinfo = device_get_ivars(pf);
4466 KASSERT(pf_dinfo->cfg.iov != NULL,
4467 ("SR-IOV PF %s has no IOV state", device_get_nameunit(pf)));
4468 vf_dinfo->cfg.iov = pf_dinfo->cfg.iov;
4469 vf_dinfo->cfg.flags |= PCICFG_VF;
4470 pci_add_child(bus, vf_dinfo);
4471
4472 return (vf_dinfo->cfg.dev);
4473 }
4474
4475 device_t
pci_create_iov_child_method(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4476 pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid,
4477 uint16_t vid, uint16_t did)
4478 {
4479
4480 return (pci_add_iov_child(bus, pf, rid, vid, did));
4481 }
4482 #else
4483 device_t
pci_iov_get_pf(device_t dev __unused)4484 pci_iov_get_pf(device_t dev __unused)
4485 {
4486
4487 return (NULL);
4488 }
4489 #endif
4490
4491 static int
pcie_mps_bytes(uint16_t mps)4492 pcie_mps_bytes(uint16_t mps)
4493 {
4494
4495 return (128 << (mps >> 5));
4496 }
4497
4498 static bool
pcie_mps_limit_value(uint16_t * mps)4499 pcie_mps_limit_value(uint16_t *mps)
4500 {
4501
4502 if (pci_mps_limit == 0)
4503 return (false);
4504 if (pci_mps_limit < 128 || pci_mps_limit > 4096 ||
4505 !powerof2(pci_mps_limit)) {
4506 if (!pci_mps_limit_warned) {
4507 printf("pci: invalid hw.pci.mps_limit=%d; ignoring\n",
4508 pci_mps_limit);
4509 pci_mps_limit_warned = true;
4510 }
4511 return (false);
4512 }
4513 *mps = (fls(pci_mps_limit) - 8) << 5;
4514 return (true);
4515 }
4516
4517 /* Return the smallest configured MPS above dev, if the walk reaches a root. */
4518 static bool
pcie_path_mps(device_t dev,uint16_t * mpsp)4519 pcie_path_mps(device_t dev, uint16_t *mpsp)
4520 {
4521 struct pci_devinfo *dinfo;
4522 device_t bus, pcib, start;
4523 uint16_t mps;
4524 bool found;
4525
4526 start = dev;
4527 found = false;
4528 for (;;) {
4529 bus = device_get_parent(dev);
4530 if (bus == NULL)
4531 break;
4532 pcib = device_get_parent(bus);
4533 if (pcib == NULL || !is_pci_device(pcib))
4534 break;
4535 /*
4536 * A PCI function may expose a host bridge for a synthetic PCI
4537 * domain. Its Device Control belongs to the parent domain and
4538 * does not describe an upstream link in the synthetic hierarchy.
4539 */
4540 if (pci_get_domain(pcib) != pci_get_domain(dev))
4541 break;
4542 dinfo = device_get_ivars(pcib);
4543 if (dinfo->cfg.pcie.pcie_location != 0) {
4544 mps = pcie_read_config(pcib, PCIER_DEVICE_CTL, 2) &
4545 PCIEM_CTL_MAX_PAYLOAD;
4546 if (!found || mps < *mpsp)
4547 *mpsp = mps;
4548 found = true;
4549 if (dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
4550 return (true);
4551 }
4552 dev = pcib;
4553 }
4554 if (found && bootverbose)
4555 device_printf(start,
4556 "PCIe MPS path walk did not reach a Root Port\n");
4557 return (false);
4558 }
4559
4560 static bool
pcie_mps_first_warning(device_t dev)4561 pcie_mps_first_warning(device_t dev)
4562 {
4563 struct pci_devinfo *dinfo;
4564
4565 dinfo = device_get_ivars(dev);
4566 if ((dinfo->cfg.flags & PCICFG_MPS_WARNED) != 0)
4567 return (false);
4568 dinfo->cfg.flags |= PCICFG_MPS_WARNED;
4569 return (true);
4570 }
4571
4572 static void
pcie_mps_conflict(device_t dev,uint16_t path_mps,uint16_t max_mps)4573 pcie_mps_conflict(device_t dev, uint16_t path_mps, uint16_t max_mps)
4574 {
4575
4576 if (!pcie_mps_first_warning(dev))
4577 return;
4578 device_printf(dev,
4579 "maximum supported MPS %d is below configured path MPS %d; "
4580 "cannot safely retune the shared ancestor hierarchy\n",
4581 pcie_mps_bytes(max_mps), pcie_mps_bytes(path_mps));
4582 }
4583
4584 static bool
pcie_mps_is_bridge(struct pci_devinfo * dinfo)4585 pcie_mps_is_bridge(struct pci_devinfo *dinfo)
4586 {
4587 uint8_t hdrtype;
4588
4589 hdrtype = dinfo->cfg.hdrtype & PCIM_HDRTYPE;
4590 return (hdrtype == PCIM_HDRTYPE_BRIDGE ||
4591 hdrtype == PCIM_HDRTYPE_CARDBUS);
4592 }
4593
4594 static void
pcie_mps_active_conflict(device_t dev,uint16_t path_mps,uint16_t device_mps)4595 pcie_mps_active_conflict(device_t dev, uint16_t path_mps,
4596 uint16_t device_mps)
4597 {
4598 struct pci_devinfo *dinfo;
4599 const char *action;
4600
4601 if (!pcie_mps_first_warning(dev))
4602 return;
4603 dinfo = device_get_ivars(dev);
4604 if (pci_mps_enforce && !pcie_mps_is_bridge(dinfo))
4605 action = "disabling device";
4606 else
4607 action = "leaving device unchanged";
4608 device_printf(dev,
4609 "configured MPS %d does not match path MPS %d while bus "
4610 "mastering is enabled; %s\n", pcie_mps_bytes(device_mps),
4611 pcie_mps_bytes(path_mps), action);
4612 }
4613
4614 static void
pcie_mps_mark_unreconciled(device_t dev)4615 pcie_mps_mark_unreconciled(device_t dev)
4616 {
4617 struct pci_devinfo *dinfo;
4618 uint16_t cmd;
4619
4620 dinfo = device_get_ivars(dev);
4621 if ((dinfo->cfg.flags & PCICFG_MPS_UNRECONCILED) != 0)
4622 return;
4623 dinfo->cfg.flags |= PCICFG_MPS_UNRECONCILED;
4624 if (!pci_mps_enforce)
4625 return;
4626 if (pcie_mps_is_bridge(dinfo)) {
4627 device_printf(dev,
4628 "not disabled by hw.pci.mps_enforce because it is a bridge\n");
4629 return;
4630 }
4631 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
4632 cmd &= ~(PCIM_CMD_PORTEN | PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
4633 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
4634 dinfo->cfg.cmdreg = cmd;
4635 device_disable(dev);
4636 device_printf(dev,
4637 "disabled because its MPS cannot be safely configured\n");
4638 }
4639
4640 static void
pcie_mps_unreconciled(device_t dev,uint16_t path_mps,uint16_t max_mps)4641 pcie_mps_unreconciled(device_t dev, uint16_t path_mps, uint16_t max_mps)
4642 {
4643
4644 pcie_mps_conflict(dev, path_mps, max_mps);
4645 pcie_mps_mark_unreconciled(dev);
4646 }
4647
4648 static void
pcie_mps_active_unreconciled(device_t dev,uint16_t path_mps,uint16_t device_mps)4649 pcie_mps_active_unreconciled(device_t dev, uint16_t path_mps,
4650 uint16_t device_mps)
4651 {
4652
4653 pcie_mps_active_conflict(dev, path_mps, device_mps);
4654 pcie_mps_mark_unreconciled(dev);
4655 }
4656
4657 static void
pcie_mps_mark_link_unreconciled(device_t * devlist,int count,uint16_t path_mps,bool all)4658 pcie_mps_mark_link_unreconciled(device_t *devlist, int count,
4659 uint16_t path_mps,
4660 bool all)
4661 {
4662 struct pci_devinfo *dinfo;
4663 device_t child;
4664 uint16_t mmps;
4665 int i;
4666
4667 for (i = 0; i < count; i++) {
4668 child = devlist[i];
4669 dinfo = device_get_ivars(child);
4670 if ((dinfo->cfg.flags & (PCICFG_VF |
4671 PCICFG_MPS_UNRECONCILED)) != 0 ||
4672 dinfo->cfg.pcie.pcie_location == 0)
4673 continue;
4674 if (all) {
4675 pcie_mps_mark_unreconciled(child);
4676 continue;
4677 }
4678 mmps = (pcie_read_config(child, PCIER_DEVICE_CAP, 2) &
4679 PCIEM_CAP_MAX_PAYLOAD) << 5;
4680 if (mmps < path_mps)
4681 pcie_mps_unreconciled(child, path_mps, mmps);
4682 }
4683 }
4684
4685 /*
4686 * Tune a function discovered by rescan or hot-add against the established
4687 * path. Never change a shared upstream port here: doing so requires
4688 * quiescing every driver and draining all outstanding transactions in the
4689 * hierarchy. Cold enumeration is reconciled by pcie_reconcile_link_mps().
4690 */
4691 static void
pcie_setup_mps(device_t dev)4692 pcie_setup_mps(device_t dev)
4693 {
4694 struct pci_devinfo *dinfo;
4695 device_t bus;
4696 uint16_t mmps, mps, path_mps;
4697
4698 dinfo = device_get_ivars(dev);
4699 /*
4700 * PCIe r4.0, sec 9.3.5.4 defines the VF MPS and MRRS fields as
4701 * Reserved and Preserved, with the PF settings applying to the VF.
4702 * Do not use the VF's hardwired value to configure the shared path.
4703 */
4704 if ((dinfo->cfg.flags & PCICFG_VF) != 0)
4705 return;
4706 if (dinfo->cfg.pcie.pcie_location == 0)
4707 return;
4708
4709 /* Cold enumeration is reconciled one complete link at a time. */
4710 bus = device_get_parent(dev);
4711 if (!device_is_attached(bus))
4712 return;
4713 path_mps = 0;
4714 if (!pcie_path_mps(dev, &path_mps))
4715 return;
4716
4717 mmps = (pcie_read_config(dev, PCIER_DEVICE_CAP, 2) &
4718 PCIEM_CAP_MAX_PAYLOAD) << 5;
4719 if (path_mps > mmps) {
4720 pcie_mps_unreconciled(dev, path_mps, mmps);
4721 return;
4722 }
4723 mps = pcie_read_config(dev, PCIER_DEVICE_CTL, 2) &
4724 PCIEM_CTL_MAX_PAYLOAD;
4725 if (mps == path_mps)
4726 return;
4727 if ((pci_read_config(dev, PCIR_COMMAND, 2) &
4728 PCIM_CMD_BUSMASTEREN) != 0) {
4729 pcie_mps_active_unreconciled(dev, path_mps, mps);
4730 return;
4731 }
4732 pcie_adjust_config(dev, PCIER_DEVICE_CTL, PCIEM_CTL_MAX_PAYLOAD,
4733 path_mps, 2);
4734 }
4735
4736 /*
4737 * Reconcile a newly enumerated link before attaching any child drivers. A
4738 * Root Port may be lowered because its complete downstream hierarchy is still
4739 * idle. A late reduction below a switch is not propagated through ancestors,
4740 * since sibling subtrees may already be active.
4741 */
4742 void
pcie_reconcile_link_mps(device_t bus)4743 pcie_reconcile_link_mps(device_t bus)
4744 {
4745 struct pci_devinfo *dinfo, *upinfo;
4746 device_t child, limiting, pcib, *devlist;
4747 uint16_t cap_target, lmps, mmps, mps, target, up_mmps, up_mps;
4748 int count, error, i;
4749 bool limit_requested;
4750
4751 if (!pci_enable_mps_tune)
4752 return;
4753 /* Shared-path tuning is only safe before this bus attaches children. */
4754 if (device_is_attached(bus))
4755 return;
4756 pcib = device_get_parent(bus);
4757 if (!is_pci_device(pcib))
4758 return;
4759 /*
4760 * A PCI function may provide a host bridge into a separate domain,
4761 * as Intel VMD does. Do not treat the function's host-facing PCIe
4762 * Device Control as the upstream end of a link in the child domain.
4763 */
4764 if (pci_get_domain(pcib) != pcib_get_domain(bus))
4765 return;
4766 upinfo = device_get_ivars(pcib);
4767 if (upinfo->cfg.pcie.pcie_location == 0)
4768 return;
4769 error = device_get_children(bus, &devlist, &count);
4770 if (error != 0)
4771 return;
4772
4773 up_mps = pcie_read_config(pcib, PCIER_DEVICE_CTL, 2) &
4774 PCIEM_CTL_MAX_PAYLOAD;
4775 cap_target = up_mps;
4776 limiting = NULL;
4777 up_mmps = (pcie_read_config(pcib, PCIER_DEVICE_CAP, 2) &
4778 PCIEM_CAP_MAX_PAYLOAD) << 5;
4779 if (cap_target > up_mmps) {
4780 cap_target = up_mmps;
4781 limiting = pcib;
4782 }
4783 /*
4784 * Firmware may leave Bus Master Enable set after handoff. Since no
4785 * child driver has attached during this cold pass, it is not a proxy
4786 * for a live FreeBSD consumer.
4787 */
4788 for (i = 0; i < count; i++) {
4789 child = devlist[i];
4790 dinfo = device_get_ivars(child);
4791 if ((dinfo->cfg.flags & (PCICFG_VF |
4792 PCICFG_MPS_UNRECONCILED)) != 0 ||
4793 dinfo->cfg.pcie.pcie_location == 0)
4794 continue;
4795 mmps = (pcie_read_config(child, PCIER_DEVICE_CAP, 2) &
4796 PCIEM_CAP_MAX_PAYLOAD) << 5;
4797 if (cap_target > mmps) {
4798 cap_target = mmps;
4799 limiting = child;
4800 }
4801 }
4802 target = cap_target;
4803 limit_requested = pcie_mps_limit_value(&lmps) && up_mps > lmps;
4804 if (limit_requested && target > lmps)
4805 target = lmps;
4806
4807 /*
4808 * Do not lower one link below a switch without also reconciling every
4809 * ancestor and sibling subtree. Recursive newbus attachment may already
4810 * have made another subtree live, so leave the established path intact.
4811 */
4812 if (target < up_mps &&
4813 upinfo->cfg.pcie.pcie_type != PCIEM_TYPE_ROOT_PORT) {
4814 if (cap_target < up_mps)
4815 pcie_mps_conflict(limiting, up_mps, cap_target);
4816 if (limit_requested) {
4817 device_printf(pcib,
4818 "cannot apply hw.pci.mps_limit=%d below a switch "
4819 "without retuning the shared ancestor hierarchy; "
4820 "leaving path MPS %d unchanged\n",
4821 pci_mps_limit, pcie_mps_bytes(up_mps));
4822 }
4823 pcie_mps_mark_link_unreconciled(devlist, count, up_mps,
4824 up_mps > up_mmps);
4825 /* Keep compatible functions at the established path MPS. */
4826 target = up_mps;
4827 }
4828 /* Lower downstream producers before lowering the shared Root Port. */
4829 for (i = 0; i < count; i++) {
4830 child = devlist[i];
4831 dinfo = device_get_ivars(child);
4832 if ((dinfo->cfg.flags & (PCICFG_VF |
4833 PCICFG_MPS_UNRECONCILED)) != 0 ||
4834 dinfo->cfg.pcie.pcie_location == 0)
4835 continue;
4836 mps = pcie_read_config(child, PCIER_DEVICE_CTL, 2) &
4837 PCIEM_CTL_MAX_PAYLOAD;
4838 if (mps > target)
4839 pcie_adjust_config(child, PCIER_DEVICE_CTL,
4840 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4841 }
4842 if (up_mps > target)
4843 pcie_adjust_config(pcib, PCIER_DEVICE_CTL,
4844 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4845
4846 /* Raise idle children only after the upstream port is configured. */
4847 for (i = 0; i < count; i++) {
4848 child = devlist[i];
4849 dinfo = device_get_ivars(child);
4850 if ((dinfo->cfg.flags & (PCICFG_VF |
4851 PCICFG_MPS_UNRECONCILED)) != 0 ||
4852 dinfo->cfg.pcie.pcie_location == 0)
4853 continue;
4854 mps = pcie_read_config(child, PCIER_DEVICE_CTL, 2) &
4855 PCIEM_CTL_MAX_PAYLOAD;
4856 if (mps < target)
4857 pcie_adjust_config(child, PCIER_DEVICE_CTL,
4858 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4859 }
4860 free(devlist, M_TEMP);
4861 }
4862
4863 static void
pci_add_child_clear_aer(device_t dev,struct pci_devinfo * dinfo)4864 pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo)
4865 {
4866 int aer;
4867 uint32_t r;
4868
4869 if (dinfo->cfg.pcie.pcie_location != 0 &&
4870 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) {
4871 r = PCIEM_ROOT_CTL_SERR_CORR |
4872 PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL;
4873 pcie_adjust_config(dev, PCIER_ROOT_CTL, r, 0, 2);
4874 }
4875 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
4876 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
4877 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
4878 if (r != 0 && bootverbose) {
4879 pci_printf(&dinfo->cfg,
4880 "clearing AER UC 0x%08x -> 0x%08x\n",
4881 r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS,
4882 4));
4883 }
4884
4885 r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4);
4886 r &= ~(PCIM_AER_UC_TRAINING_ERROR |
4887 PCIM_AER_UC_DL_PROTOCOL_ERROR |
4888 PCIM_AER_UC_SURPRISE_LINK_DOWN |
4889 PCIM_AER_UC_POISONED_TLP |
4890 PCIM_AER_UC_FC_PROTOCOL_ERROR |
4891 PCIM_AER_UC_COMPLETION_TIMEOUT |
4892 PCIM_AER_UC_COMPLETER_ABORT |
4893 PCIM_AER_UC_UNEXPECTED_COMPLETION |
4894 PCIM_AER_UC_RECEIVER_OVERFLOW |
4895 PCIM_AER_UC_MALFORMED_TLP |
4896 PCIM_AER_UC_ECRC_ERROR |
4897 PCIM_AER_UC_UNSUPPORTED_REQUEST |
4898 PCIM_AER_UC_ACS_VIOLATION |
4899 PCIM_AER_UC_INTERNAL_ERROR |
4900 PCIM_AER_UC_MC_BLOCKED_TLP |
4901 PCIM_AER_UC_ATOMIC_EGRESS_BLK |
4902 PCIM_AER_UC_TLP_PREFIX_BLOCKED);
4903 pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4);
4904
4905 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
4906 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
4907 if (r != 0 && bootverbose) {
4908 pci_printf(&dinfo->cfg,
4909 "clearing AER COR 0x%08x -> 0x%08x\n",
4910 r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS,
4911 4));
4912 }
4913
4914 r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4);
4915 r &= ~(PCIM_AER_COR_RECEIVER_ERROR |
4916 PCIM_AER_COR_BAD_TLP |
4917 PCIM_AER_COR_BAD_DLLP |
4918 PCIM_AER_COR_REPLAY_ROLLOVER |
4919 PCIM_AER_COR_REPLAY_TIMEOUT |
4920 PCIM_AER_COR_ADVISORY_NF_ERROR |
4921 PCIM_AER_COR_INTERNAL_ERROR |
4922 PCIM_AER_COR_HEADER_LOG_OVFLOW);
4923 pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4);
4924
4925 r = PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE |
4926 PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE;
4927 pcie_adjust_config(dev, PCIER_DEVICE_CTL, r, r, 2);
4928 }
4929 }
4930
4931 void
pci_add_child(device_t bus,struct pci_devinfo * dinfo)4932 pci_add_child(device_t bus, struct pci_devinfo *dinfo)
4933 {
4934 device_t dev;
4935
4936 dinfo->cfg.dev = dev = device_add_child(bus, NULL, DEVICE_UNIT_ANY);
4937 device_set_ivars(dev, dinfo);
4938 resource_list_init(&dinfo->resources);
4939 pci_cfg_save(dev, dinfo, 0);
4940 pci_cfg_restore(dev, dinfo);
4941 pci_clear_pme(dev);
4942 pci_print_verbose(dinfo);
4943 pci_add_resources(bus, dev, 0, 0);
4944 if (pci_enable_mps_tune)
4945 pcie_setup_mps(dev);
4946 pci_child_added(dinfo->cfg.dev);
4947
4948 if (pci_clear_aer_on_attach)
4949 pci_add_child_clear_aer(dev, dinfo);
4950
4951 EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
4952 }
4953
4954 void
pci_child_added_method(device_t dev,device_t child)4955 pci_child_added_method(device_t dev, device_t child)
4956 {
4957
4958 }
4959
4960 static int
pci_probe(device_t dev)4961 pci_probe(device_t dev)
4962 {
4963
4964 device_set_desc(dev, "PCI bus");
4965
4966 /* Allow other subclasses to override this driver. */
4967 return (BUS_PROBE_GENERIC);
4968 }
4969
4970 int
pci_attach_common(device_t dev)4971 pci_attach_common(device_t dev)
4972 {
4973 struct pci_softc *sc;
4974 int busno, domain;
4975 int rid;
4976
4977 sc = device_get_softc(dev);
4978 domain = pcib_get_domain(dev);
4979 busno = pcib_get_bus(dev);
4980 rid = 0;
4981 sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno,
4982 1, 0);
4983 if (sc->sc_bus == NULL) {
4984 device_printf(dev, "failed to allocate bus number\n");
4985 return (ENXIO);
4986 }
4987 if (bootverbose)
4988 device_printf(dev, "domain=%d, physical bus=%d\n",
4989 domain, busno);
4990 sc->sc_dma_tag = bus_get_dma_tag(dev);
4991 return (0);
4992 }
4993
4994 int
pci_attach(device_t dev)4995 pci_attach(device_t dev)
4996 {
4997 int busno, domain, error;
4998
4999 error = pci_attach_common(dev);
5000 if (error)
5001 return (error);
5002
5003 /*
5004 * Since there can be multiple independently numbered PCI
5005 * buses on systems with multiple PCI domains, we can't use
5006 * the unit number to decide which bus we are probing. We ask
5007 * the parent pcib what our domain and bus numbers are.
5008 */
5009 domain = pcib_get_domain(dev);
5010 busno = pcib_get_bus(dev);
5011 pci_add_children(dev, domain, busno);
5012 bus_attach_children(dev);
5013 return (0);
5014 }
5015
5016 int
pci_detach(device_t dev)5017 pci_detach(device_t dev)
5018 {
5019 struct pci_softc *sc;
5020 int error;
5021
5022 error = bus_generic_detach(dev);
5023 if (error)
5024 return (error);
5025 sc = device_get_softc(dev);
5026 error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus);
5027 return (error);
5028 }
5029
5030 static void
pci_hint_device_unit(device_t dev,device_t child,const char * name,int * unitp)5031 pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp)
5032 {
5033 int line, unit;
5034 const char *at;
5035 char me1[24], me2[32];
5036 uint8_t b, s, f;
5037 uint32_t d;
5038 device_location_cache_t *cache;
5039
5040 d = pci_get_domain(child);
5041 b = pci_get_bus(child);
5042 s = pci_get_slot(child);
5043 f = pci_get_function(child);
5044 snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f);
5045 snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f);
5046 line = 0;
5047 cache = dev_wired_cache_init();
5048 while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) {
5049 resource_string_value(name, unit, "at", &at);
5050 if (strcmp(at, me1) == 0 || strcmp(at, me2) == 0) {
5051 *unitp = unit;
5052 break;
5053 }
5054 if (dev_wired_cache_match(cache, child, at)) {
5055 *unitp = unit;
5056 break;
5057 }
5058 }
5059 dev_wired_cache_fini(cache);
5060 }
5061
5062 static void
pci_set_power_child(device_t dev,device_t child,int state)5063 pci_set_power_child(device_t dev, device_t child, int state)
5064 {
5065 device_t pcib;
5066 int dstate;
5067
5068 /*
5069 * Set the device to the given state. If the firmware suggests
5070 * a different power state, use it instead. If power management
5071 * is not present, the firmware is responsible for managing
5072 * device power. Skip children who aren't attached since they
5073 * are handled separately.
5074 */
5075 pcib = device_get_parent(dev);
5076 dstate = state;
5077 if (device_is_attached(child) &&
5078 PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0)
5079 pci_set_powerstate(child, dstate);
5080 }
5081
5082 int
pci_suspend_child(device_t dev,device_t child)5083 pci_suspend_child(device_t dev, device_t child)
5084 {
5085 struct pci_devinfo *dinfo;
5086 struct resource_list_entry *rle;
5087 int error;
5088
5089 dinfo = device_get_ivars(child);
5090
5091 /*
5092 * Save the PCI configuration space for the child and set the
5093 * device in the appropriate power state for this sleep state.
5094 */
5095 pci_cfg_save(child, dinfo, 0);
5096
5097 /* Suspend devices before potentially powering them down. */
5098 error = bus_generic_suspend_child(dev, child);
5099
5100 if (error)
5101 return (error);
5102
5103 if (pci_do_power_suspend) {
5104 /*
5105 * Make sure this device's interrupt handler is not invoked
5106 * in the case the device uses a shared interrupt that can
5107 * be raised by some other device.
5108 * This is applicable only to regular (legacy) PCI interrupts
5109 * as MSI/MSI-X interrupts are never shared.
5110 */
5111 rle = resource_list_find(&dinfo->resources,
5112 SYS_RES_IRQ, 0);
5113 if (rle != NULL && rle->res != NULL)
5114 (void)bus_suspend_intr(child, rle->res);
5115 pci_set_power_child(dev, child, PCI_POWERSTATE_D3);
5116 }
5117
5118 return (0);
5119 }
5120
5121 int
pci_resume_child(device_t dev,device_t child)5122 pci_resume_child(device_t dev, device_t child)
5123 {
5124 struct pci_devinfo *dinfo;
5125 struct resource_list_entry *rle;
5126
5127 if (pci_do_power_resume)
5128 pci_set_power_child(dev, child, PCI_POWERSTATE_D0);
5129
5130 dinfo = device_get_ivars(child);
5131 pci_cfg_restore(child, dinfo);
5132 pci_clear_pme(child);
5133 if (!device_is_attached(child))
5134 pci_cfg_save(child, dinfo, 1);
5135
5136 bus_generic_resume_child(dev, child);
5137
5138 /*
5139 * Allow interrupts only after fully resuming the driver and hardware.
5140 */
5141 if (pci_do_power_suspend) {
5142 /* See pci_suspend_child for details. */
5143 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
5144 if (rle != NULL && rle->res != NULL)
5145 (void)bus_resume_intr(child, rle->res);
5146 }
5147
5148 return (0);
5149 }
5150
5151 int
pci_resume(device_t dev)5152 pci_resume(device_t dev)
5153 {
5154 device_t child, *devlist;
5155 int error, i, numdevs;
5156
5157 if ((error = device_get_children(dev, &devlist, &numdevs)) != 0)
5158 return (error);
5159
5160 /*
5161 * Resume critical devices first, then everything else later.
5162 */
5163 for (i = 0; i < numdevs; i++) {
5164 child = devlist[i];
5165 switch (pci_get_class(child)) {
5166 case PCIC_DISPLAY:
5167 case PCIC_MEMORY:
5168 case PCIC_BRIDGE:
5169 case PCIC_BASEPERIPH:
5170 BUS_RESUME_CHILD(dev, child);
5171 break;
5172 }
5173 }
5174 for (i = 0; i < numdevs; i++) {
5175 child = devlist[i];
5176 switch (pci_get_class(child)) {
5177 case PCIC_DISPLAY:
5178 case PCIC_MEMORY:
5179 case PCIC_BRIDGE:
5180 case PCIC_BASEPERIPH:
5181 break;
5182 default:
5183 BUS_RESUME_CHILD(dev, child);
5184 }
5185 }
5186 free(devlist, M_TEMP);
5187 return (0);
5188 }
5189
5190 static void
pci_load_vendor_data(void)5191 pci_load_vendor_data(void)
5192 {
5193 caddr_t data;
5194 void *ptr;
5195 size_t sz;
5196
5197 data = preload_search_by_type("pci_vendor_data");
5198 if (data != NULL) {
5199 ptr = preload_fetch_addr(data);
5200 sz = preload_fetch_size(data);
5201 if (ptr != NULL && sz != 0) {
5202 pci_vendordata = ptr;
5203 pci_vendordata_size = sz;
5204 /* terminate the database */
5205 pci_vendordata[pci_vendordata_size] = '\n';
5206 }
5207 }
5208 }
5209
5210 void
pci_driver_added(device_t dev,driver_t * driver)5211 pci_driver_added(device_t dev, driver_t *driver)
5212 {
5213 int numdevs;
5214 device_t *devlist;
5215 device_t child;
5216 struct pci_devinfo *dinfo;
5217 int i;
5218
5219 if (bootverbose)
5220 device_printf(dev, "driver added\n");
5221 DEVICE_IDENTIFY(driver, dev);
5222 if (device_get_children(dev, &devlist, &numdevs) != 0)
5223 return;
5224 for (i = 0; i < numdevs; i++) {
5225 child = devlist[i];
5226 if (device_get_state(child) != DS_NOTPRESENT)
5227 continue;
5228 dinfo = device_get_ivars(child);
5229 pci_print_verbose(dinfo);
5230 if (bootverbose)
5231 pci_printf(&dinfo->cfg, "reprobing on driver added\n");
5232 pci_cfg_restore(child, dinfo);
5233 if (device_probe_and_attach(child) != 0)
5234 pci_child_detached(dev, child);
5235 }
5236 free(devlist, M_TEMP);
5237 }
5238
5239 int
pci_setup_intr(device_t dev,device_t child,struct resource * irq,int flags,driver_filter_t * filter,driver_intr_t * intr,void * arg,void ** cookiep)5240 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
5241 driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep)
5242 {
5243 struct pci_devinfo *dinfo;
5244 struct msix_table_entry *mte;
5245 struct msix_vector *mv;
5246 uint64_t addr;
5247 uint32_t data;
5248 void *cookie;
5249 int error, rid;
5250
5251 error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr,
5252 arg, &cookie);
5253 if (error)
5254 return (error);
5255
5256 /* If this is not a direct child, just bail out. */
5257 if (device_get_parent(child) != dev) {
5258 *cookiep = cookie;
5259 return(0);
5260 }
5261
5262 rid = rman_get_rid(irq);
5263 if (rid == 0) {
5264 /* Make sure that INTx is enabled */
5265 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
5266 } else {
5267 /*
5268 * Check to see if the interrupt is MSI or MSI-X.
5269 * Ask our parent to map the MSI and give
5270 * us the address and data register values.
5271 * If we fail for some reason, teardown the
5272 * interrupt handler.
5273 */
5274 dinfo = device_get_ivars(child);
5275 if (dinfo->cfg.msi.msi_alloc > 0) {
5276 if (dinfo->cfg.msi.msi_addr == 0) {
5277 KASSERT(dinfo->cfg.msi.msi_handlers == 0,
5278 ("MSI has handlers, but vectors not mapped"));
5279 error = PCIB_MAP_MSI(device_get_parent(dev),
5280 child, rman_get_start(irq), &addr, &data);
5281 if (error)
5282 goto bad;
5283 dinfo->cfg.msi.msi_addr = addr;
5284 dinfo->cfg.msi.msi_data = data;
5285 }
5286 if (dinfo->cfg.msi.msi_handlers == 0)
5287 pci_enable_msi(child, dinfo->cfg.msi.msi_addr,
5288 dinfo->cfg.msi.msi_data);
5289 dinfo->cfg.msi.msi_handlers++;
5290 } else {
5291 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
5292 ("No MSI or MSI-X interrupts allocated"));
5293 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
5294 ("MSI-X index too high"));
5295 mte = &dinfo->cfg.msix.msix_table[rid - 1];
5296 KASSERT(mte->mte_vector != 0, ("no message vector"));
5297 mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
5298 KASSERT(mv->mv_irq == rman_get_start(irq),
5299 ("IRQ mismatch"));
5300 if (mv->mv_address == 0) {
5301 KASSERT(mte->mte_handlers == 0,
5302 ("MSI-X table entry has handlers, but vector not mapped"));
5303 error = PCIB_MAP_MSI(device_get_parent(dev),
5304 child, rman_get_start(irq), &addr, &data);
5305 if (error)
5306 goto bad;
5307 mv->mv_address = addr;
5308 mv->mv_data = data;
5309 }
5310
5311 /*
5312 * The MSIX table entry must be made valid by
5313 * incrementing the mte_handlers before
5314 * calling pci_enable_msix() and
5315 * pci_resume_msix(). Else the MSIX rewrite
5316 * table quirk will not work as expected.
5317 */
5318 mte->mte_handlers++;
5319 if (mte->mte_handlers == 1) {
5320 pci_enable_msix(child, rid - 1, mv->mv_address,
5321 mv->mv_data);
5322 pci_unmask_msix(child, rid - 1);
5323 }
5324 }
5325
5326 /*
5327 * Make sure that INTx is disabled if we are using MSI/MSI-X,
5328 * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG,
5329 * in which case we "enable" INTx so MSI/MSI-X actually works.
5330 */
5331 if (!pci_has_quirk(pci_get_devid(child),
5332 PCI_QUIRK_MSI_INTX_BUG))
5333 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
5334 else
5335 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
5336 bad:
5337 if (error) {
5338 (void)bus_generic_teardown_intr(dev, child, irq,
5339 cookie);
5340 return (error);
5341 }
5342 }
5343 *cookiep = cookie;
5344 return (0);
5345 }
5346
5347 int
pci_teardown_intr(device_t dev,device_t child,struct resource * irq,void * cookie)5348 pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
5349 void *cookie)
5350 {
5351 struct msix_table_entry *mte;
5352 struct resource_list_entry *rle;
5353 struct pci_devinfo *dinfo;
5354 int error, rid;
5355
5356 if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
5357 return (EINVAL);
5358
5359 /* If this isn't a direct child, just bail out */
5360 if (device_get_parent(child) != dev)
5361 return(bus_generic_teardown_intr(dev, child, irq, cookie));
5362
5363 rid = rman_get_rid(irq);
5364 if (rid == 0) {
5365 /* Mask INTx */
5366 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
5367 } else {
5368 /*
5369 * Check to see if the interrupt is MSI or MSI-X. If so,
5370 * decrement the appropriate handlers count and mask the
5371 * MSI-X message, or disable MSI messages if the count
5372 * drops to 0.
5373 */
5374 dinfo = device_get_ivars(child);
5375 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
5376 if (rle->res != irq)
5377 return (EINVAL);
5378 if (dinfo->cfg.msi.msi_alloc > 0) {
5379 KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
5380 ("MSI-X index too high"));
5381 if (dinfo->cfg.msi.msi_handlers == 0)
5382 return (EINVAL);
5383 dinfo->cfg.msi.msi_handlers--;
5384 if (dinfo->cfg.msi.msi_handlers == 0)
5385 pci_disable_msi(child);
5386 } else {
5387 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
5388 ("No MSI or MSI-X interrupts allocated"));
5389 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
5390 ("MSI-X index too high"));
5391 mte = &dinfo->cfg.msix.msix_table[rid - 1];
5392 if (mte->mte_handlers == 0)
5393 return (EINVAL);
5394 mte->mte_handlers--;
5395 if (mte->mte_handlers == 0)
5396 pci_mask_msix(child, rid - 1);
5397 }
5398 }
5399 error = bus_generic_teardown_intr(dev, child, irq, cookie);
5400 if (rid > 0)
5401 KASSERT(error == 0,
5402 ("%s: generic teardown failed for MSI/MSI-X", __func__));
5403 return (error);
5404 }
5405
5406 int
pci_print_child(device_t dev,device_t child)5407 pci_print_child(device_t dev, device_t child)
5408 {
5409 struct pci_devinfo *dinfo;
5410 struct resource_list *rl;
5411 int retval = 0;
5412
5413 dinfo = device_get_ivars(child);
5414 rl = &dinfo->resources;
5415
5416 retval += bus_print_child_header(dev, child);
5417
5418 retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
5419 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
5420 retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
5421 if (device_get_flags(dev))
5422 retval += printf(" flags %#x", device_get_flags(dev));
5423
5424 retval += printf(" at device %d.%d", pci_get_slot(child),
5425 pci_get_function(child));
5426
5427 retval += bus_print_child_domain(dev, child);
5428 retval += bus_print_child_footer(dev, child);
5429
5430 return (retval);
5431 }
5432
5433 static const struct
5434 {
5435 int class;
5436 int subclass;
5437 int report; /* 0 = bootverbose, 1 = always */
5438 const char *desc;
5439 } pci_nomatch_tab[] = {
5440 {PCIC_OLD, -1, 1, "old"},
5441 {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"},
5442 {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"},
5443 {PCIC_STORAGE, -1, 1, "mass storage"},
5444 {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"},
5445 {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"},
5446 {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"},
5447 {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"},
5448 {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"},
5449 {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"},
5450 {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"},
5451 {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"},
5452 {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"},
5453 {PCIC_NETWORK, -1, 1, "network"},
5454 {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"},
5455 {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"},
5456 {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"},
5457 {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"},
5458 {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"},
5459 {PCIC_DISPLAY, -1, 1, "display"},
5460 {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"},
5461 {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"},
5462 {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"},
5463 {PCIC_MULTIMEDIA, -1, 1, "multimedia"},
5464 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"},
5465 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"},
5466 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"},
5467 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"},
5468 {PCIC_MEMORY, -1, 1, "memory"},
5469 {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"},
5470 {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"},
5471 {PCIC_BRIDGE, -1, 1, "bridge"},
5472 {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"},
5473 {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"},
5474 {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"},
5475 {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"},
5476 {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"},
5477 {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"},
5478 {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"},
5479 {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"},
5480 {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"},
5481 {PCIC_SIMPLECOMM, -1, 1, "simple comms"},
5482 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */
5483 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"},
5484 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"},
5485 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"},
5486 {PCIC_BASEPERIPH, -1, 0, "base peripheral"},
5487 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"},
5488 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"},
5489 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"},
5490 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"},
5491 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"},
5492 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"},
5493 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"},
5494 {PCIC_INPUTDEV, -1, 1, "input device"},
5495 {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"},
5496 {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"},
5497 {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"},
5498 {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"},
5499 {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"},
5500 {PCIC_DOCKING, -1, 1, "docking station"},
5501 {PCIC_PROCESSOR, -1, 1, "processor"},
5502 {PCIC_SERIALBUS, -1, 1, "serial bus"},
5503 {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"},
5504 {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"},
5505 {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"},
5506 {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"},
5507 {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"},
5508 {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"},
5509 {PCIC_WIRELESS, -1, 1, "wireless controller"},
5510 {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"},
5511 {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"},
5512 {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"},
5513 {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"},
5514 {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"},
5515 {PCIC_SATCOM, -1, 1, "satellite communication"},
5516 {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"},
5517 {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"},
5518 {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"},
5519 {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"},
5520 {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"},
5521 {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"},
5522 {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"},
5523 {PCIC_DASP, -1, 0, "dasp"},
5524 {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"},
5525 {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"},
5526 {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"},
5527 {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"},
5528 {PCIC_INSTRUMENT, -1, 0, "non-essential instrumentation"},
5529 {0, 0, 0, NULL}
5530 };
5531
5532 void
pci_probe_nomatch(device_t dev,device_t child)5533 pci_probe_nomatch(device_t dev, device_t child)
5534 {
5535 int i, report;
5536 const char *cp, *scp;
5537 char *device;
5538
5539 /*
5540 * Look for a listing for this device in a loaded device database.
5541 */
5542 report = 1;
5543 if ((device = pci_describe_device(child)) != NULL) {
5544 device_printf(dev, "<%s>", device);
5545 free(device, M_DEVBUF);
5546 } else {
5547 /*
5548 * Scan the class/subclass descriptions for a general
5549 * description.
5550 */
5551 cp = "unknown";
5552 scp = NULL;
5553 for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) {
5554 if (pci_nomatch_tab[i].class == pci_get_class(child)) {
5555 if (pci_nomatch_tab[i].subclass == -1) {
5556 cp = pci_nomatch_tab[i].desc;
5557 report = pci_nomatch_tab[i].report;
5558 } else if (pci_nomatch_tab[i].subclass ==
5559 pci_get_subclass(child)) {
5560 scp = pci_nomatch_tab[i].desc;
5561 report = pci_nomatch_tab[i].report;
5562 }
5563 }
5564 }
5565 if (report || bootverbose) {
5566 device_printf(dev, "<%s%s%s>",
5567 cp ? cp : "",
5568 ((cp != NULL) && (scp != NULL)) ? ", " : "",
5569 scp ? scp : "");
5570 }
5571 }
5572 if (report || bootverbose) {
5573 printf(" at device %d.%d (no driver attached)\n",
5574 pci_get_slot(child), pci_get_function(child));
5575 }
5576 pci_cfg_save(child, device_get_ivars(child), 1);
5577 }
5578
5579 void
pci_child_detached(device_t dev,device_t child)5580 pci_child_detached(device_t dev, device_t child)
5581 {
5582 struct pci_devinfo *dinfo;
5583 struct resource_list *rl;
5584
5585 dinfo = device_get_ivars(child);
5586 rl = &dinfo->resources;
5587
5588 /*
5589 * Have to deallocate IRQs before releasing any MSI messages and
5590 * have to release MSI messages before deallocating any memory
5591 * BARs.
5592 */
5593 if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0)
5594 pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n");
5595 if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) {
5596 if (dinfo->cfg.msi.msi_alloc != 0)
5597 pci_printf(&dinfo->cfg, "Device leaked %d MSI "
5598 "vectors\n", dinfo->cfg.msi.msi_alloc);
5599 else
5600 pci_printf(&dinfo->cfg, "Device leaked %d MSI-X "
5601 "vectors\n", dinfo->cfg.msix.msix_alloc);
5602 (void)pci_release_msi(child);
5603 }
5604 if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0)
5605 pci_printf(&dinfo->cfg, "Device leaked memory resources\n");
5606 if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0)
5607 pci_printf(&dinfo->cfg, "Device leaked I/O resources\n");
5608 if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0)
5609 pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n");
5610
5611 pci_cfg_save(child, dinfo, 1);
5612 }
5613
5614 /*
5615 * Parse the PCI device database, if loaded, and return a pointer to a
5616 * description of the device.
5617 *
5618 * The database is flat text formatted as follows:
5619 *
5620 * Any line not in a valid format is ignored.
5621 * Lines are terminated with newline '\n' characters.
5622 *
5623 * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then
5624 * the vendor name.
5625 *
5626 * A DEVICE line is entered immediately below the corresponding VENDOR ID.
5627 * - devices cannot be listed without a corresponding VENDOR line.
5628 * A DEVICE line consists of a TAB, the 4 digit (hex) device code,
5629 * another TAB, then the device name.
5630 */
5631
5632 /*
5633 * Assuming (ptr) points to the beginning of a line in the database,
5634 * return the vendor or device and description of the next entry.
5635 * The value of (vendor) or (device) inappropriate for the entry type
5636 * is set to -1. Returns nonzero at the end of the database.
5637 *
5638 * Note that this is slightly unrobust in the face of corrupt data;
5639 * we attempt to safeguard against this by spamming the end of the
5640 * database with a newline when we initialise.
5641 */
5642 static int
pci_describe_parse_line(char ** ptr,int * vendor,int * device,char ** desc)5643 pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc)
5644 {
5645 char *cp = *ptr;
5646 int left;
5647
5648 *device = -1;
5649 *vendor = -1;
5650 **desc = '\0';
5651 for (;;) {
5652 left = pci_vendordata_size - (cp - pci_vendordata);
5653 if (left <= 0) {
5654 *ptr = cp;
5655 return(1);
5656 }
5657
5658 /* vendor entry? */
5659 if (*cp != '\t' &&
5660 sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2)
5661 break;
5662 /* device entry? */
5663 if (*cp == '\t' &&
5664 sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2)
5665 break;
5666
5667 /* skip to next line */
5668 while (*cp != '\n' && left > 0) {
5669 cp++;
5670 left--;
5671 }
5672 if (*cp == '\n') {
5673 cp++;
5674 left--;
5675 }
5676 }
5677 /* skip to next line */
5678 while (*cp != '\n' && left > 0) {
5679 cp++;
5680 left--;
5681 }
5682 if (*cp == '\n' && left > 0)
5683 cp++;
5684 *ptr = cp;
5685 return(0);
5686 }
5687
5688 static char *
pci_describe_device(device_t dev)5689 pci_describe_device(device_t dev)
5690 {
5691 int vendor, device;
5692 char *desc, *vp, *dp, *line;
5693
5694 desc = vp = dp = NULL;
5695
5696 /*
5697 * If we have no vendor data, we can't do anything.
5698 */
5699 if (pci_vendordata == NULL)
5700 goto out;
5701
5702 /*
5703 * Scan the vendor data looking for this device
5704 */
5705 line = pci_vendordata;
5706 if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5707 goto out;
5708 for (;;) {
5709 if (pci_describe_parse_line(&line, &vendor, &device, &vp))
5710 goto out;
5711 if (vendor == pci_get_vendor(dev))
5712 break;
5713 }
5714 if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5715 goto out;
5716 for (;;) {
5717 if (pci_describe_parse_line(&line, &vendor, &device, &dp)) {
5718 *dp = 0;
5719 break;
5720 }
5721 if (vendor != -1) {
5722 *dp = 0;
5723 break;
5724 }
5725 if (device == pci_get_device(dev))
5726 break;
5727 }
5728 if (dp[0] == '\0')
5729 snprintf(dp, 80, "0x%x", pci_get_device(dev));
5730 if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) !=
5731 NULL)
5732 sprintf(desc, "%s, %s", vp, dp);
5733 out:
5734 if (vp != NULL)
5735 free(vp, M_DEVBUF);
5736 if (dp != NULL)
5737 free(dp, M_DEVBUF);
5738 return(desc);
5739 }
5740
5741 int
pci_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)5742 pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
5743 {
5744 struct pci_devinfo *dinfo;
5745 pcicfgregs *cfg;
5746
5747 dinfo = device_get_ivars(child);
5748 cfg = &dinfo->cfg;
5749
5750 switch (which) {
5751 case PCI_IVAR_ETHADDR:
5752 /*
5753 * The generic accessor doesn't deal with failure, so
5754 * we set the return value, then return an error.
5755 */
5756 *((uint8_t **) result) = NULL;
5757 return (EINVAL);
5758 case PCI_IVAR_SUBVENDOR:
5759 *result = cfg->subvendor;
5760 break;
5761 case PCI_IVAR_SUBDEVICE:
5762 *result = cfg->subdevice;
5763 break;
5764 case PCI_IVAR_VENDOR:
5765 *result = cfg->vendor;
5766 break;
5767 case PCI_IVAR_DEVICE:
5768 *result = cfg->device;
5769 break;
5770 case PCI_IVAR_DEVID:
5771 *result = (cfg->device << 16) | cfg->vendor;
5772 break;
5773 case PCI_IVAR_CLASS:
5774 *result = cfg->baseclass;
5775 break;
5776 case PCI_IVAR_SUBCLASS:
5777 *result = cfg->subclass;
5778 break;
5779 case PCI_IVAR_PROGIF:
5780 *result = cfg->progif;
5781 break;
5782 case PCI_IVAR_REVID:
5783 *result = cfg->revid;
5784 break;
5785 case PCI_IVAR_INTPIN:
5786 *result = cfg->intpin;
5787 break;
5788 case PCI_IVAR_IRQ:
5789 *result = cfg->intline;
5790 break;
5791 case PCI_IVAR_DOMAIN:
5792 *result = cfg->domain;
5793 break;
5794 case PCI_IVAR_BUS:
5795 *result = cfg->bus;
5796 break;
5797 case PCI_IVAR_SLOT:
5798 *result = cfg->slot;
5799 break;
5800 case PCI_IVAR_FUNCTION:
5801 *result = cfg->func;
5802 break;
5803 case PCI_IVAR_CMDREG:
5804 *result = cfg->cmdreg;
5805 break;
5806 case PCI_IVAR_CACHELNSZ:
5807 *result = cfg->cachelnsz;
5808 break;
5809 case PCI_IVAR_MINGNT:
5810 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5811 *result = -1;
5812 return (EINVAL);
5813 }
5814 *result = cfg->mingnt;
5815 break;
5816 case PCI_IVAR_MAXLAT:
5817 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5818 *result = -1;
5819 return (EINVAL);
5820 }
5821 *result = cfg->maxlat;
5822 break;
5823 case PCI_IVAR_LATTIMER:
5824 *result = cfg->lattimer;
5825 break;
5826 default:
5827 return (ENOENT);
5828 }
5829 return (0);
5830 }
5831
5832 int
pci_write_ivar(device_t dev,device_t child,int which,uintptr_t value)5833 pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
5834 {
5835 struct pci_devinfo *dinfo;
5836
5837 dinfo = device_get_ivars(child);
5838
5839 switch (which) {
5840 case PCI_IVAR_INTPIN:
5841 dinfo->cfg.intpin = value;
5842 return (0);
5843 case PCI_IVAR_ETHADDR:
5844 case PCI_IVAR_SUBVENDOR:
5845 case PCI_IVAR_SUBDEVICE:
5846 case PCI_IVAR_VENDOR:
5847 case PCI_IVAR_DEVICE:
5848 case PCI_IVAR_DEVID:
5849 case PCI_IVAR_CLASS:
5850 case PCI_IVAR_SUBCLASS:
5851 case PCI_IVAR_PROGIF:
5852 case PCI_IVAR_REVID:
5853 case PCI_IVAR_IRQ:
5854 case PCI_IVAR_DOMAIN:
5855 case PCI_IVAR_BUS:
5856 case PCI_IVAR_SLOT:
5857 case PCI_IVAR_FUNCTION:
5858 return (EINVAL); /* disallow for now */
5859
5860 default:
5861 return (ENOENT);
5862 }
5863 }
5864
5865 #include "opt_ddb.h"
5866 #ifdef DDB
5867 #include <ddb/ddb.h>
5868 #include <sys/cons.h>
5869
5870 /*
5871 * List resources based on pci map registers, used for within ddb
5872 */
5873
DB_SHOW_COMMAND_FLAGS(pciregs,db_pci_dump,DB_CMD_MEMSAFE)5874 DB_SHOW_COMMAND_FLAGS(pciregs, db_pci_dump, DB_CMD_MEMSAFE)
5875 {
5876 struct pci_devinfo *dinfo;
5877 struct devlist *devlist_head;
5878 struct pci_conf *p;
5879 const char *name;
5880 int i, error, none_count;
5881
5882 none_count = 0;
5883 /* get the head of the device queue */
5884 devlist_head = &pci_devq;
5885
5886 /*
5887 * Go through the list of devices and print out devices
5888 */
5889 for (error = 0, i = 0,
5890 dinfo = STAILQ_FIRST(devlist_head);
5891 (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit;
5892 dinfo = STAILQ_NEXT(dinfo, pci_links), i++) {
5893 /* Populate pd_name and pd_unit */
5894 name = NULL;
5895 if (dinfo->cfg.dev)
5896 name = device_get_name(dinfo->cfg.dev);
5897
5898 p = &dinfo->conf;
5899 db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x "
5900 "chip=0x%08x rev=0x%02x hdr=0x%02x\n",
5901 (name && *name) ? name : "none",
5902 (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) :
5903 none_count++,
5904 p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev,
5905 p->pc_sel.pc_func, (p->pc_class << 16) |
5906 (p->pc_subclass << 8) | p->pc_progif,
5907 (p->pc_subdevice << 16) | p->pc_subvendor,
5908 (p->pc_device << 16) | p->pc_vendor,
5909 p->pc_revid, p->pc_hdr);
5910 }
5911 }
5912 #endif /* DDB */
5913
5914 struct resource *
pci_reserve_map(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int num,u_int flags)5915 pci_reserve_map(device_t dev, device_t child, int type, int rid,
5916 rman_res_t start, rman_res_t end, rman_res_t count, u_int num,
5917 u_int flags)
5918 {
5919 struct pci_devinfo *dinfo = device_get_ivars(child);
5920 struct resource_list *rl = &dinfo->resources;
5921 struct resource *res;
5922 struct pci_map *pm;
5923 uint16_t cmd;
5924 pci_addr_t map, testval;
5925 int mapsize;
5926
5927 res = NULL;
5928
5929 /* If rid is managed by EA, ignore it */
5930 if (pci_ea_is_enabled(child, rid))
5931 goto out;
5932
5933 pm = pci_find_bar(child, rid);
5934 if (pm != NULL) {
5935 /* This is a BAR that we failed to allocate earlier. */
5936 mapsize = pm->pm_size;
5937 map = pm->pm_value;
5938 } else {
5939 /*
5940 * Weed out the bogons, and figure out how large the
5941 * BAR/map is. BARs that read back 0 here are bogus
5942 * and unimplemented. Note: atapci in legacy mode are
5943 * special and handled elsewhere in the code. If you
5944 * have a atapci device in legacy mode and it fails
5945 * here, that other code is broken.
5946 */
5947 pci_read_bar(child, rid, &map, &testval, NULL);
5948
5949 /*
5950 * Determine the size of the BAR and ignore BARs with a size
5951 * of 0. Device ROM BARs use a different mask value.
5952 */
5953 if (PCIR_IS_BIOS(&dinfo->cfg, rid))
5954 mapsize = pci_romsize(testval);
5955 else
5956 mapsize = pci_mapsize(testval);
5957 if (mapsize == 0)
5958 goto out;
5959 pm = pci_add_bar(child, rid, map, mapsize);
5960 }
5961
5962 if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, rid)) {
5963 if (type != SYS_RES_MEMORY) {
5964 if (bootverbose)
5965 device_printf(dev,
5966 "child %s requested type %d for rid %#x,"
5967 " but the BAR says it is an memio\n",
5968 device_get_nameunit(child), type, rid);
5969 goto out;
5970 }
5971 } else {
5972 if (type != SYS_RES_IOPORT) {
5973 if (bootverbose)
5974 device_printf(dev,
5975 "child %s requested type %d for rid %#x,"
5976 " but the BAR says it is an ioport\n",
5977 device_get_nameunit(child), type, rid);
5978 goto out;
5979 }
5980 }
5981
5982 /*
5983 * For real BARs, we need to override the size that
5984 * the driver requests, because that's what the BAR
5985 * actually uses and we would otherwise have a
5986 * situation where we might allocate the excess to
5987 * another driver, which won't work.
5988 */
5989 count = ((pci_addr_t)1 << mapsize) * num;
5990 if (RF_ALIGNMENT(flags) < mapsize)
5991 flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize);
5992 if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH))
5993 flags |= RF_PREFETCHABLE;
5994
5995 /*
5996 * Allocate enough resource, and then write back the
5997 * appropriate BAR for that resource.
5998 */
5999 resource_list_add(rl, type, rid, start, end, count);
6000 res = resource_list_reserve(rl, dev, child, type, rid, start, end,
6001 count, flags & ~RF_ACTIVE);
6002 if (res == NULL) {
6003 resource_list_delete(rl, type, rid);
6004 device_printf(child,
6005 "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n",
6006 count, rid, type, start, end);
6007 goto out;
6008 }
6009 if (bootverbose)
6010 device_printf(child,
6011 "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n",
6012 count, rid, type, rman_get_start(res));
6013
6014 /* Disable decoding via the CMD register before updating the BAR */
6015 cmd = pci_read_config(child, PCIR_COMMAND, 2);
6016 pci_write_config(child, PCIR_COMMAND,
6017 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
6018
6019 map = rman_get_start(res);
6020 pci_write_bar(child, pm, map);
6021
6022 /* Restore the original value of the CMD register */
6023 pci_write_config(child, PCIR_COMMAND, cmd, 2);
6024 out:
6025 return (res);
6026 }
6027
6028 struct resource *
pci_alloc_multi_resource(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_long num,u_int flags)6029 pci_alloc_multi_resource(device_t dev, device_t child, int type, int rid,
6030 rman_res_t start, rman_res_t end, rman_res_t count, u_long num,
6031 u_int flags)
6032 {
6033 struct pci_devinfo *dinfo;
6034 struct resource_list *rl;
6035 struct resource_list_entry *rle;
6036 struct resource *res;
6037 pcicfgregs *cfg;
6038
6039 /*
6040 * Perform lazy resource allocation
6041 */
6042 dinfo = device_get_ivars(child);
6043 rl = &dinfo->resources;
6044 cfg = &dinfo->cfg;
6045 switch (type) {
6046 case PCI_RES_BUS:
6047 return (pci_alloc_secbus(dev, child, rid, start, end, count,
6048 flags));
6049 case SYS_RES_IRQ:
6050 /*
6051 * Can't alloc legacy interrupt once MSI messages have
6052 * been allocated.
6053 */
6054 if (rid == 0 && (cfg->msi.msi_alloc > 0 ||
6055 cfg->msix.msix_alloc > 0))
6056 return (NULL);
6057
6058 /*
6059 * If the child device doesn't have an interrupt
6060 * routed and is deserving of an interrupt, try to
6061 * assign it one.
6062 */
6063 if (rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) &&
6064 (cfg->intpin != 0))
6065 pci_assign_interrupt(dev, child, 0);
6066 break;
6067 case SYS_RES_IOPORT:
6068 case SYS_RES_MEMORY:
6069 /*
6070 * PCI-PCI bridge I/O window resources are not BARs.
6071 * For those allocations just pass the request up the
6072 * tree.
6073 */
6074 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) {
6075 switch (rid) {
6076 case PCIR_IOBASEL_1:
6077 case PCIR_MEMBASE_1:
6078 case PCIR_PMBASEL_1:
6079 /*
6080 * XXX: Should we bother creating a resource
6081 * list entry?
6082 */
6083 return (bus_generic_alloc_resource(dev, child,
6084 type, rid, start, end, count, flags));
6085 }
6086 }
6087 /* Reserve resources for this BAR if needed. */
6088 rle = resource_list_find(rl, type, rid);
6089 if (rle == NULL) {
6090 res = pci_reserve_map(dev, child, type, rid, start, end,
6091 count, num, flags);
6092 if (res == NULL)
6093 return (NULL);
6094 }
6095 }
6096 return (resource_list_alloc(rl, dev, child, type, rid,
6097 start, end, count, flags));
6098 }
6099
6100 struct resource *
pci_alloc_resource(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)6101 pci_alloc_resource(device_t dev, device_t child, int type, int rid,
6102 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
6103 {
6104 #ifdef PCI_IOV
6105 struct pci_devinfo *dinfo;
6106 #endif
6107
6108 if (device_get_parent(child) != dev)
6109 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
6110 type, rid, start, end, count, flags));
6111
6112 #ifdef PCI_IOV
6113 dinfo = device_get_ivars(child);
6114 if (dinfo->cfg.flags & PCICFG_VF) {
6115 switch (type) {
6116 /* VFs can't have I/O BARs. */
6117 case SYS_RES_IOPORT:
6118 return (NULL);
6119 case SYS_RES_MEMORY:
6120 return (pci_vf_alloc_mem_resource(dev, child, rid,
6121 start, end, count, flags));
6122 }
6123
6124 /* Fall through for other types of resource allocations. */
6125 }
6126 #endif
6127
6128 return (pci_alloc_multi_resource(dev, child, type, rid, start, end,
6129 count, 1, flags));
6130 }
6131
6132 int
pci_release_resource(device_t dev,device_t child,struct resource * r)6133 pci_release_resource(device_t dev, device_t child, struct resource *r)
6134 {
6135 struct pci_devinfo *dinfo;
6136 struct resource_list *rl;
6137 pcicfgregs *cfg __unused;
6138
6139 if (device_get_parent(child) != dev)
6140 return (bus_generic_release_resource(dev, child, r));
6141
6142 dinfo = device_get_ivars(child);
6143 cfg = &dinfo->cfg;
6144
6145 #ifdef PCI_IOV
6146 if (cfg->flags & PCICFG_VF) {
6147 switch (rman_get_type(r)) {
6148 /* VFs can't have I/O BARs. */
6149 case SYS_RES_IOPORT:
6150 return (EDOOFUS);
6151 case SYS_RES_MEMORY:
6152 return (pci_vf_release_mem_resource(dev, child, r));
6153 }
6154
6155 /* Fall through for other types of resource allocations. */
6156 }
6157 #endif
6158
6159 /*
6160 * PCI-PCI bridge I/O window resources are not BARs. For
6161 * those allocations just pass the request up the tree.
6162 */
6163 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE &&
6164 (rman_get_type(r) == SYS_RES_IOPORT ||
6165 rman_get_type(r) == SYS_RES_MEMORY)) {
6166 switch (rman_get_rid(r)) {
6167 case PCIR_IOBASEL_1:
6168 case PCIR_MEMBASE_1:
6169 case PCIR_PMBASEL_1:
6170 return (bus_generic_release_resource(dev, child, r));
6171 }
6172 }
6173
6174 rl = &dinfo->resources;
6175 return (resource_list_release(rl, dev, child, r));
6176 }
6177
6178 int
pci_activate_resource(device_t dev,device_t child,struct resource * r)6179 pci_activate_resource(device_t dev, device_t child, struct resource *r)
6180 {
6181 struct pci_devinfo *dinfo;
6182 int error, rid, type;
6183
6184 if (device_get_parent(child) != dev)
6185 return (bus_generic_activate_resource(dev, child, r));
6186
6187 dinfo = device_get_ivars(child);
6188 #ifdef PCI_IOV
6189 if (dinfo->cfg.flags & PCICFG_VF) {
6190 switch (rman_get_type(r)) {
6191 /* VFs can't have I/O BARs. */
6192 case SYS_RES_IOPORT:
6193 error = EINVAL;
6194 break;
6195 case SYS_RES_MEMORY:
6196 error = pci_vf_activate_mem_resource(dev, child, r);
6197 break;
6198 default:
6199 error = bus_generic_activate_resource(dev, child, r);
6200 break;
6201 }
6202 } else
6203 #endif
6204 error = bus_generic_activate_resource(dev, child, r);
6205 if (error)
6206 return (error);
6207
6208 rid = rman_get_rid(r);
6209 type = rman_get_type(r);
6210
6211 /* Device ROMs need their decoding explicitly enabled. */
6212 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
6213 pci_write_bar(child, pci_find_bar(child, rid),
6214 rman_get_start(r) | PCIM_BIOS_ENABLE);
6215
6216 /* Enable decoding in the command register when activating BARs. */
6217 switch (type) {
6218 case SYS_RES_IOPORT:
6219 case SYS_RES_MEMORY:
6220 error = PCI_ENABLE_IO(dev, child, type);
6221 break;
6222 }
6223 return (error);
6224 }
6225
6226 int
pci_deactivate_resource(device_t dev,device_t child,struct resource * r)6227 pci_deactivate_resource(device_t dev, device_t child, struct resource *r)
6228 {
6229 struct pci_devinfo *dinfo;
6230 int error, rid, type;
6231
6232 if (device_get_parent(child) != dev)
6233 return (bus_generic_deactivate_resource(dev, child, r));
6234
6235 dinfo = device_get_ivars(child);
6236 #ifdef PCI_IOV
6237 if (dinfo->cfg.flags & PCICFG_VF) {
6238 switch (rman_get_type(r)) {
6239 /* VFs can't have I/O BARs. */
6240 case SYS_RES_IOPORT:
6241 error = EINVAL;
6242 break;
6243 case SYS_RES_MEMORY:
6244 error = pci_vf_deactivate_mem_resource(dev, child, r);
6245 break;
6246 default:
6247 error = bus_generic_deactivate_resource(dev, child, r);
6248 break;
6249 }
6250 } else
6251 #endif
6252 error = bus_generic_deactivate_resource(dev, child, r);
6253 if (error)
6254 return (error);
6255
6256 /* Disable decoding for device ROMs. */
6257 rid = rman_get_rid(r);
6258 type = rman_get_type(r);
6259 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
6260 pci_write_bar(child, pci_find_bar(child, rid),
6261 rman_get_start(r));
6262 return (0);
6263 }
6264
6265 int
pci_adjust_resource(device_t dev,device_t child,struct resource * r,rman_res_t start,rman_res_t end)6266 pci_adjust_resource(device_t dev, device_t child, struct resource *r,
6267 rman_res_t start, rman_res_t end)
6268 {
6269 #ifdef PCI_IOV
6270 struct pci_devinfo *dinfo;
6271
6272 if (device_get_parent(child) != dev)
6273 return (bus_generic_adjust_resource(dev, child, r, start,
6274 end));
6275
6276 dinfo = device_get_ivars(child);
6277 if (dinfo->cfg.flags & PCICFG_VF) {
6278 switch (rman_get_type(r)) {
6279 /* VFs can't have I/O BARs. */
6280 case SYS_RES_IOPORT:
6281 return (EINVAL);
6282 case SYS_RES_MEMORY:
6283 return (pci_vf_adjust_mem_resource(dev, child, r,
6284 start, end));
6285 }
6286
6287 /* Fall through for other types of resource allocations. */
6288 }
6289 #endif
6290
6291 return (bus_generic_adjust_resource(dev, child, r, start, end));
6292 }
6293
6294 int
pci_map_resource(device_t dev,device_t child,struct resource * r,struct resource_map_request * argsp,struct resource_map * map)6295 pci_map_resource(device_t dev, device_t child, struct resource *r,
6296 struct resource_map_request *argsp, struct resource_map *map)
6297 {
6298 #ifdef PCI_IOV
6299 struct pci_devinfo *dinfo;
6300
6301 if (device_get_parent(child) != dev)
6302 return (bus_generic_map_resource(dev, child, r, argsp,
6303 map));
6304
6305 dinfo = device_get_ivars(child);
6306 if (dinfo->cfg.flags & PCICFG_VF) {
6307 switch (rman_get_type(r)) {
6308 /* VFs can't have I/O BARs. */
6309 case SYS_RES_IOPORT:
6310 return (EINVAL);
6311 case SYS_RES_MEMORY:
6312 return (pci_vf_map_mem_resource(dev, child, r, argsp,
6313 map));
6314 }
6315
6316 /* Fall through for other types of resource allocations. */
6317 }
6318 #endif
6319
6320 return (bus_generic_map_resource(dev, child, r, argsp, map));
6321 }
6322
6323 int
pci_unmap_resource(device_t dev,device_t child,struct resource * r,struct resource_map * map)6324 pci_unmap_resource(device_t dev, device_t child, struct resource *r,
6325 struct resource_map *map)
6326 {
6327 #ifdef PCI_IOV
6328 struct pci_devinfo *dinfo;
6329
6330 if (device_get_parent(child) != dev)
6331 return (bus_generic_unmap_resource(dev, child, r, map));
6332
6333 dinfo = device_get_ivars(child);
6334 if (dinfo->cfg.flags & PCICFG_VF) {
6335 switch (rman_get_type(r)) {
6336 /* VFs can't have I/O BARs. */
6337 case SYS_RES_IOPORT:
6338 return (EINVAL);
6339 case SYS_RES_MEMORY:
6340 return (pci_vf_unmap_mem_resource(dev, child, r, map));
6341 }
6342
6343 /* Fall through for other types of resource allocations. */
6344 }
6345 #endif
6346
6347 return (bus_generic_unmap_resource(dev, child, r, map));
6348 }
6349
6350 void
pci_child_deleted(device_t dev,device_t child)6351 pci_child_deleted(device_t dev, device_t child)
6352 {
6353 struct resource_list_entry *rle;
6354 struct resource_list *rl;
6355 struct pci_devinfo *dinfo;
6356
6357 dinfo = device_get_ivars(child);
6358 rl = &dinfo->resources;
6359
6360 EVENTHANDLER_INVOKE(pci_delete_device, child);
6361
6362 /* Turn off access to resources we're about to free */
6363 if (bus_child_present(child) != 0) {
6364 pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
6365 PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
6366
6367 pci_disable_busmaster(child);
6368 }
6369
6370 /* Free all allocated resources */
6371 STAILQ_FOREACH(rle, rl, link) {
6372 if (rle->res) {
6373 if (rman_get_flags(rle->res) & RF_ACTIVE ||
6374 resource_list_busy(rl, rle->type, rle->rid)) {
6375 pci_printf(&dinfo->cfg,
6376 "Resource still owned, oops. "
6377 "(type=%d, rid=%d, addr=%lx)\n",
6378 rle->type, rle->rid,
6379 rman_get_start(rle->res));
6380 bus_release_resource(child, rle->type, rle->rid,
6381 rle->res);
6382 }
6383 resource_list_unreserve(rl, dev, child, rle->type,
6384 rle->rid);
6385 }
6386 }
6387 resource_list_free(rl);
6388
6389 pci_freecfg(dinfo);
6390 }
6391
6392 void
pci_delete_resource(device_t dev,device_t child,int type,int rid)6393 pci_delete_resource(device_t dev, device_t child, int type, int rid)
6394 {
6395 struct pci_devinfo *dinfo;
6396 struct resource_list *rl;
6397 struct resource_list_entry *rle;
6398
6399 if (device_get_parent(child) != dev)
6400 return;
6401
6402 dinfo = device_get_ivars(child);
6403 rl = &dinfo->resources;
6404 rle = resource_list_find(rl, type, rid);
6405 if (rle == NULL)
6406 return;
6407
6408 if (rle->res) {
6409 if (rman_get_flags(rle->res) & RF_ACTIVE ||
6410 resource_list_busy(rl, type, rid)) {
6411 device_printf(dev, "delete_resource: "
6412 "Resource still owned by child, oops. "
6413 "(type=%d, rid=%d, addr=%jx)\n",
6414 type, rid, rman_get_start(rle->res));
6415 return;
6416 }
6417 resource_list_unreserve(rl, dev, child, type, rid);
6418 }
6419 resource_list_delete(rl, type, rid);
6420 }
6421
6422 struct resource_list *
pci_get_resource_list(device_t dev,device_t child)6423 pci_get_resource_list (device_t dev, device_t child)
6424 {
6425 struct pci_devinfo *dinfo = device_get_ivars(child);
6426
6427 return (&dinfo->resources);
6428 }
6429
6430 #ifdef IOMMU
6431 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)6432 pci_get_dma_tag(device_t bus, device_t dev)
6433 {
6434 bus_dma_tag_t tag;
6435 struct pci_softc *sc;
6436
6437 if (device_get_parent(dev) == bus) {
6438 /* try iommu and return if it works */
6439 tag = iommu_get_dma_tag(bus, dev);
6440 } else
6441 tag = NULL;
6442 if (tag == NULL) {
6443 sc = device_get_softc(bus);
6444 tag = sc->sc_dma_tag;
6445 }
6446 return (tag);
6447 }
6448 #else
6449 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)6450 pci_get_dma_tag(device_t bus, device_t dev)
6451 {
6452 struct pci_softc *sc = device_get_softc(bus);
6453
6454 return (sc->sc_dma_tag);
6455 }
6456 #endif
6457
6458 uint32_t
pci_read_config_method(device_t dev,device_t child,int reg,int width)6459 pci_read_config_method(device_t dev, device_t child, int reg, int width)
6460 {
6461 struct pci_devinfo *dinfo = device_get_ivars(child);
6462 pcicfgregs *cfg = &dinfo->cfg;
6463
6464 #ifdef PCI_IOV
6465 /*
6466 * SR-IOV VFs don't implement the VID or DID registers, so we have to
6467 * emulate them here.
6468 */
6469 if (cfg->flags & PCICFG_VF) {
6470 if (reg == PCIR_VENDOR) {
6471 switch (width) {
6472 case 4:
6473 return (cfg->device << 16 | cfg->vendor);
6474 case 2:
6475 return (cfg->vendor);
6476 case 1:
6477 return (cfg->vendor & 0xff);
6478 default:
6479 return (0xffffffff);
6480 }
6481 } else if (reg == PCIR_DEVICE) {
6482 switch (width) {
6483 /* Note that an unaligned 4-byte read is an error. */
6484 case 2:
6485 return (cfg->device);
6486 case 1:
6487 return (cfg->device & 0xff);
6488 default:
6489 return (0xffffffff);
6490 }
6491 }
6492 }
6493 #endif
6494
6495 return (PCIB_READ_CONFIG(device_get_parent(dev),
6496 cfg->bus, cfg->slot, cfg->func, reg, width));
6497 }
6498
6499 void
pci_write_config_method(device_t dev,device_t child,int reg,uint32_t val,int width)6500 pci_write_config_method(device_t dev, device_t child, int reg,
6501 uint32_t val, int width)
6502 {
6503 struct pci_devinfo *dinfo = device_get_ivars(child);
6504 pcicfgregs *cfg = &dinfo->cfg;
6505
6506 PCIB_WRITE_CONFIG(device_get_parent(dev),
6507 cfg->bus, cfg->slot, cfg->func, reg, val, width);
6508 }
6509
6510 int
pci_child_location_method(device_t dev,device_t child,struct sbuf * sb)6511 pci_child_location_method(device_t dev, device_t child, struct sbuf *sb)
6512 {
6513
6514 sbuf_printf(sb, "slot=%d function=%d dbsf=pci%d:%d:%d:%d",
6515 pci_get_slot(child), pci_get_function(child), pci_get_domain(child),
6516 pci_get_bus(child), pci_get_slot(child), pci_get_function(child));
6517 return (0);
6518 }
6519
6520 int
pci_child_pnpinfo_method(device_t dev,device_t child,struct sbuf * sb)6521 pci_child_pnpinfo_method(device_t dev, device_t child, struct sbuf *sb)
6522 {
6523 struct pci_devinfo *dinfo;
6524 pcicfgregs *cfg;
6525
6526 dinfo = device_get_ivars(child);
6527 cfg = &dinfo->cfg;
6528 sbuf_printf(sb, "vendor=0x%04x device=0x%04x subvendor=0x%04x "
6529 "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device,
6530 cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass,
6531 cfg->progif);
6532 return (0);
6533 }
6534
6535 int
pci_get_device_path_method(device_t bus,device_t child,const char * locator,struct sbuf * sb)6536 pci_get_device_path_method(device_t bus, device_t child, const char *locator,
6537 struct sbuf *sb)
6538 {
6539 device_t parent = device_get_parent(bus);
6540 int rv;
6541
6542 if (strcmp(locator, BUS_LOCATOR_UEFI) == 0) {
6543 rv = bus_generic_get_device_path(parent, bus, locator, sb);
6544 if (rv == 0) {
6545 sbuf_printf(sb, "/Pci(0x%x,0x%x)", pci_get_slot(child),
6546 pci_get_function(child));
6547 }
6548 return (0);
6549 }
6550 return (bus_generic_get_device_path(bus, child, locator, sb));
6551 }
6552
6553 int
pci_assign_interrupt_method(device_t dev,device_t child)6554 pci_assign_interrupt_method(device_t dev, device_t child)
6555 {
6556 struct pci_devinfo *dinfo = device_get_ivars(child);
6557 pcicfgregs *cfg = &dinfo->cfg;
6558
6559 return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child,
6560 cfg->intpin));
6561 }
6562
6563 static void
pci_lookup(void * arg,const char * name,device_t * dev)6564 pci_lookup(void *arg, const char *name, device_t *dev)
6565 {
6566 long val;
6567 char *end;
6568 int domain, bus, slot, func;
6569
6570 if (*dev != NULL)
6571 return;
6572
6573 /*
6574 * Accept pciconf-style selectors of either pciD:B:S:F or
6575 * pciB:S:F. In the latter case, the domain is assumed to
6576 * be zero.
6577 */
6578 if (strncmp(name, "pci", 3) != 0)
6579 return;
6580 val = strtol(name + 3, &end, 10);
6581 if (val < 0 || val > INT_MAX || *end != ':')
6582 return;
6583 domain = val;
6584 val = strtol(end + 1, &end, 10);
6585 if (val < 0 || val > INT_MAX || *end != ':')
6586 return;
6587 bus = val;
6588 val = strtol(end + 1, &end, 10);
6589 if (val < 0 || val > INT_MAX)
6590 return;
6591 slot = val;
6592 if (*end == ':') {
6593 val = strtol(end + 1, &end, 10);
6594 if (val < 0 || val > INT_MAX || *end != '\0')
6595 return;
6596 func = val;
6597 } else if (*end == '\0') {
6598 func = slot;
6599 slot = bus;
6600 bus = domain;
6601 domain = 0;
6602 } else
6603 return;
6604
6605 if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX ||
6606 func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX))
6607 return;
6608
6609 *dev = pci_find_dbsf(domain, bus, slot, func);
6610 }
6611
6612 static int
pci_modevent(module_t mod,int what,void * arg)6613 pci_modevent(module_t mod, int what, void *arg)
6614 {
6615 static struct cdev *pci_cdev;
6616 static eventhandler_tag tag;
6617
6618 switch (what) {
6619 case MOD_LOAD:
6620 STAILQ_INIT(&pci_devq);
6621 pci_generation = 0;
6622 pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644,
6623 "pci");
6624 pci_load_vendor_data();
6625 tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL,
6626 1000);
6627 break;
6628
6629 case MOD_UNLOAD:
6630 if (tag != NULL)
6631 EVENTHANDLER_DEREGISTER(dev_lookup, tag);
6632 destroy_dev(pci_cdev);
6633 break;
6634 }
6635
6636 return (0);
6637 }
6638
6639 static void
pci_cfg_restore_pcie(device_t dev,struct pci_devinfo * dinfo)6640 pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo)
6641 {
6642 #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2)
6643 struct pcicfg_pcie *cfg;
6644 int version, pos;
6645
6646 cfg = &dinfo->cfg.pcie;
6647 pos = cfg->pcie_location;
6648
6649 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6650
6651 WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl);
6652
6653 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6654 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6655 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6656 WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl);
6657
6658 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6659 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6660 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6661 WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl);
6662
6663 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6664 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6665 WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl);
6666
6667 if (version > 1) {
6668 WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2);
6669 WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2);
6670 WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2);
6671 }
6672 #undef WREG
6673 }
6674
6675 static void
pci_cfg_restore_pcix(device_t dev,struct pci_devinfo * dinfo)6676 pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo)
6677 {
6678 pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND,
6679 dinfo->cfg.pcix.pcix_command, 2);
6680 }
6681
6682 void
pci_cfg_restore(device_t dev,struct pci_devinfo * dinfo)6683 pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
6684 {
6685
6686 /*
6687 * Restore the device to full power mode. We must do this
6688 * before we restore the registers because moving from D3 to
6689 * D0 will cause the chip's BARs and some other registers to
6690 * be reset to some unknown power on reset values. Cut down
6691 * the noise on boot by doing nothing if we are already in
6692 * state D0.
6693 */
6694 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0)
6695 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6696 pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1);
6697 pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1);
6698 pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1);
6699 pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1);
6700 pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
6701 pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
6702 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6703 case PCIM_HDRTYPE_NORMAL:
6704 pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1);
6705 pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1);
6706 break;
6707 case PCIM_HDRTYPE_BRIDGE:
6708 pci_write_config(dev, PCIR_SECLAT_1,
6709 dinfo->cfg.bridge.br_seclat, 1);
6710 pci_write_config(dev, PCIR_SUBBUS_1,
6711 dinfo->cfg.bridge.br_subbus, 1);
6712 pci_write_config(dev, PCIR_SECBUS_1,
6713 dinfo->cfg.bridge.br_secbus, 1);
6714 pci_write_config(dev, PCIR_PRIBUS_1,
6715 dinfo->cfg.bridge.br_pribus, 1);
6716 pci_write_config(dev, PCIR_BRIDGECTL_1,
6717 dinfo->cfg.bridge.br_control, 2);
6718 break;
6719 case PCIM_HDRTYPE_CARDBUS:
6720 pci_write_config(dev, PCIR_SECLAT_2,
6721 dinfo->cfg.bridge.br_seclat, 1);
6722 pci_write_config(dev, PCIR_SUBBUS_2,
6723 dinfo->cfg.bridge.br_subbus, 1);
6724 pci_write_config(dev, PCIR_SECBUS_2,
6725 dinfo->cfg.bridge.br_secbus, 1);
6726 pci_write_config(dev, PCIR_PRIBUS_2,
6727 dinfo->cfg.bridge.br_pribus, 1);
6728 pci_write_config(dev, PCIR_BRIDGECTL_2,
6729 dinfo->cfg.bridge.br_control, 2);
6730 break;
6731 }
6732 pci_restore_bars(dev);
6733
6734 if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE)
6735 pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2);
6736
6737 /*
6738 * Restore extended capabilities for PCI-Express and PCI-X
6739 */
6740 if (dinfo->cfg.pcie.pcie_location != 0)
6741 pci_cfg_restore_pcie(dev, dinfo);
6742 if (dinfo->cfg.pcix.pcix_location != 0)
6743 pci_cfg_restore_pcix(dev, dinfo);
6744
6745 /* Restore MSI and MSI-X configurations if they are present. */
6746 if (dinfo->cfg.msi.msi_location != 0)
6747 pci_resume_msi(dev);
6748 if (dinfo->cfg.msix.msix_location != 0)
6749 pci_resume_msix(dev);
6750
6751 #ifdef PCI_IOV
6752 /* The SR-IOV capability is implemented only by PFs. */
6753 if (dinfo->cfg.iov != NULL &&
6754 (dinfo->cfg.flags & PCICFG_VF) == 0)
6755 pci_iov_cfg_restore(dev, dinfo);
6756 #endif
6757 }
6758
6759 static void
pci_cfg_save_pcie(device_t dev,struct pci_devinfo * dinfo)6760 pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo)
6761 {
6762 #define RREG(n) pci_read_config(dev, pos + (n), 2)
6763 struct pcicfg_pcie *cfg;
6764 int version, pos;
6765
6766 cfg = &dinfo->cfg.pcie;
6767 pos = cfg->pcie_location;
6768
6769 cfg->pcie_flags = RREG(PCIER_FLAGS);
6770
6771 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6772
6773 cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL);
6774
6775 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6776 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6777 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6778 cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL);
6779
6780 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6781 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6782 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6783 cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL);
6784
6785 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6786 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6787 cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL);
6788
6789 if (version > 1) {
6790 cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2);
6791 cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2);
6792 cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2);
6793 }
6794 #undef RREG
6795 }
6796
6797 static void
pci_cfg_save_pcix(device_t dev,struct pci_devinfo * dinfo)6798 pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo)
6799 {
6800 dinfo->cfg.pcix.pcix_command = pci_read_config(dev,
6801 dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2);
6802 }
6803
6804 void
pci_cfg_save(device_t dev,struct pci_devinfo * dinfo,int setstate)6805 pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
6806 {
6807 uint32_t cls;
6808 int ps;
6809
6810 /*
6811 * Some drivers apparently write to these registers w/o updating our
6812 * cached copy. No harm happens if we update the copy, so do so here
6813 * so we can restore them. The COMMAND register is modified by the
6814 * bus w/o updating the cache. This should represent the normally
6815 * writable portion of the 'defined' part of type 0/1/2 headers.
6816 */
6817 dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2);
6818 dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2);
6819 dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2);
6820 dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1);
6821 dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1);
6822 dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
6823 dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
6824 dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1);
6825 dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1);
6826 dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1);
6827 dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1);
6828 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6829 case PCIM_HDRTYPE_NORMAL:
6830 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2);
6831 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2);
6832 dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1);
6833 dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1);
6834 break;
6835 case PCIM_HDRTYPE_BRIDGE:
6836 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6837 PCIR_SECLAT_1, 1);
6838 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6839 PCIR_SUBBUS_1, 1);
6840 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6841 PCIR_SECBUS_1, 1);
6842 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6843 PCIR_PRIBUS_1, 1);
6844 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6845 PCIR_BRIDGECTL_1, 2);
6846 break;
6847 case PCIM_HDRTYPE_CARDBUS:
6848 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6849 PCIR_SECLAT_2, 1);
6850 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6851 PCIR_SUBBUS_2, 1);
6852 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6853 PCIR_SECBUS_2, 1);
6854 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6855 PCIR_PRIBUS_2, 1);
6856 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6857 PCIR_BRIDGECTL_2, 2);
6858 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2);
6859 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2);
6860 break;
6861 }
6862
6863 if (dinfo->cfg.pcie.pcie_location != 0)
6864 pci_cfg_save_pcie(dev, dinfo);
6865
6866 if (dinfo->cfg.pcix.pcix_location != 0)
6867 pci_cfg_save_pcix(dev, dinfo);
6868
6869 #ifdef PCI_IOV
6870 /* The SR-IOV capability is implemented only by PFs. */
6871 if (dinfo->cfg.iov != NULL &&
6872 (dinfo->cfg.flags & PCICFG_VF) == 0)
6873 pci_iov_cfg_save(dev, dinfo);
6874 #endif
6875
6876 /*
6877 * don't set the state for display devices, base peripherals and
6878 * memory devices since bad things happen when they are powered down.
6879 * We should (a) have drivers that can easily detach and (b) use
6880 * generic drivers for these devices so that some device actually
6881 * attaches. We need to make sure that when we implement (a) we don't
6882 * power the device down on a reattach.
6883 */
6884 cls = pci_get_class(dev);
6885 if (!setstate)
6886 return;
6887 switch (pci_do_power_nodriver)
6888 {
6889 case 0: /* NO powerdown at all */
6890 return;
6891 case 1: /* Conservative about what to power down */
6892 if (cls == PCIC_STORAGE)
6893 return;
6894 /*FALLTHROUGH*/
6895 case 2: /* Aggressive about what to power down */
6896 if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY ||
6897 cls == PCIC_BASEPERIPH)
6898 return;
6899 /*FALLTHROUGH*/
6900 case 3: /* Power down everything */
6901 break;
6902 }
6903 /*
6904 * PCI spec says we can only go into D3 state from D0 state.
6905 * Transition from D[12] into D0 before going to D3 state.
6906 */
6907 ps = pci_get_powerstate(dev);
6908 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6909 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6910 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
6911 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6912 }
6913
6914 /* Wrapper APIs suitable for device driver use. */
6915 void
pci_save_state(device_t dev)6916 pci_save_state(device_t dev)
6917 {
6918 struct pci_devinfo *dinfo;
6919
6920 dinfo = device_get_ivars(dev);
6921 pci_cfg_save(dev, dinfo, 0);
6922 }
6923
6924 void
pci_restore_state(device_t dev)6925 pci_restore_state(device_t dev)
6926 {
6927 struct pci_devinfo *dinfo;
6928
6929 dinfo = device_get_ivars(dev);
6930 pci_cfg_restore(dev, dinfo);
6931 }
6932
6933 static int
pci_get_id_method(device_t dev,device_t child,enum pci_id_type type,uintptr_t * id)6934 pci_get_id_method(device_t dev, device_t child, enum pci_id_type type,
6935 uintptr_t *id)
6936 {
6937
6938 return (PCIB_GET_ID(device_get_parent(dev), child, type, id));
6939 }
6940
6941 /* Find the upstream port of a given PCI device in a root complex. */
6942 device_t
pci_find_pcie_root_port(device_t dev)6943 pci_find_pcie_root_port(device_t dev)
6944 {
6945 struct pci_devinfo *dinfo;
6946 device_t pcib, bus;
6947
6948 KASSERT(is_pci_device(dev),
6949 ("%s: non-pci device %s", __func__, device_get_nameunit(dev)));
6950
6951 /*
6952 * Walk the bridge hierarchy until we find a PCI-e root
6953 * port or a non-PCI device.
6954 */
6955 for (;;) {
6956 bus = device_get_parent(dev);
6957 KASSERT(bus != NULL, ("%s: null parent of %s", __func__,
6958 device_get_nameunit(dev)));
6959
6960 pcib = device_get_parent(bus);
6961 KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__,
6962 device_get_nameunit(bus)));
6963
6964 if (!is_pci_device(pcib))
6965 return (NULL);
6966
6967 dinfo = device_get_ivars(pcib);
6968 if (dinfo->cfg.pcie.pcie_location != 0 &&
6969 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
6970 return (pcib);
6971
6972 dev = pcib;
6973 }
6974 }
6975
6976 /*
6977 * Wait for pending transactions to complete on a PCI-express function.
6978 *
6979 * The maximum delay is specified in milliseconds in max_delay. Note
6980 * that this function may sleep.
6981 *
6982 * Returns true if the function is idle and false if the timeout is
6983 * exceeded. If dev is not a PCI-express function, this returns true.
6984 */
6985 bool
pcie_wait_for_pending_transactions(device_t dev,u_int max_delay)6986 pcie_wait_for_pending_transactions(device_t dev, u_int max_delay)
6987 {
6988 struct pci_devinfo *dinfo = device_get_ivars(dev);
6989 uint16_t sta;
6990 int cap;
6991
6992 cap = dinfo->cfg.pcie.pcie_location;
6993 if (cap == 0)
6994 return (true);
6995
6996 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6997 while (sta & PCIEM_STA_TRANSACTION_PND) {
6998 if (max_delay == 0)
6999 return (false);
7000
7001 /* Poll once every 100 milliseconds up to the timeout. */
7002 if (max_delay > 100) {
7003 pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK);
7004 max_delay -= 100;
7005 } else {
7006 pause_sbt("pcietp", max_delay * SBT_1MS, 0,
7007 C_HARDCLOCK);
7008 max_delay = 0;
7009 }
7010 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
7011 }
7012
7013 return (true);
7014 }
7015
7016 /*
7017 * Determine the maximum Completion Timeout in microseconds.
7018 *
7019 * For non-PCI-express functions this returns 0.
7020 */
7021 int
pcie_get_max_completion_timeout(device_t dev)7022 pcie_get_max_completion_timeout(device_t dev)
7023 {
7024 struct pci_devinfo *dinfo = device_get_ivars(dev);
7025 int cap;
7026
7027 cap = dinfo->cfg.pcie.pcie_location;
7028 if (cap == 0)
7029 return (0);
7030
7031 /*
7032 * Functions using the 1.x spec use the default timeout range of
7033 * 50 microseconds to 50 milliseconds. Functions that do not
7034 * support programmable timeouts also use this range.
7035 */
7036 if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 ||
7037 (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) &
7038 PCIEM_CAP2_COMP_TIMO_RANGES) == 0)
7039 return (50 * 1000);
7040
7041 switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) &
7042 PCIEM_CTL2_COMP_TIMO_VAL) {
7043 case PCIEM_CTL2_COMP_TIMO_100US:
7044 return (100);
7045 case PCIEM_CTL2_COMP_TIMO_10MS:
7046 return (10 * 1000);
7047 case PCIEM_CTL2_COMP_TIMO_55MS:
7048 return (55 * 1000);
7049 case PCIEM_CTL2_COMP_TIMO_210MS:
7050 return (210 * 1000);
7051 case PCIEM_CTL2_COMP_TIMO_900MS:
7052 return (900 * 1000);
7053 case PCIEM_CTL2_COMP_TIMO_3500MS:
7054 return (3500 * 1000);
7055 case PCIEM_CTL2_COMP_TIMO_13S:
7056 return (13 * 1000 * 1000);
7057 case PCIEM_CTL2_COMP_TIMO_64S:
7058 return (64 * 1000 * 1000);
7059 default:
7060 return (50 * 1000);
7061 }
7062 }
7063
7064 void
pcie_apei_error(device_t dev,int sev,uint8_t * aerp)7065 pcie_apei_error(device_t dev, int sev, uint8_t *aerp)
7066 {
7067 struct pci_devinfo *dinfo = device_get_ivars(dev);
7068 const char *s;
7069 int aer;
7070 uint32_t r, r1;
7071 uint16_t rs;
7072
7073 if (sev == PCIEM_STA_CORRECTABLE_ERROR)
7074 s = "Correctable";
7075 else if (sev == PCIEM_STA_NON_FATAL_ERROR)
7076 s = "Uncorrectable (Non-Fatal)";
7077 else
7078 s = "Uncorrectable (Fatal)";
7079 device_printf(dev, "%s PCIe error reported by APEI\n", s);
7080 if (aerp) {
7081 if (sev == PCIEM_STA_CORRECTABLE_ERROR) {
7082 r = le32dec(aerp + PCIR_AER_COR_STATUS);
7083 r1 = le32dec(aerp + PCIR_AER_COR_MASK);
7084 } else {
7085 r = le32dec(aerp + PCIR_AER_UC_STATUS);
7086 r1 = le32dec(aerp + PCIR_AER_UC_MASK);
7087 }
7088 device_printf(dev, "status 0x%08x mask 0x%08x", r, r1);
7089 if (sev != PCIEM_STA_CORRECTABLE_ERROR) {
7090 r = le32dec(aerp + PCIR_AER_UC_SEVERITY);
7091 rs = le16dec(aerp + PCIR_AER_CAP_CONTROL);
7092 printf(" severity 0x%08x first %d\n",
7093 r, rs & 0x1f);
7094 } else
7095 printf("\n");
7096 }
7097
7098 /* As kind of recovery just report and clear the error statuses. */
7099 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
7100 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7101 if (r != 0) {
7102 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
7103 device_printf(dev, "Clearing UC AER errors 0x%08x\n", r);
7104 }
7105
7106 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7107 if (r != 0) {
7108 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
7109 device_printf(dev, "Clearing COR AER errors 0x%08x\n", r);
7110 }
7111 }
7112 if (dinfo->cfg.pcie.pcie_location != 0) {
7113 rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
7114 PCIER_DEVICE_STA, 2);
7115 if ((rs & (PCIEM_STA_CORRECTABLE_ERROR |
7116 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
7117 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
7118 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
7119 PCIER_DEVICE_STA, rs, 2);
7120 device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs);
7121 }
7122 }
7123 }
7124
7125 /*
7126 * Return true if the device supports FLR, taking both its advertised
7127 * capability and the PCI quirk policy into account.
7128 */
7129 bool
pcie_flr_supported(device_t dev)7130 pcie_flr_supported(device_t dev)
7131 {
7132 struct pci_devinfo *dinfo = device_get_ivars(dev);
7133 int cap;
7134
7135 cap = dinfo->cfg.pcie.pcie_location;
7136 if (cap == 0)
7137 return (false);
7138
7139 if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR) &&
7140 !pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_ENABLE_FLR))
7141 return (false);
7142 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_FLR))
7143 return (false);
7144 return (true);
7145 }
7146
7147 /*
7148 * Perform a Function Level Reset (FLR) on a device.
7149 *
7150 * This function first waits for any pending transactions to complete
7151 * within the timeout specified by max_delay. If transactions are
7152 * still pending, the function will return false without attempting a
7153 * reset.
7154 *
7155 * If dev is not a PCI-express function, or neither advertises FLR nor
7156 * has a quirk enabling FLR, this function returns false.
7157 *
7158 * Note that no registers are saved or restored. The caller is
7159 * responsible for saving and restoring any registers including
7160 * PCI-standard registers via pci_save_state() and
7161 * pci_restore_state().
7162 */
7163 bool
pcie_flr(device_t dev,u_int max_delay,bool force)7164 pcie_flr(device_t dev, u_int max_delay, bool force)
7165 {
7166 struct pci_devinfo *dinfo = device_get_ivars(dev);
7167 uint16_t cmd, ctl;
7168 int compl_delay;
7169 int cap;
7170
7171 if (!pcie_flr_supported(dev))
7172 return (false);
7173
7174 cap = dinfo->cfg.pcie.pcie_location;
7175
7176 /*
7177 * Disable busmastering to prevent generation of new
7178 * transactions while waiting for the device to go idle. If
7179 * the idle timeout fails, the command register is restored
7180 * which will re-enable busmastering.
7181 */
7182 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
7183 pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2);
7184 if (!pcie_wait_for_pending_transactions(dev, max_delay)) {
7185 if (!force) {
7186 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
7187 return (false);
7188 }
7189 pci_printf(&dinfo->cfg,
7190 "Resetting with transactions pending after %d ms\n",
7191 max_delay);
7192
7193 /*
7194 * Extend the post-FLR delay to cover the maximum
7195 * Completion Timeout delay of anything in flight
7196 * during the FLR delay. Enforce a minimum delay of
7197 * at least 10ms.
7198 */
7199 compl_delay = pcie_get_max_completion_timeout(dev) / 1000;
7200 if (compl_delay < 10)
7201 compl_delay = 10;
7202 } else
7203 compl_delay = 0;
7204
7205 /* Initiate the reset. */
7206 ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
7207 pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl |
7208 PCIEM_CTL_INITIATE_FLR, 2);
7209
7210 /* Wait for 100ms. */
7211 pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK);
7212
7213 if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) &
7214 PCIEM_STA_TRANSACTION_PND)
7215 pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
7216 return (true);
7217 }
7218
7219 /*
7220 * Attempt a power-management reset by cycling the device in/out of D3
7221 * state. PCI spec says we can only go into D3 state from D0 state.
7222 * Transition from D[12] into D0 before going to D3 state.
7223 */
7224 int
pci_power_reset(device_t dev)7225 pci_power_reset(device_t dev)
7226 {
7227 int ps;
7228
7229 ps = pci_get_powerstate(dev);
7230 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
7231 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
7232 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
7233 pci_set_powerstate(dev, ps);
7234 return (0);
7235 }
7236
7237 /*
7238 * Try link drop and retrain of the downstream port of upstream
7239 * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must
7240 * cause Conventional Hot reset of the device in the slot.
7241 * Alternative, for PCIe, could be the secondary bus reset initiatied
7242 * on the upstream switch PCIR_BRIDGECTL_1, bit 6.
7243 */
7244 int
pcie_link_reset(device_t port,int pcie_location)7245 pcie_link_reset(device_t port, int pcie_location)
7246 {
7247 uint16_t v;
7248
7249 v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2);
7250 v |= PCIEM_LINK_CTL_LINK_DIS;
7251 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
7252 pause_sbt("pcier1", mstosbt(20), 0, 0);
7253 v &= ~PCIEM_LINK_CTL_LINK_DIS;
7254 v |= PCIEM_LINK_CTL_RETRAIN_LINK;
7255 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
7256 pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */
7257 v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2);
7258 return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0);
7259 }
7260
7261 static int
pci_reset_post(device_t dev,device_t child)7262 pci_reset_post(device_t dev, device_t child)
7263 {
7264
7265 if (dev == device_get_parent(child))
7266 pci_restore_state(child);
7267 return (0);
7268 }
7269
7270 static int
pci_reset_prepare(device_t dev,device_t child)7271 pci_reset_prepare(device_t dev, device_t child)
7272 {
7273
7274 if (dev == device_get_parent(child))
7275 pci_save_state(child);
7276 return (0);
7277 }
7278
7279 static int
pci_reset_child(device_t dev,device_t child,int flags)7280 pci_reset_child(device_t dev, device_t child, int flags)
7281 {
7282 int error;
7283
7284 if (dev == NULL || device_get_parent(child) != dev)
7285 return (0);
7286 if ((flags & DEVF_RESET_DETACH) != 0) {
7287 error = device_get_state(child) == DS_ATTACHED ?
7288 device_detach(child) : 0;
7289 } else {
7290 error = BUS_SUSPEND_CHILD(dev, child);
7291 }
7292 if (error == 0) {
7293 if (!pcie_flr(child, 1000, false)) {
7294 error = BUS_RESET_PREPARE(dev, child);
7295 if (error == 0)
7296 pci_power_reset(child);
7297 BUS_RESET_POST(dev, child);
7298 }
7299 if ((flags & DEVF_RESET_DETACH) != 0)
7300 device_probe_and_attach(child);
7301 else
7302 BUS_RESUME_CHILD(dev, child);
7303 }
7304 return (error);
7305 }
7306
7307 const struct pci_device_table *
pci_match_device(device_t child,const struct pci_device_table * id,size_t nelt)7308 pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt)
7309 {
7310 bool match;
7311 uint16_t vendor, device, subvendor, subdevice, class, subclass, revid;
7312
7313 vendor = pci_get_vendor(child);
7314 device = pci_get_device(child);
7315 subvendor = pci_get_subvendor(child);
7316 subdevice = pci_get_subdevice(child);
7317 class = pci_get_class(child);
7318 subclass = pci_get_subclass(child);
7319 revid = pci_get_revid(child);
7320 while (nelt-- > 0) {
7321 match = true;
7322 if (id->match_flag_vendor)
7323 match &= vendor == id->vendor;
7324 if (id->match_flag_device)
7325 match &= device == id->device;
7326 if (id->match_flag_subvendor)
7327 match &= subvendor == id->subvendor;
7328 if (id->match_flag_subdevice)
7329 match &= subdevice == id->subdevice;
7330 if (id->match_flag_class)
7331 match &= class == id->class_id;
7332 if (id->match_flag_subclass)
7333 match &= subclass == id->subclass;
7334 if (id->match_flag_revid)
7335 match &= revid == id->revid;
7336 if (match)
7337 return (id);
7338 id++;
7339 }
7340 return (NULL);
7341 }
7342
7343 static void
pci_print_faulted_dev_name(const struct pci_devinfo * dinfo)7344 pci_print_faulted_dev_name(const struct pci_devinfo *dinfo)
7345 {
7346 const char *dev_name;
7347 device_t dev;
7348
7349 dev = dinfo->cfg.dev;
7350 printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus,
7351 dinfo->cfg.slot, dinfo->cfg.func);
7352 dev_name = device_get_name(dev);
7353 if (dev_name != NULL)
7354 printf(" (%s%d)", dev_name, device_get_unit(dev));
7355 }
7356
7357 void
pci_print_faulted_dev(void)7358 pci_print_faulted_dev(void)
7359 {
7360 struct pci_devinfo *dinfo;
7361 device_t dev;
7362 int aer, i;
7363 uint32_t r1, r2;
7364 uint16_t status;
7365
7366 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
7367 dev = dinfo->cfg.dev;
7368 status = pci_read_config(dev, PCIR_STATUS, 2);
7369 status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
7370 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
7371 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
7372 if (status != 0) {
7373 pci_print_faulted_dev_name(dinfo);
7374 printf(" error 0x%04x\n", status);
7375 }
7376 if (dinfo->cfg.pcie.pcie_location != 0) {
7377 status = pci_read_config(dev,
7378 dinfo->cfg.pcie.pcie_location +
7379 PCIER_DEVICE_STA, 2);
7380 if ((status & (PCIEM_STA_CORRECTABLE_ERROR |
7381 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
7382 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
7383 pci_print_faulted_dev_name(dinfo);
7384 printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n",
7385 pci_read_config(dev,
7386 dinfo->cfg.pcie.pcie_location +
7387 PCIER_DEVICE_CTL, 2),
7388 status);
7389 }
7390 }
7391 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
7392 r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7393 r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7394 if (r1 != 0 || r2 != 0) {
7395 pci_print_faulted_dev_name(dinfo);
7396 printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n"
7397 " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n",
7398 r1, pci_read_config(dev, aer +
7399 PCIR_AER_UC_MASK, 4),
7400 pci_read_config(dev, aer +
7401 PCIR_AER_UC_SEVERITY, 4),
7402 r2, pci_read_config(dev, aer +
7403 PCIR_AER_COR_MASK, 4),
7404 pci_read_config(dev, aer +
7405 PCIR_AER_CAP_CONTROL, 4));
7406 for (i = 0; i < 4; i++) {
7407 r1 = pci_read_config(dev, aer +
7408 PCIR_AER_HEADER_LOG + i * 4, 4);
7409 printf(" HL%d: 0x%08x\n", i, r1);
7410 }
7411 }
7412 }
7413 }
7414 }
7415
7416 bool
is_pci_device(device_t dev)7417 is_pci_device(device_t dev)
7418 {
7419 devclass_t pci_class;
7420
7421 if (device_get_parent(dev) == NULL)
7422 return (false);
7423 pci_class = devclass_find("pci");
7424 return (device_get_devclass(device_get_parent(dev)) == pci_class);
7425 }
7426
7427 #ifdef DDB
DB_SHOW_COMMAND_FLAGS(pcierr,pci_print_faulted_dev_db,DB_CMD_MEMSAFE)7428 DB_SHOW_COMMAND_FLAGS(pcierr, pci_print_faulted_dev_db, DB_CMD_MEMSAFE)
7429 {
7430
7431 pci_print_faulted_dev();
7432 }
7433
7434 static void
db_clear_pcie_errors(const struct pci_devinfo * dinfo)7435 db_clear_pcie_errors(const struct pci_devinfo *dinfo)
7436 {
7437 device_t dev;
7438 int aer;
7439 uint32_t r;
7440
7441 dev = dinfo->cfg.dev;
7442 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
7443 PCIER_DEVICE_STA, 2);
7444 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
7445 PCIER_DEVICE_STA, r, 2);
7446
7447 if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0)
7448 return;
7449 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7450 if (r != 0)
7451 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
7452 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7453 if (r != 0)
7454 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
7455 }
7456
DB_COMMAND_FLAGS(pci_clearerr,db_pci_clearerr,DB_CMD_MEMSAFE)7457 DB_COMMAND_FLAGS(pci_clearerr, db_pci_clearerr, DB_CMD_MEMSAFE)
7458 {
7459 struct pci_devinfo *dinfo;
7460 device_t dev;
7461 uint16_t status, status1;
7462
7463 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
7464 dev = dinfo->cfg.dev;
7465 status1 = status = pci_read_config(dev, PCIR_STATUS, 2);
7466 status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
7467 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
7468 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
7469 if (status1 != 0) {
7470 status &= ~status1;
7471 pci_write_config(dev, PCIR_STATUS, status, 2);
7472 }
7473 if (dinfo->cfg.pcie.pcie_location != 0)
7474 db_clear_pcie_errors(dinfo);
7475 }
7476 }
7477 #endif
7478