1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
6 * Copyright (c) 2000, BSDi
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice unmodified, this list of conditions, and the following
14 * disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include "opt_acpi.h"
33 #include "opt_iommu.h"
34 #include "opt_bus.h"
35
36 #include <sys/param.h>
37 #include <sys/conf.h>
38 #include <sys/endian.h>
39 #include <sys/eventhandler.h>
40 #include <sys/fcntl.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/linker.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/queue.h>
47 #include <sys/sbuf.h>
48 #include <sys/stdarg.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 #include <sys/taskqueue.h>
52 #include <sys/tree.h>
53
54 #include <vm/vm.h>
55 #include <vm/pmap.h>
56 #include <vm/vm_extern.h>
57
58 #include <sys/bus.h>
59 #include <machine/bus.h>
60 #include <sys/rman.h>
61 #include <machine/resource.h>
62
63 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
64 #include <machine/intr_machdep.h>
65 #endif
66
67 #include <sys/pciio.h>
68 #include <dev/pci/pcireg.h>
69 #include <dev/pci/pcivar.h>
70 #include <dev/pci/pci_private.h>
71
72 #ifdef PCI_IOV
73 #include <sys/nv.h>
74 #include <dev/pci/pci_iov_private.h>
75 #endif
76
77 #include <dev/usb/controller/xhcireg.h>
78 #include <dev/usb/controller/ehcireg.h>
79 #include <dev/usb/controller/ohcireg.h>
80 #include <dev/usb/controller/uhcireg.h>
81
82 #include <dev/iommu/iommu.h>
83
84 #include "pcib_if.h"
85 #include "pci_if.h"
86
87 #define PCIR_IS_BIOS(cfg, reg) \
88 (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \
89 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))
90
91 static device_probe_t pci_probe;
92
93 static bus_reset_post_t pci_reset_post;
94 static bus_reset_prepare_t pci_reset_prepare;
95 static bus_reset_child_t pci_reset_child;
96 static bus_hint_device_unit_t pci_hint_device_unit;
97 static bus_remap_intr_t pci_remap_intr_method;
98
99 static pci_get_id_t pci_get_id_method;
100
101 static int pci_has_quirk(uint32_t devid, int quirk);
102 static pci_addr_t pci_mapbase(uint64_t mapreg);
103 static const char *pci_maptype(uint64_t mapreg);
104 static int pci_maprange(uint64_t mapreg);
105 static pci_addr_t pci_rombase(uint64_t mapreg);
106 static int pci_romsize(uint64_t testval);
107 static void pci_fixancient(pcicfgregs *cfg);
108 static int pci_printf(pcicfgregs *cfg, const char *fmt, ...);
109
110 static int pci_porten(device_t dev);
111 static int pci_memen(device_t dev);
112 static void pci_assign_interrupt(device_t bus, device_t dev,
113 int force_route);
114 static int pci_add_map(device_t bus, device_t dev, int reg,
115 struct resource_list *rl, int force, int prefetch);
116 static void pci_load_vendor_data(void);
117 static int pci_describe_parse_line(char **ptr, int *vendor,
118 int *device, char **desc);
119 static char *pci_describe_device(device_t dev);
120 static int pci_modevent(module_t mod, int what, void *arg);
121 static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
122 pcicfgregs *cfg);
123 static void pci_read_cap(device_t pcib, pcicfgregs *cfg);
124 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg,
125 int reg, uint32_t *data);
126 #if 0
127 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
128 int reg, uint32_t data);
129 #endif
130 static void pci_read_vpd(device_t pcib, pcicfgregs *cfg);
131 static void pci_mask_msix(device_t dev, u_int index);
132 static void pci_unmask_msix(device_t dev, u_int index);
133 static int pci_msi_blacklisted(void);
134 static int pci_msix_blacklisted(void);
135 static void pci_resume_msi(device_t dev);
136 static void pci_resume_msix(device_t dev);
137 static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d,
138 int b, int s, int f, uint16_t vid, uint16_t did);
139
140 static device_method_t pci_methods[] = {
141 /* Device interface */
142 DEVMETHOD(device_probe, pci_probe),
143 DEVMETHOD(device_attach, pci_attach),
144 DEVMETHOD(device_detach, pci_detach),
145 DEVMETHOD(device_shutdown, bus_generic_shutdown),
146 DEVMETHOD(device_suspend, bus_generic_suspend),
147 DEVMETHOD(device_resume, pci_resume),
148
149 /* Bus interface */
150 DEVMETHOD(bus_print_child, pci_print_child),
151 DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch),
152 DEVMETHOD(bus_read_ivar, pci_read_ivar),
153 DEVMETHOD(bus_write_ivar, pci_write_ivar),
154 DEVMETHOD(bus_driver_added, pci_driver_added),
155 DEVMETHOD(bus_setup_intr, pci_setup_intr),
156 DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
157 DEVMETHOD(bus_reset_prepare, pci_reset_prepare),
158 DEVMETHOD(bus_reset_post, pci_reset_post),
159 DEVMETHOD(bus_reset_child, pci_reset_child),
160
161 DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag),
162 DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
163 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
164 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
165 DEVMETHOD(bus_delete_resource, pci_delete_resource),
166 DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
167 DEVMETHOD(bus_adjust_resource, pci_adjust_resource),
168 DEVMETHOD(bus_release_resource, pci_release_resource),
169 DEVMETHOD(bus_activate_resource, pci_activate_resource),
170 DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
171 DEVMETHOD(bus_map_resource, pci_map_resource),
172 DEVMETHOD(bus_unmap_resource, pci_unmap_resource),
173 DEVMETHOD(bus_child_deleted, pci_child_deleted),
174 DEVMETHOD(bus_child_detached, pci_child_detached),
175 DEVMETHOD(bus_child_pnpinfo, pci_child_pnpinfo_method),
176 DEVMETHOD(bus_child_location, pci_child_location_method),
177 DEVMETHOD(bus_get_device_path, pci_get_device_path_method),
178 DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit),
179 DEVMETHOD(bus_remap_intr, pci_remap_intr_method),
180 DEVMETHOD(bus_suspend_child, pci_suspend_child),
181 DEVMETHOD(bus_resume_child, pci_resume_child),
182 DEVMETHOD(bus_rescan, pci_rescan_method),
183
184 /* PCI interface */
185 DEVMETHOD(pci_read_config, pci_read_config_method),
186 DEVMETHOD(pci_write_config, pci_write_config_method),
187 DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method),
188 DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method),
189 DEVMETHOD(pci_enable_io, pci_enable_io_method),
190 DEVMETHOD(pci_disable_io, pci_disable_io_method),
191 DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method),
192 DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method),
193 DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method),
194 DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method),
195 DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method),
196 DEVMETHOD(pci_find_cap, pci_find_cap_method),
197 DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method),
198 DEVMETHOD(pci_find_extcap, pci_find_extcap_method),
199 DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method),
200 DEVMETHOD(pci_find_htcap, pci_find_htcap_method),
201 DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method),
202 DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method),
203 DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method),
204 DEVMETHOD(pci_enable_msi, pci_enable_msi_method),
205 DEVMETHOD(pci_enable_msix, pci_enable_msix_method),
206 DEVMETHOD(pci_disable_msi, pci_disable_msi_method),
207 DEVMETHOD(pci_remap_msix, pci_remap_msix_method),
208 DEVMETHOD(pci_release_msi, pci_release_msi_method),
209 DEVMETHOD(pci_msi_count, pci_msi_count_method),
210 DEVMETHOD(pci_msix_count, pci_msix_count_method),
211 DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method),
212 DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method),
213 DEVMETHOD(pci_get_id, pci_get_id_method),
214 DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method),
215 DEVMETHOD(pci_child_added, pci_child_added_method),
216 #ifdef PCI_IOV
217 DEVMETHOD(pci_iov_attach, pci_iov_attach_method),
218 DEVMETHOD(pci_iov_detach, pci_iov_detach_method),
219 DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method),
220 #endif
221
222 DEVMETHOD_END
223 };
224
225 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));
226
227 EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_modevent, NULL, BUS_PASS_BUS);
228 MODULE_VERSION(pci, 1);
229
230 static char *pci_vendordata;
231 static size_t pci_vendordata_size;
232
233 struct pci_quirk {
234 uint32_t devid; /* Vendor/device of the card */
235 int type;
236 #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */
237 #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */
238 #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */
239 #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */
240 #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */
241 #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */
242 #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */
243 #define PCI_QUIRK_DISABLE_FLR 8 /* Function-Level Reset (FLR) not working. */
244 #define PCI_QUIRK_ENABLE_FLR 9 /* FLR works but is not advertised. */
245 int arg1;
246 int arg2;
247 };
248
249 static const struct pci_quirk pci_quirks[] = {
250 /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
251 { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 },
252 { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 },
253 /* As does the Serverworks OSB4 (the SMBus mapping register) */
254 { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 },
255
256 /*
257 * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge
258 * or the CMIC-SL (AKA ServerWorks GC_LE).
259 */
260 { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
261 { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
262
263 /*
264 * MSI doesn't work on earlier Intel chipsets including
265 * E7500, E7501, E7505, 845, 865, 875/E7210, and 855.
266 */
267 { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
268 { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
269 { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
270 { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
271 { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
272 { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
273 { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
274
275 /*
276 * MSI doesn't work with devices behind the AMD 8131 HT-PCIX
277 * bridge.
278 */
279 { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 },
280
281 /*
282 * Some virtualization environments emulate an older chipset
283 * but support MSI just fine. QEMU uses the Intel 82440.
284 */
285 { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 },
286
287 /*
288 * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus
289 * controller depending on SoftPciRst register (PM_IO 0x55 [7]).
290 * It prevents us from attaching hpet(4) when the bit is unset.
291 * Note this quirk only affects SB600 revision A13 and earlier.
292 * For SB600 A21 and later, firmware must set the bit to hide it.
293 * For SB700 and later, it is unused and hardcoded to zero.
294 */
295 { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 },
296
297 /*
298 * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have
299 * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit
300 * of the command register is set.
301 */
302 { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
303 { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
304 { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
305 { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
306 { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
307
308 /*
309 * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't
310 * issue MSI interrupts with PCIM_CMD_INTxDIS set either.
311 */
312 { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */
313 { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */
314 { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */
315 { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */
316 { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */
317 { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */
318
319 /*
320 * HPE Gen 10 VGA has a memory range that can't be allocated in the
321 * expected place.
322 */
323 { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 },
324
325 /*
326 * The Intel 82599 VF implements FLR without advertising it; see
327 * 82599 Specification Update, erratum 35.
328 */
329 { 0x10ed8086, PCI_QUIRK_ENABLE_FLR, 0, 0 },
330
331 /*
332 * With some MediaTek mt76 WiFi FLR does not work despite advertised.
333 */
334 { 0x061614c3, PCI_QUIRK_DISABLE_FLR, 0, 0 }, /* mt76 7922 */
335
336 /* end of table */
337 { 0 }
338 };
339
340 /* map register information */
341 #define PCI_MAPMEM 0x01 /* memory map */
342 #define PCI_MAPMEMP 0x02 /* prefetchable memory map */
343 #define PCI_MAPPORT 0x04 /* port map */
344
345 struct devlist pci_devq;
346 uint32_t pci_generation;
347 uint32_t pci_numdevs = 0;
348 static int pcie_chipset, pcix_chipset;
349
350 /* sysctl vars */
351 SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
352 "PCI bus tuning parameters");
353
354 static int pci_enable_io_modes = 1;
355 SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN,
356 &pci_enable_io_modes, 1,
357 "Enable I/O and memory bits in the config register. Some BIOSes do not"
358 " enable these bits correctly. We'd like to do this all the time, but"
359 " there are some peripherals that this causes problems with.");
360
361 static int pci_do_realloc_bars = 1;
362 SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN,
363 &pci_do_realloc_bars, 0,
364 "Attempt to allocate a new range for any BARs whose original "
365 "firmware-assigned ranges fail to allocate during the initial device scan.");
366
367 static int pci_do_power_nodriver = 0;
368 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN,
369 &pci_do_power_nodriver, 0,
370 "Place a function into D3 state when no driver attaches to it. 0 means"
371 " disable. 1 means conservatively place function into D3 state. 2 means"
372 " aggressively place function into D3 state. 3 means put absolutely"
373 " everything in D3 state.");
374
375 int pci_do_power_resume = 1;
376 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN,
377 &pci_do_power_resume, 1,
378 "Transition from D3 -> D0 on resume.");
379
380 int pci_do_power_suspend = 1;
381 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN,
382 &pci_do_power_suspend, 1,
383 "Transition from D0 -> D3 on suspend.");
384
385 static int pci_do_msi = 1;
386 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1,
387 "Enable support for MSI interrupts");
388
389 static int pci_do_msix = 1;
390 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1,
391 "Enable support for MSI-X interrupts");
392
393 static int pci_msix_rewrite_table = 0;
394 SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
395 &pci_msix_rewrite_table, 0,
396 "Rewrite entire MSI-X table when updating MSI-X entries");
397
398 static int pci_honor_msi_blacklist = 1;
399 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN,
400 &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");
401
402 #if defined(__i386__) || defined(__amd64__)
403 static int pci_usb_takeover = 1;
404 #else
405 static int pci_usb_takeover = 0;
406 #endif
407 SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN,
408 &pci_usb_takeover, 1,
409 "Enable early takeover of USB controllers. Disable this if you depend on"
410 " BIOS emulation of USB devices, that is you use USB devices (like"
411 " keyboard or mouse) but do not load USB drivers");
412
413 static int pci_clear_bars;
414 SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0,
415 "Ignore firmware-assigned resources for BARs.");
416
417 static int pci_clear_buses;
418 SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
419 "Ignore firmware-assigned bus numbers.");
420
421 static int pci_enable_ari = 1;
422 SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
423 0, "Enable support for PCIe Alternative RID Interpretation");
424
425 /*
426 * Some x86 firmware only enables PCIe hotplug if we claim to support aspm,
427 * however enabling it breaks some arm64 firmware as it powers off devices.
428 */
429 #if defined(__i386__) || defined(__amd64__)
430 int pci_enable_aspm = 1;
431 #else
432 int pci_enable_aspm = 0;
433 #endif
434 SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm,
435 0, "Enable support for PCIe Active State Power Management");
436
437 static int pci_clear_aer_on_attach = 0;
438 SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN,
439 &pci_clear_aer_on_attach, 0,
440 "Clear port and device AER state on driver attach");
441
442 static bool pci_enable_mps_tune = true;
443 SYSCTL_BOOL(_hw_pci, OID_AUTO, enable_mps_tune, CTLFLAG_RWTUN,
444 &pci_enable_mps_tune, 1,
445 "Enable tuning of MPS(maximum payload size)." );
446
447 static int pci_mps_limit;
448 SYSCTL_INT(_hw_pci, OID_AUTO, mps_limit, CTLFLAG_RDTUN, &pci_mps_limit, 0,
449 "Limit PCIe MPS to this many bytes (power of two from 128 to 4096)");
450 static bool pci_mps_limit_warned;
451
452 static bool pci_mps_enforce;
453 SYSCTL_BOOL(_hw_pci, OID_AUTO, mps_enforce, CTLFLAG_RDTUN,
454 &pci_mps_enforce, 0,
455 "Disable PCIe endpoints with an MPS incompatible with their shared path");
456
457 static bool pci_intx_reroute = true;
458 SYSCTL_BOOL(_hw_pci, OID_AUTO, intx_reroute, CTLFLAG_RWTUN,
459 &pci_intx_reroute, 0, "Re-route INTx interrupts when scanning devices");
460
461 static int
pci_has_quirk(uint32_t devid,int quirk)462 pci_has_quirk(uint32_t devid, int quirk)
463 {
464 const struct pci_quirk *q;
465
466 for (q = &pci_quirks[0]; q->devid; q++) {
467 if (q->devid == devid && q->type == quirk)
468 return (1);
469 }
470 return (0);
471 }
472
473 /* Find a device_t by bus/slot/function in domain 0 */
474
475 device_t
pci_find_bsf(uint8_t bus,uint8_t slot,uint8_t func)476 pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func)
477 {
478
479 return (pci_find_dbsf(0, bus, slot, func));
480 }
481
482 /* Find a device_t by domain/bus/slot/function */
483
484 device_t
pci_find_dbsf(uint32_t domain,uint8_t bus,uint8_t slot,uint8_t func)485 pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func)
486 {
487 struct pci_devinfo *dinfo = NULL;
488
489 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
490 if ((dinfo->cfg.domain == domain) &&
491 (dinfo->cfg.bus == bus) &&
492 (dinfo->cfg.slot == slot) &&
493 (dinfo->cfg.func == func)) {
494 break;
495 }
496 }
497
498 return (dinfo != NULL ? dinfo->cfg.dev : NULL);
499 }
500
501 /* Find a device_t by vendor/device ID */
502
503 device_t
pci_find_device(uint16_t vendor,uint16_t device)504 pci_find_device(uint16_t vendor, uint16_t device)
505 {
506 struct pci_devinfo *dinfo;
507
508 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
509 if ((dinfo->cfg.vendor == vendor) &&
510 (dinfo->cfg.device == device)) {
511 return (dinfo->cfg.dev);
512 }
513 }
514
515 return (NULL);
516 }
517
518 device_t
pci_find_class(uint8_t class,uint8_t subclass)519 pci_find_class(uint8_t class, uint8_t subclass)
520 {
521 struct pci_devinfo *dinfo;
522
523 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
524 if (dinfo->cfg.baseclass == class &&
525 dinfo->cfg.subclass == subclass) {
526 return (dinfo->cfg.dev);
527 }
528 }
529
530 return (NULL);
531 }
532
533 device_t
pci_find_class_from(uint8_t class,uint8_t subclass,device_t from)534 pci_find_class_from(uint8_t class, uint8_t subclass, device_t from)
535 {
536 struct pci_devinfo *dinfo;
537 bool found = false;
538
539 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
540 if (from != NULL && found == false) {
541 if (from != dinfo->cfg.dev)
542 continue;
543 found = true;
544 continue;
545 }
546 if (dinfo->cfg.baseclass == class &&
547 dinfo->cfg.subclass == subclass) {
548 return (dinfo->cfg.dev);
549 }
550 }
551
552 return (NULL);
553 }
554
555 device_t
pci_find_base_class_from(uint8_t class,device_t from)556 pci_find_base_class_from(uint8_t class, device_t from)
557 {
558 struct pci_devinfo *dinfo;
559 bool found = false;
560
561 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
562 if (from != NULL && found == false) {
563 if (from != dinfo->cfg.dev)
564 continue;
565 found = true;
566 continue;
567 }
568 if (dinfo->cfg.baseclass == class) {
569 return (dinfo->cfg.dev);
570 }
571 }
572
573 return (NULL);
574 }
575
576 static int
pci_printf(pcicfgregs * cfg,const char * fmt,...)577 pci_printf(pcicfgregs *cfg, const char *fmt, ...)
578 {
579 va_list ap;
580 int retval;
581
582 retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot,
583 cfg->func);
584 va_start(ap, fmt);
585 retval += vprintf(fmt, ap);
586 va_end(ap);
587 return (retval);
588 }
589
590 /* return base address of memory or port map */
591
592 static pci_addr_t
pci_mapbase(uint64_t mapreg)593 pci_mapbase(uint64_t mapreg)
594 {
595
596 if (PCI_BAR_MEM(mapreg))
597 return (mapreg & PCIM_BAR_MEM_BASE);
598 else
599 return (mapreg & PCIM_BAR_IO_BASE);
600 }
601
602 /* return map type of memory or port map */
603
604 static const char *
pci_maptype(uint64_t mapreg)605 pci_maptype(uint64_t mapreg)
606 {
607
608 if (PCI_BAR_IO(mapreg))
609 return ("I/O Port");
610 if (mapreg & PCIM_BAR_MEM_PREFETCH)
611 return ("Prefetchable Memory");
612 return ("Memory");
613 }
614
615 /* return log2 of map size decoded for memory or port map */
616
617 int
pci_mapsize(uint64_t testval)618 pci_mapsize(uint64_t testval)
619 {
620 int ln2size;
621
622 testval = pci_mapbase(testval);
623 ln2size = 0;
624 if (testval != 0) {
625 while ((testval & 1) == 0)
626 {
627 ln2size++;
628 testval >>= 1;
629 }
630 }
631 return (ln2size);
632 }
633
634 /* return base address of device ROM */
635
636 static pci_addr_t
pci_rombase(uint64_t mapreg)637 pci_rombase(uint64_t mapreg)
638 {
639
640 return (mapreg & PCIM_BIOS_ADDR_MASK);
641 }
642
643 /* return log2 of map size decided for device ROM */
644
645 static int
pci_romsize(uint64_t testval)646 pci_romsize(uint64_t testval)
647 {
648 int ln2size;
649
650 testval = pci_rombase(testval);
651 ln2size = 0;
652 if (testval != 0) {
653 while ((testval & 1) == 0)
654 {
655 ln2size++;
656 testval >>= 1;
657 }
658 }
659 return (ln2size);
660 }
661
662 /* return log2 of address range supported by map register */
663
664 static int
pci_maprange(uint64_t mapreg)665 pci_maprange(uint64_t mapreg)
666 {
667 int ln2range = 0;
668
669 if (PCI_BAR_IO(mapreg))
670 ln2range = 32;
671 else
672 switch (mapreg & PCIM_BAR_MEM_TYPE) {
673 case PCIM_BAR_MEM_32:
674 ln2range = 32;
675 break;
676 case PCIM_BAR_MEM_1MB:
677 ln2range = 20;
678 break;
679 case PCIM_BAR_MEM_64:
680 ln2range = 64;
681 break;
682 }
683 return (ln2range);
684 }
685
686 /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */
687
688 static void
pci_fixancient(pcicfgregs * cfg)689 pci_fixancient(pcicfgregs *cfg)
690 {
691 if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
692 return;
693
694 /* PCI to PCI bridges use header type 1 */
695 if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI)
696 cfg->hdrtype = PCIM_HDRTYPE_BRIDGE;
697 }
698
699 /* extract header type specific config data */
700
701 static void
pci_hdrtypedata(device_t pcib,int b,int s,int f,pcicfgregs * cfg)702 pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg)
703 {
704 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
705 switch (cfg->hdrtype & PCIM_HDRTYPE) {
706 case PCIM_HDRTYPE_NORMAL:
707 cfg->subvendor = REG(PCIR_SUBVEND_0, 2);
708 cfg->subdevice = REG(PCIR_SUBDEV_0, 2);
709 cfg->mingnt = REG(PCIR_MINGNT, 1);
710 cfg->maxlat = REG(PCIR_MAXLAT, 1);
711 cfg->nummaps = PCI_MAXMAPS_0;
712 break;
713 case PCIM_HDRTYPE_BRIDGE:
714 cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1);
715 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1);
716 cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1);
717 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1);
718 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2);
719 cfg->nummaps = PCI_MAXMAPS_1;
720 break;
721 case PCIM_HDRTYPE_CARDBUS:
722 cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1);
723 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1);
724 cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1);
725 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1);
726 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2);
727 cfg->subvendor = REG(PCIR_SUBVEND_2, 2);
728 cfg->subdevice = REG(PCIR_SUBDEV_2, 2);
729 cfg->nummaps = PCI_MAXMAPS_2;
730 break;
731 }
732 #undef REG
733 }
734
735 /* read configuration header into pcicfgregs structure */
736 struct pci_devinfo *
pci_read_device(device_t pcib,device_t bus,int d,int b,int s,int f)737 pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f)
738 {
739 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
740 uint16_t vid, did;
741
742 vid = REG(PCIR_VENDOR, 2);
743 if (vid == PCIV_INVALID)
744 return (NULL);
745
746 did = REG(PCIR_DEVICE, 2);
747
748 return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did));
749 }
750
751 struct pci_devinfo *
pci_alloc_devinfo_method(device_t dev)752 pci_alloc_devinfo_method(device_t dev)
753 {
754
755 return (malloc(sizeof(struct pci_devinfo), M_DEVBUF,
756 M_WAITOK | M_ZERO));
757 }
758
759 static struct pci_devinfo *
pci_fill_devinfo(device_t pcib,device_t bus,int d,int b,int s,int f,uint16_t vid,uint16_t did)760 pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f,
761 uint16_t vid, uint16_t did)
762 {
763 struct pci_devinfo *devlist_entry;
764 pcicfgregs *cfg;
765
766 devlist_entry = PCI_ALLOC_DEVINFO(bus);
767
768 cfg = &devlist_entry->cfg;
769
770 cfg->domain = d;
771 cfg->bus = b;
772 cfg->slot = s;
773 cfg->func = f;
774 cfg->vendor = vid;
775 cfg->device = did;
776 cfg->cmdreg = REG(PCIR_COMMAND, 2);
777 cfg->statreg = REG(PCIR_STATUS, 2);
778 cfg->baseclass = REG(PCIR_CLASS, 1);
779 cfg->subclass = REG(PCIR_SUBCLASS, 1);
780 cfg->progif = REG(PCIR_PROGIF, 1);
781 cfg->revid = REG(PCIR_REVID, 1);
782 cfg->hdrtype = REG(PCIR_HDRTYPE, 1);
783 cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1);
784 cfg->lattimer = REG(PCIR_LATTIMER, 1);
785 cfg->intpin = REG(PCIR_INTPIN, 1);
786 cfg->intline = REG(PCIR_INTLINE, 1);
787
788 cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0;
789 cfg->hdrtype &= ~PCIM_MFDEV;
790 STAILQ_INIT(&cfg->maps);
791
792 cfg->iov = NULL;
793
794 pci_fixancient(cfg);
795 pci_hdrtypedata(pcib, b, s, f, cfg);
796
797 if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)
798 pci_read_cap(pcib, cfg);
799
800 STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links);
801
802 devlist_entry->conf.pc_sel.pc_domain = cfg->domain;
803 devlist_entry->conf.pc_sel.pc_bus = cfg->bus;
804 devlist_entry->conf.pc_sel.pc_dev = cfg->slot;
805 devlist_entry->conf.pc_sel.pc_func = cfg->func;
806 devlist_entry->conf.pc_hdr = cfg->hdrtype;
807
808 devlist_entry->conf.pc_subvendor = cfg->subvendor;
809 devlist_entry->conf.pc_subdevice = cfg->subdevice;
810 devlist_entry->conf.pc_vendor = cfg->vendor;
811 devlist_entry->conf.pc_device = cfg->device;
812
813 devlist_entry->conf.pc_class = cfg->baseclass;
814 devlist_entry->conf.pc_subclass = cfg->subclass;
815 devlist_entry->conf.pc_progif = cfg->progif;
816 devlist_entry->conf.pc_revid = cfg->revid;
817
818 devlist_entry->conf.pc_secbus = cfg->bridge.br_secbus;
819 devlist_entry->conf.pc_subbus = cfg->bridge.br_subbus;
820
821 pci_numdevs++;
822 pci_generation++;
823
824 return (devlist_entry);
825 }
826 #undef REG
827
828 static void
pci_ea_fill_info(device_t pcib,pcicfgregs * cfg)829 pci_ea_fill_info(device_t pcib, pcicfgregs *cfg)
830 {
831 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \
832 cfg->ea.ea_location + (n), w)
833 int num_ent;
834 int ptr;
835 int a, b;
836 uint32_t val;
837 int ent_size;
838 uint32_t dw[4];
839 uint64_t base, max_offset;
840 struct pci_ea_entry *eae;
841
842 if (cfg->ea.ea_location == 0)
843 return;
844
845 STAILQ_INIT(&cfg->ea.ea_entries);
846
847 /* Determine the number of entries */
848 num_ent = REG(PCIR_EA_NUM_ENT, 2);
849 num_ent &= PCIM_EA_NUM_ENT_MASK;
850
851 /* Find the first entry to care of */
852 ptr = PCIR_EA_FIRST_ENT;
853
854 /* Skip DWORD 2 for type 1 functions */
855 if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE)
856 ptr += 4;
857
858 for (a = 0; a < num_ent; a++) {
859 eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO);
860 eae->eae_cfg_offset = cfg->ea.ea_location + ptr;
861
862 /* Read a number of dwords in the entry */
863 val = REG(ptr, 4);
864 ptr += 4;
865 ent_size = (val & PCIM_EA_ES);
866
867 for (b = 0; b < ent_size; b++) {
868 dw[b] = REG(ptr, 4);
869 ptr += 4;
870 }
871
872 eae->eae_flags = val;
873 eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET;
874
875 base = dw[0] & PCIM_EA_FIELD_MASK;
876 max_offset = dw[1] | ~PCIM_EA_FIELD_MASK;
877 b = 2;
878 if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) {
879 base |= (uint64_t)dw[b] << 32UL;
880 b++;
881 }
882 if (((dw[1] & PCIM_EA_IS_64) != 0)
883 && (b < ent_size)) {
884 max_offset |= (uint64_t)dw[b] << 32UL;
885 b++;
886 }
887
888 eae->eae_base = base;
889 eae->eae_max_offset = max_offset;
890
891 STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link);
892
893 if (bootverbose) {
894 printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n",
895 cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags,
896 (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset);
897 }
898 }
899 }
900 #undef REG
901
902 static void
pci_read_cap(device_t pcib,pcicfgregs * cfg)903 pci_read_cap(device_t pcib, pcicfgregs *cfg)
904 {
905 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w)
906 #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w)
907 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
908 uint64_t addr;
909 #endif
910 uint32_t val;
911 int ptr, nextptr, ptrptr;
912
913 switch (cfg->hdrtype & PCIM_HDRTYPE) {
914 case PCIM_HDRTYPE_NORMAL:
915 case PCIM_HDRTYPE_BRIDGE:
916 ptrptr = PCIR_CAP_PTR;
917 break;
918 case PCIM_HDRTYPE_CARDBUS:
919 ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */
920 break;
921 default:
922 return; /* no extended capabilities support */
923 }
924 nextptr = REG(ptrptr, 1); /* sanity check? */
925
926 /*
927 * Read capability entries.
928 */
929 while (nextptr != 0) {
930 /* Sanity check */
931 if (nextptr > 255) {
932 printf("illegal PCI extended capability offset %d\n",
933 nextptr);
934 return;
935 }
936 /* Find the next entry */
937 ptr = nextptr;
938 nextptr = REG(ptr + PCICAP_NEXTPTR, 1);
939
940 /* Process this entry */
941 switch (REG(ptr + PCICAP_ID, 1)) {
942 case PCIY_PMG: /* PCI power management */
943 cfg->pp.pp_location = ptr;
944 cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2);
945 break;
946 case PCIY_HT: /* HyperTransport */
947 /* Determine HT-specific capability type. */
948 val = REG(ptr + PCIR_HT_COMMAND, 2);
949
950 if ((val & 0xe000) == PCIM_HTCAP_SLAVE)
951 cfg->ht.ht_slave = ptr;
952
953 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
954 switch (val & PCIM_HTCMD_CAP_MASK) {
955 case PCIM_HTCAP_MSI_MAPPING:
956 if (!(val & PCIM_HTCMD_MSI_FIXED)) {
957 /* Sanity check the mapping window. */
958 addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI,
959 4);
960 addr <<= 32;
961 addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO,
962 4);
963 if (addr != MSI_INTEL_ADDR_BASE)
964 device_printf(pcib,
965 "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n",
966 cfg->domain, cfg->bus,
967 cfg->slot, cfg->func,
968 (long long)addr);
969 } else
970 addr = MSI_INTEL_ADDR_BASE;
971
972 cfg->ht.ht_msimap = ptr;
973 cfg->ht.ht_msictrl = val;
974 cfg->ht.ht_msiaddr = addr;
975 break;
976 }
977 #endif
978 break;
979 case PCIY_MSI: /* PCI MSI */
980 cfg->msi.msi_location = ptr;
981 cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2);
982 break;
983 case PCIY_MSIX: /* PCI MSI-X */
984 cfg->msix.msix_location = ptr;
985 cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2);
986 val = REG(ptr + PCIR_MSIX_TABLE, 4);
987 cfg->msix.msix_table_bar = PCIR_BAR(val &
988 PCIM_MSIX_BIR_MASK);
989 cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK;
990 val = REG(ptr + PCIR_MSIX_PBA, 4);
991 cfg->msix.msix_pba_bar = PCIR_BAR(val &
992 PCIM_MSIX_BIR_MASK);
993 cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
994 break;
995 case PCIY_VPD: /* PCI Vital Product Data */
996 cfg->vpd.vpd_reg = ptr;
997 break;
998 case PCIY_SUBVENDOR:
999 /* Should always be true. */
1000 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
1001 PCIM_HDRTYPE_BRIDGE) {
1002 val = REG(ptr + PCIR_SUBVENDCAP_ID, 4);
1003 cfg->subvendor = val & 0xffff;
1004 cfg->subdevice = val >> 16;
1005 }
1006 break;
1007 case PCIY_PCIX: /* PCI-X */
1008 /*
1009 * Assume we have a PCI-X chipset if we have
1010 * at least one PCI-PCI bridge with a PCI-X
1011 * capability. Note that some systems with
1012 * PCI-express or HT chipsets might match on
1013 * this check as well.
1014 */
1015 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
1016 PCIM_HDRTYPE_BRIDGE)
1017 pcix_chipset = 1;
1018 cfg->pcix.pcix_location = ptr;
1019 break;
1020 case PCIY_EXPRESS: /* PCI-express */
1021 /*
1022 * Assume we have a PCI-express chipset if we have
1023 * at least one PCI-express device.
1024 */
1025 pcie_chipset = 1;
1026 cfg->pcie.pcie_location = ptr;
1027 val = REG(ptr + PCIER_FLAGS, 2);
1028 cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE;
1029 break;
1030 case PCIY_EA: /* Enhanced Allocation */
1031 cfg->ea.ea_location = ptr;
1032 pci_ea_fill_info(pcib, cfg);
1033 break;
1034 default:
1035 break;
1036 }
1037 }
1038
1039 #if defined(__powerpc__)
1040 /*
1041 * Enable the MSI mapping window for all HyperTransport
1042 * slaves. PCI-PCI bridges have their windows enabled via
1043 * PCIB_MAP_MSI().
1044 */
1045 if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 &&
1046 !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
1047 device_printf(pcib,
1048 "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n",
1049 cfg->domain, cfg->bus, cfg->slot, cfg->func);
1050 cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
1051 WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl,
1052 2);
1053 }
1054 #endif
1055 /* REG and WREG use carry through to next functions */
1056 }
1057
1058 /*
1059 * PCI Vital Product Data
1060 */
1061
1062 #define PCI_VPD_TIMEOUT 1000000
1063
1064 static int
pci_read_vpd_reg(device_t pcib,pcicfgregs * cfg,int reg,uint32_t * data)1065 pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data)
1066 {
1067 int count = PCI_VPD_TIMEOUT;
1068
1069 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1070
1071 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2);
1072
1073 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) {
1074 if (--count < 0)
1075 return (ENXIO);
1076 DELAY(1); /* limit looping */
1077 }
1078 *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4));
1079
1080 return (0);
1081 }
1082
1083 #if 0
1084 static int
1085 pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data)
1086 {
1087 int count = PCI_VPD_TIMEOUT;
1088
1089 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1090
1091 WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4);
1092 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2);
1093 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) {
1094 if (--count < 0)
1095 return (ENXIO);
1096 DELAY(1); /* limit looping */
1097 }
1098
1099 return (0);
1100 }
1101 #endif
1102
1103 #undef PCI_VPD_TIMEOUT
1104
1105 struct vpd_readstate {
1106 device_t pcib;
1107 pcicfgregs *cfg;
1108 uint32_t val;
1109 int bytesinval;
1110 int off;
1111 uint8_t cksum;
1112 };
1113
1114 /* return 0 and one byte in *data if no read error, -1 else */
1115 static int
vpd_nextbyte(struct vpd_readstate * vrs,uint8_t * data)1116 vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data)
1117 {
1118 uint32_t reg;
1119 uint8_t byte;
1120
1121 if (vrs->bytesinval == 0) {
1122 if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®))
1123 return (-1);
1124 vrs->val = le32toh(reg);
1125 vrs->off += 4;
1126 byte = vrs->val & 0xff;
1127 vrs->bytesinval = 3;
1128 } else {
1129 vrs->val = vrs->val >> 8;
1130 byte = vrs->val & 0xff;
1131 vrs->bytesinval--;
1132 }
1133
1134 vrs->cksum += byte;
1135 *data = byte;
1136 return (0);
1137 }
1138
1139 /* return 0 on match, -1 and "unget" byte on no match */
1140 static int
vpd_expectbyte(struct vpd_readstate * vrs,uint8_t expected)1141 vpd_expectbyte(struct vpd_readstate *vrs, uint8_t expected)
1142 {
1143 uint8_t data;
1144
1145 if (vpd_nextbyte(vrs, &data) != 0)
1146 return (-1);
1147
1148 if (data == expected)
1149 return (0);
1150
1151 vrs->cksum -= data;
1152 vrs->val = (vrs->val << 8) + data;
1153 vrs->bytesinval++;
1154 return (-1);
1155 }
1156
1157 /* return size if tag matches, -1 on no match, -2 on read error */
1158 static int
vpd_read_tag_size(struct vpd_readstate * vrs,uint8_t vpd_tag)1159 vpd_read_tag_size(struct vpd_readstate *vrs, uint8_t vpd_tag)
1160 {
1161 uint8_t byte1, byte2;
1162
1163 if (vpd_expectbyte(vrs, vpd_tag) != 0)
1164 return (-1);
1165
1166 if ((vpd_tag & 0x80) == 0)
1167 return (vpd_tag & 0x07);
1168
1169 if (vpd_nextbyte(vrs, &byte1) != 0)
1170 return (-2);
1171 if (vpd_nextbyte(vrs, &byte2) != 0)
1172 return (-2);
1173
1174 return ((byte2 << 8) + byte1);
1175 }
1176
1177 /* (re)allocate buffer in multiples of 8 elements */
1178 static void*
alloc_buffer(void * buffer,size_t element_size,int needed)1179 alloc_buffer(void* buffer, size_t element_size, int needed)
1180 {
1181 int alloc, new_alloc;
1182
1183 alloc = roundup2(needed, 8);
1184 new_alloc = roundup2(needed + 1, 8);
1185 if (alloc != new_alloc) {
1186 buffer = reallocf(buffer,
1187 new_alloc * element_size, M_DEVBUF, M_WAITOK | M_ZERO);
1188 }
1189
1190 return (buffer);
1191 }
1192
1193 /* read VPD keyword and return element size, return -1 on read error */
1194 static int
vpd_read_elem_head(struct vpd_readstate * vrs,char keyword[2])1195 vpd_read_elem_head(struct vpd_readstate *vrs, char keyword[2])
1196 {
1197 uint8_t data;
1198
1199 if (vpd_nextbyte(vrs, &keyword[0]) != 0)
1200 return (-1);
1201 if (vpd_nextbyte(vrs, &keyword[1]) != 0)
1202 return (-1);
1203 if (vpd_nextbyte(vrs, &data) != 0)
1204 return (-1);
1205
1206 return (data);
1207 }
1208
1209 /* read VPD data element of given size into allocated buffer */
1210 static char *
vpd_read_value(struct vpd_readstate * vrs,int size)1211 vpd_read_value(struct vpd_readstate *vrs, int size)
1212 {
1213 int i;
1214 char char1;
1215 char *value;
1216
1217 value = malloc(size + 1, M_DEVBUF, M_WAITOK);
1218 for (i = 0; i < size; i++) {
1219 if (vpd_nextbyte(vrs, &char1) != 0) {
1220 free(value, M_DEVBUF);
1221 return (NULL);
1222 }
1223 value[i] = char1;
1224 }
1225 value[size] = '\0';
1226
1227 return (value);
1228 }
1229
1230 /* read VPD into *keyword and *value, return length of data element */
1231 static int
vpd_read_elem_data(struct vpd_readstate * vrs,char keyword[2],char ** value,int maxlen)1232 vpd_read_elem_data(struct vpd_readstate *vrs, char keyword[2], char **value, int maxlen)
1233 {
1234 int len;
1235
1236 len = vpd_read_elem_head(vrs, keyword);
1237 if (len < 0 || len > maxlen)
1238 return (-1);
1239 *value = vpd_read_value(vrs, len);
1240
1241 return (len);
1242 }
1243
1244 /* subtract all data following first byte from checksum of RV element */
1245 static void
vpd_fixup_cksum(struct vpd_readstate * vrs,char * rvstring,int len)1246 vpd_fixup_cksum(struct vpd_readstate *vrs, char *rvstring, int len)
1247 {
1248 int i;
1249 uint8_t fixup;
1250
1251 fixup = 0;
1252 for (i = 1; i < len; i++)
1253 fixup += rvstring[i];
1254 vrs->cksum -= fixup;
1255 }
1256
1257 /* fetch one read-only element and return size of heading + data */
1258 static int
next_vpd_ro_elem(struct vpd_readstate * vrs,int maxsize)1259 next_vpd_ro_elem(struct vpd_readstate *vrs, int maxsize)
1260 {
1261 struct pcicfg_vpd *vpd;
1262 pcicfgregs *cfg;
1263 struct vpd_readonly *vpd_ros;
1264 int len;
1265
1266 cfg = vrs->cfg;
1267 vpd = &cfg->vpd;
1268
1269 if (maxsize < 3)
1270 return (-1);
1271 vpd->vpd_ros = alloc_buffer(vpd->vpd_ros, sizeof(*vpd->vpd_ros), vpd->vpd_rocnt);
1272 vpd_ros = &vpd->vpd_ros[vpd->vpd_rocnt];
1273 maxsize -= 3;
1274 len = vpd_read_elem_data(vrs, vpd_ros->keyword, &vpd_ros->value, maxsize);
1275 if (vpd_ros->value == NULL)
1276 return (-1);
1277 vpd_ros->len = len;
1278 if (vpd_ros->keyword[0] == 'R' && vpd_ros->keyword[1] == 'V') {
1279 vpd_fixup_cksum(vrs, vpd_ros->value, len);
1280 if (vrs->cksum != 0) {
1281 pci_printf(cfg,
1282 "invalid VPD checksum %#hhx\n", vrs->cksum);
1283 return (-1);
1284 }
1285 }
1286 vpd->vpd_rocnt++;
1287
1288 return (len + 3);
1289 }
1290
1291 /* fetch one writable element and return size of heading + data */
1292 static int
next_vpd_rw_elem(struct vpd_readstate * vrs,int maxsize)1293 next_vpd_rw_elem(struct vpd_readstate *vrs, int maxsize)
1294 {
1295 struct pcicfg_vpd *vpd;
1296 pcicfgregs *cfg;
1297 struct vpd_write *vpd_w;
1298 int len;
1299
1300 cfg = vrs->cfg;
1301 vpd = &cfg->vpd;
1302
1303 if (maxsize < 3)
1304 return (-1);
1305 vpd->vpd_w = alloc_buffer(vpd->vpd_w, sizeof(*vpd->vpd_w), vpd->vpd_wcnt);
1306 if (vpd->vpd_w == NULL) {
1307 pci_printf(cfg, "out of memory");
1308 return (-1);
1309 }
1310 vpd_w = &vpd->vpd_w[vpd->vpd_wcnt];
1311 maxsize -= 3;
1312 vpd_w->start = vrs->off + 3 - vrs->bytesinval;
1313 len = vpd_read_elem_data(vrs, vpd_w->keyword, &vpd_w->value, maxsize);
1314 if (vpd_w->value == NULL)
1315 return (-1);
1316 vpd_w->len = len;
1317 vpd->vpd_wcnt++;
1318
1319 return (len + 3);
1320 }
1321
1322 /* free all memory allocated for VPD data */
1323 static void
vpd_free(struct pcicfg_vpd * vpd)1324 vpd_free(struct pcicfg_vpd *vpd)
1325 {
1326 int i;
1327
1328 free(vpd->vpd_ident, M_DEVBUF);
1329 for (i = 0; i < vpd->vpd_rocnt; i++)
1330 free(vpd->vpd_ros[i].value, M_DEVBUF);
1331 free(vpd->vpd_ros, M_DEVBUF);
1332 vpd->vpd_rocnt = 0;
1333 for (i = 0; i < vpd->vpd_wcnt; i++)
1334 free(vpd->vpd_w[i].value, M_DEVBUF);
1335 free(vpd->vpd_w, M_DEVBUF);
1336 vpd->vpd_wcnt = 0;
1337 }
1338
1339 #define VPD_TAG_END ((0x0f << 3) | 0) /* small tag, len == 0 */
1340 #define VPD_TAG_IDENT (0x02 | 0x80) /* large tag */
1341 #define VPD_TAG_RO (0x10 | 0x80) /* large tag */
1342 #define VPD_TAG_RW (0x11 | 0x80) /* large tag */
1343
1344 static int
pci_parse_vpd(device_t pcib,pcicfgregs * cfg)1345 pci_parse_vpd(device_t pcib, pcicfgregs *cfg)
1346 {
1347 struct vpd_readstate vrs;
1348 int cksumvalid;
1349 int size, elem_size;
1350
1351 /* init vpd reader */
1352 vrs.bytesinval = 0;
1353 vrs.off = 0;
1354 vrs.pcib = pcib;
1355 vrs.cfg = cfg;
1356 vrs.cksum = 0;
1357
1358 /* read VPD ident element - mandatory */
1359 size = vpd_read_tag_size(&vrs, VPD_TAG_IDENT);
1360 if (size <= 0) {
1361 pci_printf(cfg, "no VPD ident found\n");
1362 return (0);
1363 }
1364 cfg->vpd.vpd_ident = vpd_read_value(&vrs, size);
1365 if (cfg->vpd.vpd_ident == NULL) {
1366 pci_printf(cfg, "error accessing VPD ident data\n");
1367 return (0);
1368 }
1369
1370 /* read VPD RO elements - mandatory */
1371 size = vpd_read_tag_size(&vrs, VPD_TAG_RO);
1372 if (size <= 0) {
1373 pci_printf(cfg, "no read-only VPD data found\n");
1374 return (0);
1375 }
1376 while (size > 0) {
1377 elem_size = next_vpd_ro_elem(&vrs, size);
1378 if (elem_size < 0) {
1379 pci_printf(cfg, "error accessing read-only VPD data\n");
1380 return (-1);
1381 }
1382 size -= elem_size;
1383 }
1384 cksumvalid = (vrs.cksum == 0);
1385 if (!cksumvalid)
1386 return (-1);
1387
1388 /* read VPD RW elements - optional */
1389 size = vpd_read_tag_size(&vrs, VPD_TAG_RW);
1390 if (size == -2)
1391 return (-1);
1392 while (size > 0) {
1393 elem_size = next_vpd_rw_elem(&vrs, size);
1394 if (elem_size < 0) {
1395 pci_printf(cfg, "error accessing writeable VPD data\n");
1396 return (-1);
1397 }
1398 size -= elem_size;
1399 }
1400
1401 /* read empty END tag - mandatory */
1402 size = vpd_read_tag_size(&vrs, VPD_TAG_END);
1403 if (size != 0) {
1404 pci_printf(cfg, "No valid VPD end tag found\n");
1405 }
1406 return (0);
1407 }
1408
1409 static void
pci_read_vpd(device_t pcib,pcicfgregs * cfg)1410 pci_read_vpd(device_t pcib, pcicfgregs *cfg)
1411 {
1412 int status;
1413
1414 status = pci_parse_vpd(pcib, cfg);
1415 if (status < 0)
1416 vpd_free(&cfg->vpd);
1417 cfg->vpd.vpd_cached = 1;
1418 #undef REG
1419 #undef WREG
1420 }
1421
1422 int
pci_get_vpd_ident_method(device_t dev,device_t child,const char ** identptr)1423 pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr)
1424 {
1425 struct pci_devinfo *dinfo = device_get_ivars(child);
1426 pcicfgregs *cfg = &dinfo->cfg;
1427
1428 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1429 pci_read_vpd(device_get_parent(dev), cfg);
1430
1431 *identptr = cfg->vpd.vpd_ident;
1432
1433 if (*identptr == NULL)
1434 return (ENXIO);
1435
1436 return (0);
1437 }
1438
1439 int
pci_get_vpd_readonly_method(device_t dev,device_t child,const char * kw,const char ** vptr)1440 pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
1441 const char **vptr)
1442 {
1443 struct pci_devinfo *dinfo = device_get_ivars(child);
1444 pcicfgregs *cfg = &dinfo->cfg;
1445 int i;
1446
1447 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1448 pci_read_vpd(device_get_parent(dev), cfg);
1449
1450 for (i = 0; i < cfg->vpd.vpd_rocnt; i++)
1451 if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword,
1452 sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) {
1453 *vptr = cfg->vpd.vpd_ros[i].value;
1454 return (0);
1455 }
1456
1457 *vptr = NULL;
1458 return (ENXIO);
1459 }
1460
1461 struct pcicfg_vpd *
pci_fetch_vpd_list(device_t dev)1462 pci_fetch_vpd_list(device_t dev)
1463 {
1464 struct pci_devinfo *dinfo = device_get_ivars(dev);
1465 pcicfgregs *cfg = &dinfo->cfg;
1466
1467 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1468 pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg);
1469 return (&cfg->vpd);
1470 }
1471
1472 /*
1473 * Find the requested HyperTransport capability and return the offset
1474 * in configuration space via the pointer provided. The function
1475 * returns 0 on success and an error code otherwise.
1476 */
1477 int
pci_find_htcap_method(device_t dev,device_t child,int capability,int * capreg)1478 pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg)
1479 {
1480 int ptr, error;
1481 uint16_t val;
1482
1483 error = pci_find_cap(child, PCIY_HT, &ptr);
1484 if (error)
1485 return (error);
1486
1487 /*
1488 * Traverse the capabilities list checking each HT capability
1489 * to see if it matches the requested HT capability.
1490 */
1491 for (;;) {
1492 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1493 if (capability == PCIM_HTCAP_SLAVE ||
1494 capability == PCIM_HTCAP_HOST)
1495 val &= 0xe000;
1496 else
1497 val &= PCIM_HTCMD_CAP_MASK;
1498 if (val == capability) {
1499 if (capreg != NULL)
1500 *capreg = ptr;
1501 return (0);
1502 }
1503
1504 /* Skip to the next HT capability. */
1505 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1506 break;
1507 }
1508
1509 return (ENOENT);
1510 }
1511
1512 /*
1513 * Find the next requested HyperTransport capability after start and return
1514 * the offset in configuration space via the pointer provided. The function
1515 * returns 0 on success and an error code otherwise.
1516 */
1517 int
pci_find_next_htcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1518 pci_find_next_htcap_method(device_t dev, device_t child, int capability,
1519 int start, int *capreg)
1520 {
1521 int ptr;
1522 uint16_t val;
1523
1524 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT,
1525 ("start capability is not HyperTransport capability"));
1526 ptr = start;
1527
1528 /*
1529 * Traverse the capabilities list checking each HT capability
1530 * to see if it matches the requested HT capability.
1531 */
1532 for (;;) {
1533 /* Skip to the next HT capability. */
1534 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1535 break;
1536
1537 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1538 if (capability == PCIM_HTCAP_SLAVE ||
1539 capability == PCIM_HTCAP_HOST)
1540 val &= 0xe000;
1541 else
1542 val &= PCIM_HTCMD_CAP_MASK;
1543 if (val == capability) {
1544 if (capreg != NULL)
1545 *capreg = ptr;
1546 return (0);
1547 }
1548 }
1549
1550 return (ENOENT);
1551 }
1552
1553 /*
1554 * Find the requested capability and return the offset in
1555 * configuration space via the pointer provided. The function returns
1556 * 0 on success and an error code otherwise.
1557 */
1558 int
pci_find_cap_method(device_t dev,device_t child,int capability,int * capreg)1559 pci_find_cap_method(device_t dev, device_t child, int capability,
1560 int *capreg)
1561 {
1562 struct pci_devinfo *dinfo = device_get_ivars(child);
1563 pcicfgregs *cfg = &dinfo->cfg;
1564 uint32_t status;
1565 uint8_t ptr;
1566 int cnt;
1567
1568 /*
1569 * Check the CAP_LIST bit of the PCI status register first.
1570 */
1571 status = pci_read_config(child, PCIR_STATUS, 2);
1572 if (!(status & PCIM_STATUS_CAPPRESENT))
1573 return (ENXIO);
1574
1575 /*
1576 * Determine the start pointer of the capabilities list.
1577 */
1578 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1579 case PCIM_HDRTYPE_NORMAL:
1580 case PCIM_HDRTYPE_BRIDGE:
1581 ptr = PCIR_CAP_PTR;
1582 break;
1583 case PCIM_HDRTYPE_CARDBUS:
1584 ptr = PCIR_CAP_PTR_2;
1585 break;
1586 default:
1587 /* XXX: panic? */
1588 return (ENXIO); /* no extended capabilities support */
1589 }
1590 ptr = pci_read_config(child, ptr, 1);
1591
1592 /*
1593 * Traverse the capabilities list. Limit by total theoretical
1594 * maximum number of caps: capability needs at least id and
1595 * next registers, and any type X header cannot contain caps.
1596 */
1597 for (cnt = 0; ptr != 0 && cnt < (PCIE_REGMAX - 0x40) / 2; cnt++) {
1598 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1599 if (capreg != NULL)
1600 *capreg = ptr;
1601 return (0);
1602 }
1603 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1604 }
1605
1606 return (ENOENT);
1607 }
1608
1609 /*
1610 * Find the next requested capability after start and return the offset in
1611 * configuration space via the pointer provided. The function returns
1612 * 0 on success and an error code otherwise.
1613 */
1614 int
pci_find_next_cap_method(device_t dev,device_t child,int capability,int start,int * capreg)1615 pci_find_next_cap_method(device_t dev, device_t child, int capability,
1616 int start, int *capreg)
1617 {
1618 uint8_t ptr;
1619
1620 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability,
1621 ("start capability is not expected capability"));
1622
1623 ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1);
1624 while (ptr != 0) {
1625 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1626 if (capreg != NULL)
1627 *capreg = ptr;
1628 return (0);
1629 }
1630 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1631 }
1632
1633 return (ENOENT);
1634 }
1635
1636 /*
1637 * Find the requested extended capability and return the offset in
1638 * configuration space via the pointer provided. The function returns
1639 * 0 on success and an error code otherwise.
1640 */
1641 int
pci_find_extcap_method(device_t dev,device_t child,int capability,int * capreg)1642 pci_find_extcap_method(device_t dev, device_t child, int capability,
1643 int *capreg)
1644 {
1645 struct pci_devinfo *dinfo = device_get_ivars(child);
1646 pcicfgregs *cfg = &dinfo->cfg;
1647 uint32_t ecap;
1648 uint16_t ptr;
1649
1650 /* Only supported for PCI-express devices. */
1651 if (cfg->pcie.pcie_location == 0)
1652 return (ENXIO);
1653
1654 ptr = PCIR_EXTCAP;
1655 ecap = pci_read_config(child, ptr, 4);
1656 if (ecap == 0xffffffff || ecap == 0)
1657 return (ENOENT);
1658 for (;;) {
1659 if (PCI_EXTCAP_ID(ecap) == capability) {
1660 if (capreg != NULL)
1661 *capreg = ptr;
1662 return (0);
1663 }
1664 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1665 if (ptr == 0)
1666 break;
1667 ecap = pci_read_config(child, ptr, 4);
1668 }
1669
1670 return (ENOENT);
1671 }
1672
1673 /*
1674 * Find the next requested extended capability after start and return the
1675 * offset in configuration space via the pointer provided. The function
1676 * returns 0 on success and an error code otherwise.
1677 */
1678 int
pci_find_next_extcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1679 pci_find_next_extcap_method(device_t dev, device_t child, int capability,
1680 int start, int *capreg)
1681 {
1682 struct pci_devinfo *dinfo = device_get_ivars(child);
1683 pcicfgregs *cfg = &dinfo->cfg;
1684 uint32_t ecap;
1685 uint16_t ptr;
1686
1687 /* Only supported for PCI-express devices. */
1688 if (cfg->pcie.pcie_location == 0)
1689 return (ENXIO);
1690
1691 ecap = pci_read_config(child, start, 4);
1692 KASSERT(PCI_EXTCAP_ID(ecap) == capability,
1693 ("start extended capability is not expected capability"));
1694 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1695 while (ptr != 0) {
1696 ecap = pci_read_config(child, ptr, 4);
1697 if (PCI_EXTCAP_ID(ecap) == capability) {
1698 if (capreg != NULL)
1699 *capreg = ptr;
1700 return (0);
1701 }
1702 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1703 }
1704
1705 return (ENOENT);
1706 }
1707
1708 /*
1709 * Support for MSI-X message interrupts.
1710 */
1711 static void
pci_write_msix_entry(device_t dev,u_int index,uint64_t address,uint32_t data)1712 pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data)
1713 {
1714 struct pci_devinfo *dinfo = device_get_ivars(dev);
1715 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1716 uint32_t offset;
1717
1718 KASSERT(msix->msix_table_len > index, ("bogus index"));
1719 offset = msix->msix_table_offset + index * 16;
1720 bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
1721 bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
1722 bus_write_4(msix->msix_table_res, offset + 8, data);
1723 }
1724
1725 void
pci_enable_msix_method(device_t dev,device_t child,u_int index,uint64_t address,uint32_t data)1726 pci_enable_msix_method(device_t dev, device_t child, u_int index,
1727 uint64_t address, uint32_t data)
1728 {
1729
1730 if (pci_msix_rewrite_table) {
1731 struct pci_devinfo *dinfo = device_get_ivars(child);
1732 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1733
1734 /*
1735 * Some VM hosts require MSIX to be disabled in the
1736 * control register before updating the MSIX table
1737 * entries are allowed. It is not enough to only
1738 * disable MSIX while updating a single entry. MSIX
1739 * must be disabled while updating all entries in the
1740 * table.
1741 */
1742 pci_write_config(child,
1743 msix->msix_location + PCIR_MSIX_CTRL,
1744 msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
1745 pci_resume_msix(child);
1746 } else
1747 pci_write_msix_entry(child, index, address, data);
1748
1749 /* Enable MSI -> HT mapping. */
1750 pci_ht_map_msi(child, address);
1751 }
1752
1753 void
pci_mask_msix(device_t dev,u_int index)1754 pci_mask_msix(device_t dev, u_int index)
1755 {
1756 struct pci_devinfo *dinfo = device_get_ivars(dev);
1757 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1758 uint32_t offset, val;
1759
1760 KASSERT(PCI_MSIX_MSGNUM(msix->msix_ctrl) > index, ("bogus index"));
1761 offset = msix->msix_table_offset + index * 16 + 12;
1762 val = bus_read_4(msix->msix_table_res, offset);
1763 val |= PCIM_MSIX_VCTRL_MASK;
1764
1765 /*
1766 * Some devices (e.g. Samsung PM961) do not support reads of this
1767 * register, so always write the new value.
1768 */
1769 bus_write_4(msix->msix_table_res, offset, val);
1770 }
1771
1772 void
pci_unmask_msix(device_t dev,u_int index)1773 pci_unmask_msix(device_t dev, u_int index)
1774 {
1775 struct pci_devinfo *dinfo = device_get_ivars(dev);
1776 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1777 uint32_t offset, val;
1778
1779 KASSERT(PCI_MSIX_MSGNUM(msix->msix_ctrl) > index, ("bogus index"));
1780 offset = msix->msix_table_offset + index * 16 + 12;
1781 val = bus_read_4(msix->msix_table_res, offset);
1782 val &= ~PCIM_MSIX_VCTRL_MASK;
1783
1784 /*
1785 * Some devices (e.g. Samsung PM961) do not support reads of this
1786 * register, so always write the new value.
1787 */
1788 bus_write_4(msix->msix_table_res, offset, val);
1789 }
1790
1791 int
pci_pending_msix(device_t dev,u_int index)1792 pci_pending_msix(device_t dev, u_int index)
1793 {
1794 struct pci_devinfo *dinfo = device_get_ivars(dev);
1795 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1796 uint32_t offset, bit;
1797
1798 KASSERT(msix->msix_table_len > index, ("bogus index"));
1799 offset = msix->msix_pba_offset + (index / 32) * 4;
1800 bit = 1 << index % 32;
1801 return (bus_read_4(msix->msix_pba_res, offset) & bit);
1802 }
1803
1804 /*
1805 * Restore MSI-X registers and table during resume. If MSI-X is
1806 * enabled then walk the virtual table to restore the actual MSI-X
1807 * table.
1808 */
1809 static void
pci_resume_msix(device_t dev)1810 pci_resume_msix(device_t dev)
1811 {
1812 struct pci_devinfo *dinfo = device_get_ivars(dev);
1813 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1814 struct msix_table_entry *mte;
1815 struct msix_vector *mv;
1816 u_int i, msgnum;
1817
1818 if (msix->msix_alloc > 0) {
1819 msgnum = PCI_MSIX_MSGNUM(msix->msix_ctrl);
1820
1821 /* First, mask all vectors. */
1822 for (i = 0; i < msgnum; i++)
1823 pci_mask_msix(dev, i);
1824
1825 /* Second, program any messages with at least one handler. */
1826 for (i = 0; i < msix->msix_table_len; i++) {
1827 mte = &msix->msix_table[i];
1828 if (mte->mte_vector == 0 || mte->mte_handlers == 0)
1829 continue;
1830 mv = &msix->msix_vectors[mte->mte_vector - 1];
1831 pci_write_msix_entry(dev, i, mv->mv_address,
1832 mv->mv_data);
1833 pci_unmask_msix(dev, i);
1834 }
1835 }
1836 pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
1837 msix->msix_ctrl, 2);
1838 }
1839
1840 /*
1841 * Attempt to allocate *count MSI-X messages. The actual number allocated is
1842 * returned in *count. After this function returns, each message will be
1843 * available to the driver as SYS_RES_IRQ resources starting at rid 1.
1844 */
1845 int
pci_alloc_msix_method(device_t dev,device_t child,int * count)1846 pci_alloc_msix_method(device_t dev, device_t child, int *count)
1847 {
1848 struct pci_devinfo *dinfo = device_get_ivars(child);
1849 pcicfgregs *cfg = &dinfo->cfg;
1850 struct resource_list_entry *rle;
1851 u_int actual, i, max;
1852 int error, irq;
1853 uint16_t ctrl, msgnum;
1854
1855 /* Don't let count == 0 get us into trouble. */
1856 if (*count < 1)
1857 return (EINVAL);
1858
1859 /* If rid 0 is allocated, then fail. */
1860 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
1861 if (rle != NULL && rle->res != NULL)
1862 return (ENXIO);
1863
1864 /* Already have allocated messages? */
1865 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
1866 return (ENXIO);
1867
1868 /* If MSI-X is blacklisted for this system, fail. */
1869 if (pci_msix_blacklisted())
1870 return (ENXIO);
1871
1872 /* MSI-X capability present? */
1873 if (cfg->msix.msix_location == 0 || !pci_do_msix)
1874 return (ENODEV);
1875
1876 /* Make sure the appropriate BARs are mapped. */
1877 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1878 cfg->msix.msix_table_bar);
1879 if (rle == NULL || rle->res == NULL ||
1880 !(rman_get_flags(rle->res) & RF_ACTIVE))
1881 return (ENXIO);
1882 cfg->msix.msix_table_res = rle->res;
1883 if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
1884 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1885 cfg->msix.msix_pba_bar);
1886 if (rle == NULL || rle->res == NULL ||
1887 !(rman_get_flags(rle->res) & RF_ACTIVE))
1888 return (ENXIO);
1889 }
1890 cfg->msix.msix_pba_res = rle->res;
1891
1892 ctrl = pci_read_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1893 2);
1894 msgnum = PCI_MSIX_MSGNUM(ctrl);
1895 if (bootverbose)
1896 device_printf(child,
1897 "attempting to allocate %d MSI-X vectors (%d supported)\n",
1898 *count, msgnum);
1899 max = min(*count, msgnum);
1900 for (i = 0; i < max; i++) {
1901 /* Allocate a message. */
1902 error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
1903 if (error) {
1904 if (i == 0)
1905 return (error);
1906 break;
1907 }
1908 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
1909 irq, 1);
1910 }
1911 actual = i;
1912
1913 if (bootverbose) {
1914 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
1915 if (actual == 1)
1916 device_printf(child, "using IRQ %ju for MSI-X\n",
1917 rle->start);
1918 else {
1919 bool run;
1920
1921 /*
1922 * Be fancy and try to print contiguous runs of
1923 * IRQ values as ranges. 'irq' is the previous IRQ.
1924 * 'run' is true if we are in a range.
1925 */
1926 device_printf(child, "using IRQs %ju", rle->start);
1927 irq = rle->start;
1928 run = false;
1929 for (i = 1; i < actual; i++) {
1930 rle = resource_list_find(&dinfo->resources,
1931 SYS_RES_IRQ, i + 1);
1932
1933 /* Still in a run? */
1934 if (rle->start == irq + 1) {
1935 run = true;
1936 irq++;
1937 continue;
1938 }
1939
1940 /* Finish previous range. */
1941 if (run) {
1942 printf("-%d", irq);
1943 run = false;
1944 }
1945
1946 /* Start new range. */
1947 printf(",%ju", rle->start);
1948 irq = rle->start;
1949 }
1950
1951 /* Unfinished range? */
1952 if (run)
1953 printf("-%d", irq);
1954 printf(" for MSI-X\n");
1955 }
1956 }
1957
1958 /*
1959 * Mask all vectors. Note that the message index assertion in
1960 * pci_mask_msix requires msix_ctrl to be set.
1961 */
1962 cfg->msix.msix_ctrl = ctrl;
1963 for (i = 0; i < msgnum; i++)
1964 pci_mask_msix(child, i);
1965
1966 /* Allocate and initialize vector data and virtual table. */
1967 cfg->msix.msix_vectors = mallocarray(actual, sizeof(struct msix_vector),
1968 M_DEVBUF, M_WAITOK | M_ZERO);
1969 cfg->msix.msix_table = mallocarray(actual,
1970 sizeof(struct msix_table_entry), M_DEVBUF, M_WAITOK | M_ZERO);
1971 for (i = 0; i < actual; i++) {
1972 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
1973 cfg->msix.msix_vectors[i].mv_irq = rle->start;
1974 cfg->msix.msix_table[i].mte_vector = i + 1;
1975 }
1976
1977 /* Update control register to enable MSI-X. */
1978 ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1979 pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1980 ctrl, 2);
1981 cfg->msix.msix_ctrl = ctrl;
1982
1983 /* Update counts of alloc'd messages. */
1984 cfg->msix.msix_alloc = actual;
1985 cfg->msix.msix_table_len = actual;
1986 *count = actual;
1987 return (0);
1988 }
1989
1990 /*
1991 * By default, pci_alloc_msix() will assign the allocated IRQ
1992 * resources consecutively to the first N messages in the MSI-X table.
1993 * However, device drivers may want to use different layouts if they
1994 * either receive fewer messages than they asked for, or they wish to
1995 * populate the MSI-X table sparsely. This method allows the driver
1996 * to specify what layout it wants. It must be called after a
1997 * successful pci_alloc_msix() but before any of the associated
1998 * SYS_RES_IRQ resources are allocated via bus_alloc_resource().
1999 *
2000 * The 'vectors' array contains 'count' message vectors. The array
2001 * maps directly to the MSI-X table in that index 0 in the array
2002 * specifies the vector for the first message in the MSI-X table, etc.
2003 * The vector value in each array index can either be 0 to indicate
2004 * that no vector should be assigned to a message slot, or it can be a
2005 * number from 1 to N (where N is the count returned from a
2006 * succcessful call to pci_alloc_msix()) to indicate which message
2007 * vector (IRQ) to be used for the corresponding message.
2008 *
2009 * On successful return, each message with a non-zero vector will have
2010 * an associated SYS_RES_IRQ whose rid is equal to the array index +
2011 * 1. Additionally, if any of the IRQs allocated via the previous
2012 * call to pci_alloc_msix() are not used in the mapping, those IRQs
2013 * will be freed back to the system automatically.
2014 *
2015 * For example, suppose a driver has a MSI-X table with 6 messages and
2016 * asks for 6 messages, but pci_alloc_msix() only returns a count of
2017 * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and
2018 * C. After the call to pci_alloc_msix(), the device will be setup to
2019 * have an MSI-X table of ABC--- (where - means no vector assigned).
2020 * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 },
2021 * then the MSI-X table will look like A-AB-B, and the 'C' vector will
2022 * be freed back to the system. This device will also have valid
2023 * SYS_RES_IRQ rids of 1, 3, 4, and 6.
2024 *
2025 * In any case, the SYS_RES_IRQ rid X will always map to the message
2026 * at MSI-X table index X - 1 and will only be valid if a vector is
2027 * assigned to that table entry.
2028 */
2029 int
pci_remap_msix_method(device_t dev,device_t child,int count,const u_int * vectors)2030 pci_remap_msix_method(device_t dev, device_t child, int count,
2031 const u_int *vectors)
2032 {
2033 struct pci_devinfo *dinfo = device_get_ivars(child);
2034 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2035 struct resource_list_entry *rle;
2036 u_int i, irq, j;
2037 bool *used;
2038
2039 /*
2040 * Have to have at least one message in the table but the
2041 * table can't be bigger than the actual MSI-X table in the
2042 * device.
2043 */
2044 if (count < 1 || count > PCI_MSIX_MSGNUM(msix->msix_ctrl))
2045 return (EINVAL);
2046
2047 /* Sanity check the vectors. */
2048 for (i = 0; i < count; i++)
2049 if (vectors[i] > msix->msix_alloc)
2050 return (EINVAL);
2051
2052 /*
2053 * Make sure there aren't any holes in the vectors to be used.
2054 * It's a big pain to support it, and it doesn't really make
2055 * sense anyway. Also, at least one vector must be used.
2056 */
2057 used = mallocarray(msix->msix_alloc, sizeof(*used), M_DEVBUF, M_WAITOK |
2058 M_ZERO);
2059 for (i = 0; i < count; i++)
2060 if (vectors[i] != 0)
2061 used[vectors[i] - 1] = true;
2062 for (i = 0; i < msix->msix_alloc - 1; i++)
2063 if (!used[i] && used[i + 1]) {
2064 free(used, M_DEVBUF);
2065 return (EINVAL);
2066 }
2067 if (!used[0]) {
2068 free(used, M_DEVBUF);
2069 return (EINVAL);
2070 }
2071
2072 /* Make sure none of the resources are allocated. */
2073 for (i = 0; i < msix->msix_table_len; i++) {
2074 if (msix->msix_table[i].mte_vector == 0)
2075 continue;
2076 if (msix->msix_table[i].mte_handlers > 0) {
2077 free(used, M_DEVBUF);
2078 return (EBUSY);
2079 }
2080 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2081 KASSERT(rle != NULL, ("missing resource"));
2082 if (rle->res != NULL) {
2083 free(used, M_DEVBUF);
2084 return (EBUSY);
2085 }
2086 }
2087
2088 /* Free the existing resource list entries. */
2089 for (i = 0; i < msix->msix_table_len; i++) {
2090 if (msix->msix_table[i].mte_vector == 0)
2091 continue;
2092 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2093 }
2094
2095 /*
2096 * Build the new virtual table keeping track of which vectors are
2097 * used.
2098 */
2099 free(msix->msix_table, M_DEVBUF);
2100 msix->msix_table = mallocarray(count, sizeof(struct msix_table_entry),
2101 M_DEVBUF, M_WAITOK | M_ZERO);
2102 for (i = 0; i < count; i++)
2103 msix->msix_table[i].mte_vector = vectors[i];
2104 msix->msix_table_len = count;
2105
2106 /* Free any unused IRQs and resize the vectors array if necessary. */
2107 j = msix->msix_alloc - 1;
2108 if (!used[j]) {
2109 struct msix_vector *vec;
2110
2111 while (!used[j]) {
2112 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2113 msix->msix_vectors[j].mv_irq);
2114 j--;
2115 }
2116 vec = mallocarray(j + 1, sizeof(struct msix_vector), M_DEVBUF,
2117 M_WAITOK);
2118 bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) *
2119 (j + 1));
2120 free(msix->msix_vectors, M_DEVBUF);
2121 msix->msix_vectors = vec;
2122 msix->msix_alloc = j + 1;
2123 }
2124 free(used, M_DEVBUF);
2125
2126 /* Map the IRQs onto the rids. */
2127 for (i = 0; i < count; i++) {
2128 if (vectors[i] == 0)
2129 continue;
2130 irq = msix->msix_vectors[vectors[i] - 1].mv_irq;
2131 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
2132 irq, 1);
2133 }
2134
2135 if (bootverbose) {
2136 device_printf(child, "Remapped MSI-X IRQs as: ");
2137 for (i = 0; i < count; i++) {
2138 if (i != 0)
2139 printf(", ");
2140 if (vectors[i] == 0)
2141 printf("---");
2142 else
2143 printf("%d",
2144 msix->msix_vectors[vectors[i] - 1].mv_irq);
2145 }
2146 printf("\n");
2147 }
2148
2149 return (0);
2150 }
2151
2152 static int
pci_release_msix(device_t dev,device_t child)2153 pci_release_msix(device_t dev, device_t child)
2154 {
2155 struct pci_devinfo *dinfo = device_get_ivars(child);
2156 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2157 struct resource_list_entry *rle;
2158 u_int i;
2159
2160 /* Do we have any messages to release? */
2161 if (msix->msix_alloc == 0)
2162 return (ENODEV);
2163
2164 /* Make sure none of the resources are allocated. */
2165 for (i = 0; i < msix->msix_table_len; i++) {
2166 if (msix->msix_table[i].mte_vector == 0)
2167 continue;
2168 if (msix->msix_table[i].mte_handlers > 0)
2169 return (EBUSY);
2170 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2171 KASSERT(rle != NULL, ("missing resource"));
2172 if (rle->res != NULL)
2173 return (EBUSY);
2174 }
2175
2176 /* Update control register to disable MSI-X. */
2177 msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
2178 pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
2179 msix->msix_ctrl, 2);
2180
2181 /* Free the resource list entries. */
2182 for (i = 0; i < msix->msix_table_len; i++) {
2183 if (msix->msix_table[i].mte_vector == 0)
2184 continue;
2185 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2186 }
2187 free(msix->msix_table, M_DEVBUF);
2188 msix->msix_table_len = 0;
2189
2190 /* Release the IRQs. */
2191 for (i = 0; i < msix->msix_alloc; i++)
2192 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2193 msix->msix_vectors[i].mv_irq);
2194 free(msix->msix_vectors, M_DEVBUF);
2195 msix->msix_alloc = 0;
2196 return (0);
2197 }
2198
2199 /*
2200 * Return the max supported MSI-X messages this device supports.
2201 * Basically, assuming the MD code can alloc messages, this function
2202 * should return the maximum value that pci_alloc_msix() can return.
2203 * Thus, it is subject to the tunables, etc.
2204 */
2205 int
pci_msix_count_method(device_t dev,device_t child)2206 pci_msix_count_method(device_t dev, device_t child)
2207 {
2208 struct pci_devinfo *dinfo = device_get_ivars(child);
2209 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2210 uint16_t ctrl;
2211
2212 if (pci_do_msix && msix->msix_location != 0) {
2213 ctrl = pci_read_config(child, msix->msix_location +
2214 PCIR_MSI_CTRL, 2);
2215 return (PCI_MSIX_MSGNUM(ctrl));
2216 }
2217 return (0);
2218 }
2219
2220 int
pci_msix_pba_bar_method(device_t dev,device_t child)2221 pci_msix_pba_bar_method(device_t dev, device_t child)
2222 {
2223 struct pci_devinfo *dinfo = device_get_ivars(child);
2224 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2225
2226 if (pci_do_msix && msix->msix_location != 0)
2227 return (msix->msix_pba_bar);
2228 return (-1);
2229 }
2230
2231 int
pci_msix_table_bar_method(device_t dev,device_t child)2232 pci_msix_table_bar_method(device_t dev, device_t child)
2233 {
2234 struct pci_devinfo *dinfo = device_get_ivars(child);
2235 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2236
2237 if (pci_do_msix && msix->msix_location != 0)
2238 return (msix->msix_table_bar);
2239 return (-1);
2240 }
2241
2242 /*
2243 * HyperTransport MSI mapping control
2244 */
2245 void
pci_ht_map_msi(device_t dev,uint64_t addr)2246 pci_ht_map_msi(device_t dev, uint64_t addr)
2247 {
2248 struct pci_devinfo *dinfo = device_get_ivars(dev);
2249 struct pcicfg_ht *ht = &dinfo->cfg.ht;
2250
2251 if (!ht->ht_msimap)
2252 return;
2253
2254 if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) &&
2255 ht->ht_msiaddr >> 20 == addr >> 20) {
2256 /* Enable MSI -> HT mapping. */
2257 ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
2258 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2259 ht->ht_msictrl, 2);
2260 }
2261
2262 if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
2263 /* Disable MSI -> HT mapping. */
2264 ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
2265 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2266 ht->ht_msictrl, 2);
2267 }
2268 }
2269
2270 int
pci_get_relaxed_ordering_enabled(device_t dev)2271 pci_get_relaxed_ordering_enabled(device_t dev)
2272 {
2273 struct pci_devinfo *dinfo = device_get_ivars(dev);
2274 int cap;
2275 uint16_t val;
2276
2277 cap = dinfo->cfg.pcie.pcie_location;
2278 if (cap == 0)
2279 return (0);
2280 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2281 val &= PCIEM_CTL_RELAXED_ORD_ENABLE;
2282 return (val != 0);
2283 }
2284
2285 int
pci_get_max_payload(device_t dev)2286 pci_get_max_payload(device_t dev)
2287 {
2288 struct pci_devinfo *dinfo = device_get_ivars(dev);
2289 int cap;
2290 uint16_t val;
2291
2292 cap = dinfo->cfg.pcie.pcie_location;
2293 if (cap == 0)
2294 return (0);
2295 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2296 val &= PCIEM_CTL_MAX_PAYLOAD;
2297 val >>= 5;
2298 return (1 << (val + 7));
2299 }
2300
2301 int
pci_get_max_read_req(device_t dev)2302 pci_get_max_read_req(device_t dev)
2303 {
2304 struct pci_devinfo *dinfo = device_get_ivars(dev);
2305 int cap;
2306 uint16_t val;
2307
2308 cap = dinfo->cfg.pcie.pcie_location;
2309 if (cap == 0)
2310 return (0);
2311 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2312 val &= PCIEM_CTL_MAX_READ_REQUEST;
2313 val >>= 12;
2314 return (1 << (val + 7));
2315 }
2316
2317 int
pci_set_max_read_req(device_t dev,int size)2318 pci_set_max_read_req(device_t dev, int size)
2319 {
2320 struct pci_devinfo *dinfo = device_get_ivars(dev);
2321 int cap;
2322 uint16_t val;
2323
2324 cap = dinfo->cfg.pcie.pcie_location;
2325 if (cap == 0)
2326 return (0);
2327 if (size < 128)
2328 size = 128;
2329 if (size > 4096)
2330 size = 4096;
2331 size = (1 << (fls(size) - 1));
2332 val = (fls(size) - 8) << 12;
2333 pcie_adjust_config(dev, PCIER_DEVICE_CTL,
2334 PCIEM_CTL_MAX_READ_REQUEST, val, 2);
2335 return (size);
2336 }
2337
2338 uint32_t
pcie_read_config(device_t dev,int reg,int width)2339 pcie_read_config(device_t dev, int reg, int width)
2340 {
2341 struct pci_devinfo *dinfo = device_get_ivars(dev);
2342 int cap;
2343
2344 cap = dinfo->cfg.pcie.pcie_location;
2345 if (cap == 0) {
2346 if (width == 2)
2347 return (0xffff);
2348 return (0xffffffff);
2349 }
2350
2351 return (pci_read_config(dev, cap + reg, width));
2352 }
2353
2354 void
pcie_write_config(device_t dev,int reg,uint32_t value,int width)2355 pcie_write_config(device_t dev, int reg, uint32_t value, int width)
2356 {
2357 struct pci_devinfo *dinfo = device_get_ivars(dev);
2358 int cap;
2359
2360 cap = dinfo->cfg.pcie.pcie_location;
2361 if (cap == 0)
2362 return;
2363 pci_write_config(dev, cap + reg, value, width);
2364 }
2365
2366 /*
2367 * Adjusts a PCI-e capability register by clearing the bits in mask
2368 * and setting the bits in (value & mask). Bits not set in mask are
2369 * not adjusted.
2370 *
2371 * Returns the old value on success or all ones on failure.
2372 */
2373 uint32_t
pcie_adjust_config(device_t dev,int reg,uint32_t mask,uint32_t value,int width)2374 pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
2375 int width)
2376 {
2377 struct pci_devinfo *dinfo = device_get_ivars(dev);
2378 uint16_t *saved;
2379 uint32_t old, new;
2380 int cap;
2381
2382 cap = dinfo->cfg.pcie.pcie_location;
2383 if (cap == 0) {
2384 if (width == 2)
2385 return (0xffff);
2386 return (0xffffffff);
2387 }
2388
2389 old = pci_read_config(dev, cap + reg, width);
2390 new = old & ~mask;
2391 new |= (value & mask);
2392 pci_write_config(dev, cap + reg, new, width);
2393 /* Apply only the requested policy bits to the saved restore image. */
2394 if (width == 2) {
2395 saved = NULL;
2396 switch (reg) {
2397 case PCIER_DEVICE_CTL:
2398 saved = &dinfo->cfg.pcie.pcie_device_ctl;
2399 break;
2400 case PCIER_ROOT_CTL:
2401 saved = &dinfo->cfg.pcie.pcie_root_ctl;
2402 break;
2403 }
2404 if (saved != NULL) {
2405 *saved &= ~(uint16_t)mask;
2406 *saved |= (uint16_t)(value & mask);
2407 }
2408 }
2409 return (old);
2410 }
2411
2412 /*
2413 * Support for MSI message signalled interrupts.
2414 */
2415 void
pci_enable_msi_method(device_t dev,device_t child,uint64_t address,uint16_t data)2416 pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
2417 uint16_t data)
2418 {
2419 struct pci_devinfo *dinfo = device_get_ivars(child);
2420 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2421
2422 /* Write data and address values. */
2423 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
2424 address & 0xffffffff, 4);
2425 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2426 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
2427 address >> 32, 4);
2428 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
2429 data, 2);
2430 } else
2431 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
2432 2);
2433
2434 /* Enable MSI in the control register. */
2435 msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE;
2436 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2437 msi->msi_ctrl, 2);
2438
2439 /* Enable MSI -> HT mapping. */
2440 pci_ht_map_msi(child, address);
2441 }
2442
2443 void
pci_disable_msi_method(device_t dev,device_t child)2444 pci_disable_msi_method(device_t dev, device_t child)
2445 {
2446 struct pci_devinfo *dinfo = device_get_ivars(child);
2447 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2448
2449 /* Disable MSI -> HT mapping. */
2450 pci_ht_map_msi(child, 0);
2451
2452 /* Disable MSI in the control register. */
2453 msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
2454 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2455 msi->msi_ctrl, 2);
2456 }
2457
2458 /*
2459 * Restore MSI registers during resume. If MSI is enabled then
2460 * restore the data and address registers in addition to the control
2461 * register.
2462 */
2463 static void
pci_resume_msi(device_t dev)2464 pci_resume_msi(device_t dev)
2465 {
2466 struct pci_devinfo *dinfo = device_get_ivars(dev);
2467 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2468 uint64_t address;
2469 uint16_t data;
2470
2471 if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) {
2472 address = msi->msi_addr;
2473 data = msi->msi_data;
2474 pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
2475 address & 0xffffffff, 4);
2476 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2477 pci_write_config(dev, msi->msi_location +
2478 PCIR_MSI_ADDR_HIGH, address >> 32, 4);
2479 pci_write_config(dev, msi->msi_location +
2480 PCIR_MSI_DATA_64BIT, data, 2);
2481 } else
2482 pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA,
2483 data, 2);
2484 }
2485 pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
2486 2);
2487 }
2488
2489 static int
pci_remap_intr_method(device_t bus,device_t dev,u_int irq)2490 pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
2491 {
2492 struct pci_devinfo *dinfo = device_get_ivars(dev);
2493 pcicfgregs *cfg = &dinfo->cfg;
2494 struct resource_list_entry *rle;
2495 struct msix_table_entry *mte;
2496 struct msix_vector *mv;
2497 uint64_t addr;
2498 uint32_t data;
2499 u_int i, j;
2500 int error;
2501
2502 /*
2503 * Handle MSI first. We try to find this IRQ among our list
2504 * of MSI IRQs. If we find it, we request updated address and
2505 * data registers and apply the results.
2506 */
2507 if (cfg->msi.msi_alloc > 0) {
2508 /* If we don't have any active handlers, nothing to do. */
2509 if (cfg->msi.msi_handlers == 0)
2510 return (0);
2511 for (i = 0; i < cfg->msi.msi_alloc; i++) {
2512 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
2513 i + 1);
2514 if (rle->start == irq) {
2515 error = PCIB_MAP_MSI(device_get_parent(bus),
2516 dev, irq, &addr, &data);
2517 if (error)
2518 return (error);
2519 pci_disable_msi(dev);
2520 dinfo->cfg.msi.msi_addr = addr;
2521 dinfo->cfg.msi.msi_data = data;
2522 pci_enable_msi(dev, addr, data);
2523 return (0);
2524 }
2525 }
2526 return (ENOENT);
2527 }
2528
2529 /*
2530 * For MSI-X, we check to see if we have this IRQ. If we do,
2531 * we request the updated mapping info. If that works, we go
2532 * through all the slots that use this IRQ and update them.
2533 */
2534 if (cfg->msix.msix_alloc > 0) {
2535 bool found = false;
2536
2537 for (i = 0; i < cfg->msix.msix_alloc; i++) {
2538 mv = &cfg->msix.msix_vectors[i];
2539 if (mv->mv_irq == irq) {
2540 error = PCIB_MAP_MSI(device_get_parent(bus),
2541 dev, irq, &addr, &data);
2542 if (error)
2543 return (error);
2544 mv->mv_address = addr;
2545 mv->mv_data = data;
2546 for (j = 0; j < cfg->msix.msix_table_len; j++) {
2547 mte = &cfg->msix.msix_table[j];
2548 if (mte->mte_vector != i + 1)
2549 continue;
2550 if (mte->mte_handlers == 0)
2551 continue;
2552 pci_mask_msix(dev, j);
2553 pci_enable_msix(dev, j, addr, data);
2554 pci_unmask_msix(dev, j);
2555 }
2556 found = true;
2557 }
2558 }
2559 return (found ? 0 : ENOENT);
2560 }
2561
2562 return (ENOENT);
2563 }
2564
2565 /*
2566 * Returns true if the specified device is blacklisted because MSI
2567 * doesn't work.
2568 */
2569 int
pci_msi_device_blacklisted(device_t dev)2570 pci_msi_device_blacklisted(device_t dev)
2571 {
2572
2573 if (!pci_honor_msi_blacklist)
2574 return (0);
2575
2576 return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
2577 }
2578
2579 /*
2580 * Determine if MSI is blacklisted globally on this system. Currently,
2581 * we just check for blacklisted chipsets as represented by the
2582 * host-PCI bridge at device 0:0:0. In the future, it may become
2583 * necessary to check other system attributes, such as the kenv values
2584 * that give the motherboard manufacturer and model number.
2585 */
2586 static int
pci_msi_blacklisted(void)2587 pci_msi_blacklisted(void)
2588 {
2589 device_t dev;
2590
2591 if (!pci_honor_msi_blacklist)
2592 return (0);
2593
2594 /* Blacklist all non-PCI-express and non-PCI-X chipsets. */
2595 if (!(pcie_chipset || pcix_chipset)) {
2596 if (vm_guest != VM_GUEST_NO) {
2597 /*
2598 * Whitelist older chipsets in virtual
2599 * machines known to support MSI.
2600 */
2601 dev = pci_find_bsf(0, 0, 0);
2602 if (dev != NULL)
2603 return (!pci_has_quirk(pci_get_devid(dev),
2604 PCI_QUIRK_ENABLE_MSI_VM));
2605 }
2606 return (1);
2607 }
2608
2609 dev = pci_find_bsf(0, 0, 0);
2610 if (dev != NULL)
2611 return (pci_msi_device_blacklisted(dev));
2612 return (0);
2613 }
2614
2615 /*
2616 * Returns true if the specified device is blacklisted because MSI-X
2617 * doesn't work. Note that this assumes that if MSI doesn't work,
2618 * MSI-X doesn't either.
2619 */
2620 int
pci_msix_device_blacklisted(device_t dev)2621 pci_msix_device_blacklisted(device_t dev)
2622 {
2623
2624 if (!pci_honor_msi_blacklist)
2625 return (0);
2626
2627 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
2628 return (1);
2629
2630 return (pci_msi_device_blacklisted(dev));
2631 }
2632
2633 /*
2634 * Determine if MSI-X is blacklisted globally on this system. If MSI
2635 * is blacklisted, assume that MSI-X is as well. Check for additional
2636 * chipsets where MSI works but MSI-X does not.
2637 */
2638 static int
pci_msix_blacklisted(void)2639 pci_msix_blacklisted(void)
2640 {
2641 device_t dev;
2642
2643 if (!pci_honor_msi_blacklist)
2644 return (0);
2645
2646 dev = pci_find_bsf(0, 0, 0);
2647 if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
2648 PCI_QUIRK_DISABLE_MSIX))
2649 return (1);
2650
2651 return (pci_msi_blacklisted());
2652 }
2653
2654 /*
2655 * Attempt to allocate *count MSI messages. The actual number allocated is
2656 * returned in *count. After this function returns, each message will be
2657 * available to the driver as SYS_RES_IRQ resources starting at a rid 1.
2658 */
2659 int
pci_alloc_msi_method(device_t dev,device_t child,int * count)2660 pci_alloc_msi_method(device_t dev, device_t child, int *count)
2661 {
2662 struct pci_devinfo *dinfo = device_get_ivars(child);
2663 pcicfgregs *cfg = &dinfo->cfg;
2664 struct resource_list_entry *rle;
2665 u_int actual, i;
2666 int error, irqs[32];
2667 uint16_t ctrl, msgnum;
2668
2669 /* Don't let count == 0 get us into trouble. */
2670 if (*count < 1)
2671 return (EINVAL);
2672
2673 /* If rid 0 is allocated, then fail. */
2674 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
2675 if (rle != NULL && rle->res != NULL)
2676 return (ENXIO);
2677
2678 /* Already have allocated messages? */
2679 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
2680 return (ENXIO);
2681
2682 /* If MSI is blacklisted for this system, fail. */
2683 if (pci_msi_blacklisted())
2684 return (ENXIO);
2685
2686 /* MSI capability present? */
2687 if (cfg->msi.msi_location == 0 || !pci_do_msi)
2688 return (ENODEV);
2689
2690 ctrl = pci_read_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, 2);
2691 msgnum = PCI_MSI_MSGNUM(ctrl);
2692 if (bootverbose)
2693 device_printf(child,
2694 "attempting to allocate %d MSI vectors (%u supported)\n",
2695 *count, msgnum);
2696
2697 /* Don't ask for more than the device supports. */
2698 actual = min(*count, msgnum);
2699
2700 /* Don't ask for more than 32 messages. */
2701 actual = min(actual, 32);
2702
2703 /* MSI requires power of 2 number of messages. */
2704 if (!powerof2(actual))
2705 return (EINVAL);
2706
2707 for (;;) {
2708 /* Try to allocate N messages. */
2709 error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual,
2710 actual, irqs);
2711 if (error == 0)
2712 break;
2713 if (actual == 1)
2714 return (error);
2715
2716 /* Try N / 2. */
2717 actual >>= 1;
2718 }
2719
2720 /*
2721 * We now have N actual messages mapped onto SYS_RES_IRQ
2722 * resources in the irqs[] array, so add new resources
2723 * starting at rid 1.
2724 */
2725 for (i = 0; i < actual; i++)
2726 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1,
2727 irqs[i], irqs[i], 1);
2728
2729 if (bootverbose) {
2730 if (actual == 1)
2731 device_printf(child, "using IRQ %d for MSI\n", irqs[0]);
2732 else {
2733 bool run;
2734
2735 /*
2736 * Be fancy and try to print contiguous runs
2737 * of IRQ values as ranges. 'run' is true if
2738 * we are in a range.
2739 */
2740 device_printf(child, "using IRQs %d", irqs[0]);
2741 run = false;
2742 for (i = 1; i < actual; i++) {
2743 /* Still in a run? */
2744 if (irqs[i] == irqs[i - 1] + 1) {
2745 run = true;
2746 continue;
2747 }
2748
2749 /* Finish previous range. */
2750 if (run) {
2751 printf("-%d", irqs[i - 1]);
2752 run = false;
2753 }
2754
2755 /* Start new range. */
2756 printf(",%d", irqs[i]);
2757 }
2758
2759 /* Unfinished range? */
2760 if (run)
2761 printf("-%d", irqs[actual - 1]);
2762 printf(" for MSI\n");
2763 }
2764 }
2765
2766 /* Update control register with actual count. */
2767 ctrl &= ~PCIM_MSICTRL_MME_MASK;
2768 ctrl |= (ffs(actual) - 1) << 4;
2769 cfg->msi.msi_ctrl = ctrl;
2770 pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2);
2771
2772 /* Update counts of alloc'd messages. */
2773 cfg->msi.msi_alloc = actual;
2774 cfg->msi.msi_handlers = 0;
2775 *count = actual;
2776 return (0);
2777 }
2778
2779 /* Release the MSI messages associated with this device. */
2780 int
pci_release_msi_method(device_t dev,device_t child)2781 pci_release_msi_method(device_t dev, device_t child)
2782 {
2783 struct pci_devinfo *dinfo = device_get_ivars(child);
2784 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2785 struct resource_list_entry *rle;
2786 u_int i, irqs[32];
2787 int error;
2788
2789 /* Try MSI-X first. */
2790 error = pci_release_msix(dev, child);
2791 if (error != ENODEV)
2792 return (error);
2793
2794 /* Do we have any messages to release? */
2795 if (msi->msi_alloc == 0)
2796 return (ENODEV);
2797 KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));
2798
2799 /* Make sure none of the resources are allocated. */
2800 if (msi->msi_handlers > 0)
2801 return (EBUSY);
2802 for (i = 0; i < msi->msi_alloc; i++) {
2803 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2804 KASSERT(rle != NULL, ("missing MSI resource"));
2805 if (rle->res != NULL)
2806 return (EBUSY);
2807 irqs[i] = rle->start;
2808 }
2809
2810 /* Update control register with 0 count. */
2811 KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
2812 ("%s: MSI still enabled", __func__));
2813 msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
2814 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2815 msi->msi_ctrl, 2);
2816
2817 /* Release the messages. */
2818 PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs);
2819 for (i = 0; i < msi->msi_alloc; i++)
2820 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2821
2822 /* Update alloc count. */
2823 msi->msi_alloc = 0;
2824 msi->msi_addr = 0;
2825 msi->msi_data = 0;
2826 return (0);
2827 }
2828
2829 /*
2830 * Return the max supported MSI messages this device supports.
2831 * Basically, assuming the MD code can alloc messages, this function
2832 * should return the maximum value that pci_alloc_msi() can return.
2833 * Thus, it is subject to the tunables, etc.
2834 */
2835 int
pci_msi_count_method(device_t dev,device_t child)2836 pci_msi_count_method(device_t dev, device_t child)
2837 {
2838 struct pci_devinfo *dinfo = device_get_ivars(child);
2839 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2840 uint16_t ctrl;
2841
2842 if (pci_do_msi && msi->msi_location != 0) {
2843 ctrl = pci_read_config(child, msi->msi_location + PCIR_MSI_CTRL,
2844 2);
2845 return (PCI_MSI_MSGNUM(ctrl));
2846 }
2847 return (0);
2848 }
2849
2850 /* free pcicfgregs structure and all depending data structures */
2851
2852 int
pci_freecfg(struct pci_devinfo * dinfo)2853 pci_freecfg(struct pci_devinfo *dinfo)
2854 {
2855 struct devlist *devlist_head;
2856 struct pci_map *pm, *next;
2857
2858 devlist_head = &pci_devq;
2859
2860 if (dinfo->cfg.vpd.vpd_reg)
2861 vpd_free(&dinfo->cfg.vpd);
2862
2863 STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) {
2864 free(pm, M_DEVBUF);
2865 }
2866 STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links);
2867 free(dinfo, M_DEVBUF);
2868
2869 /* increment the generation count */
2870 pci_generation++;
2871
2872 /* we're losing one device */
2873 pci_numdevs--;
2874 return (0);
2875 }
2876
2877 /*
2878 * PCI power manangement
2879 */
2880 int
pci_set_powerstate_method(device_t dev,device_t child,int state)2881 pci_set_powerstate_method(device_t dev, device_t child, int state)
2882 {
2883 struct pci_devinfo *dinfo = device_get_ivars(child);
2884 pcicfgregs *cfg = &dinfo->cfg;
2885 uint16_t status;
2886 int oldstate, highest, delay;
2887
2888 if (cfg->pp.pp_location == 0)
2889 return (EOPNOTSUPP);
2890
2891 /*
2892 * Optimize a no state change request away. While it would be OK to
2893 * write to the hardware in theory, some devices have shown odd
2894 * behavior when going from D3 -> D3.
2895 */
2896 oldstate = pci_get_powerstate(child);
2897 if (oldstate == state)
2898 return (0);
2899
2900 /*
2901 * The PCI power management specification states that after a state
2902 * transition between PCI power states, system software must
2903 * guarantee a minimal delay before the function accesses the device.
2904 * Compute the worst case delay that we need to guarantee before we
2905 * access the device. Many devices will be responsive much more
2906 * quickly than this delay, but there are some that don't respond
2907 * instantly to state changes. Transitions to/from D3 state require
2908 * 10ms, while D2 requires 200us, and D0/1 require none. The delay
2909 * is done below with DELAY rather than a sleeper function because
2910 * this function can be called from contexts where we cannot sleep.
2911 */
2912 highest = (oldstate > state) ? oldstate : state;
2913 if (highest == PCI_POWERSTATE_D3)
2914 delay = 10000;
2915 else if (highest == PCI_POWERSTATE_D2)
2916 delay = 200;
2917 else
2918 delay = 0;
2919 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_location +
2920 PCIR_POWER_STATUS, 2) & ~PCIM_PSTAT_DMASK;
2921 switch (state) {
2922 case PCI_POWERSTATE_D0:
2923 status |= PCIM_PSTAT_D0;
2924 break;
2925 case PCI_POWERSTATE_D1:
2926 if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0)
2927 return (EOPNOTSUPP);
2928 status |= PCIM_PSTAT_D1;
2929 break;
2930 case PCI_POWERSTATE_D2:
2931 if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0)
2932 return (EOPNOTSUPP);
2933 status |= PCIM_PSTAT_D2;
2934 break;
2935 case PCI_POWERSTATE_D3:
2936 status |= PCIM_PSTAT_D3;
2937 break;
2938 default:
2939 return (EINVAL);
2940 }
2941
2942 if (bootverbose)
2943 pci_printf(cfg, "Transition from %s to %s\n",
2944 pci_powerstate_to_str(oldstate),
2945 pci_powerstate_to_str(state));
2946
2947 PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_location + PCIR_POWER_STATUS,
2948 status, 2);
2949 if (delay)
2950 DELAY(delay);
2951 return (0);
2952 }
2953
2954 int
pci_get_powerstate_method(device_t dev,device_t child)2955 pci_get_powerstate_method(device_t dev, device_t child)
2956 {
2957 struct pci_devinfo *dinfo = device_get_ivars(child);
2958 pcicfgregs *cfg = &dinfo->cfg;
2959 uint16_t status;
2960 int result;
2961
2962 if (cfg->pp.pp_location != 0) {
2963 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_location +
2964 PCIR_POWER_STATUS, 2);
2965 switch (status & PCIM_PSTAT_DMASK) {
2966 case PCIM_PSTAT_D0:
2967 result = PCI_POWERSTATE_D0;
2968 break;
2969 case PCIM_PSTAT_D1:
2970 result = PCI_POWERSTATE_D1;
2971 break;
2972 case PCIM_PSTAT_D2:
2973 result = PCI_POWERSTATE_D2;
2974 break;
2975 case PCIM_PSTAT_D3:
2976 result = PCI_POWERSTATE_D3;
2977 break;
2978 default:
2979 result = PCI_POWERSTATE_UNKNOWN;
2980 break;
2981 }
2982 } else {
2983 /* No support, device is always at D0 */
2984 result = PCI_POWERSTATE_D0;
2985 }
2986 return (result);
2987 }
2988
2989 /* Clear any active PME# and disable PME# generation. */
2990 void
pci_clear_pme(device_t dev)2991 pci_clear_pme(device_t dev)
2992 {
2993 struct pci_devinfo *dinfo = device_get_ivars(dev);
2994 pcicfgregs *cfg = &dinfo->cfg;
2995 uint16_t status;
2996
2997 if (cfg->pp.pp_location != 0) {
2998 status = pci_read_config(dev, dinfo->cfg.pp.pp_location +
2999 PCIR_POWER_STATUS, 2);
3000 status &= ~PCIM_PSTAT_PMEENABLE;
3001 status |= PCIM_PSTAT_PME;
3002 pci_write_config(dev, dinfo->cfg.pp.pp_location +
3003 PCIR_POWER_STATUS, status, 2);
3004 }
3005 }
3006
3007 /* Clear any active PME# and enable PME# generation. */
3008 void
pci_enable_pme(device_t dev)3009 pci_enable_pme(device_t dev)
3010 {
3011 struct pci_devinfo *dinfo = device_get_ivars(dev);
3012 pcicfgregs *cfg = &dinfo->cfg;
3013 uint16_t status;
3014
3015 if (cfg->pp.pp_location != 0) {
3016 status = pci_read_config(dev, dinfo->cfg.pp.pp_location +
3017 PCIR_POWER_STATUS, 2);
3018 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
3019 pci_write_config(dev, dinfo->cfg.pp.pp_location +
3020 PCIR_POWER_STATUS, status, 2);
3021 }
3022 }
3023
3024 bool
pci_has_pm(device_t dev)3025 pci_has_pm(device_t dev)
3026 {
3027 struct pci_devinfo *dinfo = device_get_ivars(dev);
3028 pcicfgregs *cfg = &dinfo->cfg;
3029
3030 return (cfg->pp.pp_location != 0);
3031 }
3032
3033 /*
3034 * Some convenience functions for PCI device drivers.
3035 */
3036
3037 static __inline void
pci_set_command_bit(device_t dev,device_t child,uint16_t bit)3038 pci_set_command_bit(device_t dev, device_t child, uint16_t bit)
3039 {
3040 uint16_t command;
3041
3042 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
3043 command |= bit;
3044 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
3045 }
3046
3047 static __inline void
pci_clear_command_bit(device_t dev,device_t child,uint16_t bit)3048 pci_clear_command_bit(device_t dev, device_t child, uint16_t bit)
3049 {
3050 uint16_t command;
3051
3052 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
3053 command &= ~bit;
3054 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
3055 }
3056
3057 int
pci_enable_busmaster_method(device_t dev,device_t child)3058 pci_enable_busmaster_method(device_t dev, device_t child)
3059 {
3060 pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
3061 return (0);
3062 }
3063
3064 int
pci_disable_busmaster_method(device_t dev,device_t child)3065 pci_disable_busmaster_method(device_t dev, device_t child)
3066 {
3067 pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
3068 return (0);
3069 }
3070
3071 int
pci_enable_io_method(device_t dev,device_t child,int space)3072 pci_enable_io_method(device_t dev, device_t child, int space)
3073 {
3074 uint16_t bit;
3075
3076 switch(space) {
3077 case SYS_RES_IOPORT:
3078 bit = PCIM_CMD_PORTEN;
3079 break;
3080 case SYS_RES_MEMORY:
3081 bit = PCIM_CMD_MEMEN;
3082 break;
3083 default:
3084 return (EINVAL);
3085 }
3086 pci_set_command_bit(dev, child, bit);
3087 return (0);
3088 }
3089
3090 int
pci_disable_io_method(device_t dev,device_t child,int space)3091 pci_disable_io_method(device_t dev, device_t child, int space)
3092 {
3093 uint16_t bit;
3094
3095 switch(space) {
3096 case SYS_RES_IOPORT:
3097 bit = PCIM_CMD_PORTEN;
3098 break;
3099 case SYS_RES_MEMORY:
3100 bit = PCIM_CMD_MEMEN;
3101 break;
3102 default:
3103 return (EINVAL);
3104 }
3105 pci_clear_command_bit(dev, child, bit);
3106 return (0);
3107 }
3108
3109 /*
3110 * New style pci driver. Parent device is either a pci-host-bridge or a
3111 * pci-pci-bridge. Both kinds are represented by instances of pcib.
3112 */
3113
3114 void
pci_print_verbose(struct pci_devinfo * dinfo)3115 pci_print_verbose(struct pci_devinfo *dinfo)
3116 {
3117
3118 if (bootverbose) {
3119 pcicfgregs *cfg = &dinfo->cfg;
3120
3121 printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n",
3122 cfg->vendor, cfg->device, cfg->revid);
3123 printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n",
3124 cfg->domain, cfg->bus, cfg->slot, cfg->func);
3125 printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n",
3126 cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype,
3127 cfg->mfdev);
3128 printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n",
3129 cfg->cmdreg, cfg->statreg, cfg->cachelnsz);
3130 printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n",
3131 cfg->lattimer, cfg->lattimer * 30, cfg->mingnt,
3132 cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250);
3133 if (cfg->intpin > 0)
3134 printf("\tintpin=%c, irq=%d\n",
3135 cfg->intpin +'a' -1, cfg->intline);
3136 if (cfg->pp.pp_location) {
3137 uint16_t status;
3138
3139 status = pci_read_config(cfg->dev, cfg->pp.pp_location +
3140 PCIR_POWER_STATUS, 2);
3141 printf("\tpowerspec %d supports D0%s%s D3 current D%d\n",
3142 cfg->pp.pp_cap & PCIM_PCAP_SPEC,
3143 cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "",
3144 cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "",
3145 status & PCIM_PSTAT_DMASK);
3146 }
3147 if (cfg->msi.msi_location) {
3148 uint16_t ctrl, msgnum;
3149
3150 ctrl = cfg->msi.msi_ctrl;
3151 msgnum = PCI_MSI_MSGNUM(ctrl);
3152 printf("\tMSI supports %d message%s%s%s\n",
3153 msgnum, (msgnum == 1) ? "" : "s",
3154 (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "",
3155 (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":"");
3156 }
3157 if (cfg->msix.msix_location) {
3158 uint16_t msgnum;
3159
3160 msgnum = PCI_MSIX_MSGNUM(cfg->msix.msix_ctrl);
3161 printf("\tMSI-X supports %d message%s ",
3162 msgnum, (msgnum == 1) ? "" : "s");
3163 if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar)
3164 printf("in map 0x%x\n",
3165 cfg->msix.msix_table_bar);
3166 else
3167 printf("in maps 0x%x and 0x%x\n",
3168 cfg->msix.msix_table_bar,
3169 cfg->msix.msix_pba_bar);
3170 }
3171 }
3172 }
3173
3174 static int
pci_porten(device_t dev)3175 pci_porten(device_t dev)
3176 {
3177 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0;
3178 }
3179
3180 static int
pci_memen(device_t dev)3181 pci_memen(device_t dev)
3182 {
3183 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0;
3184 }
3185
3186 void
pci_read_bar(device_t dev,int reg,pci_addr_t * mapp,pci_addr_t * testvalp,int * bar64)3187 pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp,
3188 int *bar64)
3189 {
3190 struct pci_devinfo *dinfo;
3191 pci_addr_t map, testval;
3192 int ln2range;
3193 uint16_t cmd;
3194
3195 /*
3196 * The device ROM BAR is special. It is always a 32-bit
3197 * memory BAR. Bit 0 is special and should not be set when
3198 * sizing the BAR.
3199 */
3200 dinfo = device_get_ivars(dev);
3201 if (PCIR_IS_BIOS(&dinfo->cfg, reg)) {
3202 map = pci_read_config(dev, reg, 4);
3203 pci_write_config(dev, reg, 0xfffffffe, 4);
3204 testval = pci_read_config(dev, reg, 4);
3205 pci_write_config(dev, reg, map, 4);
3206 *mapp = map;
3207 *testvalp = testval;
3208 if (bar64 != NULL)
3209 *bar64 = 0;
3210 return;
3211 }
3212
3213 map = pci_read_config(dev, reg, 4);
3214 ln2range = pci_maprange(map);
3215 if (ln2range == 64)
3216 map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3217
3218 /*
3219 * Disable decoding via the command register before
3220 * determining the BAR's length since we will be placing it in
3221 * a weird state.
3222 */
3223 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3224 pci_write_config(dev, PCIR_COMMAND,
3225 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
3226
3227 /*
3228 * Determine the BAR's length by writing all 1's. The bottom
3229 * log_2(size) bits of the BAR will stick as 0 when we read
3230 * the value back.
3231 *
3232 * NB: according to the PCI Local Bus Specification, rev. 3.0:
3233 * "Software writes 0FFFFFFFFh to both registers, reads them back,
3234 * and combines the result into a 64-bit value." (section 6.2.5.1)
3235 *
3236 * Writes to both registers must be performed before attempting to
3237 * read back the size value.
3238 */
3239 testval = 0;
3240 pci_write_config(dev, reg, 0xffffffff, 4);
3241 if (ln2range == 64) {
3242 pci_write_config(dev, reg + 4, 0xffffffff, 4);
3243 testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3244 }
3245 testval |= pci_read_config(dev, reg, 4);
3246
3247 /*
3248 * Restore the original value of the BAR. We may have reprogrammed
3249 * the BAR of the low-level console device and when booting verbose,
3250 * we need the console device addressable.
3251 */
3252 pci_write_config(dev, reg, map, 4);
3253 if (ln2range == 64)
3254 pci_write_config(dev, reg + 4, map >> 32, 4);
3255 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3256
3257 *mapp = map;
3258 *testvalp = testval;
3259 if (bar64 != NULL)
3260 *bar64 = (ln2range == 64);
3261 }
3262
3263 static void
pci_write_bar(device_t dev,struct pci_map * pm,pci_addr_t base)3264 pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base)
3265 {
3266 struct pci_devinfo *dinfo;
3267 int ln2range;
3268
3269 /* The device ROM BAR is always a 32-bit memory BAR. */
3270 dinfo = device_get_ivars(dev);
3271 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3272 ln2range = 32;
3273 else
3274 ln2range = pci_maprange(pm->pm_value);
3275 pci_write_config(dev, pm->pm_reg, base, 4);
3276 if (ln2range == 64)
3277 pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4);
3278 pm->pm_value = pci_read_config(dev, pm->pm_reg, 4);
3279 if (ln2range == 64)
3280 pm->pm_value |= (pci_addr_t)pci_read_config(dev,
3281 pm->pm_reg + 4, 4) << 32;
3282 }
3283
3284 struct pci_map *
pci_find_bar(device_t dev,int reg)3285 pci_find_bar(device_t dev, int reg)
3286 {
3287 struct pci_devinfo *dinfo;
3288 struct pci_map *pm;
3289
3290 dinfo = device_get_ivars(dev);
3291 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3292 if (pm->pm_reg == reg)
3293 return (pm);
3294 }
3295 return (NULL);
3296 }
3297
3298 struct pci_map *
pci_first_bar(device_t dev)3299 pci_first_bar(device_t dev)
3300 {
3301 struct pci_devinfo *dinfo;
3302
3303 dinfo = device_get_ivars(dev);
3304 return (STAILQ_FIRST(&dinfo->cfg.maps));
3305 }
3306
3307 struct pci_map *
pci_next_bar(struct pci_map * pm)3308 pci_next_bar(struct pci_map *pm)
3309 {
3310 return (STAILQ_NEXT(pm, pm_link));
3311 }
3312
3313 int
pci_bar_enabled(device_t dev,struct pci_map * pm)3314 pci_bar_enabled(device_t dev, struct pci_map *pm)
3315 {
3316 struct pci_devinfo *dinfo;
3317 uint16_t cmd;
3318
3319 dinfo = device_get_ivars(dev);
3320 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) &&
3321 !(pm->pm_value & PCIM_BIOS_ENABLE))
3322 return (0);
3323 #ifdef PCI_IOV
3324 if ((dinfo->cfg.flags & PCICFG_VF) != 0) {
3325 struct pcicfg_iov *iov;
3326
3327 iov = dinfo->cfg.iov;
3328 cmd = pci_read_config(iov->iov_pf,
3329 iov->iov_pos + PCIR_SRIOV_CTL, 2);
3330 return ((cmd & PCIM_SRIOV_VF_MSE) != 0);
3331 }
3332 #endif
3333 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3334 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value))
3335 return ((cmd & PCIM_CMD_MEMEN) != 0);
3336 else
3337 return ((cmd & PCIM_CMD_PORTEN) != 0);
3338 }
3339
3340 struct pci_map *
pci_add_bar(device_t dev,int reg,pci_addr_t value,pci_addr_t size)3341 pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
3342 {
3343 struct pci_devinfo *dinfo;
3344 struct pci_map *pm, *prev;
3345
3346 dinfo = device_get_ivars(dev);
3347 pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO);
3348 pm->pm_reg = reg;
3349 pm->pm_value = value;
3350 pm->pm_size = size;
3351 STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) {
3352 KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x",
3353 reg));
3354 if (STAILQ_NEXT(prev, pm_link) == NULL ||
3355 STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg)
3356 break;
3357 }
3358 if (prev != NULL)
3359 STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link);
3360 else
3361 STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link);
3362 return (pm);
3363 }
3364
3365 static void
pci_restore_bars(device_t dev)3366 pci_restore_bars(device_t dev)
3367 {
3368 struct pci_devinfo *dinfo;
3369 struct pci_map *pm;
3370 int ln2range;
3371
3372 dinfo = device_get_ivars(dev);
3373 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3374 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3375 ln2range = 32;
3376 else
3377 ln2range = pci_maprange(pm->pm_value);
3378 pci_write_config(dev, pm->pm_reg, pm->pm_value, 4);
3379 if (ln2range == 64)
3380 pci_write_config(dev, pm->pm_reg + 4,
3381 pm->pm_value >> 32, 4);
3382 }
3383 }
3384
3385 /*
3386 * Add a resource based on a pci map register. Return 1 if the map
3387 * register is a 32bit map register or 2 if it is a 64bit register.
3388 */
3389 static int
pci_add_map(device_t bus,device_t dev,int reg,struct resource_list * rl,int force,int prefetch)3390 pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
3391 int force, int prefetch)
3392 {
3393 struct pci_map *pm;
3394 pci_addr_t base, map, testval;
3395 pci_addr_t start, end, count;
3396 int barlen, basezero, flags, maprange, mapsize, type;
3397 uint16_t cmd;
3398 struct resource *res;
3399
3400 /*
3401 * The BAR may already exist if the device is a CardBus card
3402 * whose CIS is stored in this BAR.
3403 */
3404 pm = pci_find_bar(dev, reg);
3405 if (pm != NULL) {
3406 maprange = pci_maprange(pm->pm_value);
3407 barlen = maprange == 64 ? 2 : 1;
3408 return (barlen);
3409 }
3410
3411 pci_read_bar(dev, reg, &map, &testval, NULL);
3412 if (PCI_BAR_MEM(map)) {
3413 type = SYS_RES_MEMORY;
3414 if (map & PCIM_BAR_MEM_PREFETCH)
3415 prefetch = 1;
3416 } else
3417 type = SYS_RES_IOPORT;
3418 mapsize = pci_mapsize(testval);
3419 base = pci_mapbase(map);
3420 #ifdef __PCI_BAR_ZERO_VALID
3421 basezero = 0;
3422 #else
3423 basezero = base == 0;
3424 #endif
3425 maprange = pci_maprange(map);
3426 barlen = maprange == 64 ? 2 : 1;
3427
3428 /*
3429 * For I/O registers, if bottom bit is set, and the next bit up
3430 * isn't clear, we know we have a BAR that doesn't conform to the
3431 * spec, so ignore it. Also, sanity check the size of the data
3432 * areas to the type of memory involved. Memory must be at least
3433 * 16 bytes in size, while I/O ranges must be at least 4.
3434 */
3435 if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0)
3436 return (barlen);
3437 if ((type == SYS_RES_MEMORY && mapsize < 4) ||
3438 (type == SYS_RES_IOPORT && mapsize < 2))
3439 return (barlen);
3440
3441 /* Save a record of this BAR. */
3442 pm = pci_add_bar(dev, reg, map, mapsize);
3443 if (bootverbose) {
3444 printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d",
3445 reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize);
3446 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3447 printf(", port disabled\n");
3448 else if (type == SYS_RES_MEMORY && !pci_memen(dev))
3449 printf(", memory disabled\n");
3450 else
3451 printf(", enabled\n");
3452 }
3453
3454 /*
3455 * If base is 0, then we have problems if this architecture does
3456 * not allow that. It is best to ignore such entries for the
3457 * moment. These will be allocated later if the driver specifically
3458 * requests them. However, some removable buses look better when
3459 * all resources are allocated, so allow '0' to be overridden.
3460 *
3461 * Similarly treat maps whose values is the same as the test value
3462 * read back. These maps have had all f's written to them by the
3463 * BIOS in an attempt to disable the resources.
3464 */
3465 if (!force && (basezero || map == testval))
3466 return (barlen);
3467 if ((u_long)base != base) {
3468 device_printf(bus,
3469 "pci%d:%d:%d:%d bar %#x too many address bits",
3470 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
3471 pci_get_function(dev), reg);
3472 return (barlen);
3473 }
3474
3475 /*
3476 * This code theoretically does the right thing, but has
3477 * undesirable side effects in some cases where peripherals
3478 * respond oddly to having these bits enabled. Let the user
3479 * be able to turn them off (since pci_enable_io_modes is 1 by
3480 * default).
3481 */
3482 if (pci_enable_io_modes) {
3483 /* Turn on resources that have been left off by a lazy BIOS */
3484 if (type == SYS_RES_IOPORT && !pci_porten(dev)) {
3485 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3486 cmd |= PCIM_CMD_PORTEN;
3487 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3488 }
3489 if (type == SYS_RES_MEMORY && !pci_memen(dev)) {
3490 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3491 cmd |= PCIM_CMD_MEMEN;
3492 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3493 }
3494 } else {
3495 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3496 return (barlen);
3497 if (type == SYS_RES_MEMORY && !pci_memen(dev))
3498 return (barlen);
3499 }
3500
3501 count = (pci_addr_t)1 << mapsize;
3502 flags = RF_ALIGNMENT_LOG2(mapsize);
3503 if (prefetch)
3504 flags |= RF_PREFETCHABLE;
3505 if (basezero || base == pci_mapbase(testval) || pci_clear_bars) {
3506 start = 0; /* Let the parent decide. */
3507 end = ~0;
3508 } else {
3509 start = base;
3510 end = base + count - 1;
3511 }
3512 resource_list_add(rl, type, reg, start, end, count);
3513
3514 /*
3515 * Try to allocate the resource for this BAR from our parent
3516 * so that this resource range is already reserved. The
3517 * driver for this device will later inherit this resource in
3518 * pci_alloc_resource().
3519 */
3520 res = resource_list_reserve(rl, bus, dev, type, reg, start, end, count,
3521 flags);
3522 if ((pci_do_realloc_bars
3523 || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR))
3524 && res == NULL && (start != 0 || end != ~0)) {
3525 /*
3526 * If the allocation fails, try to allocate a resource for
3527 * this BAR using any available range. The firmware felt
3528 * it was important enough to assign a resource, so don't
3529 * disable decoding if we can help it.
3530 */
3531 resource_list_delete(rl, type, reg);
3532 resource_list_add(rl, type, reg, 0, ~0, count);
3533 res = resource_list_reserve(rl, bus, dev, type, reg, 0, ~0,
3534 count, flags);
3535 }
3536 if (res == NULL) {
3537 /*
3538 * If the allocation fails, delete the resource list entry
3539 * and disable decoding for this device.
3540 *
3541 * If the driver requests this resource in the future,
3542 * pci_reserve_map() will try to allocate a fresh
3543 * resource range.
3544 */
3545 resource_list_delete(rl, type, reg);
3546 pci_disable_io(dev, type);
3547 if (bootverbose)
3548 device_printf(bus,
3549 "pci%d:%d:%d:%d bar %#x failed to allocate\n",
3550 pci_get_domain(dev), pci_get_bus(dev),
3551 pci_get_slot(dev), pci_get_function(dev), reg);
3552 } else {
3553 start = rman_get_start(res);
3554 pci_write_bar(dev, pm, start);
3555 }
3556 return (barlen);
3557 }
3558
3559 /*
3560 * For ATA devices we need to decide early what addressing mode to use.
3561 * Legacy demands that the primary and secondary ATA ports sits on the
3562 * same addresses that old ISA hardware did. This dictates that we use
3563 * those addresses and ignore the BAR's if we cannot set PCI native
3564 * addressing mode.
3565 */
3566 static void
pci_ata_maps(device_t bus,device_t dev,struct resource_list * rl,int force,uint32_t prefetchmask)3567 pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
3568 uint32_t prefetchmask)
3569 {
3570 int rid, type, progif;
3571 #if 0
3572 /* if this device supports PCI native addressing use it */
3573 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3574 if ((progif & 0x8a) == 0x8a) {
3575 if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) &&
3576 pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) {
3577 printf("Trying ATA native PCI addressing mode\n");
3578 pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1);
3579 }
3580 }
3581 #endif
3582 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3583 type = SYS_RES_IOPORT;
3584 if (progif & PCIP_STORAGE_IDE_MODEPRIM) {
3585 pci_add_map(bus, dev, PCIR_BAR(0), rl, force,
3586 prefetchmask & (1 << 0));
3587 pci_add_map(bus, dev, PCIR_BAR(1), rl, force,
3588 prefetchmask & (1 << 1));
3589 } else {
3590 rid = PCIR_BAR(0);
3591 resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
3592 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x1f0,
3593 0x1f7, 8, 0);
3594 rid = PCIR_BAR(1);
3595 resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
3596 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x3f6,
3597 0x3f6, 1, 0);
3598 }
3599 if (progif & PCIP_STORAGE_IDE_MODESEC) {
3600 pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
3601 prefetchmask & (1 << 2));
3602 pci_add_map(bus, dev, PCIR_BAR(3), rl, force,
3603 prefetchmask & (1 << 3));
3604 } else {
3605 rid = PCIR_BAR(2);
3606 resource_list_add(rl, type, rid, 0x170, 0x177, 8);
3607 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x170,
3608 0x177, 8, 0);
3609 rid = PCIR_BAR(3);
3610 resource_list_add(rl, type, rid, 0x376, 0x376, 1);
3611 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x376,
3612 0x376, 1, 0);
3613 }
3614 pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
3615 prefetchmask & (1 << 4));
3616 pci_add_map(bus, dev, PCIR_BAR(5), rl, force,
3617 prefetchmask & (1 << 5));
3618 }
3619
3620 static void
pci_assign_interrupt(device_t bus,device_t dev,int force_route)3621 pci_assign_interrupt(device_t bus, device_t dev, int force_route)
3622 {
3623 struct pci_devinfo *dinfo = device_get_ivars(dev);
3624 pcicfgregs *cfg = &dinfo->cfg;
3625 char tunable_name[64];
3626 int irq;
3627
3628 /* Has to have an intpin to have an interrupt. */
3629 if (cfg->intpin == 0)
3630 return;
3631
3632 /* Let the user override the IRQ with a tunable. */
3633 irq = PCI_INVALID_IRQ;
3634 snprintf(tunable_name, sizeof(tunable_name),
3635 "hw.pci%d.%d.%d.INT%c.irq",
3636 cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
3637 if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0))
3638 irq = PCI_INVALID_IRQ;
3639
3640 /*
3641 * If we didn't get an IRQ via the tunable, then we either use the
3642 * IRQ value in the intline register or we ask the bus to route an
3643 * interrupt for us. If force_route is true, then we only use the
3644 * value in the intline register if the bus was unable to assign an
3645 * IRQ.
3646 */
3647 if (!PCI_INTERRUPT_VALID(irq)) {
3648 if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route)
3649 irq = PCI_ASSIGN_INTERRUPT(bus, dev);
3650 if (!PCI_INTERRUPT_VALID(irq))
3651 irq = cfg->intline;
3652 }
3653
3654 /* If after all that we don't have an IRQ, just bail. */
3655 if (!PCI_INTERRUPT_VALID(irq))
3656 return;
3657
3658 /* Update the config register if it changed. */
3659 if (irq != cfg->intline) {
3660 cfg->intline = irq;
3661 pci_write_config(dev, PCIR_INTLINE, irq, 1);
3662 }
3663
3664 /* Add this IRQ as rid 0 interrupt resource. */
3665 resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1);
3666 }
3667
3668 /* Perform early OHCI takeover from SMM. */
3669 static void
ohci_early_takeover(device_t self)3670 ohci_early_takeover(device_t self)
3671 {
3672 struct resource *res;
3673 uint32_t ctl;
3674 int rid;
3675 int i;
3676
3677 rid = PCIR_BAR(0);
3678 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3679 if (res == NULL)
3680 return;
3681
3682 ctl = bus_read_4(res, OHCI_CONTROL);
3683 if (ctl & OHCI_IR) {
3684 if (bootverbose)
3685 printf("ohci early: "
3686 "SMM active, request owner change\n");
3687 bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR);
3688 for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) {
3689 DELAY(1000);
3690 ctl = bus_read_4(res, OHCI_CONTROL);
3691 }
3692 if (ctl & OHCI_IR) {
3693 if (bootverbose)
3694 printf("ohci early: "
3695 "SMM does not respond, resetting\n");
3696 bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET);
3697 }
3698 /* Disable interrupts */
3699 bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS);
3700 }
3701
3702 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3703 }
3704
3705 /* Perform early UHCI takeover from SMM. */
3706 static void
uhci_early_takeover(device_t self)3707 uhci_early_takeover(device_t self)
3708 {
3709 struct resource *res;
3710 int rid;
3711
3712 /*
3713 * Set the PIRQD enable bit and switch off all the others. We don't
3714 * want legacy support to interfere with us XXX Does this also mean
3715 * that the BIOS won't touch the keyboard anymore if it is connected
3716 * to the ports of the root hub?
3717 */
3718 pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);
3719
3720 /* Disable interrupts */
3721 rid = PCI_UHCI_BASE_REG;
3722 res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE);
3723 if (res != NULL) {
3724 bus_write_2(res, UHCI_INTR, 0);
3725 bus_release_resource(self, SYS_RES_IOPORT, rid, res);
3726 }
3727 }
3728
3729 /* Perform early EHCI takeover from SMM. */
3730 static void
ehci_early_takeover(device_t self)3731 ehci_early_takeover(device_t self)
3732 {
3733 struct resource *res;
3734 uint32_t cparams;
3735 uint32_t eec;
3736 uint8_t eecp;
3737 uint8_t bios_sem;
3738 uint8_t offs;
3739 int rid;
3740 int i;
3741
3742 rid = PCIR_BAR(0);
3743 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3744 if (res == NULL)
3745 return;
3746
3747 cparams = bus_read_4(res, EHCI_HCCPARAMS);
3748
3749 /* Synchronise with the BIOS if it owns the controller. */
3750 for (eecp = EHCI_HCC_EECP(cparams); eecp != 0;
3751 eecp = EHCI_EECP_NEXT(eec)) {
3752 eec = pci_read_config(self, eecp, 4);
3753 if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) {
3754 continue;
3755 }
3756 bios_sem = pci_read_config(self, eecp +
3757 EHCI_LEGSUP_BIOS_SEM, 1);
3758 if (bios_sem == 0) {
3759 continue;
3760 }
3761 if (bootverbose)
3762 printf("ehci early: "
3763 "SMM active, request owner change\n");
3764
3765 pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1);
3766
3767 for (i = 0; (i < 100) && (bios_sem != 0); i++) {
3768 DELAY(1000);
3769 bios_sem = pci_read_config(self, eecp +
3770 EHCI_LEGSUP_BIOS_SEM, 1);
3771 }
3772
3773 if (bios_sem != 0) {
3774 if (bootverbose)
3775 printf("ehci early: "
3776 "SMM does not respond\n");
3777 }
3778 /* Disable interrupts */
3779 offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION));
3780 bus_write_4(res, offs + EHCI_USBINTR, 0);
3781 }
3782 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3783 }
3784
3785 /* Perform early XHCI takeover from SMM. */
3786 static void
xhci_early_takeover(device_t self)3787 xhci_early_takeover(device_t self)
3788 {
3789 struct resource *res;
3790 uint32_t cparams;
3791 uint32_t eec;
3792 uint32_t eecp;
3793 uint8_t bios_sem;
3794 uint8_t offs;
3795 int rid;
3796 int i;
3797
3798 rid = PCIR_BAR(0);
3799 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3800 if (res == NULL)
3801 return;
3802
3803 cparams = bus_read_4(res, XHCI_HCCPARAMS1);
3804
3805 eec = -1;
3806
3807 /* Synchronise with the BIOS if it owns the controller. */
3808 for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec);
3809 eecp += XHCI_XECP_NEXT(eec) << 2) {
3810 eec = bus_read_4(res, eecp);
3811
3812 if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY)
3813 continue;
3814
3815 bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM);
3816 if (bios_sem == 0)
3817 continue;
3818
3819 if (bootverbose)
3820 printf("xhci early: "
3821 "SMM active, request owner change\n");
3822
3823 bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1);
3824
3825 /* wait a maximum of 5 second */
3826
3827 for (i = 0; (i < 5000) && (bios_sem != 0); i++) {
3828 DELAY(1000);
3829 bios_sem = bus_read_1(res, eecp +
3830 XHCI_XECP_BIOS_SEM);
3831 }
3832
3833 if (bios_sem != 0) {
3834 if (bootverbose)
3835 printf("xhci early: "
3836 "SMM does not respond\n");
3837 }
3838
3839 /* Disable interrupts */
3840 offs = bus_read_1(res, XHCI_CAPLENGTH);
3841 bus_write_4(res, offs + XHCI_USBCMD, 0);
3842 bus_read_4(res, offs + XHCI_USBSTS);
3843 }
3844 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3845 }
3846
3847 static void
pci_reserve_secbus(device_t bus,device_t dev,pcicfgregs * cfg,struct resource_list * rl)3848 pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg,
3849 struct resource_list *rl)
3850 {
3851 struct resource *res;
3852 char *cp;
3853 rman_res_t start, end, count;
3854 int sec_bus, sec_reg, sub_bus, sub_reg, sup_bus;
3855
3856 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3857 case PCIM_HDRTYPE_BRIDGE:
3858 sec_reg = PCIR_SECBUS_1;
3859 sub_reg = PCIR_SUBBUS_1;
3860 break;
3861 case PCIM_HDRTYPE_CARDBUS:
3862 sec_reg = PCIR_SECBUS_2;
3863 sub_reg = PCIR_SUBBUS_2;
3864 break;
3865 default:
3866 return;
3867 }
3868
3869 /*
3870 * If the existing bus range is valid, attempt to reserve it
3871 * from our parent. If this fails for any reason, clear the
3872 * secbus and subbus registers.
3873 *
3874 * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus?
3875 * This would at least preserve the existing sec_bus if it is
3876 * valid.
3877 */
3878 sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1);
3879 sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1);
3880
3881 /* Quirk handling. */
3882 switch (pci_get_devid(dev)) {
3883 case 0x12258086: /* Intel 82454KX/GX (Orion) */
3884 sup_bus = pci_read_config(dev, 0x41, 1);
3885 if (sup_bus != 0xff) {
3886 sec_bus = sup_bus + 1;
3887 sub_bus = sup_bus + 1;
3888 PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1);
3889 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3890 }
3891 break;
3892
3893 case 0x00dd10de:
3894 /* Compaq R3000 BIOS sets wrong subordinate bus number. */
3895 if ((cp = kern_getenv("smbios.planar.maker")) == NULL)
3896 break;
3897 if (strncmp(cp, "Compal", 6) != 0) {
3898 freeenv(cp);
3899 break;
3900 }
3901 freeenv(cp);
3902 if ((cp = kern_getenv("smbios.planar.product")) == NULL)
3903 break;
3904 if (strncmp(cp, "08A0", 4) != 0) {
3905 freeenv(cp);
3906 break;
3907 }
3908 freeenv(cp);
3909 if (sub_bus < 0xa) {
3910 sub_bus = 0xa;
3911 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3912 }
3913 break;
3914 }
3915
3916 if (bootverbose)
3917 printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus);
3918 if (sec_bus > 0 && sub_bus >= sec_bus) {
3919 start = sec_bus;
3920 end = sub_bus;
3921 count = end - start + 1;
3922
3923 resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count);
3924
3925 /*
3926 * If requested, clear secondary bus registers in
3927 * bridge devices to force a complete renumbering
3928 * rather than reserving the existing range. However,
3929 * preserve the existing size.
3930 */
3931 if (pci_clear_buses)
3932 goto clear;
3933
3934 res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, 0,
3935 start, end, count, 0);
3936 if (res != NULL)
3937 return;
3938
3939 if (bootverbose)
3940 device_printf(bus,
3941 "pci%d:%d:%d:%d secbus failed to allocate\n",
3942 pci_get_domain(dev), pci_get_bus(dev),
3943 pci_get_slot(dev), pci_get_function(dev));
3944 }
3945
3946 clear:
3947 PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1);
3948 PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1);
3949 }
3950
3951 static struct resource *
pci_alloc_secbus(device_t dev,device_t child,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)3952 pci_alloc_secbus(device_t dev, device_t child, int rid, rman_res_t start,
3953 rman_res_t end, rman_res_t count, u_int flags)
3954 {
3955 struct pci_devinfo *dinfo;
3956 pcicfgregs *cfg;
3957 struct resource_list *rl;
3958 struct resource *res;
3959 int sec_reg, sub_reg;
3960
3961 dinfo = device_get_ivars(child);
3962 cfg = &dinfo->cfg;
3963 rl = &dinfo->resources;
3964 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3965 case PCIM_HDRTYPE_BRIDGE:
3966 sec_reg = PCIR_SECBUS_1;
3967 sub_reg = PCIR_SUBBUS_1;
3968 break;
3969 case PCIM_HDRTYPE_CARDBUS:
3970 sec_reg = PCIR_SECBUS_2;
3971 sub_reg = PCIR_SUBBUS_2;
3972 break;
3973 default:
3974 return (NULL);
3975 }
3976
3977 if (rid != 0)
3978 return (NULL);
3979
3980 if (resource_list_find(rl, PCI_RES_BUS, rid) == NULL)
3981 resource_list_add(rl, PCI_RES_BUS, rid, start, end, count);
3982 if (!resource_list_reserved(rl, PCI_RES_BUS, rid)) {
3983 res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid,
3984 start, end, count, flags & ~RF_ACTIVE);
3985 if (res == NULL) {
3986 resource_list_delete(rl, PCI_RES_BUS, rid);
3987 device_printf(child, "allocating %ju bus%s failed\n",
3988 count, count == 1 ? "" : "es");
3989 return (NULL);
3990 }
3991 if (bootverbose)
3992 device_printf(child,
3993 "Lazy allocation of %ju bus%s at %ju\n", count,
3994 count == 1 ? "" : "es", rman_get_start(res));
3995 PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1);
3996 PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1);
3997 }
3998 return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start,
3999 end, count, flags));
4000 }
4001
4002 static int
pci_ea_bei_to_rid(device_t dev,int bei)4003 pci_ea_bei_to_rid(device_t dev, int bei)
4004 {
4005 #ifdef PCI_IOV
4006 struct pci_devinfo *dinfo;
4007 int iov_pos;
4008 struct pcicfg_iov *iov;
4009
4010 dinfo = device_get_ivars(dev);
4011 iov = dinfo->cfg.iov;
4012 if (iov != NULL)
4013 iov_pos = iov->iov_pos;
4014 else
4015 iov_pos = 0;
4016 #endif
4017
4018 /* Check if matches BAR */
4019 if ((bei >= PCIM_EA_BEI_BAR_0) &&
4020 (bei <= PCIM_EA_BEI_BAR_5))
4021 return (PCIR_BAR(bei));
4022
4023 /* Check ROM */
4024 if (bei == PCIM_EA_BEI_ROM)
4025 return (PCIR_BIOS);
4026
4027 #ifdef PCI_IOV
4028 /* Check if matches VF_BAR */
4029 if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) &&
4030 (bei <= PCIM_EA_BEI_VF_BAR_5))
4031 return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) +
4032 iov_pos);
4033 #endif
4034
4035 return (-1);
4036 }
4037
4038 int
pci_ea_is_enabled(device_t dev,int rid)4039 pci_ea_is_enabled(device_t dev, int rid)
4040 {
4041 struct pci_ea_entry *ea;
4042 struct pci_devinfo *dinfo;
4043
4044 dinfo = device_get_ivars(dev);
4045
4046 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
4047 if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid)
4048 return ((ea->eae_flags & PCIM_EA_ENABLE) > 0);
4049 }
4050
4051 return (0);
4052 }
4053
4054 void
pci_add_resources_ea(device_t bus,device_t dev,int alloc_iov)4055 pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
4056 {
4057 struct pci_ea_entry *ea;
4058 struct pci_devinfo *dinfo;
4059 pci_addr_t start, end, count;
4060 struct resource_list *rl;
4061 int type, flags, rid;
4062 struct resource *res;
4063 uint32_t tmp;
4064 #ifdef PCI_IOV
4065 struct pcicfg_iov *iov;
4066 #endif
4067
4068 dinfo = device_get_ivars(dev);
4069 rl = &dinfo->resources;
4070 flags = 0;
4071
4072 #ifdef PCI_IOV
4073 iov = dinfo->cfg.iov;
4074 #endif
4075
4076 if (dinfo->cfg.ea.ea_location == 0)
4077 return;
4078
4079 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
4080 /*
4081 * TODO: Ignore EA-BAR if is not enabled.
4082 * Currently the EA implementation supports
4083 * only situation, where EA structure contains
4084 * predefined entries. In case they are not enabled
4085 * leave them unallocated and proceed with
4086 * a legacy-BAR mechanism.
4087 */
4088 if ((ea->eae_flags & PCIM_EA_ENABLE) == 0)
4089 continue;
4090
4091 switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) {
4092 case PCIM_EA_P_MEM_PREFETCH:
4093 case PCIM_EA_P_VF_MEM_PREFETCH:
4094 flags = RF_PREFETCHABLE;
4095 /* FALLTHROUGH */
4096 case PCIM_EA_P_VF_MEM:
4097 case PCIM_EA_P_MEM:
4098 type = SYS_RES_MEMORY;
4099 break;
4100 case PCIM_EA_P_IO:
4101 type = SYS_RES_IOPORT;
4102 break;
4103 default:
4104 continue;
4105 }
4106
4107 if (alloc_iov != 0) {
4108 #ifdef PCI_IOV
4109 /* Allocating IOV, confirm BEI matches */
4110 if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) ||
4111 (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5))
4112 continue;
4113 #else
4114 continue;
4115 #endif
4116 } else {
4117 /* Allocating BAR, confirm BEI matches */
4118 if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) ||
4119 (ea->eae_bei > PCIM_EA_BEI_BAR_5)) &&
4120 (ea->eae_bei != PCIM_EA_BEI_ROM))
4121 continue;
4122 }
4123
4124 rid = pci_ea_bei_to_rid(dev, ea->eae_bei);
4125 if (rid < 0)
4126 continue;
4127
4128 /* Skip resources already allocated by EA */
4129 if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) ||
4130 (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL))
4131 continue;
4132
4133 start = ea->eae_base;
4134 count = ea->eae_max_offset + 1;
4135 #ifdef PCI_IOV
4136 if (iov != NULL)
4137 count = count * iov->iov_num_vfs;
4138 #endif
4139 end = start + count - 1;
4140 if (count == 0)
4141 continue;
4142
4143 resource_list_add(rl, type, rid, start, end, count);
4144 res = resource_list_reserve(rl, bus, dev, type, rid, start, end, count,
4145 flags);
4146 if (res == NULL) {
4147 resource_list_delete(rl, type, rid);
4148
4149 /*
4150 * Failed to allocate using EA, disable entry.
4151 * Another attempt to allocation will be performed
4152 * further, but this time using legacy BAR registers
4153 */
4154 tmp = pci_read_config(dev, ea->eae_cfg_offset, 4);
4155 tmp &= ~PCIM_EA_ENABLE;
4156 pci_write_config(dev, ea->eae_cfg_offset, tmp, 4);
4157
4158 /*
4159 * Disabling entry might fail in case it is hardwired.
4160 * Read flags again to match current status.
4161 */
4162 ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4);
4163
4164 continue;
4165 }
4166
4167 /* As per specification, fill BAR with zeros */
4168 pci_write_config(dev, rid, 0, 4);
4169 }
4170 }
4171
4172 void
pci_add_resources(device_t bus,device_t dev,int force,uint32_t prefetchmask)4173 pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
4174 {
4175 struct pci_devinfo *dinfo;
4176 pcicfgregs *cfg;
4177 struct resource_list *rl;
4178 const struct pci_quirk *q;
4179 uint32_t devid;
4180 int i;
4181
4182 dinfo = device_get_ivars(dev);
4183 cfg = &dinfo->cfg;
4184 rl = &dinfo->resources;
4185 devid = (cfg->device << 16) | cfg->vendor;
4186
4187 /* Allocate resources using Enhanced Allocation */
4188 pci_add_resources_ea(bus, dev, 0);
4189
4190 /* ATA devices needs special map treatment */
4191 if ((pci_get_class(dev) == PCIC_STORAGE) &&
4192 (pci_get_subclass(dev) == PCIS_STORAGE_IDE) &&
4193 ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) ||
4194 (!pci_read_config(dev, PCIR_BAR(0), 4) &&
4195 !pci_read_config(dev, PCIR_BAR(2), 4))) )
4196 pci_ata_maps(bus, dev, rl, force, prefetchmask);
4197 else
4198 for (i = 0; i < cfg->nummaps;) {
4199 /* Skip resources already managed by EA */
4200 if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) ||
4201 (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) ||
4202 pci_ea_is_enabled(dev, PCIR_BAR(i))) {
4203 i++;
4204 continue;
4205 }
4206
4207 /*
4208 * Skip quirked resources.
4209 */
4210 for (q = &pci_quirks[0]; q->devid != 0; q++)
4211 if (q->devid == devid &&
4212 q->type == PCI_QUIRK_UNMAP_REG &&
4213 q->arg1 == PCIR_BAR(i))
4214 break;
4215 if (q->devid != 0) {
4216 i++;
4217 continue;
4218 }
4219 i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force,
4220 prefetchmask & (1 << i));
4221 }
4222
4223 /*
4224 * Add additional, quirked resources.
4225 */
4226 for (q = &pci_quirks[0]; q->devid != 0; q++)
4227 if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG)
4228 pci_add_map(bus, dev, q->arg1, rl, force, 0);
4229
4230 if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline) &&
4231 pci_intx_reroute) {
4232 /*
4233 * Try to re-route interrupts. Sometimes the BIOS or
4234 * firmware may leave bogus values in these registers.
4235 * If the re-route fails, then just stick with what we
4236 * have.
4237 */
4238 pci_assign_interrupt(bus, dev, 1);
4239 }
4240
4241 if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS &&
4242 pci_get_subclass(dev) == PCIS_SERIALBUS_USB) {
4243 if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI)
4244 xhci_early_takeover(dev);
4245 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI)
4246 ehci_early_takeover(dev);
4247 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI)
4248 ohci_early_takeover(dev);
4249 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI)
4250 uhci_early_takeover(dev);
4251 }
4252
4253 /*
4254 * Reserve resources for secondary bus ranges behind bridge
4255 * devices.
4256 */
4257 pci_reserve_secbus(bus, dev, cfg, rl);
4258 }
4259
4260 static struct pci_devinfo *
pci_identify_function(device_t pcib,device_t dev,int domain,int busno,int slot,int func)4261 pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
4262 int slot, int func)
4263 {
4264 struct pci_devinfo *dinfo;
4265
4266 dinfo = pci_read_device(pcib, dev, domain, busno, slot, func);
4267 if (dinfo != NULL)
4268 pci_add_child(dev, dinfo);
4269
4270 return (dinfo);
4271 }
4272
4273 void
pci_add_children(device_t dev,int domain,int busno)4274 pci_add_children(device_t dev, int domain, int busno)
4275 {
4276 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4277 device_t pcib = device_get_parent(dev);
4278 struct pci_devinfo *dinfo;
4279 int maxslots;
4280 int s, f, pcifunchigh;
4281 uint8_t hdrtype;
4282 int first_func;
4283
4284 /*
4285 * Try to detect a device at slot 0, function 0. If it exists, try to
4286 * enable ARI. We must enable ARI before detecting the rest of the
4287 * functions on this bus as ARI changes the set of slots and functions
4288 * that are legal on this bus.
4289 */
4290 dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0);
4291 if (dinfo != NULL && pci_enable_ari)
4292 PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);
4293
4294 /*
4295 * Start looking for new devices on slot 0 at function 1 because we
4296 * just identified the device at slot 0, function 0.
4297 */
4298 first_func = 1;
4299
4300 maxslots = PCIB_MAXSLOTS(pcib);
4301 for (s = 0; s <= maxslots; s++, first_func = 0) {
4302 pcifunchigh = 0;
4303 f = 0;
4304 DELAY(1);
4305
4306 /* If function 0 is not present, skip to the next slot. */
4307 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4308 continue;
4309 hdrtype = REG(PCIR_HDRTYPE, 1);
4310 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4311 continue;
4312 if (hdrtype & PCIM_MFDEV)
4313 pcifunchigh = PCIB_MAXFUNCS(pcib);
4314 for (f = first_func; f <= pcifunchigh; f++)
4315 pci_identify_function(pcib, dev, domain, busno, s, f);
4316 }
4317 pcie_reconcile_link_mps(dev);
4318 #undef REG
4319 }
4320
4321 int
pci_rescan_method(device_t dev)4322 pci_rescan_method(device_t dev)
4323 {
4324 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4325 device_t pcib = device_get_parent(dev);
4326 device_t child, *devlist, *unchanged;
4327 int devcount, error, i, j, maxslots, oldcount;
4328 int busno, domain, s, f, pcifunchigh;
4329 uint8_t hdrtype;
4330
4331 /* No need to check for ARI on a rescan. */
4332 error = device_get_children(dev, &devlist, &devcount);
4333 if (error)
4334 return (error);
4335 if (devcount != 0) {
4336 unchanged = malloc(devcount * sizeof(device_t), M_TEMP,
4337 M_NOWAIT | M_ZERO);
4338 if (unchanged == NULL) {
4339 free(devlist, M_TEMP);
4340 return (ENOMEM);
4341 }
4342 } else
4343 unchanged = NULL;
4344
4345 domain = pcib_get_domain(dev);
4346 busno = pcib_get_bus(dev);
4347 maxslots = PCIB_MAXSLOTS(pcib);
4348 for (s = 0; s <= maxslots; s++) {
4349 /* If function 0 is not present, skip to the next slot. */
4350 f = 0;
4351 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4352 continue;
4353 pcifunchigh = 0;
4354 hdrtype = REG(PCIR_HDRTYPE, 1);
4355 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4356 continue;
4357 if (hdrtype & PCIM_MFDEV)
4358 pcifunchigh = PCIB_MAXFUNCS(pcib);
4359 for (f = 0; f <= pcifunchigh; f++) {
4360 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4361 continue;
4362
4363 /*
4364 * Found a valid function. Check if a
4365 * device_t for this device already exists.
4366 */
4367 for (i = 0; i < devcount; i++) {
4368 child = devlist[i];
4369 if (child == NULL)
4370 continue;
4371 if (pci_get_slot(child) == s &&
4372 pci_get_function(child) == f) {
4373 unchanged[i] = child;
4374 goto next_func;
4375 }
4376 }
4377
4378 pci_identify_function(pcib, dev, domain, busno, s, f);
4379 next_func:;
4380 }
4381 }
4382
4383 /* Remove devices that are no longer present. */
4384 for (i = 0; i < devcount; i++) {
4385 if (unchanged[i] != NULL)
4386 continue;
4387 device_delete_child(dev, devlist[i]);
4388 }
4389
4390 free(devlist, M_TEMP);
4391 oldcount = devcount;
4392
4393 /* Try to attach the devices just added. */
4394 error = device_get_children(dev, &devlist, &devcount);
4395 if (error) {
4396 free(unchanged, M_TEMP);
4397 return (error);
4398 }
4399
4400 for (i = 0; i < devcount; i++) {
4401 for (j = 0; j < oldcount; j++) {
4402 if (devlist[i] == unchanged[j])
4403 goto next_device;
4404 }
4405
4406 device_probe_and_attach(devlist[i]);
4407 next_device:;
4408 }
4409
4410 free(unchanged, M_TEMP);
4411 free(devlist, M_TEMP);
4412 return (0);
4413 #undef REG
4414 }
4415
4416 #ifdef PCI_IOV
4417 device_t
pci_iov_get_pf(device_t dev)4418 pci_iov_get_pf(device_t dev)
4419 {
4420 struct pci_devinfo *dinfo;
4421
4422 dinfo = device_get_ivars(dev);
4423 if (dinfo == NULL || (dinfo->cfg.flags & PCICFG_VF) == 0 ||
4424 dinfo->cfg.iov == NULL)
4425 return (NULL);
4426 return (dinfo->cfg.iov->iov_pf);
4427 }
4428
4429 device_t
pci_add_iov_child(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4430 pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid,
4431 uint16_t did)
4432 {
4433 struct pci_devinfo *pf_dinfo;
4434 struct pci_devinfo *vf_dinfo;
4435 device_t pcib;
4436 int busno, slot, func;
4437
4438 pcib = device_get_parent(bus);
4439
4440 PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func);
4441
4442 vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno,
4443 slot, func, vid, did);
4444
4445 /* Make the VF-to-PF relationship available to child-added callbacks. */
4446 pf_dinfo = device_get_ivars(pf);
4447 KASSERT(pf_dinfo->cfg.iov != NULL,
4448 ("SR-IOV PF %s has no IOV state", device_get_nameunit(pf)));
4449 vf_dinfo->cfg.iov = pf_dinfo->cfg.iov;
4450 vf_dinfo->cfg.flags |= PCICFG_VF;
4451 pci_add_child(bus, vf_dinfo);
4452
4453 return (vf_dinfo->cfg.dev);
4454 }
4455
4456 device_t
pci_create_iov_child_method(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4457 pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid,
4458 uint16_t vid, uint16_t did)
4459 {
4460
4461 return (pci_add_iov_child(bus, pf, rid, vid, did));
4462 }
4463 #else
4464 device_t
pci_iov_get_pf(device_t dev __unused)4465 pci_iov_get_pf(device_t dev __unused)
4466 {
4467
4468 return (NULL);
4469 }
4470 #endif
4471
4472 static int
pcie_mps_bytes(uint16_t mps)4473 pcie_mps_bytes(uint16_t mps)
4474 {
4475
4476 return (128 << (mps >> 5));
4477 }
4478
4479 static bool
pcie_mps_limit_value(uint16_t * mps)4480 pcie_mps_limit_value(uint16_t *mps)
4481 {
4482
4483 if (pci_mps_limit == 0)
4484 return (false);
4485 if (pci_mps_limit < 128 || pci_mps_limit > 4096 ||
4486 !powerof2(pci_mps_limit)) {
4487 if (!pci_mps_limit_warned) {
4488 printf("pci: invalid hw.pci.mps_limit=%d; ignoring\n",
4489 pci_mps_limit);
4490 pci_mps_limit_warned = true;
4491 }
4492 return (false);
4493 }
4494 *mps = (fls(pci_mps_limit) - 8) << 5;
4495 return (true);
4496 }
4497
4498 /* Return the smallest configured MPS above dev, if the walk reaches a root. */
4499 static bool
pcie_path_mps(device_t dev,uint16_t * mpsp)4500 pcie_path_mps(device_t dev, uint16_t *mpsp)
4501 {
4502 struct pci_devinfo *dinfo;
4503 device_t bus, pcib, start;
4504 uint16_t mps;
4505 bool found;
4506
4507 start = dev;
4508 found = false;
4509 for (;;) {
4510 bus = device_get_parent(dev);
4511 if (bus == NULL)
4512 break;
4513 pcib = device_get_parent(bus);
4514 if (pcib == NULL || !is_pci_device(pcib))
4515 break;
4516 /*
4517 * A PCI function may expose a host bridge for a synthetic PCI
4518 * domain. Its Device Control belongs to the parent domain and
4519 * does not describe an upstream link in the synthetic hierarchy.
4520 */
4521 if (pci_get_domain(pcib) != pci_get_domain(dev))
4522 break;
4523 dinfo = device_get_ivars(pcib);
4524 if (dinfo->cfg.pcie.pcie_location != 0) {
4525 mps = pcie_read_config(pcib, PCIER_DEVICE_CTL, 2) &
4526 PCIEM_CTL_MAX_PAYLOAD;
4527 if (!found || mps < *mpsp)
4528 *mpsp = mps;
4529 found = true;
4530 if (dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
4531 return (true);
4532 }
4533 dev = pcib;
4534 }
4535 if (found && bootverbose)
4536 device_printf(start,
4537 "PCIe MPS path walk did not reach a Root Port\n");
4538 return (false);
4539 }
4540
4541 static bool
pcie_mps_first_warning(device_t dev)4542 pcie_mps_first_warning(device_t dev)
4543 {
4544 struct pci_devinfo *dinfo;
4545
4546 dinfo = device_get_ivars(dev);
4547 if ((dinfo->cfg.flags & PCICFG_MPS_WARNED) != 0)
4548 return (false);
4549 dinfo->cfg.flags |= PCICFG_MPS_WARNED;
4550 return (true);
4551 }
4552
4553 static void
pcie_mps_conflict(device_t dev,uint16_t path_mps,uint16_t max_mps)4554 pcie_mps_conflict(device_t dev, uint16_t path_mps, uint16_t max_mps)
4555 {
4556
4557 if (!pcie_mps_first_warning(dev))
4558 return;
4559 device_printf(dev,
4560 "maximum supported MPS %d is below configured path MPS %d; "
4561 "cannot safely retune the shared ancestor hierarchy\n",
4562 pcie_mps_bytes(max_mps), pcie_mps_bytes(path_mps));
4563 }
4564
4565 static bool
pcie_mps_is_bridge(struct pci_devinfo * dinfo)4566 pcie_mps_is_bridge(struct pci_devinfo *dinfo)
4567 {
4568 uint8_t hdrtype;
4569
4570 hdrtype = dinfo->cfg.hdrtype & PCIM_HDRTYPE;
4571 return (hdrtype == PCIM_HDRTYPE_BRIDGE ||
4572 hdrtype == PCIM_HDRTYPE_CARDBUS);
4573 }
4574
4575 static void
pcie_mps_active_conflict(device_t dev,uint16_t path_mps,uint16_t device_mps)4576 pcie_mps_active_conflict(device_t dev, uint16_t path_mps,
4577 uint16_t device_mps)
4578 {
4579 struct pci_devinfo *dinfo;
4580 const char *action;
4581
4582 if (!pcie_mps_first_warning(dev))
4583 return;
4584 dinfo = device_get_ivars(dev);
4585 if (pci_mps_enforce && !pcie_mps_is_bridge(dinfo))
4586 action = "disabling device";
4587 else
4588 action = "leaving device unchanged";
4589 device_printf(dev,
4590 "configured MPS %d does not match path MPS %d while bus "
4591 "mastering is enabled; %s\n", pcie_mps_bytes(device_mps),
4592 pcie_mps_bytes(path_mps), action);
4593 }
4594
4595 static void
pcie_mps_mark_unreconciled(device_t dev)4596 pcie_mps_mark_unreconciled(device_t dev)
4597 {
4598 struct pci_devinfo *dinfo;
4599 uint16_t cmd;
4600
4601 dinfo = device_get_ivars(dev);
4602 if ((dinfo->cfg.flags & PCICFG_MPS_UNRECONCILED) != 0)
4603 return;
4604 dinfo->cfg.flags |= PCICFG_MPS_UNRECONCILED;
4605 if (!pci_mps_enforce)
4606 return;
4607 if (pcie_mps_is_bridge(dinfo)) {
4608 device_printf(dev,
4609 "not disabled by hw.pci.mps_enforce because it is a bridge\n");
4610 return;
4611 }
4612 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
4613 cmd &= ~(PCIM_CMD_PORTEN | PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
4614 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
4615 dinfo->cfg.cmdreg = cmd;
4616 device_disable(dev);
4617 device_printf(dev,
4618 "disabled because its MPS cannot be safely configured\n");
4619 }
4620
4621 static void
pcie_mps_unreconciled(device_t dev,uint16_t path_mps,uint16_t max_mps)4622 pcie_mps_unreconciled(device_t dev, uint16_t path_mps, uint16_t max_mps)
4623 {
4624
4625 pcie_mps_conflict(dev, path_mps, max_mps);
4626 pcie_mps_mark_unreconciled(dev);
4627 }
4628
4629 static void
pcie_mps_active_unreconciled(device_t dev,uint16_t path_mps,uint16_t device_mps)4630 pcie_mps_active_unreconciled(device_t dev, uint16_t path_mps,
4631 uint16_t device_mps)
4632 {
4633
4634 pcie_mps_active_conflict(dev, path_mps, device_mps);
4635 pcie_mps_mark_unreconciled(dev);
4636 }
4637
4638 static void
pcie_mps_mark_link_unreconciled(device_t * devlist,int count,uint16_t path_mps,bool all)4639 pcie_mps_mark_link_unreconciled(device_t *devlist, int count,
4640 uint16_t path_mps,
4641 bool all)
4642 {
4643 struct pci_devinfo *dinfo;
4644 device_t child;
4645 uint16_t mmps;
4646 int i;
4647
4648 for (i = 0; i < count; i++) {
4649 child = devlist[i];
4650 dinfo = device_get_ivars(child);
4651 if ((dinfo->cfg.flags & (PCICFG_VF |
4652 PCICFG_MPS_UNRECONCILED)) != 0 ||
4653 dinfo->cfg.pcie.pcie_location == 0)
4654 continue;
4655 if (all) {
4656 pcie_mps_mark_unreconciled(child);
4657 continue;
4658 }
4659 mmps = (pcie_read_config(child, PCIER_DEVICE_CAP, 2) &
4660 PCIEM_CAP_MAX_PAYLOAD) << 5;
4661 if (mmps < path_mps)
4662 pcie_mps_unreconciled(child, path_mps, mmps);
4663 }
4664 }
4665
4666 /*
4667 * Tune a function discovered by rescan or hot-add against the established
4668 * path. Never change a shared upstream port here: doing so requires
4669 * quiescing every driver and draining all outstanding transactions in the
4670 * hierarchy. Cold enumeration is reconciled by pcie_reconcile_link_mps().
4671 */
4672 static void
pcie_setup_mps(device_t dev)4673 pcie_setup_mps(device_t dev)
4674 {
4675 struct pci_devinfo *dinfo;
4676 device_t bus;
4677 uint16_t mmps, mps, path_mps;
4678
4679 dinfo = device_get_ivars(dev);
4680 /*
4681 * PCIe r4.0, sec 9.3.5.4 defines the VF MPS and MRRS fields as
4682 * Reserved and Preserved, with the PF settings applying to the VF.
4683 * Do not use the VF's hardwired value to configure the shared path.
4684 */
4685 if ((dinfo->cfg.flags & PCICFG_VF) != 0)
4686 return;
4687 if (dinfo->cfg.pcie.pcie_location == 0)
4688 return;
4689
4690 /* Cold enumeration is reconciled one complete link at a time. */
4691 bus = device_get_parent(dev);
4692 if (!device_is_attached(bus))
4693 return;
4694 path_mps = 0;
4695 if (!pcie_path_mps(dev, &path_mps))
4696 return;
4697
4698 mmps = (pcie_read_config(dev, PCIER_DEVICE_CAP, 2) &
4699 PCIEM_CAP_MAX_PAYLOAD) << 5;
4700 if (path_mps > mmps) {
4701 pcie_mps_unreconciled(dev, path_mps, mmps);
4702 return;
4703 }
4704 mps = pcie_read_config(dev, PCIER_DEVICE_CTL, 2) &
4705 PCIEM_CTL_MAX_PAYLOAD;
4706 if (mps == path_mps)
4707 return;
4708 if ((pci_read_config(dev, PCIR_COMMAND, 2) &
4709 PCIM_CMD_BUSMASTEREN) != 0) {
4710 pcie_mps_active_unreconciled(dev, path_mps, mps);
4711 return;
4712 }
4713 pcie_adjust_config(dev, PCIER_DEVICE_CTL, PCIEM_CTL_MAX_PAYLOAD,
4714 path_mps, 2);
4715 }
4716
4717 /*
4718 * Reconcile a newly enumerated link before attaching any child drivers. A
4719 * Root Port may be lowered because its complete downstream hierarchy is still
4720 * idle. A late reduction below a switch is not propagated through ancestors,
4721 * since sibling subtrees may already be active.
4722 */
4723 void
pcie_reconcile_link_mps(device_t bus)4724 pcie_reconcile_link_mps(device_t bus)
4725 {
4726 struct pci_devinfo *dinfo, *upinfo;
4727 device_t child, limiting, pcib, *devlist;
4728 uint16_t cap_target, lmps, mmps, mps, target, up_mmps, up_mps;
4729 int count, error, i;
4730 bool limit_requested;
4731
4732 if (!pci_enable_mps_tune)
4733 return;
4734 /* Shared-path tuning is only safe before this bus attaches children. */
4735 if (device_is_attached(bus))
4736 return;
4737 pcib = device_get_parent(bus);
4738 if (!is_pci_device(pcib))
4739 return;
4740 /*
4741 * A PCI function may provide a host bridge into a separate domain,
4742 * as Intel VMD does. Do not treat the function's host-facing PCIe
4743 * Device Control as the upstream end of a link in the child domain.
4744 */
4745 if (pci_get_domain(pcib) != pcib_get_domain(bus))
4746 return;
4747 upinfo = device_get_ivars(pcib);
4748 if (upinfo->cfg.pcie.pcie_location == 0)
4749 return;
4750 error = device_get_children(bus, &devlist, &count);
4751 if (error != 0)
4752 return;
4753
4754 up_mps = pcie_read_config(pcib, PCIER_DEVICE_CTL, 2) &
4755 PCIEM_CTL_MAX_PAYLOAD;
4756 cap_target = up_mps;
4757 limiting = NULL;
4758 up_mmps = (pcie_read_config(pcib, PCIER_DEVICE_CAP, 2) &
4759 PCIEM_CAP_MAX_PAYLOAD) << 5;
4760 if (cap_target > up_mmps) {
4761 cap_target = up_mmps;
4762 limiting = pcib;
4763 }
4764 /*
4765 * Firmware may leave Bus Master Enable set after handoff. Since no
4766 * child driver has attached during this cold pass, it is not a proxy
4767 * for a live FreeBSD consumer.
4768 */
4769 for (i = 0; i < count; i++) {
4770 child = devlist[i];
4771 dinfo = device_get_ivars(child);
4772 if ((dinfo->cfg.flags & (PCICFG_VF |
4773 PCICFG_MPS_UNRECONCILED)) != 0 ||
4774 dinfo->cfg.pcie.pcie_location == 0)
4775 continue;
4776 mmps = (pcie_read_config(child, PCIER_DEVICE_CAP, 2) &
4777 PCIEM_CAP_MAX_PAYLOAD) << 5;
4778 if (cap_target > mmps) {
4779 cap_target = mmps;
4780 limiting = child;
4781 }
4782 }
4783 target = cap_target;
4784 limit_requested = pcie_mps_limit_value(&lmps) && up_mps > lmps;
4785 if (limit_requested && target > lmps)
4786 target = lmps;
4787
4788 /*
4789 * Do not lower one link below a switch without also reconciling every
4790 * ancestor and sibling subtree. Recursive newbus attachment may already
4791 * have made another subtree live, so leave the established path intact.
4792 */
4793 if (target < up_mps &&
4794 upinfo->cfg.pcie.pcie_type != PCIEM_TYPE_ROOT_PORT) {
4795 if (cap_target < up_mps)
4796 pcie_mps_conflict(limiting, up_mps, cap_target);
4797 if (limit_requested) {
4798 device_printf(pcib,
4799 "cannot apply hw.pci.mps_limit=%d below a switch "
4800 "without retuning the shared ancestor hierarchy; "
4801 "leaving path MPS %d unchanged\n",
4802 pci_mps_limit, pcie_mps_bytes(up_mps));
4803 }
4804 pcie_mps_mark_link_unreconciled(devlist, count, up_mps,
4805 up_mps > up_mmps);
4806 /* Keep compatible functions at the established path MPS. */
4807 target = up_mps;
4808 }
4809 /* Lower downstream producers before lowering the shared Root Port. */
4810 for (i = 0; i < count; i++) {
4811 child = devlist[i];
4812 dinfo = device_get_ivars(child);
4813 if ((dinfo->cfg.flags & (PCICFG_VF |
4814 PCICFG_MPS_UNRECONCILED)) != 0 ||
4815 dinfo->cfg.pcie.pcie_location == 0)
4816 continue;
4817 mps = pcie_read_config(child, PCIER_DEVICE_CTL, 2) &
4818 PCIEM_CTL_MAX_PAYLOAD;
4819 if (mps > target)
4820 pcie_adjust_config(child, PCIER_DEVICE_CTL,
4821 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4822 }
4823 if (up_mps > target)
4824 pcie_adjust_config(pcib, PCIER_DEVICE_CTL,
4825 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4826
4827 /* Raise idle children only after the upstream port is configured. */
4828 for (i = 0; i < count; i++) {
4829 child = devlist[i];
4830 dinfo = device_get_ivars(child);
4831 if ((dinfo->cfg.flags & (PCICFG_VF |
4832 PCICFG_MPS_UNRECONCILED)) != 0 ||
4833 dinfo->cfg.pcie.pcie_location == 0)
4834 continue;
4835 mps = pcie_read_config(child, PCIER_DEVICE_CTL, 2) &
4836 PCIEM_CTL_MAX_PAYLOAD;
4837 if (mps < target)
4838 pcie_adjust_config(child, PCIER_DEVICE_CTL,
4839 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4840 }
4841 free(devlist, M_TEMP);
4842 }
4843
4844 static void
pci_add_child_clear_aer(device_t dev,struct pci_devinfo * dinfo)4845 pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo)
4846 {
4847 int aer;
4848 uint32_t r;
4849
4850 if (dinfo->cfg.pcie.pcie_location != 0 &&
4851 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) {
4852 r = PCIEM_ROOT_CTL_SERR_CORR |
4853 PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL;
4854 pcie_adjust_config(dev, PCIER_ROOT_CTL, r, 0, 2);
4855 }
4856 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
4857 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
4858 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
4859 if (r != 0 && bootverbose) {
4860 pci_printf(&dinfo->cfg,
4861 "clearing AER UC 0x%08x -> 0x%08x\n",
4862 r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS,
4863 4));
4864 }
4865
4866 r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4);
4867 r &= ~(PCIM_AER_UC_TRAINING_ERROR |
4868 PCIM_AER_UC_DL_PROTOCOL_ERROR |
4869 PCIM_AER_UC_SURPRISE_LINK_DOWN |
4870 PCIM_AER_UC_POISONED_TLP |
4871 PCIM_AER_UC_FC_PROTOCOL_ERROR |
4872 PCIM_AER_UC_COMPLETION_TIMEOUT |
4873 PCIM_AER_UC_COMPLETER_ABORT |
4874 PCIM_AER_UC_UNEXPECTED_COMPLETION |
4875 PCIM_AER_UC_RECEIVER_OVERFLOW |
4876 PCIM_AER_UC_MALFORMED_TLP |
4877 PCIM_AER_UC_ECRC_ERROR |
4878 PCIM_AER_UC_UNSUPPORTED_REQUEST |
4879 PCIM_AER_UC_ACS_VIOLATION |
4880 PCIM_AER_UC_INTERNAL_ERROR |
4881 PCIM_AER_UC_MC_BLOCKED_TLP |
4882 PCIM_AER_UC_ATOMIC_EGRESS_BLK |
4883 PCIM_AER_UC_TLP_PREFIX_BLOCKED);
4884 pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4);
4885
4886 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
4887 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
4888 if (r != 0 && bootverbose) {
4889 pci_printf(&dinfo->cfg,
4890 "clearing AER COR 0x%08x -> 0x%08x\n",
4891 r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS,
4892 4));
4893 }
4894
4895 r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4);
4896 r &= ~(PCIM_AER_COR_RECEIVER_ERROR |
4897 PCIM_AER_COR_BAD_TLP |
4898 PCIM_AER_COR_BAD_DLLP |
4899 PCIM_AER_COR_REPLAY_ROLLOVER |
4900 PCIM_AER_COR_REPLAY_TIMEOUT |
4901 PCIM_AER_COR_ADVISORY_NF_ERROR |
4902 PCIM_AER_COR_INTERNAL_ERROR |
4903 PCIM_AER_COR_HEADER_LOG_OVFLOW);
4904 pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4);
4905
4906 r = PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE |
4907 PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE;
4908 pcie_adjust_config(dev, PCIER_DEVICE_CTL, r, r, 2);
4909 }
4910 }
4911
4912 void
pci_add_child(device_t bus,struct pci_devinfo * dinfo)4913 pci_add_child(device_t bus, struct pci_devinfo *dinfo)
4914 {
4915 device_t dev;
4916
4917 dinfo->cfg.dev = dev = device_add_child(bus, NULL, DEVICE_UNIT_ANY);
4918 device_set_ivars(dev, dinfo);
4919 resource_list_init(&dinfo->resources);
4920 pci_cfg_save(dev, dinfo, 0);
4921 pci_cfg_restore(dev, dinfo);
4922 pci_clear_pme(dev);
4923 pci_print_verbose(dinfo);
4924 pci_add_resources(bus, dev, 0, 0);
4925 if (pci_enable_mps_tune)
4926 pcie_setup_mps(dev);
4927 pci_child_added(dinfo->cfg.dev);
4928
4929 if (pci_clear_aer_on_attach)
4930 pci_add_child_clear_aer(dev, dinfo);
4931
4932 EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
4933 }
4934
4935 void
pci_child_added_method(device_t dev,device_t child)4936 pci_child_added_method(device_t dev, device_t child)
4937 {
4938
4939 }
4940
4941 static int
pci_probe(device_t dev)4942 pci_probe(device_t dev)
4943 {
4944
4945 device_set_desc(dev, "PCI bus");
4946
4947 /* Allow other subclasses to override this driver. */
4948 return (BUS_PROBE_GENERIC);
4949 }
4950
4951 int
pci_attach_common(device_t dev)4952 pci_attach_common(device_t dev)
4953 {
4954 struct pci_softc *sc;
4955 int busno, domain;
4956 int rid;
4957
4958 sc = device_get_softc(dev);
4959 domain = pcib_get_domain(dev);
4960 busno = pcib_get_bus(dev);
4961 rid = 0;
4962 sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno,
4963 1, 0);
4964 if (sc->sc_bus == NULL) {
4965 device_printf(dev, "failed to allocate bus number\n");
4966 return (ENXIO);
4967 }
4968 if (bootverbose)
4969 device_printf(dev, "domain=%d, physical bus=%d\n",
4970 domain, busno);
4971 sc->sc_dma_tag = bus_get_dma_tag(dev);
4972 return (0);
4973 }
4974
4975 int
pci_attach(device_t dev)4976 pci_attach(device_t dev)
4977 {
4978 int busno, domain, error;
4979
4980 error = pci_attach_common(dev);
4981 if (error)
4982 return (error);
4983
4984 /*
4985 * Since there can be multiple independently numbered PCI
4986 * buses on systems with multiple PCI domains, we can't use
4987 * the unit number to decide which bus we are probing. We ask
4988 * the parent pcib what our domain and bus numbers are.
4989 */
4990 domain = pcib_get_domain(dev);
4991 busno = pcib_get_bus(dev);
4992 pci_add_children(dev, domain, busno);
4993 bus_attach_children(dev);
4994 return (0);
4995 }
4996
4997 int
pci_detach(device_t dev)4998 pci_detach(device_t dev)
4999 {
5000 struct pci_softc *sc;
5001 int error;
5002
5003 error = bus_generic_detach(dev);
5004 if (error)
5005 return (error);
5006 sc = device_get_softc(dev);
5007 error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus);
5008 return (error);
5009 }
5010
5011 static void
pci_hint_device_unit(device_t dev,device_t child,const char * name,int * unitp)5012 pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp)
5013 {
5014 int line, unit;
5015 const char *at;
5016 char me1[24], me2[32];
5017 uint8_t b, s, f;
5018 uint32_t d;
5019 device_location_cache_t *cache;
5020
5021 d = pci_get_domain(child);
5022 b = pci_get_bus(child);
5023 s = pci_get_slot(child);
5024 f = pci_get_function(child);
5025 snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f);
5026 snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f);
5027 line = 0;
5028 cache = dev_wired_cache_init();
5029 while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) {
5030 resource_string_value(name, unit, "at", &at);
5031 if (strcmp(at, me1) == 0 || strcmp(at, me2) == 0) {
5032 *unitp = unit;
5033 break;
5034 }
5035 if (dev_wired_cache_match(cache, child, at)) {
5036 *unitp = unit;
5037 break;
5038 }
5039 }
5040 dev_wired_cache_fini(cache);
5041 }
5042
5043 static void
pci_set_power_child(device_t dev,device_t child,int state)5044 pci_set_power_child(device_t dev, device_t child, int state)
5045 {
5046 device_t pcib;
5047 int dstate;
5048
5049 /*
5050 * Set the device to the given state. If the firmware suggests
5051 * a different power state, use it instead. If power management
5052 * is not present, the firmware is responsible for managing
5053 * device power. Skip children who aren't attached since they
5054 * are handled separately.
5055 */
5056 pcib = device_get_parent(dev);
5057 dstate = state;
5058 if (device_is_attached(child) &&
5059 PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0)
5060 pci_set_powerstate(child, dstate);
5061 }
5062
5063 int
pci_suspend_child(device_t dev,device_t child)5064 pci_suspend_child(device_t dev, device_t child)
5065 {
5066 struct pci_devinfo *dinfo;
5067 struct resource_list_entry *rle;
5068 int error;
5069
5070 dinfo = device_get_ivars(child);
5071
5072 /*
5073 * Save the PCI configuration space for the child and set the
5074 * device in the appropriate power state for this sleep state.
5075 */
5076 pci_cfg_save(child, dinfo, 0);
5077
5078 /* Suspend devices before potentially powering them down. */
5079 error = bus_generic_suspend_child(dev, child);
5080
5081 if (error)
5082 return (error);
5083
5084 if (pci_do_power_suspend) {
5085 /*
5086 * Make sure this device's interrupt handler is not invoked
5087 * in the case the device uses a shared interrupt that can
5088 * be raised by some other device.
5089 * This is applicable only to regular (legacy) PCI interrupts
5090 * as MSI/MSI-X interrupts are never shared.
5091 */
5092 rle = resource_list_find(&dinfo->resources,
5093 SYS_RES_IRQ, 0);
5094 if (rle != NULL && rle->res != NULL)
5095 (void)bus_suspend_intr(child, rle->res);
5096 pci_set_power_child(dev, child, PCI_POWERSTATE_D3);
5097 }
5098
5099 return (0);
5100 }
5101
5102 int
pci_resume_child(device_t dev,device_t child)5103 pci_resume_child(device_t dev, device_t child)
5104 {
5105 struct pci_devinfo *dinfo;
5106 struct resource_list_entry *rle;
5107
5108 if (pci_do_power_resume)
5109 pci_set_power_child(dev, child, PCI_POWERSTATE_D0);
5110
5111 dinfo = device_get_ivars(child);
5112 pci_cfg_restore(child, dinfo);
5113 pci_clear_pme(child);
5114 if (!device_is_attached(child))
5115 pci_cfg_save(child, dinfo, 1);
5116
5117 bus_generic_resume_child(dev, child);
5118
5119 /*
5120 * Allow interrupts only after fully resuming the driver and hardware.
5121 */
5122 if (pci_do_power_suspend) {
5123 /* See pci_suspend_child for details. */
5124 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
5125 if (rle != NULL && rle->res != NULL)
5126 (void)bus_resume_intr(child, rle->res);
5127 }
5128
5129 return (0);
5130 }
5131
5132 int
pci_resume(device_t dev)5133 pci_resume(device_t dev)
5134 {
5135 device_t child, *devlist;
5136 int error, i, numdevs;
5137
5138 if ((error = device_get_children(dev, &devlist, &numdevs)) != 0)
5139 return (error);
5140
5141 /*
5142 * Resume critical devices first, then everything else later.
5143 */
5144 for (i = 0; i < numdevs; i++) {
5145 child = devlist[i];
5146 switch (pci_get_class(child)) {
5147 case PCIC_DISPLAY:
5148 case PCIC_MEMORY:
5149 case PCIC_BRIDGE:
5150 case PCIC_BASEPERIPH:
5151 BUS_RESUME_CHILD(dev, child);
5152 break;
5153 }
5154 }
5155 for (i = 0; i < numdevs; i++) {
5156 child = devlist[i];
5157 switch (pci_get_class(child)) {
5158 case PCIC_DISPLAY:
5159 case PCIC_MEMORY:
5160 case PCIC_BRIDGE:
5161 case PCIC_BASEPERIPH:
5162 break;
5163 default:
5164 BUS_RESUME_CHILD(dev, child);
5165 }
5166 }
5167 free(devlist, M_TEMP);
5168 return (0);
5169 }
5170
5171 static void
pci_load_vendor_data(void)5172 pci_load_vendor_data(void)
5173 {
5174 caddr_t data;
5175 void *ptr;
5176 size_t sz;
5177
5178 data = preload_search_by_type("pci_vendor_data");
5179 if (data != NULL) {
5180 ptr = preload_fetch_addr(data);
5181 sz = preload_fetch_size(data);
5182 if (ptr != NULL && sz != 0) {
5183 pci_vendordata = ptr;
5184 pci_vendordata_size = sz;
5185 /* terminate the database */
5186 pci_vendordata[pci_vendordata_size] = '\n';
5187 }
5188 }
5189 }
5190
5191 void
pci_driver_added(device_t dev,driver_t * driver)5192 pci_driver_added(device_t dev, driver_t *driver)
5193 {
5194 int numdevs;
5195 device_t *devlist;
5196 device_t child;
5197 struct pci_devinfo *dinfo;
5198 int i;
5199
5200 if (bootverbose)
5201 device_printf(dev, "driver added\n");
5202 DEVICE_IDENTIFY(driver, dev);
5203 if (device_get_children(dev, &devlist, &numdevs) != 0)
5204 return;
5205 for (i = 0; i < numdevs; i++) {
5206 child = devlist[i];
5207 if (device_get_state(child) != DS_NOTPRESENT)
5208 continue;
5209 dinfo = device_get_ivars(child);
5210 pci_print_verbose(dinfo);
5211 if (bootverbose)
5212 pci_printf(&dinfo->cfg, "reprobing on driver added\n");
5213 pci_cfg_restore(child, dinfo);
5214 if (device_probe_and_attach(child) != 0)
5215 pci_child_detached(dev, child);
5216 }
5217 free(devlist, M_TEMP);
5218 }
5219
5220 int
pci_setup_intr(device_t dev,device_t child,struct resource * irq,int flags,driver_filter_t * filter,driver_intr_t * intr,void * arg,void ** cookiep)5221 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
5222 driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep)
5223 {
5224 struct pci_devinfo *dinfo;
5225 struct msix_table_entry *mte;
5226 struct msix_vector *mv;
5227 uint64_t addr;
5228 uint32_t data;
5229 void *cookie;
5230 int error, rid;
5231
5232 error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr,
5233 arg, &cookie);
5234 if (error)
5235 return (error);
5236
5237 /* If this is not a direct child, just bail out. */
5238 if (device_get_parent(child) != dev) {
5239 *cookiep = cookie;
5240 return(0);
5241 }
5242
5243 rid = rman_get_rid(irq);
5244 if (rid == 0) {
5245 /* Make sure that INTx is enabled */
5246 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
5247 } else {
5248 /*
5249 * Check to see if the interrupt is MSI or MSI-X.
5250 * Ask our parent to map the MSI and give
5251 * us the address and data register values.
5252 * If we fail for some reason, teardown the
5253 * interrupt handler.
5254 */
5255 dinfo = device_get_ivars(child);
5256 if (dinfo->cfg.msi.msi_alloc > 0) {
5257 if (dinfo->cfg.msi.msi_addr == 0) {
5258 KASSERT(dinfo->cfg.msi.msi_handlers == 0,
5259 ("MSI has handlers, but vectors not mapped"));
5260 error = PCIB_MAP_MSI(device_get_parent(dev),
5261 child, rman_get_start(irq), &addr, &data);
5262 if (error)
5263 goto bad;
5264 dinfo->cfg.msi.msi_addr = addr;
5265 dinfo->cfg.msi.msi_data = data;
5266 }
5267 if (dinfo->cfg.msi.msi_handlers == 0)
5268 pci_enable_msi(child, dinfo->cfg.msi.msi_addr,
5269 dinfo->cfg.msi.msi_data);
5270 dinfo->cfg.msi.msi_handlers++;
5271 } else {
5272 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
5273 ("No MSI or MSI-X interrupts allocated"));
5274 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
5275 ("MSI-X index too high"));
5276 mte = &dinfo->cfg.msix.msix_table[rid - 1];
5277 KASSERT(mte->mte_vector != 0, ("no message vector"));
5278 mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
5279 KASSERT(mv->mv_irq == rman_get_start(irq),
5280 ("IRQ mismatch"));
5281 if (mv->mv_address == 0) {
5282 KASSERT(mte->mte_handlers == 0,
5283 ("MSI-X table entry has handlers, but vector not mapped"));
5284 error = PCIB_MAP_MSI(device_get_parent(dev),
5285 child, rman_get_start(irq), &addr, &data);
5286 if (error)
5287 goto bad;
5288 mv->mv_address = addr;
5289 mv->mv_data = data;
5290 }
5291
5292 /*
5293 * The MSIX table entry must be made valid by
5294 * incrementing the mte_handlers before
5295 * calling pci_enable_msix() and
5296 * pci_resume_msix(). Else the MSIX rewrite
5297 * table quirk will not work as expected.
5298 */
5299 mte->mte_handlers++;
5300 if (mte->mte_handlers == 1) {
5301 pci_enable_msix(child, rid - 1, mv->mv_address,
5302 mv->mv_data);
5303 pci_unmask_msix(child, rid - 1);
5304 }
5305 }
5306
5307 /*
5308 * Make sure that INTx is disabled if we are using MSI/MSI-X,
5309 * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG,
5310 * in which case we "enable" INTx so MSI/MSI-X actually works.
5311 */
5312 if (!pci_has_quirk(pci_get_devid(child),
5313 PCI_QUIRK_MSI_INTX_BUG))
5314 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
5315 else
5316 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
5317 bad:
5318 if (error) {
5319 (void)bus_generic_teardown_intr(dev, child, irq,
5320 cookie);
5321 return (error);
5322 }
5323 }
5324 *cookiep = cookie;
5325 return (0);
5326 }
5327
5328 int
pci_teardown_intr(device_t dev,device_t child,struct resource * irq,void * cookie)5329 pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
5330 void *cookie)
5331 {
5332 struct msix_table_entry *mte;
5333 struct resource_list_entry *rle;
5334 struct pci_devinfo *dinfo;
5335 int error, rid;
5336
5337 if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
5338 return (EINVAL);
5339
5340 /* If this isn't a direct child, just bail out */
5341 if (device_get_parent(child) != dev)
5342 return(bus_generic_teardown_intr(dev, child, irq, cookie));
5343
5344 rid = rman_get_rid(irq);
5345 if (rid == 0) {
5346 /* Mask INTx */
5347 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
5348 } else {
5349 /*
5350 * Check to see if the interrupt is MSI or MSI-X. If so,
5351 * decrement the appropriate handlers count and mask the
5352 * MSI-X message, or disable MSI messages if the count
5353 * drops to 0.
5354 */
5355 dinfo = device_get_ivars(child);
5356 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
5357 if (rle->res != irq)
5358 return (EINVAL);
5359 if (dinfo->cfg.msi.msi_alloc > 0) {
5360 KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
5361 ("MSI-X index too high"));
5362 if (dinfo->cfg.msi.msi_handlers == 0)
5363 return (EINVAL);
5364 dinfo->cfg.msi.msi_handlers--;
5365 if (dinfo->cfg.msi.msi_handlers == 0)
5366 pci_disable_msi(child);
5367 } else {
5368 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
5369 ("No MSI or MSI-X interrupts allocated"));
5370 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
5371 ("MSI-X index too high"));
5372 mte = &dinfo->cfg.msix.msix_table[rid - 1];
5373 if (mte->mte_handlers == 0)
5374 return (EINVAL);
5375 mte->mte_handlers--;
5376 if (mte->mte_handlers == 0)
5377 pci_mask_msix(child, rid - 1);
5378 }
5379 }
5380 error = bus_generic_teardown_intr(dev, child, irq, cookie);
5381 if (rid > 0)
5382 KASSERT(error == 0,
5383 ("%s: generic teardown failed for MSI/MSI-X", __func__));
5384 return (error);
5385 }
5386
5387 int
pci_print_child(device_t dev,device_t child)5388 pci_print_child(device_t dev, device_t child)
5389 {
5390 struct pci_devinfo *dinfo;
5391 struct resource_list *rl;
5392 int retval = 0;
5393
5394 dinfo = device_get_ivars(child);
5395 rl = &dinfo->resources;
5396
5397 retval += bus_print_child_header(dev, child);
5398
5399 retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
5400 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
5401 retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
5402 if (device_get_flags(dev))
5403 retval += printf(" flags %#x", device_get_flags(dev));
5404
5405 retval += printf(" at device %d.%d", pci_get_slot(child),
5406 pci_get_function(child));
5407
5408 retval += bus_print_child_domain(dev, child);
5409 retval += bus_print_child_footer(dev, child);
5410
5411 return (retval);
5412 }
5413
5414 static const struct
5415 {
5416 int class;
5417 int subclass;
5418 int report; /* 0 = bootverbose, 1 = always */
5419 const char *desc;
5420 } pci_nomatch_tab[] = {
5421 {PCIC_OLD, -1, 1, "old"},
5422 {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"},
5423 {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"},
5424 {PCIC_STORAGE, -1, 1, "mass storage"},
5425 {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"},
5426 {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"},
5427 {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"},
5428 {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"},
5429 {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"},
5430 {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"},
5431 {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"},
5432 {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"},
5433 {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"},
5434 {PCIC_NETWORK, -1, 1, "network"},
5435 {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"},
5436 {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"},
5437 {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"},
5438 {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"},
5439 {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"},
5440 {PCIC_DISPLAY, -1, 1, "display"},
5441 {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"},
5442 {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"},
5443 {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"},
5444 {PCIC_MULTIMEDIA, -1, 1, "multimedia"},
5445 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"},
5446 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"},
5447 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"},
5448 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"},
5449 {PCIC_MEMORY, -1, 1, "memory"},
5450 {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"},
5451 {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"},
5452 {PCIC_BRIDGE, -1, 1, "bridge"},
5453 {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"},
5454 {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"},
5455 {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"},
5456 {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"},
5457 {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"},
5458 {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"},
5459 {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"},
5460 {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"},
5461 {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"},
5462 {PCIC_SIMPLECOMM, -1, 1, "simple comms"},
5463 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */
5464 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"},
5465 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"},
5466 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"},
5467 {PCIC_BASEPERIPH, -1, 0, "base peripheral"},
5468 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"},
5469 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"},
5470 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"},
5471 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"},
5472 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"},
5473 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"},
5474 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"},
5475 {PCIC_INPUTDEV, -1, 1, "input device"},
5476 {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"},
5477 {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"},
5478 {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"},
5479 {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"},
5480 {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"},
5481 {PCIC_DOCKING, -1, 1, "docking station"},
5482 {PCIC_PROCESSOR, -1, 1, "processor"},
5483 {PCIC_SERIALBUS, -1, 1, "serial bus"},
5484 {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"},
5485 {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"},
5486 {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"},
5487 {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"},
5488 {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"},
5489 {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"},
5490 {PCIC_WIRELESS, -1, 1, "wireless controller"},
5491 {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"},
5492 {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"},
5493 {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"},
5494 {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"},
5495 {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"},
5496 {PCIC_SATCOM, -1, 1, "satellite communication"},
5497 {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"},
5498 {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"},
5499 {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"},
5500 {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"},
5501 {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"},
5502 {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"},
5503 {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"},
5504 {PCIC_DASP, -1, 0, "dasp"},
5505 {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"},
5506 {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"},
5507 {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"},
5508 {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"},
5509 {PCIC_INSTRUMENT, -1, 0, "non-essential instrumentation"},
5510 {0, 0, 0, NULL}
5511 };
5512
5513 void
pci_probe_nomatch(device_t dev,device_t child)5514 pci_probe_nomatch(device_t dev, device_t child)
5515 {
5516 int i, report;
5517 const char *cp, *scp;
5518 char *device;
5519
5520 /*
5521 * Look for a listing for this device in a loaded device database.
5522 */
5523 report = 1;
5524 if ((device = pci_describe_device(child)) != NULL) {
5525 device_printf(dev, "<%s>", device);
5526 free(device, M_DEVBUF);
5527 } else {
5528 /*
5529 * Scan the class/subclass descriptions for a general
5530 * description.
5531 */
5532 cp = "unknown";
5533 scp = NULL;
5534 for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) {
5535 if (pci_nomatch_tab[i].class == pci_get_class(child)) {
5536 if (pci_nomatch_tab[i].subclass == -1) {
5537 cp = pci_nomatch_tab[i].desc;
5538 report = pci_nomatch_tab[i].report;
5539 } else if (pci_nomatch_tab[i].subclass ==
5540 pci_get_subclass(child)) {
5541 scp = pci_nomatch_tab[i].desc;
5542 report = pci_nomatch_tab[i].report;
5543 }
5544 }
5545 }
5546 if (report || bootverbose) {
5547 device_printf(dev, "<%s%s%s>",
5548 cp ? cp : "",
5549 ((cp != NULL) && (scp != NULL)) ? ", " : "",
5550 scp ? scp : "");
5551 }
5552 }
5553 if (report || bootverbose) {
5554 printf(" at device %d.%d (no driver attached)\n",
5555 pci_get_slot(child), pci_get_function(child));
5556 }
5557 pci_cfg_save(child, device_get_ivars(child), 1);
5558 }
5559
5560 void
pci_child_detached(device_t dev,device_t child)5561 pci_child_detached(device_t dev, device_t child)
5562 {
5563 struct pci_devinfo *dinfo;
5564 struct resource_list *rl;
5565
5566 dinfo = device_get_ivars(child);
5567 rl = &dinfo->resources;
5568
5569 /*
5570 * Have to deallocate IRQs before releasing any MSI messages and
5571 * have to release MSI messages before deallocating any memory
5572 * BARs.
5573 */
5574 if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0)
5575 pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n");
5576 if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) {
5577 if (dinfo->cfg.msi.msi_alloc != 0)
5578 pci_printf(&dinfo->cfg, "Device leaked %d MSI "
5579 "vectors\n", dinfo->cfg.msi.msi_alloc);
5580 else
5581 pci_printf(&dinfo->cfg, "Device leaked %d MSI-X "
5582 "vectors\n", dinfo->cfg.msix.msix_alloc);
5583 (void)pci_release_msi(child);
5584 }
5585 if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0)
5586 pci_printf(&dinfo->cfg, "Device leaked memory resources\n");
5587 if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0)
5588 pci_printf(&dinfo->cfg, "Device leaked I/O resources\n");
5589 if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0)
5590 pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n");
5591
5592 pci_cfg_save(child, dinfo, 1);
5593 }
5594
5595 /*
5596 * Parse the PCI device database, if loaded, and return a pointer to a
5597 * description of the device.
5598 *
5599 * The database is flat text formatted as follows:
5600 *
5601 * Any line not in a valid format is ignored.
5602 * Lines are terminated with newline '\n' characters.
5603 *
5604 * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then
5605 * the vendor name.
5606 *
5607 * A DEVICE line is entered immediately below the corresponding VENDOR ID.
5608 * - devices cannot be listed without a corresponding VENDOR line.
5609 * A DEVICE line consists of a TAB, the 4 digit (hex) device code,
5610 * another TAB, then the device name.
5611 */
5612
5613 /*
5614 * Assuming (ptr) points to the beginning of a line in the database,
5615 * return the vendor or device and description of the next entry.
5616 * The value of (vendor) or (device) inappropriate for the entry type
5617 * is set to -1. Returns nonzero at the end of the database.
5618 *
5619 * Note that this is slightly unrobust in the face of corrupt data;
5620 * we attempt to safeguard against this by spamming the end of the
5621 * database with a newline when we initialise.
5622 */
5623 static int
pci_describe_parse_line(char ** ptr,int * vendor,int * device,char ** desc)5624 pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc)
5625 {
5626 char *cp = *ptr;
5627 int left;
5628
5629 *device = -1;
5630 *vendor = -1;
5631 **desc = '\0';
5632 for (;;) {
5633 left = pci_vendordata_size - (cp - pci_vendordata);
5634 if (left <= 0) {
5635 *ptr = cp;
5636 return(1);
5637 }
5638
5639 /* vendor entry? */
5640 if (*cp != '\t' &&
5641 sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2)
5642 break;
5643 /* device entry? */
5644 if (*cp == '\t' &&
5645 sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2)
5646 break;
5647
5648 /* skip to next line */
5649 while (*cp != '\n' && left > 0) {
5650 cp++;
5651 left--;
5652 }
5653 if (*cp == '\n') {
5654 cp++;
5655 left--;
5656 }
5657 }
5658 /* skip to next line */
5659 while (*cp != '\n' && left > 0) {
5660 cp++;
5661 left--;
5662 }
5663 if (*cp == '\n' && left > 0)
5664 cp++;
5665 *ptr = cp;
5666 return(0);
5667 }
5668
5669 static char *
pci_describe_device(device_t dev)5670 pci_describe_device(device_t dev)
5671 {
5672 int vendor, device;
5673 char *desc, *vp, *dp, *line;
5674
5675 desc = vp = dp = NULL;
5676
5677 /*
5678 * If we have no vendor data, we can't do anything.
5679 */
5680 if (pci_vendordata == NULL)
5681 goto out;
5682
5683 /*
5684 * Scan the vendor data looking for this device
5685 */
5686 line = pci_vendordata;
5687 if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5688 goto out;
5689 for (;;) {
5690 if (pci_describe_parse_line(&line, &vendor, &device, &vp))
5691 goto out;
5692 if (vendor == pci_get_vendor(dev))
5693 break;
5694 }
5695 if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5696 goto out;
5697 for (;;) {
5698 if (pci_describe_parse_line(&line, &vendor, &device, &dp)) {
5699 *dp = 0;
5700 break;
5701 }
5702 if (vendor != -1) {
5703 *dp = 0;
5704 break;
5705 }
5706 if (device == pci_get_device(dev))
5707 break;
5708 }
5709 if (dp[0] == '\0')
5710 snprintf(dp, 80, "0x%x", pci_get_device(dev));
5711 if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) !=
5712 NULL)
5713 sprintf(desc, "%s, %s", vp, dp);
5714 out:
5715 if (vp != NULL)
5716 free(vp, M_DEVBUF);
5717 if (dp != NULL)
5718 free(dp, M_DEVBUF);
5719 return(desc);
5720 }
5721
5722 int
pci_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)5723 pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
5724 {
5725 struct pci_devinfo *dinfo;
5726 pcicfgregs *cfg;
5727
5728 dinfo = device_get_ivars(child);
5729 cfg = &dinfo->cfg;
5730
5731 switch (which) {
5732 case PCI_IVAR_ETHADDR:
5733 /*
5734 * The generic accessor doesn't deal with failure, so
5735 * we set the return value, then return an error.
5736 */
5737 *((uint8_t **) result) = NULL;
5738 return (EINVAL);
5739 case PCI_IVAR_SUBVENDOR:
5740 *result = cfg->subvendor;
5741 break;
5742 case PCI_IVAR_SUBDEVICE:
5743 *result = cfg->subdevice;
5744 break;
5745 case PCI_IVAR_VENDOR:
5746 *result = cfg->vendor;
5747 break;
5748 case PCI_IVAR_DEVICE:
5749 *result = cfg->device;
5750 break;
5751 case PCI_IVAR_DEVID:
5752 *result = (cfg->device << 16) | cfg->vendor;
5753 break;
5754 case PCI_IVAR_CLASS:
5755 *result = cfg->baseclass;
5756 break;
5757 case PCI_IVAR_SUBCLASS:
5758 *result = cfg->subclass;
5759 break;
5760 case PCI_IVAR_PROGIF:
5761 *result = cfg->progif;
5762 break;
5763 case PCI_IVAR_REVID:
5764 *result = cfg->revid;
5765 break;
5766 case PCI_IVAR_INTPIN:
5767 *result = cfg->intpin;
5768 break;
5769 case PCI_IVAR_IRQ:
5770 *result = cfg->intline;
5771 break;
5772 case PCI_IVAR_DOMAIN:
5773 *result = cfg->domain;
5774 break;
5775 case PCI_IVAR_BUS:
5776 *result = cfg->bus;
5777 break;
5778 case PCI_IVAR_SLOT:
5779 *result = cfg->slot;
5780 break;
5781 case PCI_IVAR_FUNCTION:
5782 *result = cfg->func;
5783 break;
5784 case PCI_IVAR_CMDREG:
5785 *result = cfg->cmdreg;
5786 break;
5787 case PCI_IVAR_CACHELNSZ:
5788 *result = cfg->cachelnsz;
5789 break;
5790 case PCI_IVAR_MINGNT:
5791 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5792 *result = -1;
5793 return (EINVAL);
5794 }
5795 *result = cfg->mingnt;
5796 break;
5797 case PCI_IVAR_MAXLAT:
5798 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5799 *result = -1;
5800 return (EINVAL);
5801 }
5802 *result = cfg->maxlat;
5803 break;
5804 case PCI_IVAR_LATTIMER:
5805 *result = cfg->lattimer;
5806 break;
5807 default:
5808 return (ENOENT);
5809 }
5810 return (0);
5811 }
5812
5813 int
pci_write_ivar(device_t dev,device_t child,int which,uintptr_t value)5814 pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
5815 {
5816 struct pci_devinfo *dinfo;
5817
5818 dinfo = device_get_ivars(child);
5819
5820 switch (which) {
5821 case PCI_IVAR_INTPIN:
5822 dinfo->cfg.intpin = value;
5823 return (0);
5824 case PCI_IVAR_ETHADDR:
5825 case PCI_IVAR_SUBVENDOR:
5826 case PCI_IVAR_SUBDEVICE:
5827 case PCI_IVAR_VENDOR:
5828 case PCI_IVAR_DEVICE:
5829 case PCI_IVAR_DEVID:
5830 case PCI_IVAR_CLASS:
5831 case PCI_IVAR_SUBCLASS:
5832 case PCI_IVAR_PROGIF:
5833 case PCI_IVAR_REVID:
5834 case PCI_IVAR_IRQ:
5835 case PCI_IVAR_DOMAIN:
5836 case PCI_IVAR_BUS:
5837 case PCI_IVAR_SLOT:
5838 case PCI_IVAR_FUNCTION:
5839 return (EINVAL); /* disallow for now */
5840
5841 default:
5842 return (ENOENT);
5843 }
5844 }
5845
5846 #include "opt_ddb.h"
5847 #ifdef DDB
5848 #include <ddb/ddb.h>
5849 #include <sys/cons.h>
5850
5851 /*
5852 * List resources based on pci map registers, used for within ddb
5853 */
5854
DB_SHOW_COMMAND_FLAGS(pciregs,db_pci_dump,DB_CMD_MEMSAFE)5855 DB_SHOW_COMMAND_FLAGS(pciregs, db_pci_dump, DB_CMD_MEMSAFE)
5856 {
5857 struct pci_devinfo *dinfo;
5858 struct devlist *devlist_head;
5859 struct pci_conf *p;
5860 const char *name;
5861 int i, error, none_count;
5862
5863 none_count = 0;
5864 /* get the head of the device queue */
5865 devlist_head = &pci_devq;
5866
5867 /*
5868 * Go through the list of devices and print out devices
5869 */
5870 for (error = 0, i = 0,
5871 dinfo = STAILQ_FIRST(devlist_head);
5872 (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit;
5873 dinfo = STAILQ_NEXT(dinfo, pci_links), i++) {
5874 /* Populate pd_name and pd_unit */
5875 name = NULL;
5876 if (dinfo->cfg.dev)
5877 name = device_get_name(dinfo->cfg.dev);
5878
5879 p = &dinfo->conf;
5880 db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x "
5881 "chip=0x%08x rev=0x%02x hdr=0x%02x\n",
5882 (name && *name) ? name : "none",
5883 (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) :
5884 none_count++,
5885 p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev,
5886 p->pc_sel.pc_func, (p->pc_class << 16) |
5887 (p->pc_subclass << 8) | p->pc_progif,
5888 (p->pc_subdevice << 16) | p->pc_subvendor,
5889 (p->pc_device << 16) | p->pc_vendor,
5890 p->pc_revid, p->pc_hdr);
5891 }
5892 }
5893 #endif /* DDB */
5894
5895 struct resource *
pci_reserve_map(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int num,u_int flags)5896 pci_reserve_map(device_t dev, device_t child, int type, int rid,
5897 rman_res_t start, rman_res_t end, rman_res_t count, u_int num,
5898 u_int flags)
5899 {
5900 struct pci_devinfo *dinfo = device_get_ivars(child);
5901 struct resource_list *rl = &dinfo->resources;
5902 struct resource *res;
5903 struct pci_map *pm;
5904 uint16_t cmd;
5905 pci_addr_t map, testval;
5906 int mapsize;
5907
5908 res = NULL;
5909
5910 /* If rid is managed by EA, ignore it */
5911 if (pci_ea_is_enabled(child, rid))
5912 goto out;
5913
5914 pm = pci_find_bar(child, rid);
5915 if (pm != NULL) {
5916 /* This is a BAR that we failed to allocate earlier. */
5917 mapsize = pm->pm_size;
5918 map = pm->pm_value;
5919 } else {
5920 /*
5921 * Weed out the bogons, and figure out how large the
5922 * BAR/map is. BARs that read back 0 here are bogus
5923 * and unimplemented. Note: atapci in legacy mode are
5924 * special and handled elsewhere in the code. If you
5925 * have a atapci device in legacy mode and it fails
5926 * here, that other code is broken.
5927 */
5928 pci_read_bar(child, rid, &map, &testval, NULL);
5929
5930 /*
5931 * Determine the size of the BAR and ignore BARs with a size
5932 * of 0. Device ROM BARs use a different mask value.
5933 */
5934 if (PCIR_IS_BIOS(&dinfo->cfg, rid))
5935 mapsize = pci_romsize(testval);
5936 else
5937 mapsize = pci_mapsize(testval);
5938 if (mapsize == 0)
5939 goto out;
5940 pm = pci_add_bar(child, rid, map, mapsize);
5941 }
5942
5943 if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, rid)) {
5944 if (type != SYS_RES_MEMORY) {
5945 if (bootverbose)
5946 device_printf(dev,
5947 "child %s requested type %d for rid %#x,"
5948 " but the BAR says it is an memio\n",
5949 device_get_nameunit(child), type, rid);
5950 goto out;
5951 }
5952 } else {
5953 if (type != SYS_RES_IOPORT) {
5954 if (bootverbose)
5955 device_printf(dev,
5956 "child %s requested type %d for rid %#x,"
5957 " but the BAR says it is an ioport\n",
5958 device_get_nameunit(child), type, rid);
5959 goto out;
5960 }
5961 }
5962
5963 /*
5964 * For real BARs, we need to override the size that
5965 * the driver requests, because that's what the BAR
5966 * actually uses and we would otherwise have a
5967 * situation where we might allocate the excess to
5968 * another driver, which won't work.
5969 */
5970 count = ((pci_addr_t)1 << mapsize) * num;
5971 if (RF_ALIGNMENT(flags) < mapsize)
5972 flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize);
5973 if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH))
5974 flags |= RF_PREFETCHABLE;
5975
5976 /*
5977 * Allocate enough resource, and then write back the
5978 * appropriate BAR for that resource.
5979 */
5980 resource_list_add(rl, type, rid, start, end, count);
5981 res = resource_list_reserve(rl, dev, child, type, rid, start, end,
5982 count, flags & ~RF_ACTIVE);
5983 if (res == NULL) {
5984 resource_list_delete(rl, type, rid);
5985 device_printf(child,
5986 "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n",
5987 count, rid, type, start, end);
5988 goto out;
5989 }
5990 if (bootverbose)
5991 device_printf(child,
5992 "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n",
5993 count, rid, type, rman_get_start(res));
5994
5995 /* Disable decoding via the CMD register before updating the BAR */
5996 cmd = pci_read_config(child, PCIR_COMMAND, 2);
5997 pci_write_config(child, PCIR_COMMAND,
5998 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
5999
6000 map = rman_get_start(res);
6001 pci_write_bar(child, pm, map);
6002
6003 /* Restore the original value of the CMD register */
6004 pci_write_config(child, PCIR_COMMAND, cmd, 2);
6005 out:
6006 return (res);
6007 }
6008
6009 struct resource *
pci_alloc_multi_resource(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_long num,u_int flags)6010 pci_alloc_multi_resource(device_t dev, device_t child, int type, int rid,
6011 rman_res_t start, rman_res_t end, rman_res_t count, u_long num,
6012 u_int flags)
6013 {
6014 struct pci_devinfo *dinfo;
6015 struct resource_list *rl;
6016 struct resource_list_entry *rle;
6017 struct resource *res;
6018 pcicfgregs *cfg;
6019
6020 /*
6021 * Perform lazy resource allocation
6022 */
6023 dinfo = device_get_ivars(child);
6024 rl = &dinfo->resources;
6025 cfg = &dinfo->cfg;
6026 switch (type) {
6027 case PCI_RES_BUS:
6028 return (pci_alloc_secbus(dev, child, rid, start, end, count,
6029 flags));
6030 case SYS_RES_IRQ:
6031 /*
6032 * Can't alloc legacy interrupt once MSI messages have
6033 * been allocated.
6034 */
6035 if (rid == 0 && (cfg->msi.msi_alloc > 0 ||
6036 cfg->msix.msix_alloc > 0))
6037 return (NULL);
6038
6039 /*
6040 * If the child device doesn't have an interrupt
6041 * routed and is deserving of an interrupt, try to
6042 * assign it one.
6043 */
6044 if (rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) &&
6045 (cfg->intpin != 0))
6046 pci_assign_interrupt(dev, child, 0);
6047 break;
6048 case SYS_RES_IOPORT:
6049 case SYS_RES_MEMORY:
6050 /*
6051 * PCI-PCI bridge I/O window resources are not BARs.
6052 * For those allocations just pass the request up the
6053 * tree.
6054 */
6055 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) {
6056 switch (rid) {
6057 case PCIR_IOBASEL_1:
6058 case PCIR_MEMBASE_1:
6059 case PCIR_PMBASEL_1:
6060 /*
6061 * XXX: Should we bother creating a resource
6062 * list entry?
6063 */
6064 return (bus_generic_alloc_resource(dev, child,
6065 type, rid, start, end, count, flags));
6066 }
6067 }
6068 /* Reserve resources for this BAR if needed. */
6069 rle = resource_list_find(rl, type, rid);
6070 if (rle == NULL) {
6071 res = pci_reserve_map(dev, child, type, rid, start, end,
6072 count, num, flags);
6073 if (res == NULL)
6074 return (NULL);
6075 }
6076 }
6077 return (resource_list_alloc(rl, dev, child, type, rid,
6078 start, end, count, flags));
6079 }
6080
6081 struct resource *
pci_alloc_resource(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)6082 pci_alloc_resource(device_t dev, device_t child, int type, int rid,
6083 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
6084 {
6085 #ifdef PCI_IOV
6086 struct pci_devinfo *dinfo;
6087 #endif
6088
6089 if (device_get_parent(child) != dev)
6090 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
6091 type, rid, start, end, count, flags));
6092
6093 #ifdef PCI_IOV
6094 dinfo = device_get_ivars(child);
6095 if (dinfo->cfg.flags & PCICFG_VF) {
6096 switch (type) {
6097 /* VFs can't have I/O BARs. */
6098 case SYS_RES_IOPORT:
6099 return (NULL);
6100 case SYS_RES_MEMORY:
6101 return (pci_vf_alloc_mem_resource(dev, child, rid,
6102 start, end, count, flags));
6103 }
6104
6105 /* Fall through for other types of resource allocations. */
6106 }
6107 #endif
6108
6109 return (pci_alloc_multi_resource(dev, child, type, rid, start, end,
6110 count, 1, flags));
6111 }
6112
6113 int
pci_release_resource(device_t dev,device_t child,struct resource * r)6114 pci_release_resource(device_t dev, device_t child, struct resource *r)
6115 {
6116 struct pci_devinfo *dinfo;
6117 struct resource_list *rl;
6118 pcicfgregs *cfg __unused;
6119
6120 if (device_get_parent(child) != dev)
6121 return (bus_generic_release_resource(dev, child, r));
6122
6123 dinfo = device_get_ivars(child);
6124 cfg = &dinfo->cfg;
6125
6126 #ifdef PCI_IOV
6127 if (cfg->flags & PCICFG_VF) {
6128 switch (rman_get_type(r)) {
6129 /* VFs can't have I/O BARs. */
6130 case SYS_RES_IOPORT:
6131 return (EDOOFUS);
6132 case SYS_RES_MEMORY:
6133 return (pci_vf_release_mem_resource(dev, child, r));
6134 }
6135
6136 /* Fall through for other types of resource allocations. */
6137 }
6138 #endif
6139
6140 /*
6141 * PCI-PCI bridge I/O window resources are not BARs. For
6142 * those allocations just pass the request up the tree.
6143 */
6144 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE &&
6145 (rman_get_type(r) == SYS_RES_IOPORT ||
6146 rman_get_type(r) == SYS_RES_MEMORY)) {
6147 switch (rman_get_rid(r)) {
6148 case PCIR_IOBASEL_1:
6149 case PCIR_MEMBASE_1:
6150 case PCIR_PMBASEL_1:
6151 return (bus_generic_release_resource(dev, child, r));
6152 }
6153 }
6154
6155 rl = &dinfo->resources;
6156 return (resource_list_release(rl, dev, child, r));
6157 }
6158
6159 int
pci_activate_resource(device_t dev,device_t child,struct resource * r)6160 pci_activate_resource(device_t dev, device_t child, struct resource *r)
6161 {
6162 struct pci_devinfo *dinfo;
6163 int error, rid, type;
6164
6165 if (device_get_parent(child) != dev)
6166 return (bus_generic_activate_resource(dev, child, r));
6167
6168 dinfo = device_get_ivars(child);
6169 #ifdef PCI_IOV
6170 if (dinfo->cfg.flags & PCICFG_VF) {
6171 switch (rman_get_type(r)) {
6172 /* VFs can't have I/O BARs. */
6173 case SYS_RES_IOPORT:
6174 error = EINVAL;
6175 break;
6176 case SYS_RES_MEMORY:
6177 error = pci_vf_activate_mem_resource(dev, child, r);
6178 break;
6179 default:
6180 error = bus_generic_activate_resource(dev, child, r);
6181 break;
6182 }
6183 } else
6184 #endif
6185 error = bus_generic_activate_resource(dev, child, r);
6186 if (error)
6187 return (error);
6188
6189 rid = rman_get_rid(r);
6190 type = rman_get_type(r);
6191
6192 /* Device ROMs need their decoding explicitly enabled. */
6193 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
6194 pci_write_bar(child, pci_find_bar(child, rid),
6195 rman_get_start(r) | PCIM_BIOS_ENABLE);
6196
6197 /* Enable decoding in the command register when activating BARs. */
6198 switch (type) {
6199 case SYS_RES_IOPORT:
6200 case SYS_RES_MEMORY:
6201 error = PCI_ENABLE_IO(dev, child, type);
6202 break;
6203 }
6204 return (error);
6205 }
6206
6207 int
pci_deactivate_resource(device_t dev,device_t child,struct resource * r)6208 pci_deactivate_resource(device_t dev, device_t child, struct resource *r)
6209 {
6210 struct pci_devinfo *dinfo;
6211 int error, rid, type;
6212
6213 if (device_get_parent(child) != dev)
6214 return (bus_generic_deactivate_resource(dev, child, r));
6215
6216 dinfo = device_get_ivars(child);
6217 #ifdef PCI_IOV
6218 if (dinfo->cfg.flags & PCICFG_VF) {
6219 switch (rman_get_type(r)) {
6220 /* VFs can't have I/O BARs. */
6221 case SYS_RES_IOPORT:
6222 error = EINVAL;
6223 break;
6224 case SYS_RES_MEMORY:
6225 error = pci_vf_deactivate_mem_resource(dev, child, r);
6226 break;
6227 default:
6228 error = bus_generic_deactivate_resource(dev, child, r);
6229 break;
6230 }
6231 } else
6232 #endif
6233 error = bus_generic_deactivate_resource(dev, child, r);
6234 if (error)
6235 return (error);
6236
6237 /* Disable decoding for device ROMs. */
6238 rid = rman_get_rid(r);
6239 type = rman_get_type(r);
6240 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
6241 pci_write_bar(child, pci_find_bar(child, rid),
6242 rman_get_start(r));
6243 return (0);
6244 }
6245
6246 int
pci_adjust_resource(device_t dev,device_t child,struct resource * r,rman_res_t start,rman_res_t end)6247 pci_adjust_resource(device_t dev, device_t child, struct resource *r,
6248 rman_res_t start, rman_res_t end)
6249 {
6250 #ifdef PCI_IOV
6251 struct pci_devinfo *dinfo;
6252
6253 if (device_get_parent(child) != dev)
6254 return (bus_generic_adjust_resource(dev, child, r, start,
6255 end));
6256
6257 dinfo = device_get_ivars(child);
6258 if (dinfo->cfg.flags & PCICFG_VF) {
6259 switch (rman_get_type(r)) {
6260 /* VFs can't have I/O BARs. */
6261 case SYS_RES_IOPORT:
6262 return (EINVAL);
6263 case SYS_RES_MEMORY:
6264 return (pci_vf_adjust_mem_resource(dev, child, r,
6265 start, end));
6266 }
6267
6268 /* Fall through for other types of resource allocations. */
6269 }
6270 #endif
6271
6272 return (bus_generic_adjust_resource(dev, child, r, start, end));
6273 }
6274
6275 int
pci_map_resource(device_t dev,device_t child,struct resource * r,struct resource_map_request * argsp,struct resource_map * map)6276 pci_map_resource(device_t dev, device_t child, struct resource *r,
6277 struct resource_map_request *argsp, struct resource_map *map)
6278 {
6279 #ifdef PCI_IOV
6280 struct pci_devinfo *dinfo;
6281
6282 if (device_get_parent(child) != dev)
6283 return (bus_generic_map_resource(dev, child, r, argsp,
6284 map));
6285
6286 dinfo = device_get_ivars(child);
6287 if (dinfo->cfg.flags & PCICFG_VF) {
6288 switch (rman_get_type(r)) {
6289 /* VFs can't have I/O BARs. */
6290 case SYS_RES_IOPORT:
6291 return (EINVAL);
6292 case SYS_RES_MEMORY:
6293 return (pci_vf_map_mem_resource(dev, child, r, argsp,
6294 map));
6295 }
6296
6297 /* Fall through for other types of resource allocations. */
6298 }
6299 #endif
6300
6301 return (bus_generic_map_resource(dev, child, r, argsp, map));
6302 }
6303
6304 int
pci_unmap_resource(device_t dev,device_t child,struct resource * r,struct resource_map * map)6305 pci_unmap_resource(device_t dev, device_t child, struct resource *r,
6306 struct resource_map *map)
6307 {
6308 #ifdef PCI_IOV
6309 struct pci_devinfo *dinfo;
6310
6311 if (device_get_parent(child) != dev)
6312 return (bus_generic_unmap_resource(dev, child, r, map));
6313
6314 dinfo = device_get_ivars(child);
6315 if (dinfo->cfg.flags & PCICFG_VF) {
6316 switch (rman_get_type(r)) {
6317 /* VFs can't have I/O BARs. */
6318 case SYS_RES_IOPORT:
6319 return (EINVAL);
6320 case SYS_RES_MEMORY:
6321 return (pci_vf_unmap_mem_resource(dev, child, r, map));
6322 }
6323
6324 /* Fall through for other types of resource allocations. */
6325 }
6326 #endif
6327
6328 return (bus_generic_unmap_resource(dev, child, r, map));
6329 }
6330
6331 void
pci_child_deleted(device_t dev,device_t child)6332 pci_child_deleted(device_t dev, device_t child)
6333 {
6334 struct resource_list_entry *rle;
6335 struct resource_list *rl;
6336 struct pci_devinfo *dinfo;
6337
6338 dinfo = device_get_ivars(child);
6339 rl = &dinfo->resources;
6340
6341 EVENTHANDLER_INVOKE(pci_delete_device, child);
6342
6343 /* Turn off access to resources we're about to free */
6344 if (bus_child_present(child) != 0) {
6345 pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
6346 PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
6347
6348 pci_disable_busmaster(child);
6349 }
6350
6351 /* Free all allocated resources */
6352 STAILQ_FOREACH(rle, rl, link) {
6353 if (rle->res) {
6354 if (rman_get_flags(rle->res) & RF_ACTIVE ||
6355 resource_list_busy(rl, rle->type, rle->rid)) {
6356 pci_printf(&dinfo->cfg,
6357 "Resource still owned, oops. "
6358 "(type=%d, rid=%d, addr=%lx)\n",
6359 rle->type, rle->rid,
6360 rman_get_start(rle->res));
6361 bus_release_resource(child, rle->type, rle->rid,
6362 rle->res);
6363 }
6364 resource_list_unreserve(rl, dev, child, rle->type,
6365 rle->rid);
6366 }
6367 }
6368 resource_list_free(rl);
6369
6370 pci_freecfg(dinfo);
6371 }
6372
6373 void
pci_delete_resource(device_t dev,device_t child,int type,int rid)6374 pci_delete_resource(device_t dev, device_t child, int type, int rid)
6375 {
6376 struct pci_devinfo *dinfo;
6377 struct resource_list *rl;
6378 struct resource_list_entry *rle;
6379
6380 if (device_get_parent(child) != dev)
6381 return;
6382
6383 dinfo = device_get_ivars(child);
6384 rl = &dinfo->resources;
6385 rle = resource_list_find(rl, type, rid);
6386 if (rle == NULL)
6387 return;
6388
6389 if (rle->res) {
6390 if (rman_get_flags(rle->res) & RF_ACTIVE ||
6391 resource_list_busy(rl, type, rid)) {
6392 device_printf(dev, "delete_resource: "
6393 "Resource still owned by child, oops. "
6394 "(type=%d, rid=%d, addr=%jx)\n",
6395 type, rid, rman_get_start(rle->res));
6396 return;
6397 }
6398 resource_list_unreserve(rl, dev, child, type, rid);
6399 }
6400 resource_list_delete(rl, type, rid);
6401 }
6402
6403 struct resource_list *
pci_get_resource_list(device_t dev,device_t child)6404 pci_get_resource_list (device_t dev, device_t child)
6405 {
6406 struct pci_devinfo *dinfo = device_get_ivars(child);
6407
6408 return (&dinfo->resources);
6409 }
6410
6411 #ifdef IOMMU
6412 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)6413 pci_get_dma_tag(device_t bus, device_t dev)
6414 {
6415 bus_dma_tag_t tag;
6416 struct pci_softc *sc;
6417
6418 if (device_get_parent(dev) == bus) {
6419 /* try iommu and return if it works */
6420 tag = iommu_get_dma_tag(bus, dev);
6421 } else
6422 tag = NULL;
6423 if (tag == NULL) {
6424 sc = device_get_softc(bus);
6425 tag = sc->sc_dma_tag;
6426 }
6427 return (tag);
6428 }
6429 #else
6430 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)6431 pci_get_dma_tag(device_t bus, device_t dev)
6432 {
6433 struct pci_softc *sc = device_get_softc(bus);
6434
6435 return (sc->sc_dma_tag);
6436 }
6437 #endif
6438
6439 uint32_t
pci_read_config_method(device_t dev,device_t child,int reg,int width)6440 pci_read_config_method(device_t dev, device_t child, int reg, int width)
6441 {
6442 struct pci_devinfo *dinfo = device_get_ivars(child);
6443 pcicfgregs *cfg = &dinfo->cfg;
6444
6445 #ifdef PCI_IOV
6446 /*
6447 * SR-IOV VFs don't implement the VID or DID registers, so we have to
6448 * emulate them here.
6449 */
6450 if (cfg->flags & PCICFG_VF) {
6451 if (reg == PCIR_VENDOR) {
6452 switch (width) {
6453 case 4:
6454 return (cfg->device << 16 | cfg->vendor);
6455 case 2:
6456 return (cfg->vendor);
6457 case 1:
6458 return (cfg->vendor & 0xff);
6459 default:
6460 return (0xffffffff);
6461 }
6462 } else if (reg == PCIR_DEVICE) {
6463 switch (width) {
6464 /* Note that an unaligned 4-byte read is an error. */
6465 case 2:
6466 return (cfg->device);
6467 case 1:
6468 return (cfg->device & 0xff);
6469 default:
6470 return (0xffffffff);
6471 }
6472 }
6473 }
6474 #endif
6475
6476 return (PCIB_READ_CONFIG(device_get_parent(dev),
6477 cfg->bus, cfg->slot, cfg->func, reg, width));
6478 }
6479
6480 void
pci_write_config_method(device_t dev,device_t child,int reg,uint32_t val,int width)6481 pci_write_config_method(device_t dev, device_t child, int reg,
6482 uint32_t val, int width)
6483 {
6484 struct pci_devinfo *dinfo = device_get_ivars(child);
6485 pcicfgregs *cfg = &dinfo->cfg;
6486
6487 PCIB_WRITE_CONFIG(device_get_parent(dev),
6488 cfg->bus, cfg->slot, cfg->func, reg, val, width);
6489 }
6490
6491 int
pci_child_location_method(device_t dev,device_t child,struct sbuf * sb)6492 pci_child_location_method(device_t dev, device_t child, struct sbuf *sb)
6493 {
6494
6495 sbuf_printf(sb, "slot=%d function=%d dbsf=pci%d:%d:%d:%d",
6496 pci_get_slot(child), pci_get_function(child), pci_get_domain(child),
6497 pci_get_bus(child), pci_get_slot(child), pci_get_function(child));
6498 return (0);
6499 }
6500
6501 int
pci_child_pnpinfo_method(device_t dev,device_t child,struct sbuf * sb)6502 pci_child_pnpinfo_method(device_t dev, device_t child, struct sbuf *sb)
6503 {
6504 struct pci_devinfo *dinfo;
6505 pcicfgregs *cfg;
6506
6507 dinfo = device_get_ivars(child);
6508 cfg = &dinfo->cfg;
6509 sbuf_printf(sb, "vendor=0x%04x device=0x%04x subvendor=0x%04x "
6510 "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device,
6511 cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass,
6512 cfg->progif);
6513 return (0);
6514 }
6515
6516 int
pci_get_device_path_method(device_t bus,device_t child,const char * locator,struct sbuf * sb)6517 pci_get_device_path_method(device_t bus, device_t child, const char *locator,
6518 struct sbuf *sb)
6519 {
6520 device_t parent = device_get_parent(bus);
6521 int rv;
6522
6523 if (strcmp(locator, BUS_LOCATOR_UEFI) == 0) {
6524 rv = bus_generic_get_device_path(parent, bus, locator, sb);
6525 if (rv == 0) {
6526 sbuf_printf(sb, "/Pci(0x%x,0x%x)", pci_get_slot(child),
6527 pci_get_function(child));
6528 }
6529 return (0);
6530 }
6531 return (bus_generic_get_device_path(bus, child, locator, sb));
6532 }
6533
6534 int
pci_assign_interrupt_method(device_t dev,device_t child)6535 pci_assign_interrupt_method(device_t dev, device_t child)
6536 {
6537 struct pci_devinfo *dinfo = device_get_ivars(child);
6538 pcicfgregs *cfg = &dinfo->cfg;
6539
6540 return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child,
6541 cfg->intpin));
6542 }
6543
6544 static void
pci_lookup(void * arg,const char * name,device_t * dev)6545 pci_lookup(void *arg, const char *name, device_t *dev)
6546 {
6547 long val;
6548 char *end;
6549 int domain, bus, slot, func;
6550
6551 if (*dev != NULL)
6552 return;
6553
6554 /*
6555 * Accept pciconf-style selectors of either pciD:B:S:F or
6556 * pciB:S:F. In the latter case, the domain is assumed to
6557 * be zero.
6558 */
6559 if (strncmp(name, "pci", 3) != 0)
6560 return;
6561 val = strtol(name + 3, &end, 10);
6562 if (val < 0 || val > INT_MAX || *end != ':')
6563 return;
6564 domain = val;
6565 val = strtol(end + 1, &end, 10);
6566 if (val < 0 || val > INT_MAX || *end != ':')
6567 return;
6568 bus = val;
6569 val = strtol(end + 1, &end, 10);
6570 if (val < 0 || val > INT_MAX)
6571 return;
6572 slot = val;
6573 if (*end == ':') {
6574 val = strtol(end + 1, &end, 10);
6575 if (val < 0 || val > INT_MAX || *end != '\0')
6576 return;
6577 func = val;
6578 } else if (*end == '\0') {
6579 func = slot;
6580 slot = bus;
6581 bus = domain;
6582 domain = 0;
6583 } else
6584 return;
6585
6586 if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX ||
6587 func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX))
6588 return;
6589
6590 *dev = pci_find_dbsf(domain, bus, slot, func);
6591 }
6592
6593 static int
pci_modevent(module_t mod,int what,void * arg)6594 pci_modevent(module_t mod, int what, void *arg)
6595 {
6596 static struct cdev *pci_cdev;
6597 static eventhandler_tag tag;
6598
6599 switch (what) {
6600 case MOD_LOAD:
6601 STAILQ_INIT(&pci_devq);
6602 pci_generation = 0;
6603 pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644,
6604 "pci");
6605 pci_load_vendor_data();
6606 tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL,
6607 1000);
6608 break;
6609
6610 case MOD_UNLOAD:
6611 if (tag != NULL)
6612 EVENTHANDLER_DEREGISTER(dev_lookup, tag);
6613 destroy_dev(pci_cdev);
6614 break;
6615 }
6616
6617 return (0);
6618 }
6619
6620 static void
pci_cfg_restore_pcie(device_t dev,struct pci_devinfo * dinfo)6621 pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo)
6622 {
6623 #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2)
6624 struct pcicfg_pcie *cfg;
6625 int version, pos;
6626
6627 cfg = &dinfo->cfg.pcie;
6628 pos = cfg->pcie_location;
6629
6630 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6631
6632 WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl);
6633
6634 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6635 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6636 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6637 WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl);
6638
6639 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6640 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6641 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6642 WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl);
6643
6644 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6645 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6646 WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl);
6647
6648 if (version > 1) {
6649 WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2);
6650 WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2);
6651 WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2);
6652 }
6653 #undef WREG
6654 }
6655
6656 static void
pci_cfg_restore_pcix(device_t dev,struct pci_devinfo * dinfo)6657 pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo)
6658 {
6659 pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND,
6660 dinfo->cfg.pcix.pcix_command, 2);
6661 }
6662
6663 void
pci_cfg_restore(device_t dev,struct pci_devinfo * dinfo)6664 pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
6665 {
6666
6667 /*
6668 * Restore the device to full power mode. We must do this
6669 * before we restore the registers because moving from D3 to
6670 * D0 will cause the chip's BARs and some other registers to
6671 * be reset to some unknown power on reset values. Cut down
6672 * the noise on boot by doing nothing if we are already in
6673 * state D0.
6674 */
6675 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0)
6676 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6677 pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1);
6678 pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1);
6679 pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1);
6680 pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1);
6681 pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
6682 pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
6683 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6684 case PCIM_HDRTYPE_NORMAL:
6685 pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1);
6686 pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1);
6687 break;
6688 case PCIM_HDRTYPE_BRIDGE:
6689 pci_write_config(dev, PCIR_SECLAT_1,
6690 dinfo->cfg.bridge.br_seclat, 1);
6691 pci_write_config(dev, PCIR_SUBBUS_1,
6692 dinfo->cfg.bridge.br_subbus, 1);
6693 pci_write_config(dev, PCIR_SECBUS_1,
6694 dinfo->cfg.bridge.br_secbus, 1);
6695 pci_write_config(dev, PCIR_PRIBUS_1,
6696 dinfo->cfg.bridge.br_pribus, 1);
6697 pci_write_config(dev, PCIR_BRIDGECTL_1,
6698 dinfo->cfg.bridge.br_control, 2);
6699 break;
6700 case PCIM_HDRTYPE_CARDBUS:
6701 pci_write_config(dev, PCIR_SECLAT_2,
6702 dinfo->cfg.bridge.br_seclat, 1);
6703 pci_write_config(dev, PCIR_SUBBUS_2,
6704 dinfo->cfg.bridge.br_subbus, 1);
6705 pci_write_config(dev, PCIR_SECBUS_2,
6706 dinfo->cfg.bridge.br_secbus, 1);
6707 pci_write_config(dev, PCIR_PRIBUS_2,
6708 dinfo->cfg.bridge.br_pribus, 1);
6709 pci_write_config(dev, PCIR_BRIDGECTL_2,
6710 dinfo->cfg.bridge.br_control, 2);
6711 break;
6712 }
6713 pci_restore_bars(dev);
6714
6715 if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE)
6716 pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2);
6717
6718 /*
6719 * Restore extended capabilities for PCI-Express and PCI-X
6720 */
6721 if (dinfo->cfg.pcie.pcie_location != 0)
6722 pci_cfg_restore_pcie(dev, dinfo);
6723 if (dinfo->cfg.pcix.pcix_location != 0)
6724 pci_cfg_restore_pcix(dev, dinfo);
6725
6726 /* Restore MSI and MSI-X configurations if they are present. */
6727 if (dinfo->cfg.msi.msi_location != 0)
6728 pci_resume_msi(dev);
6729 if (dinfo->cfg.msix.msix_location != 0)
6730 pci_resume_msix(dev);
6731
6732 #ifdef PCI_IOV
6733 /* The SR-IOV capability is implemented only by PFs. */
6734 if (dinfo->cfg.iov != NULL &&
6735 (dinfo->cfg.flags & PCICFG_VF) == 0)
6736 pci_iov_cfg_restore(dev, dinfo);
6737 #endif
6738 }
6739
6740 static void
pci_cfg_save_pcie(device_t dev,struct pci_devinfo * dinfo)6741 pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo)
6742 {
6743 #define RREG(n) pci_read_config(dev, pos + (n), 2)
6744 struct pcicfg_pcie *cfg;
6745 int version, pos;
6746
6747 cfg = &dinfo->cfg.pcie;
6748 pos = cfg->pcie_location;
6749
6750 cfg->pcie_flags = RREG(PCIER_FLAGS);
6751
6752 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6753
6754 cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL);
6755
6756 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6757 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6758 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6759 cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL);
6760
6761 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6762 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6763 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6764 cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL);
6765
6766 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6767 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6768 cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL);
6769
6770 if (version > 1) {
6771 cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2);
6772 cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2);
6773 cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2);
6774 }
6775 #undef RREG
6776 }
6777
6778 static void
pci_cfg_save_pcix(device_t dev,struct pci_devinfo * dinfo)6779 pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo)
6780 {
6781 dinfo->cfg.pcix.pcix_command = pci_read_config(dev,
6782 dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2);
6783 }
6784
6785 void
pci_cfg_save(device_t dev,struct pci_devinfo * dinfo,int setstate)6786 pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
6787 {
6788 uint32_t cls;
6789 int ps;
6790
6791 /*
6792 * Some drivers apparently write to these registers w/o updating our
6793 * cached copy. No harm happens if we update the copy, so do so here
6794 * so we can restore them. The COMMAND register is modified by the
6795 * bus w/o updating the cache. This should represent the normally
6796 * writable portion of the 'defined' part of type 0/1/2 headers.
6797 */
6798 dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2);
6799 dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2);
6800 dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2);
6801 dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1);
6802 dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1);
6803 dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
6804 dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
6805 dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1);
6806 dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1);
6807 dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1);
6808 dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1);
6809 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6810 case PCIM_HDRTYPE_NORMAL:
6811 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2);
6812 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2);
6813 dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1);
6814 dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1);
6815 break;
6816 case PCIM_HDRTYPE_BRIDGE:
6817 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6818 PCIR_SECLAT_1, 1);
6819 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6820 PCIR_SUBBUS_1, 1);
6821 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6822 PCIR_SECBUS_1, 1);
6823 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6824 PCIR_PRIBUS_1, 1);
6825 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6826 PCIR_BRIDGECTL_1, 2);
6827 break;
6828 case PCIM_HDRTYPE_CARDBUS:
6829 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6830 PCIR_SECLAT_2, 1);
6831 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6832 PCIR_SUBBUS_2, 1);
6833 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6834 PCIR_SECBUS_2, 1);
6835 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6836 PCIR_PRIBUS_2, 1);
6837 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6838 PCIR_BRIDGECTL_2, 2);
6839 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2);
6840 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2);
6841 break;
6842 }
6843
6844 if (dinfo->cfg.pcie.pcie_location != 0)
6845 pci_cfg_save_pcie(dev, dinfo);
6846
6847 if (dinfo->cfg.pcix.pcix_location != 0)
6848 pci_cfg_save_pcix(dev, dinfo);
6849
6850 #ifdef PCI_IOV
6851 /* The SR-IOV capability is implemented only by PFs. */
6852 if (dinfo->cfg.iov != NULL &&
6853 (dinfo->cfg.flags & PCICFG_VF) == 0)
6854 pci_iov_cfg_save(dev, dinfo);
6855 #endif
6856
6857 /*
6858 * don't set the state for display devices, base peripherals and
6859 * memory devices since bad things happen when they are powered down.
6860 * We should (a) have drivers that can easily detach and (b) use
6861 * generic drivers for these devices so that some device actually
6862 * attaches. We need to make sure that when we implement (a) we don't
6863 * power the device down on a reattach.
6864 */
6865 cls = pci_get_class(dev);
6866 if (!setstate)
6867 return;
6868 switch (pci_do_power_nodriver)
6869 {
6870 case 0: /* NO powerdown at all */
6871 return;
6872 case 1: /* Conservative about what to power down */
6873 if (cls == PCIC_STORAGE)
6874 return;
6875 /*FALLTHROUGH*/
6876 case 2: /* Aggressive about what to power down */
6877 if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY ||
6878 cls == PCIC_BASEPERIPH)
6879 return;
6880 /*FALLTHROUGH*/
6881 case 3: /* Power down everything */
6882 break;
6883 }
6884 /*
6885 * PCI spec says we can only go into D3 state from D0 state.
6886 * Transition from D[12] into D0 before going to D3 state.
6887 */
6888 ps = pci_get_powerstate(dev);
6889 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6890 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6891 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
6892 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6893 }
6894
6895 /* Wrapper APIs suitable for device driver use. */
6896 void
pci_save_state(device_t dev)6897 pci_save_state(device_t dev)
6898 {
6899 struct pci_devinfo *dinfo;
6900
6901 dinfo = device_get_ivars(dev);
6902 pci_cfg_save(dev, dinfo, 0);
6903 }
6904
6905 void
pci_restore_state(device_t dev)6906 pci_restore_state(device_t dev)
6907 {
6908 struct pci_devinfo *dinfo;
6909
6910 dinfo = device_get_ivars(dev);
6911 pci_cfg_restore(dev, dinfo);
6912 }
6913
6914 static int
pci_get_id_method(device_t dev,device_t child,enum pci_id_type type,uintptr_t * id)6915 pci_get_id_method(device_t dev, device_t child, enum pci_id_type type,
6916 uintptr_t *id)
6917 {
6918
6919 return (PCIB_GET_ID(device_get_parent(dev), child, type, id));
6920 }
6921
6922 /* Find the upstream port of a given PCI device in a root complex. */
6923 device_t
pci_find_pcie_root_port(device_t dev)6924 pci_find_pcie_root_port(device_t dev)
6925 {
6926 struct pci_devinfo *dinfo;
6927 device_t pcib, bus;
6928
6929 KASSERT(is_pci_device(dev),
6930 ("%s: non-pci device %s", __func__, device_get_nameunit(dev)));
6931
6932 /*
6933 * Walk the bridge hierarchy until we find a PCI-e root
6934 * port or a non-PCI device.
6935 */
6936 for (;;) {
6937 bus = device_get_parent(dev);
6938 KASSERT(bus != NULL, ("%s: null parent of %s", __func__,
6939 device_get_nameunit(dev)));
6940
6941 pcib = device_get_parent(bus);
6942 KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__,
6943 device_get_nameunit(bus)));
6944
6945 if (!is_pci_device(pcib))
6946 return (NULL);
6947
6948 dinfo = device_get_ivars(pcib);
6949 if (dinfo->cfg.pcie.pcie_location != 0 &&
6950 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
6951 return (pcib);
6952
6953 dev = pcib;
6954 }
6955 }
6956
6957 /*
6958 * Wait for pending transactions to complete on a PCI-express function.
6959 *
6960 * The maximum delay is specified in milliseconds in max_delay. Note
6961 * that this function may sleep.
6962 *
6963 * Returns true if the function is idle and false if the timeout is
6964 * exceeded. If dev is not a PCI-express function, this returns true.
6965 */
6966 bool
pcie_wait_for_pending_transactions(device_t dev,u_int max_delay)6967 pcie_wait_for_pending_transactions(device_t dev, u_int max_delay)
6968 {
6969 struct pci_devinfo *dinfo = device_get_ivars(dev);
6970 uint16_t sta;
6971 int cap;
6972
6973 cap = dinfo->cfg.pcie.pcie_location;
6974 if (cap == 0)
6975 return (true);
6976
6977 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6978 while (sta & PCIEM_STA_TRANSACTION_PND) {
6979 if (max_delay == 0)
6980 return (false);
6981
6982 /* Poll once every 100 milliseconds up to the timeout. */
6983 if (max_delay > 100) {
6984 pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK);
6985 max_delay -= 100;
6986 } else {
6987 pause_sbt("pcietp", max_delay * SBT_1MS, 0,
6988 C_HARDCLOCK);
6989 max_delay = 0;
6990 }
6991 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6992 }
6993
6994 return (true);
6995 }
6996
6997 /*
6998 * Determine the maximum Completion Timeout in microseconds.
6999 *
7000 * For non-PCI-express functions this returns 0.
7001 */
7002 int
pcie_get_max_completion_timeout(device_t dev)7003 pcie_get_max_completion_timeout(device_t dev)
7004 {
7005 struct pci_devinfo *dinfo = device_get_ivars(dev);
7006 int cap;
7007
7008 cap = dinfo->cfg.pcie.pcie_location;
7009 if (cap == 0)
7010 return (0);
7011
7012 /*
7013 * Functions using the 1.x spec use the default timeout range of
7014 * 50 microseconds to 50 milliseconds. Functions that do not
7015 * support programmable timeouts also use this range.
7016 */
7017 if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 ||
7018 (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) &
7019 PCIEM_CAP2_COMP_TIMO_RANGES) == 0)
7020 return (50 * 1000);
7021
7022 switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) &
7023 PCIEM_CTL2_COMP_TIMO_VAL) {
7024 case PCIEM_CTL2_COMP_TIMO_100US:
7025 return (100);
7026 case PCIEM_CTL2_COMP_TIMO_10MS:
7027 return (10 * 1000);
7028 case PCIEM_CTL2_COMP_TIMO_55MS:
7029 return (55 * 1000);
7030 case PCIEM_CTL2_COMP_TIMO_210MS:
7031 return (210 * 1000);
7032 case PCIEM_CTL2_COMP_TIMO_900MS:
7033 return (900 * 1000);
7034 case PCIEM_CTL2_COMP_TIMO_3500MS:
7035 return (3500 * 1000);
7036 case PCIEM_CTL2_COMP_TIMO_13S:
7037 return (13 * 1000 * 1000);
7038 case PCIEM_CTL2_COMP_TIMO_64S:
7039 return (64 * 1000 * 1000);
7040 default:
7041 return (50 * 1000);
7042 }
7043 }
7044
7045 void
pcie_apei_error(device_t dev,int sev,uint8_t * aerp)7046 pcie_apei_error(device_t dev, int sev, uint8_t *aerp)
7047 {
7048 struct pci_devinfo *dinfo = device_get_ivars(dev);
7049 const char *s;
7050 int aer;
7051 uint32_t r, r1;
7052 uint16_t rs;
7053
7054 if (sev == PCIEM_STA_CORRECTABLE_ERROR)
7055 s = "Correctable";
7056 else if (sev == PCIEM_STA_NON_FATAL_ERROR)
7057 s = "Uncorrectable (Non-Fatal)";
7058 else
7059 s = "Uncorrectable (Fatal)";
7060 device_printf(dev, "%s PCIe error reported by APEI\n", s);
7061 if (aerp) {
7062 if (sev == PCIEM_STA_CORRECTABLE_ERROR) {
7063 r = le32dec(aerp + PCIR_AER_COR_STATUS);
7064 r1 = le32dec(aerp + PCIR_AER_COR_MASK);
7065 } else {
7066 r = le32dec(aerp + PCIR_AER_UC_STATUS);
7067 r1 = le32dec(aerp + PCIR_AER_UC_MASK);
7068 }
7069 device_printf(dev, "status 0x%08x mask 0x%08x", r, r1);
7070 if (sev != PCIEM_STA_CORRECTABLE_ERROR) {
7071 r = le32dec(aerp + PCIR_AER_UC_SEVERITY);
7072 rs = le16dec(aerp + PCIR_AER_CAP_CONTROL);
7073 printf(" severity 0x%08x first %d\n",
7074 r, rs & 0x1f);
7075 } else
7076 printf("\n");
7077 }
7078
7079 /* As kind of recovery just report and clear the error statuses. */
7080 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
7081 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7082 if (r != 0) {
7083 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
7084 device_printf(dev, "Clearing UC AER errors 0x%08x\n", r);
7085 }
7086
7087 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7088 if (r != 0) {
7089 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
7090 device_printf(dev, "Clearing COR AER errors 0x%08x\n", r);
7091 }
7092 }
7093 if (dinfo->cfg.pcie.pcie_location != 0) {
7094 rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
7095 PCIER_DEVICE_STA, 2);
7096 if ((rs & (PCIEM_STA_CORRECTABLE_ERROR |
7097 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
7098 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
7099 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
7100 PCIER_DEVICE_STA, rs, 2);
7101 device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs);
7102 }
7103 }
7104 }
7105
7106 /*
7107 * Return true if the device supports FLR, taking both its advertised
7108 * capability and the PCI quirk policy into account.
7109 */
7110 bool
pcie_flr_supported(device_t dev)7111 pcie_flr_supported(device_t dev)
7112 {
7113 struct pci_devinfo *dinfo = device_get_ivars(dev);
7114 int cap;
7115
7116 cap = dinfo->cfg.pcie.pcie_location;
7117 if (cap == 0)
7118 return (false);
7119
7120 if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR) &&
7121 !pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_ENABLE_FLR))
7122 return (false);
7123 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_FLR))
7124 return (false);
7125 return (true);
7126 }
7127
7128 /*
7129 * Perform a Function Level Reset (FLR) on a device.
7130 *
7131 * This function first waits for any pending transactions to complete
7132 * within the timeout specified by max_delay. If transactions are
7133 * still pending, the function will return false without attempting a
7134 * reset.
7135 *
7136 * If dev is not a PCI-express function, or neither advertises FLR nor
7137 * has a quirk enabling FLR, this function returns false.
7138 *
7139 * Note that no registers are saved or restored. The caller is
7140 * responsible for saving and restoring any registers including
7141 * PCI-standard registers via pci_save_state() and
7142 * pci_restore_state().
7143 */
7144 bool
pcie_flr(device_t dev,u_int max_delay,bool force)7145 pcie_flr(device_t dev, u_int max_delay, bool force)
7146 {
7147 struct pci_devinfo *dinfo = device_get_ivars(dev);
7148 uint16_t cmd, ctl;
7149 int compl_delay;
7150 int cap;
7151
7152 if (!pcie_flr_supported(dev))
7153 return (false);
7154
7155 cap = dinfo->cfg.pcie.pcie_location;
7156
7157 /*
7158 * Disable busmastering to prevent generation of new
7159 * transactions while waiting for the device to go idle. If
7160 * the idle timeout fails, the command register is restored
7161 * which will re-enable busmastering.
7162 */
7163 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
7164 pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2);
7165 if (!pcie_wait_for_pending_transactions(dev, max_delay)) {
7166 if (!force) {
7167 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
7168 return (false);
7169 }
7170 pci_printf(&dinfo->cfg,
7171 "Resetting with transactions pending after %d ms\n",
7172 max_delay);
7173
7174 /*
7175 * Extend the post-FLR delay to cover the maximum
7176 * Completion Timeout delay of anything in flight
7177 * during the FLR delay. Enforce a minimum delay of
7178 * at least 10ms.
7179 */
7180 compl_delay = pcie_get_max_completion_timeout(dev) / 1000;
7181 if (compl_delay < 10)
7182 compl_delay = 10;
7183 } else
7184 compl_delay = 0;
7185
7186 /* Initiate the reset. */
7187 ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
7188 pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl |
7189 PCIEM_CTL_INITIATE_FLR, 2);
7190
7191 /* Wait for 100ms. */
7192 pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK);
7193
7194 if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) &
7195 PCIEM_STA_TRANSACTION_PND)
7196 pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
7197 return (true);
7198 }
7199
7200 /*
7201 * Attempt a power-management reset by cycling the device in/out of D3
7202 * state. PCI spec says we can only go into D3 state from D0 state.
7203 * Transition from D[12] into D0 before going to D3 state.
7204 */
7205 int
pci_power_reset(device_t dev)7206 pci_power_reset(device_t dev)
7207 {
7208 int ps;
7209
7210 ps = pci_get_powerstate(dev);
7211 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
7212 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
7213 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
7214 pci_set_powerstate(dev, ps);
7215 return (0);
7216 }
7217
7218 /*
7219 * Try link drop and retrain of the downstream port of upstream
7220 * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must
7221 * cause Conventional Hot reset of the device in the slot.
7222 * Alternative, for PCIe, could be the secondary bus reset initiatied
7223 * on the upstream switch PCIR_BRIDGECTL_1, bit 6.
7224 */
7225 int
pcie_link_reset(device_t port,int pcie_location)7226 pcie_link_reset(device_t port, int pcie_location)
7227 {
7228 uint16_t v;
7229
7230 v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2);
7231 v |= PCIEM_LINK_CTL_LINK_DIS;
7232 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
7233 pause_sbt("pcier1", mstosbt(20), 0, 0);
7234 v &= ~PCIEM_LINK_CTL_LINK_DIS;
7235 v |= PCIEM_LINK_CTL_RETRAIN_LINK;
7236 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
7237 pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */
7238 v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2);
7239 return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0);
7240 }
7241
7242 static int
pci_reset_post(device_t dev,device_t child)7243 pci_reset_post(device_t dev, device_t child)
7244 {
7245
7246 if (dev == device_get_parent(child))
7247 pci_restore_state(child);
7248 return (0);
7249 }
7250
7251 static int
pci_reset_prepare(device_t dev,device_t child)7252 pci_reset_prepare(device_t dev, device_t child)
7253 {
7254
7255 if (dev == device_get_parent(child))
7256 pci_save_state(child);
7257 return (0);
7258 }
7259
7260 static int
pci_reset_child(device_t dev,device_t child,int flags)7261 pci_reset_child(device_t dev, device_t child, int flags)
7262 {
7263 int error;
7264
7265 if (dev == NULL || device_get_parent(child) != dev)
7266 return (0);
7267 if ((flags & DEVF_RESET_DETACH) != 0) {
7268 error = device_get_state(child) == DS_ATTACHED ?
7269 device_detach(child) : 0;
7270 } else {
7271 error = BUS_SUSPEND_CHILD(dev, child);
7272 }
7273 if (error == 0) {
7274 if (!pcie_flr(child, 1000, false)) {
7275 error = BUS_RESET_PREPARE(dev, child);
7276 if (error == 0)
7277 pci_power_reset(child);
7278 BUS_RESET_POST(dev, child);
7279 }
7280 if ((flags & DEVF_RESET_DETACH) != 0)
7281 device_probe_and_attach(child);
7282 else
7283 BUS_RESUME_CHILD(dev, child);
7284 }
7285 return (error);
7286 }
7287
7288 const struct pci_device_table *
pci_match_device(device_t child,const struct pci_device_table * id,size_t nelt)7289 pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt)
7290 {
7291 bool match;
7292 uint16_t vendor, device, subvendor, subdevice, class, subclass, revid;
7293
7294 vendor = pci_get_vendor(child);
7295 device = pci_get_device(child);
7296 subvendor = pci_get_subvendor(child);
7297 subdevice = pci_get_subdevice(child);
7298 class = pci_get_class(child);
7299 subclass = pci_get_subclass(child);
7300 revid = pci_get_revid(child);
7301 while (nelt-- > 0) {
7302 match = true;
7303 if (id->match_flag_vendor)
7304 match &= vendor == id->vendor;
7305 if (id->match_flag_device)
7306 match &= device == id->device;
7307 if (id->match_flag_subvendor)
7308 match &= subvendor == id->subvendor;
7309 if (id->match_flag_subdevice)
7310 match &= subdevice == id->subdevice;
7311 if (id->match_flag_class)
7312 match &= class == id->class_id;
7313 if (id->match_flag_subclass)
7314 match &= subclass == id->subclass;
7315 if (id->match_flag_revid)
7316 match &= revid == id->revid;
7317 if (match)
7318 return (id);
7319 id++;
7320 }
7321 return (NULL);
7322 }
7323
7324 static void
pci_print_faulted_dev_name(const struct pci_devinfo * dinfo)7325 pci_print_faulted_dev_name(const struct pci_devinfo *dinfo)
7326 {
7327 const char *dev_name;
7328 device_t dev;
7329
7330 dev = dinfo->cfg.dev;
7331 printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus,
7332 dinfo->cfg.slot, dinfo->cfg.func);
7333 dev_name = device_get_name(dev);
7334 if (dev_name != NULL)
7335 printf(" (%s%d)", dev_name, device_get_unit(dev));
7336 }
7337
7338 void
pci_print_faulted_dev(void)7339 pci_print_faulted_dev(void)
7340 {
7341 struct pci_devinfo *dinfo;
7342 device_t dev;
7343 int aer, i;
7344 uint32_t r1, r2;
7345 uint16_t status;
7346
7347 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
7348 dev = dinfo->cfg.dev;
7349 status = pci_read_config(dev, PCIR_STATUS, 2);
7350 status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
7351 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
7352 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
7353 if (status != 0) {
7354 pci_print_faulted_dev_name(dinfo);
7355 printf(" error 0x%04x\n", status);
7356 }
7357 if (dinfo->cfg.pcie.pcie_location != 0) {
7358 status = pci_read_config(dev,
7359 dinfo->cfg.pcie.pcie_location +
7360 PCIER_DEVICE_STA, 2);
7361 if ((status & (PCIEM_STA_CORRECTABLE_ERROR |
7362 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
7363 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
7364 pci_print_faulted_dev_name(dinfo);
7365 printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n",
7366 pci_read_config(dev,
7367 dinfo->cfg.pcie.pcie_location +
7368 PCIER_DEVICE_CTL, 2),
7369 status);
7370 }
7371 }
7372 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
7373 r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7374 r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7375 if (r1 != 0 || r2 != 0) {
7376 pci_print_faulted_dev_name(dinfo);
7377 printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n"
7378 " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n",
7379 r1, pci_read_config(dev, aer +
7380 PCIR_AER_UC_MASK, 4),
7381 pci_read_config(dev, aer +
7382 PCIR_AER_UC_SEVERITY, 4),
7383 r2, pci_read_config(dev, aer +
7384 PCIR_AER_COR_MASK, 4),
7385 pci_read_config(dev, aer +
7386 PCIR_AER_CAP_CONTROL, 4));
7387 for (i = 0; i < 4; i++) {
7388 r1 = pci_read_config(dev, aer +
7389 PCIR_AER_HEADER_LOG + i * 4, 4);
7390 printf(" HL%d: 0x%08x\n", i, r1);
7391 }
7392 }
7393 }
7394 }
7395 }
7396
7397 bool
is_pci_device(device_t dev)7398 is_pci_device(device_t dev)
7399 {
7400 devclass_t pci_class;
7401
7402 if (device_get_parent(dev) == NULL)
7403 return (false);
7404 pci_class = devclass_find("pci");
7405 return (device_get_devclass(device_get_parent(dev)) == pci_class);
7406 }
7407
7408 #ifdef DDB
DB_SHOW_COMMAND_FLAGS(pcierr,pci_print_faulted_dev_db,DB_CMD_MEMSAFE)7409 DB_SHOW_COMMAND_FLAGS(pcierr, pci_print_faulted_dev_db, DB_CMD_MEMSAFE)
7410 {
7411
7412 pci_print_faulted_dev();
7413 }
7414
7415 static void
db_clear_pcie_errors(const struct pci_devinfo * dinfo)7416 db_clear_pcie_errors(const struct pci_devinfo *dinfo)
7417 {
7418 device_t dev;
7419 int aer;
7420 uint32_t r;
7421
7422 dev = dinfo->cfg.dev;
7423 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
7424 PCIER_DEVICE_STA, 2);
7425 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
7426 PCIER_DEVICE_STA, r, 2);
7427
7428 if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0)
7429 return;
7430 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7431 if (r != 0)
7432 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
7433 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7434 if (r != 0)
7435 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
7436 }
7437
DB_COMMAND_FLAGS(pci_clearerr,db_pci_clearerr,DB_CMD_MEMSAFE)7438 DB_COMMAND_FLAGS(pci_clearerr, db_pci_clearerr, DB_CMD_MEMSAFE)
7439 {
7440 struct pci_devinfo *dinfo;
7441 device_t dev;
7442 uint16_t status, status1;
7443
7444 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
7445 dev = dinfo->cfg.dev;
7446 status1 = status = pci_read_config(dev, PCIR_STATUS, 2);
7447 status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
7448 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
7449 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
7450 if (status1 != 0) {
7451 status &= ~status1;
7452 pci_write_config(dev, PCIR_STATUS, status, 2);
7453 }
7454 if (dinfo->cfg.pcie.pcie_location != 0)
7455 db_clear_pcie_errors(dinfo);
7456 }
7457 }
7458 #endif
7459