1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
6 * Copyright (c) 2000, BSDi
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice unmodified, this list of conditions, and the following
14 * disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include "opt_acpi.h"
33 #include "opt_iommu.h"
34 #include "opt_bus.h"
35
36 #include <sys/param.h>
37 #include <sys/conf.h>
38 #include <sys/endian.h>
39 #include <sys/eventhandler.h>
40 #include <sys/fcntl.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/linker.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/queue.h>
47 #include <sys/sbuf.h>
48 #include <sys/stdarg.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 #include <sys/taskqueue.h>
52 #include <sys/tree.h>
53
54 #include <vm/vm.h>
55 #include <vm/pmap.h>
56 #include <vm/vm_extern.h>
57
58 #include <sys/bus.h>
59 #include <machine/bus.h>
60 #include <sys/rman.h>
61 #include <machine/resource.h>
62
63 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
64 #include <machine/intr_machdep.h>
65 #endif
66
67 #include <sys/pciio.h>
68 #include <dev/pci/pcireg.h>
69 #include <dev/pci/pcivar.h>
70 #include <dev/pci/pci_private.h>
71
72 #ifdef PCI_IOV
73 #include <sys/nv.h>
74 #include <dev/pci/pci_iov_private.h>
75 #endif
76
77 #include <dev/usb/controller/xhcireg.h>
78 #include <dev/usb/controller/ehcireg.h>
79 #include <dev/usb/controller/ohcireg.h>
80 #include <dev/usb/controller/uhcireg.h>
81
82 #include <dev/iommu/iommu.h>
83
84 #include "pcib_if.h"
85 #include "pci_if.h"
86
87 #define PCIR_IS_BIOS(cfg, reg) \
88 (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \
89 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))
90
91 static device_probe_t pci_probe;
92
93 static bus_reset_post_t pci_reset_post;
94 static bus_reset_prepare_t pci_reset_prepare;
95 static bus_reset_child_t pci_reset_child;
96 static bus_hint_device_unit_t pci_hint_device_unit;
97 static bus_remap_intr_t pci_remap_intr_method;
98
99 static pci_get_id_t pci_get_id_method;
100
101 static int pci_has_quirk(uint32_t devid, int quirk);
102 static pci_addr_t pci_mapbase(uint64_t mapreg);
103 static const char *pci_maptype(uint64_t mapreg);
104 static int pci_maprange(uint64_t mapreg);
105 static pci_addr_t pci_rombase(uint64_t mapreg);
106 static int pci_romsize(uint64_t testval);
107 static void pci_fixancient(pcicfgregs *cfg);
108 static int pci_printf(pcicfgregs *cfg, const char *fmt, ...);
109
110 static int pci_porten(device_t dev);
111 static int pci_memen(device_t dev);
112 static void pci_assign_interrupt(device_t bus, device_t dev,
113 int force_route);
114 static int pci_add_map(device_t bus, device_t dev, int reg,
115 struct resource_list *rl, int force, int prefetch);
116 static void pci_load_vendor_data(void);
117 static int pci_describe_parse_line(char **ptr, int *vendor,
118 int *device, char **desc);
119 static char *pci_describe_device(device_t dev);
120 static int pci_modevent(module_t mod, int what, void *arg);
121 static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
122 pcicfgregs *cfg);
123 static void pci_read_cap(device_t pcib, pcicfgregs *cfg);
124 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg,
125 int reg, uint32_t *data);
126 #if 0
127 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
128 int reg, uint32_t data);
129 #endif
130 static void pci_read_vpd(device_t pcib, pcicfgregs *cfg);
131 static void pci_mask_msix(device_t dev, u_int index);
132 static void pci_unmask_msix(device_t dev, u_int index);
133 static int pci_msi_blacklisted(void);
134 static int pci_msix_blacklisted(void);
135 static void pci_resume_msi(device_t dev);
136 static void pci_resume_msix(device_t dev);
137 static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d,
138 int b, int s, int f, uint16_t vid, uint16_t did);
139
140 static device_method_t pci_methods[] = {
141 /* Device interface */
142 DEVMETHOD(device_probe, pci_probe),
143 DEVMETHOD(device_attach, pci_attach),
144 DEVMETHOD(device_detach, pci_detach),
145 DEVMETHOD(device_shutdown, bus_generic_shutdown),
146 DEVMETHOD(device_suspend, bus_generic_suspend),
147 DEVMETHOD(device_resume, pci_resume),
148
149 /* Bus interface */
150 DEVMETHOD(bus_print_child, pci_print_child),
151 DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch),
152 DEVMETHOD(bus_read_ivar, pci_read_ivar),
153 DEVMETHOD(bus_write_ivar, pci_write_ivar),
154 DEVMETHOD(bus_driver_added, pci_driver_added),
155 DEVMETHOD(bus_setup_intr, pci_setup_intr),
156 DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
157 DEVMETHOD(bus_reset_prepare, pci_reset_prepare),
158 DEVMETHOD(bus_reset_post, pci_reset_post),
159 DEVMETHOD(bus_reset_child, pci_reset_child),
160
161 DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag),
162 DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
163 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
164 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
165 DEVMETHOD(bus_delete_resource, pci_delete_resource),
166 DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
167 DEVMETHOD(bus_adjust_resource, pci_adjust_resource),
168 DEVMETHOD(bus_release_resource, pci_release_resource),
169 DEVMETHOD(bus_activate_resource, pci_activate_resource),
170 DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
171 DEVMETHOD(bus_map_resource, pci_map_resource),
172 DEVMETHOD(bus_unmap_resource, pci_unmap_resource),
173 DEVMETHOD(bus_child_deleted, pci_child_deleted),
174 DEVMETHOD(bus_child_detached, pci_child_detached),
175 DEVMETHOD(bus_child_pnpinfo, pci_child_pnpinfo_method),
176 DEVMETHOD(bus_child_location, pci_child_location_method),
177 DEVMETHOD(bus_get_device_path, pci_get_device_path_method),
178 DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit),
179 DEVMETHOD(bus_remap_intr, pci_remap_intr_method),
180 DEVMETHOD(bus_suspend_child, pci_suspend_child),
181 DEVMETHOD(bus_resume_child, pci_resume_child),
182 DEVMETHOD(bus_rescan, pci_rescan_method),
183
184 /* PCI interface */
185 DEVMETHOD(pci_read_config, pci_read_config_method),
186 DEVMETHOD(pci_write_config, pci_write_config_method),
187 DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method),
188 DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method),
189 DEVMETHOD(pci_enable_io, pci_enable_io_method),
190 DEVMETHOD(pci_disable_io, pci_disable_io_method),
191 DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method),
192 DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method),
193 DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method),
194 DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method),
195 DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method),
196 DEVMETHOD(pci_find_cap, pci_find_cap_method),
197 DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method),
198 DEVMETHOD(pci_find_extcap, pci_find_extcap_method),
199 DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method),
200 DEVMETHOD(pci_find_htcap, pci_find_htcap_method),
201 DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method),
202 DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method),
203 DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method),
204 DEVMETHOD(pci_enable_msi, pci_enable_msi_method),
205 DEVMETHOD(pci_enable_msix, pci_enable_msix_method),
206 DEVMETHOD(pci_disable_msi, pci_disable_msi_method),
207 DEVMETHOD(pci_remap_msix, pci_remap_msix_method),
208 DEVMETHOD(pci_release_msi, pci_release_msi_method),
209 DEVMETHOD(pci_msi_count, pci_msi_count_method),
210 DEVMETHOD(pci_msix_count, pci_msix_count_method),
211 DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method),
212 DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method),
213 DEVMETHOD(pci_get_id, pci_get_id_method),
214 DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method),
215 DEVMETHOD(pci_child_added, pci_child_added_method),
216 #ifdef PCI_IOV
217 DEVMETHOD(pci_iov_attach, pci_iov_attach_method),
218 DEVMETHOD(pci_iov_detach, pci_iov_detach_method),
219 DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method),
220 #endif
221
222 DEVMETHOD_END
223 };
224
225 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));
226
227 EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_modevent, NULL, BUS_PASS_BUS);
228 MODULE_VERSION(pci, 1);
229
230 static char *pci_vendordata;
231 static size_t pci_vendordata_size;
232
233 struct pci_quirk {
234 uint32_t devid; /* Vendor/device of the card */
235 int type;
236 #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */
237 #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */
238 #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */
239 #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */
240 #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */
241 #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */
242 #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */
243 #define PCI_QUIRK_DISABLE_FLR 8 /* Function-Level Reset (FLR) not working. */
244 #define PCI_QUIRK_ENABLE_FLR 9 /* FLR works but is not advertised. */
245 int arg1;
246 int arg2;
247 };
248
249 static const struct pci_quirk pci_quirks[] = {
250 /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
251 { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 },
252 { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 },
253 /* As does the Serverworks OSB4 (the SMBus mapping register) */
254 { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 },
255
256 /*
257 * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge
258 * or the CMIC-SL (AKA ServerWorks GC_LE).
259 */
260 { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
261 { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
262
263 /*
264 * MSI doesn't work on earlier Intel chipsets including
265 * E7500, E7501, E7505, 845, 865, 875/E7210, and 855.
266 */
267 { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
268 { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
269 { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
270 { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
271 { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
272 { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
273 { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
274
275 /*
276 * MSI doesn't work with devices behind the AMD 8131 HT-PCIX
277 * bridge.
278 */
279 { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 },
280
281 /*
282 * Some virtualization environments emulate an older chipset
283 * but support MSI just fine. QEMU uses the Intel 82440.
284 */
285 { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 },
286
287 /*
288 * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus
289 * controller depending on SoftPciRst register (PM_IO 0x55 [7]).
290 * It prevents us from attaching hpet(4) when the bit is unset.
291 * Note this quirk only affects SB600 revision A13 and earlier.
292 * For SB600 A21 and later, firmware must set the bit to hide it.
293 * For SB700 and later, it is unused and hardcoded to zero.
294 */
295 { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 },
296
297 /*
298 * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have
299 * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit
300 * of the command register is set.
301 */
302 { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
303 { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
304 { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
305 { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
306 { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
307
308 /*
309 * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't
310 * issue MSI interrupts with PCIM_CMD_INTxDIS set either.
311 */
312 { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */
313 { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */
314 { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */
315 { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */
316 { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */
317 { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */
318
319 /*
320 * HPE Gen 10 VGA has a memory range that can't be allocated in the
321 * expected place.
322 */
323 { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 },
324
325 /*
326 * The Intel 82599 VF implements FLR without advertising it; see
327 * 82599 Specification Update, erratum 35.
328 */
329 { 0x10ed8086, PCI_QUIRK_ENABLE_FLR, 0, 0 },
330
331 /*
332 * With some MediaTek mt76 WiFi FLR does not work despite advertised.
333 */
334 { 0x061614c3, PCI_QUIRK_DISABLE_FLR, 0, 0 }, /* mt76 7922 */
335
336 /* end of table */
337 { 0 }
338 };
339
340 /* map register information */
341 #define PCI_MAPMEM 0x01 /* memory map */
342 #define PCI_MAPMEMP 0x02 /* prefetchable memory map */
343 #define PCI_MAPPORT 0x04 /* port map */
344
345 struct devlist pci_devq;
346 uint32_t pci_generation;
347 uint32_t pci_numdevs = 0;
348 static int pcie_chipset, pcix_chipset;
349
350 /* sysctl vars */
351 SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
352 "PCI bus tuning parameters");
353
354 static int pci_enable_io_modes = 1;
355 SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN,
356 &pci_enable_io_modes, 1,
357 "Enable I/O and memory bits in the config register. Some BIOSes do not"
358 " enable these bits correctly. We'd like to do this all the time, but"
359 " there are some peripherals that this causes problems with.");
360
361 static int pci_do_realloc_bars = 1;
362 SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN,
363 &pci_do_realloc_bars, 0,
364 "Attempt to allocate a new range for any BARs whose original "
365 "firmware-assigned ranges fail to allocate during the initial device scan.");
366
367 static int pci_do_power_nodriver = 0;
368 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN,
369 &pci_do_power_nodriver, 0,
370 "Place a function into D3 state when no driver attaches to it. 0 means"
371 " disable. 1 means conservatively place function into D3 state. 2 means"
372 " aggressively place function into D3 state. 3 means put absolutely"
373 " everything in D3 state.");
374
375 int pci_do_power_resume = 1;
376 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN,
377 &pci_do_power_resume, 1,
378 "Transition from D3 -> D0 on resume.");
379
380 int pci_do_power_suspend = 1;
381 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN,
382 &pci_do_power_suspend, 1,
383 "Transition from D0 -> D3 on suspend.");
384
385 static int pci_do_msi = 1;
386 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1,
387 "Enable support for MSI interrupts");
388
389 static int pci_do_msix = 1;
390 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1,
391 "Enable support for MSI-X interrupts");
392
393 static int pci_msix_rewrite_table = 0;
394 SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
395 &pci_msix_rewrite_table, 0,
396 "Rewrite entire MSI-X table when updating MSI-X entries");
397
398 static int pci_honor_msi_blacklist = 1;
399 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN,
400 &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");
401
402 #if defined(__i386__) || defined(__amd64__)
403 static int pci_usb_takeover = 1;
404 #else
405 static int pci_usb_takeover = 0;
406 #endif
407 SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN,
408 &pci_usb_takeover, 1,
409 "Enable early takeover of USB controllers. Disable this if you depend on"
410 " BIOS emulation of USB devices, that is you use USB devices (like"
411 " keyboard or mouse) but do not load USB drivers");
412
413 static int pci_clear_bars;
414 SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0,
415 "Ignore firmware-assigned resources for BARs.");
416
417 static int pci_clear_buses;
418 SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
419 "Ignore firmware-assigned bus numbers.");
420
421 static int pci_enable_ari = 1;
422 SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
423 0, "Enable support for PCIe Alternative RID Interpretation");
424
425 /*
426 * Some x86 firmware only enables PCIe hotplug if we claim to support aspm,
427 * however enabling it breaks some arm64 firmware as it powers off devices.
428 */
429 #if defined(__i386__) || defined(__amd64__)
430 int pci_enable_aspm = 1;
431 #else
432 int pci_enable_aspm = 0;
433 #endif
434 SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm,
435 0, "Enable support for PCIe Active State Power Management");
436
437 static int pci_clear_aer_on_attach = 0;
438 SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN,
439 &pci_clear_aer_on_attach, 0,
440 "Clear port and device AER state on driver attach");
441
442 static bool pci_enable_mps_tune = true;
443 SYSCTL_BOOL(_hw_pci, OID_AUTO, enable_mps_tune, CTLFLAG_RWTUN,
444 &pci_enable_mps_tune, 1,
445 "Enable tuning of MPS(maximum payload size)." );
446
447 static int pci_mps_limit;
448 SYSCTL_INT(_hw_pci, OID_AUTO, mps_limit, CTLFLAG_RDTUN, &pci_mps_limit, 0,
449 "Limit PCIe MPS to this many bytes (power of two from 128 to 4096)");
450 static bool pci_mps_limit_warned;
451
452 static bool pci_mps_enforce;
453 SYSCTL_BOOL(_hw_pci, OID_AUTO, mps_enforce, CTLFLAG_RDTUN,
454 &pci_mps_enforce, 0,
455 "Disable PCIe endpoints with an MPS incompatible with their shared path");
456
457 static bool pci_intx_reroute = true;
458 SYSCTL_BOOL(_hw_pci, OID_AUTO, intx_reroute, CTLFLAG_RWTUN,
459 &pci_intx_reroute, 0, "Re-route INTx interrupts when scanning devices");
460
461 static int
pci_has_quirk(uint32_t devid,int quirk)462 pci_has_quirk(uint32_t devid, int quirk)
463 {
464 const struct pci_quirk *q;
465
466 for (q = &pci_quirks[0]; q->devid; q++) {
467 if (q->devid == devid && q->type == quirk)
468 return (1);
469 }
470 return (0);
471 }
472
473 /* Find a device_t by bus/slot/function in domain 0 */
474
475 device_t
pci_find_bsf(uint8_t bus,uint8_t slot,uint8_t func)476 pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func)
477 {
478
479 return (pci_find_dbsf(0, bus, slot, func));
480 }
481
482 /* Find a device_t by domain/bus/slot/function */
483
484 device_t
pci_find_dbsf(uint32_t domain,uint8_t bus,uint8_t slot,uint8_t func)485 pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func)
486 {
487 struct pci_devinfo *dinfo = NULL;
488
489 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
490 if ((dinfo->cfg.domain == domain) &&
491 (dinfo->cfg.bus == bus) &&
492 (dinfo->cfg.slot == slot) &&
493 (dinfo->cfg.func == func)) {
494 break;
495 }
496 }
497
498 return (dinfo != NULL ? dinfo->cfg.dev : NULL);
499 }
500
501 /* Find a device_t by vendor/device ID */
502
503 device_t
pci_find_device(uint16_t vendor,uint16_t device)504 pci_find_device(uint16_t vendor, uint16_t device)
505 {
506 struct pci_devinfo *dinfo;
507
508 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
509 if ((dinfo->cfg.vendor == vendor) &&
510 (dinfo->cfg.device == device)) {
511 return (dinfo->cfg.dev);
512 }
513 }
514
515 return (NULL);
516 }
517
518 device_t
pci_find_class(uint8_t class,uint8_t subclass)519 pci_find_class(uint8_t class, uint8_t subclass)
520 {
521 struct pci_devinfo *dinfo;
522
523 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
524 if (dinfo->cfg.baseclass == class &&
525 dinfo->cfg.subclass == subclass) {
526 return (dinfo->cfg.dev);
527 }
528 }
529
530 return (NULL);
531 }
532
533 device_t
pci_find_class_from(uint8_t class,uint8_t subclass,device_t from)534 pci_find_class_from(uint8_t class, uint8_t subclass, device_t from)
535 {
536 struct pci_devinfo *dinfo;
537 bool found = false;
538
539 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
540 if (from != NULL && found == false) {
541 if (from != dinfo->cfg.dev)
542 continue;
543 found = true;
544 continue;
545 }
546 if (dinfo->cfg.baseclass == class &&
547 dinfo->cfg.subclass == subclass) {
548 return (dinfo->cfg.dev);
549 }
550 }
551
552 return (NULL);
553 }
554
555 device_t
pci_find_base_class_from(uint8_t class,device_t from)556 pci_find_base_class_from(uint8_t class, device_t from)
557 {
558 struct pci_devinfo *dinfo;
559 bool found = false;
560
561 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
562 if (from != NULL && found == false) {
563 if (from != dinfo->cfg.dev)
564 continue;
565 found = true;
566 continue;
567 }
568 if (dinfo->cfg.baseclass == class) {
569 return (dinfo->cfg.dev);
570 }
571 }
572
573 return (NULL);
574 }
575
576 static int
pci_printf(pcicfgregs * cfg,const char * fmt,...)577 pci_printf(pcicfgregs *cfg, const char *fmt, ...)
578 {
579 va_list ap;
580 int retval;
581
582 retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot,
583 cfg->func);
584 va_start(ap, fmt);
585 retval += vprintf(fmt, ap);
586 va_end(ap);
587 return (retval);
588 }
589
590 /* return base address of memory or port map */
591
592 static pci_addr_t
pci_mapbase(uint64_t mapreg)593 pci_mapbase(uint64_t mapreg)
594 {
595
596 if (PCI_BAR_MEM(mapreg))
597 return (mapreg & PCIM_BAR_MEM_BASE);
598 else
599 return (mapreg & PCIM_BAR_IO_BASE);
600 }
601
602 /* return map type of memory or port map */
603
604 static const char *
pci_maptype(uint64_t mapreg)605 pci_maptype(uint64_t mapreg)
606 {
607
608 if (PCI_BAR_IO(mapreg))
609 return ("I/O Port");
610 if (mapreg & PCIM_BAR_MEM_PREFETCH)
611 return ("Prefetchable Memory");
612 return ("Memory");
613 }
614
615 /* return log2 of map size decoded for memory or port map */
616
617 int
pci_mapsize(uint64_t testval)618 pci_mapsize(uint64_t testval)
619 {
620 int ln2size;
621
622 testval = pci_mapbase(testval);
623 ln2size = 0;
624 if (testval != 0) {
625 while ((testval & 1) == 0)
626 {
627 ln2size++;
628 testval >>= 1;
629 }
630 }
631 return (ln2size);
632 }
633
634 /* return base address of device ROM */
635
636 static pci_addr_t
pci_rombase(uint64_t mapreg)637 pci_rombase(uint64_t mapreg)
638 {
639
640 return (mapreg & PCIM_BIOS_ADDR_MASK);
641 }
642
643 /* return log2 of map size decided for device ROM */
644
645 static int
pci_romsize(uint64_t testval)646 pci_romsize(uint64_t testval)
647 {
648 int ln2size;
649
650 testval = pci_rombase(testval);
651 ln2size = 0;
652 if (testval != 0) {
653 while ((testval & 1) == 0)
654 {
655 ln2size++;
656 testval >>= 1;
657 }
658 }
659 return (ln2size);
660 }
661
662 /* return log2 of address range supported by map register */
663
664 static int
pci_maprange(uint64_t mapreg)665 pci_maprange(uint64_t mapreg)
666 {
667 int ln2range = 0;
668
669 if (PCI_BAR_IO(mapreg))
670 ln2range = 32;
671 else
672 switch (mapreg & PCIM_BAR_MEM_TYPE) {
673 case PCIM_BAR_MEM_32:
674 ln2range = 32;
675 break;
676 case PCIM_BAR_MEM_1MB:
677 ln2range = 20;
678 break;
679 case PCIM_BAR_MEM_64:
680 ln2range = 64;
681 break;
682 }
683 return (ln2range);
684 }
685
686 /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */
687
688 static void
pci_fixancient(pcicfgregs * cfg)689 pci_fixancient(pcicfgregs *cfg)
690 {
691 if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
692 return;
693
694 /* PCI to PCI bridges use header type 1 */
695 if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI)
696 cfg->hdrtype = PCIM_HDRTYPE_BRIDGE;
697 }
698
699 /* extract header type specific config data */
700
701 static void
pci_hdrtypedata(device_t pcib,int b,int s,int f,pcicfgregs * cfg)702 pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg)
703 {
704 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
705 switch (cfg->hdrtype & PCIM_HDRTYPE) {
706 case PCIM_HDRTYPE_NORMAL:
707 cfg->subvendor = REG(PCIR_SUBVEND_0, 2);
708 cfg->subdevice = REG(PCIR_SUBDEV_0, 2);
709 cfg->mingnt = REG(PCIR_MINGNT, 1);
710 cfg->maxlat = REG(PCIR_MAXLAT, 1);
711 cfg->nummaps = PCI_MAXMAPS_0;
712 break;
713 case PCIM_HDRTYPE_BRIDGE:
714 cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1);
715 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1);
716 cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1);
717 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1);
718 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2);
719 cfg->nummaps = PCI_MAXMAPS_1;
720 break;
721 case PCIM_HDRTYPE_CARDBUS:
722 cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1);
723 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1);
724 cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1);
725 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1);
726 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2);
727 cfg->subvendor = REG(PCIR_SUBVEND_2, 2);
728 cfg->subdevice = REG(PCIR_SUBDEV_2, 2);
729 cfg->nummaps = PCI_MAXMAPS_2;
730 break;
731 }
732 #undef REG
733 }
734
735 /* read configuration header into pcicfgregs structure */
736 struct pci_devinfo *
pci_read_device(device_t pcib,device_t bus,int d,int b,int s,int f)737 pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f)
738 {
739 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
740 uint16_t vid, did;
741
742 vid = REG(PCIR_VENDOR, 2);
743 if (vid == PCIV_INVALID)
744 return (NULL);
745
746 did = REG(PCIR_DEVICE, 2);
747
748 return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did));
749 }
750
751 struct pci_devinfo *
pci_alloc_devinfo_method(device_t dev)752 pci_alloc_devinfo_method(device_t dev)
753 {
754
755 return (malloc(sizeof(struct pci_devinfo), M_DEVBUF,
756 M_WAITOK | M_ZERO));
757 }
758
759 static struct pci_devinfo *
pci_fill_devinfo(device_t pcib,device_t bus,int d,int b,int s,int f,uint16_t vid,uint16_t did)760 pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f,
761 uint16_t vid, uint16_t did)
762 {
763 struct pci_devinfo *devlist_entry;
764 pcicfgregs *cfg;
765
766 devlist_entry = PCI_ALLOC_DEVINFO(bus);
767
768 cfg = &devlist_entry->cfg;
769
770 cfg->domain = d;
771 cfg->bus = b;
772 cfg->slot = s;
773 cfg->func = f;
774 cfg->vendor = vid;
775 cfg->device = did;
776 cfg->cmdreg = REG(PCIR_COMMAND, 2);
777 cfg->statreg = REG(PCIR_STATUS, 2);
778 cfg->baseclass = REG(PCIR_CLASS, 1);
779 cfg->subclass = REG(PCIR_SUBCLASS, 1);
780 cfg->progif = REG(PCIR_PROGIF, 1);
781 cfg->revid = REG(PCIR_REVID, 1);
782 cfg->hdrtype = REG(PCIR_HDRTYPE, 1);
783 cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1);
784 cfg->lattimer = REG(PCIR_LATTIMER, 1);
785 cfg->intpin = REG(PCIR_INTPIN, 1);
786 cfg->intline = REG(PCIR_INTLINE, 1);
787
788 cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0;
789 cfg->hdrtype &= ~PCIM_MFDEV;
790 STAILQ_INIT(&cfg->maps);
791
792 cfg->iov = NULL;
793
794 pci_fixancient(cfg);
795 pci_hdrtypedata(pcib, b, s, f, cfg);
796
797 if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)
798 pci_read_cap(pcib, cfg);
799
800 STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links);
801
802 devlist_entry->conf.pc_sel.pc_domain = cfg->domain;
803 devlist_entry->conf.pc_sel.pc_bus = cfg->bus;
804 devlist_entry->conf.pc_sel.pc_dev = cfg->slot;
805 devlist_entry->conf.pc_sel.pc_func = cfg->func;
806 devlist_entry->conf.pc_hdr = cfg->hdrtype;
807
808 devlist_entry->conf.pc_subvendor = cfg->subvendor;
809 devlist_entry->conf.pc_subdevice = cfg->subdevice;
810 devlist_entry->conf.pc_vendor = cfg->vendor;
811 devlist_entry->conf.pc_device = cfg->device;
812
813 devlist_entry->conf.pc_class = cfg->baseclass;
814 devlist_entry->conf.pc_subclass = cfg->subclass;
815 devlist_entry->conf.pc_progif = cfg->progif;
816 devlist_entry->conf.pc_revid = cfg->revid;
817
818 devlist_entry->conf.pc_secbus = cfg->bridge.br_secbus;
819 devlist_entry->conf.pc_subbus = cfg->bridge.br_subbus;
820
821 pci_numdevs++;
822 pci_generation++;
823
824 return (devlist_entry);
825 }
826 #undef REG
827
828 static void
pci_ea_fill_info(device_t pcib,pcicfgregs * cfg)829 pci_ea_fill_info(device_t pcib, pcicfgregs *cfg)
830 {
831 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \
832 cfg->ea.ea_location + (n), w)
833 int num_ent;
834 int ptr;
835 int a, b;
836 uint32_t val;
837 int ent_size;
838 uint32_t dw[4];
839 uint64_t base, max_offset;
840 struct pci_ea_entry *eae;
841
842 if (cfg->ea.ea_location == 0)
843 return;
844
845 STAILQ_INIT(&cfg->ea.ea_entries);
846
847 /* Determine the number of entries */
848 num_ent = REG(PCIR_EA_NUM_ENT, 2);
849 num_ent &= PCIM_EA_NUM_ENT_MASK;
850
851 /* Find the first entry to care of */
852 ptr = PCIR_EA_FIRST_ENT;
853
854 /* Skip DWORD 2 for type 1 functions */
855 if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE)
856 ptr += 4;
857
858 for (a = 0; a < num_ent; a++) {
859 eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO);
860 eae->eae_cfg_offset = cfg->ea.ea_location + ptr;
861
862 /* Read a number of dwords in the entry */
863 val = REG(ptr, 4);
864 ptr += 4;
865 ent_size = (val & PCIM_EA_ES);
866
867 for (b = 0; b < ent_size; b++) {
868 dw[b] = REG(ptr, 4);
869 ptr += 4;
870 }
871
872 eae->eae_flags = val;
873 eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET;
874
875 base = dw[0] & PCIM_EA_FIELD_MASK;
876 max_offset = dw[1] | ~PCIM_EA_FIELD_MASK;
877 b = 2;
878 if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) {
879 base |= (uint64_t)dw[b] << 32UL;
880 b++;
881 }
882 if (((dw[1] & PCIM_EA_IS_64) != 0)
883 && (b < ent_size)) {
884 max_offset |= (uint64_t)dw[b] << 32UL;
885 b++;
886 }
887
888 eae->eae_base = base;
889 eae->eae_max_offset = max_offset;
890
891 STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link);
892
893 if (bootverbose) {
894 printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n",
895 cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags,
896 (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset);
897 }
898 }
899 }
900 #undef REG
901
902 static void
pci_read_cap(device_t pcib,pcicfgregs * cfg)903 pci_read_cap(device_t pcib, pcicfgregs *cfg)
904 {
905 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w)
906 #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w)
907 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
908 uint64_t addr;
909 #endif
910 uint32_t val;
911 int ptr, nextptr, ptrptr;
912
913 switch (cfg->hdrtype & PCIM_HDRTYPE) {
914 case PCIM_HDRTYPE_NORMAL:
915 case PCIM_HDRTYPE_BRIDGE:
916 ptrptr = PCIR_CAP_PTR;
917 break;
918 case PCIM_HDRTYPE_CARDBUS:
919 ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */
920 break;
921 default:
922 return; /* no extended capabilities support */
923 }
924 nextptr = REG(ptrptr, 1); /* sanity check? */
925
926 /*
927 * Read capability entries.
928 */
929 while (nextptr != 0) {
930 /* Sanity check */
931 if (nextptr > 255) {
932 printf("illegal PCI extended capability offset %d\n",
933 nextptr);
934 return;
935 }
936 /* Find the next entry */
937 ptr = nextptr;
938 nextptr = REG(ptr + PCICAP_NEXTPTR, 1);
939
940 /* Process this entry */
941 switch (REG(ptr + PCICAP_ID, 1)) {
942 case PCIY_PMG: /* PCI power management */
943 cfg->pp.pp_location = ptr;
944 cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2);
945 break;
946 case PCIY_HT: /* HyperTransport */
947 /* Determine HT-specific capability type. */
948 val = REG(ptr + PCIR_HT_COMMAND, 2);
949
950 if ((val & 0xe000) == PCIM_HTCAP_SLAVE)
951 cfg->ht.ht_slave = ptr;
952
953 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
954 switch (val & PCIM_HTCMD_CAP_MASK) {
955 case PCIM_HTCAP_MSI_MAPPING:
956 if (!(val & PCIM_HTCMD_MSI_FIXED)) {
957 /* Sanity check the mapping window. */
958 addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI,
959 4);
960 addr <<= 32;
961 addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO,
962 4);
963 if (addr != MSI_INTEL_ADDR_BASE)
964 device_printf(pcib,
965 "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n",
966 cfg->domain, cfg->bus,
967 cfg->slot, cfg->func,
968 (long long)addr);
969 } else
970 addr = MSI_INTEL_ADDR_BASE;
971
972 cfg->ht.ht_msimap = ptr;
973 cfg->ht.ht_msictrl = val;
974 cfg->ht.ht_msiaddr = addr;
975 break;
976 }
977 #endif
978 break;
979 case PCIY_MSI: /* PCI MSI */
980 cfg->msi.msi_location = ptr;
981 cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2);
982 break;
983 case PCIY_MSIX: /* PCI MSI-X */
984 cfg->msix.msix_location = ptr;
985 cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2);
986 val = REG(ptr + PCIR_MSIX_TABLE, 4);
987 cfg->msix.msix_table_bar = PCIR_BAR(val &
988 PCIM_MSIX_BIR_MASK);
989 cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK;
990 val = REG(ptr + PCIR_MSIX_PBA, 4);
991 cfg->msix.msix_pba_bar = PCIR_BAR(val &
992 PCIM_MSIX_BIR_MASK);
993 cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
994 break;
995 case PCIY_VPD: /* PCI Vital Product Data */
996 cfg->vpd.vpd_reg = ptr;
997 break;
998 case PCIY_SUBVENDOR:
999 /* Should always be true. */
1000 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
1001 PCIM_HDRTYPE_BRIDGE) {
1002 val = REG(ptr + PCIR_SUBVENDCAP_ID, 4);
1003 cfg->subvendor = val & 0xffff;
1004 cfg->subdevice = val >> 16;
1005 }
1006 break;
1007 case PCIY_PCIX: /* PCI-X */
1008 /*
1009 * Assume we have a PCI-X chipset if we have
1010 * at least one PCI-PCI bridge with a PCI-X
1011 * capability. Note that some systems with
1012 * PCI-express or HT chipsets might match on
1013 * this check as well.
1014 */
1015 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
1016 PCIM_HDRTYPE_BRIDGE)
1017 pcix_chipset = 1;
1018 cfg->pcix.pcix_location = ptr;
1019 break;
1020 case PCIY_EXPRESS: /* PCI-express */
1021 /*
1022 * Assume we have a PCI-express chipset if we have
1023 * at least one PCI-express device.
1024 */
1025 pcie_chipset = 1;
1026 cfg->pcie.pcie_location = ptr;
1027 val = REG(ptr + PCIER_FLAGS, 2);
1028 cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE;
1029 break;
1030 case PCIY_EA: /* Enhanced Allocation */
1031 cfg->ea.ea_location = ptr;
1032 pci_ea_fill_info(pcib, cfg);
1033 break;
1034 default:
1035 break;
1036 }
1037 }
1038
1039 #if defined(__powerpc__)
1040 /*
1041 * Enable the MSI mapping window for all HyperTransport
1042 * slaves. PCI-PCI bridges have their windows enabled via
1043 * PCIB_MAP_MSI().
1044 */
1045 if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 &&
1046 !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
1047 device_printf(pcib,
1048 "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n",
1049 cfg->domain, cfg->bus, cfg->slot, cfg->func);
1050 cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
1051 WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl,
1052 2);
1053 }
1054 #endif
1055 /* REG and WREG use carry through to next functions */
1056 }
1057
1058 /*
1059 * PCI Vital Product Data
1060 */
1061
1062 #define PCI_VPD_TIMEOUT 1000000
1063
1064 static int
pci_read_vpd_reg(device_t pcib,pcicfgregs * cfg,int reg,uint32_t * data)1065 pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data)
1066 {
1067 int count = PCI_VPD_TIMEOUT;
1068
1069 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1070
1071 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2);
1072
1073 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) {
1074 if (--count < 0)
1075 return (ENXIO);
1076 DELAY(1); /* limit looping */
1077 }
1078 *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4));
1079
1080 return (0);
1081 }
1082
1083 #if 0
1084 static int
1085 pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data)
1086 {
1087 int count = PCI_VPD_TIMEOUT;
1088
1089 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1090
1091 WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4);
1092 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2);
1093 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) {
1094 if (--count < 0)
1095 return (ENXIO);
1096 DELAY(1); /* limit looping */
1097 }
1098
1099 return (0);
1100 }
1101 #endif
1102
1103 #undef PCI_VPD_TIMEOUT
1104
1105 struct vpd_readstate {
1106 device_t pcib;
1107 pcicfgregs *cfg;
1108 uint32_t val;
1109 int bytesinval;
1110 int off;
1111 uint8_t cksum;
1112 };
1113
1114 /* return 0 and one byte in *data if no read error, -1 else */
1115 static int
vpd_nextbyte(struct vpd_readstate * vrs,uint8_t * data)1116 vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data)
1117 {
1118 uint32_t reg;
1119 uint8_t byte;
1120
1121 if (vrs->bytesinval == 0) {
1122 if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®))
1123 return (-1);
1124 vrs->val = le32toh(reg);
1125 vrs->off += 4;
1126 byte = vrs->val & 0xff;
1127 vrs->bytesinval = 3;
1128 } else {
1129 vrs->val = vrs->val >> 8;
1130 byte = vrs->val & 0xff;
1131 vrs->bytesinval--;
1132 }
1133
1134 vrs->cksum += byte;
1135 *data = byte;
1136 return (0);
1137 }
1138
1139 /* return 0 on match, -1 and "unget" byte on no match */
1140 static int
vpd_expectbyte(struct vpd_readstate * vrs,uint8_t expected)1141 vpd_expectbyte(struct vpd_readstate *vrs, uint8_t expected)
1142 {
1143 uint8_t data;
1144
1145 if (vpd_nextbyte(vrs, &data) != 0)
1146 return (-1);
1147
1148 if (data == expected)
1149 return (0);
1150
1151 vrs->cksum -= data;
1152 vrs->val = (vrs->val << 8) + data;
1153 vrs->bytesinval++;
1154 return (-1);
1155 }
1156
1157 /* return size if tag matches, -1 on no match, -2 on read error */
1158 static int
vpd_read_tag_size(struct vpd_readstate * vrs,uint8_t vpd_tag)1159 vpd_read_tag_size(struct vpd_readstate *vrs, uint8_t vpd_tag)
1160 {
1161 uint8_t byte1, byte2;
1162
1163 if (vpd_expectbyte(vrs, vpd_tag) != 0)
1164 return (-1);
1165
1166 if ((vpd_tag & 0x80) == 0)
1167 return (vpd_tag & 0x07);
1168
1169 if (vpd_nextbyte(vrs, &byte1) != 0)
1170 return (-2);
1171 if (vpd_nextbyte(vrs, &byte2) != 0)
1172 return (-2);
1173
1174 return ((byte2 << 8) + byte1);
1175 }
1176
1177 /* (re)allocate buffer in multiples of 8 elements */
1178 static void*
alloc_buffer(void * buffer,size_t element_size,int needed)1179 alloc_buffer(void* buffer, size_t element_size, int needed)
1180 {
1181 int alloc, new_alloc;
1182
1183 alloc = roundup2(needed, 8);
1184 new_alloc = roundup2(needed + 1, 8);
1185 if (alloc != new_alloc) {
1186 buffer = reallocf(buffer,
1187 new_alloc * element_size, M_DEVBUF, M_WAITOK | M_ZERO);
1188 }
1189
1190 return (buffer);
1191 }
1192
1193 /* read VPD keyword and return element size, return -1 on read error */
1194 static int
vpd_read_elem_head(struct vpd_readstate * vrs,char keyword[2])1195 vpd_read_elem_head(struct vpd_readstate *vrs, char keyword[2])
1196 {
1197 uint8_t data;
1198
1199 if (vpd_nextbyte(vrs, &keyword[0]) != 0)
1200 return (-1);
1201 if (vpd_nextbyte(vrs, &keyword[1]) != 0)
1202 return (-1);
1203 if (vpd_nextbyte(vrs, &data) != 0)
1204 return (-1);
1205
1206 return (data);
1207 }
1208
1209 /* read VPD data element of given size into allocated buffer */
1210 static char *
vpd_read_value(struct vpd_readstate * vrs,int size)1211 vpd_read_value(struct vpd_readstate *vrs, int size)
1212 {
1213 int i;
1214 char char1;
1215 char *value;
1216
1217 value = malloc(size + 1, M_DEVBUF, M_WAITOK);
1218 for (i = 0; i < size; i++) {
1219 if (vpd_nextbyte(vrs, &char1) != 0) {
1220 free(value, M_DEVBUF);
1221 return (NULL);
1222 }
1223 value[i] = char1;
1224 }
1225 value[size] = '\0';
1226
1227 return (value);
1228 }
1229
1230 /* read VPD into *keyword and *value, return length of data element */
1231 static int
vpd_read_elem_data(struct vpd_readstate * vrs,char keyword[2],char ** value,int maxlen)1232 vpd_read_elem_data(struct vpd_readstate *vrs, char keyword[2], char **value, int maxlen)
1233 {
1234 int len;
1235
1236 len = vpd_read_elem_head(vrs, keyword);
1237 if (len < 0 || len > maxlen)
1238 return (-1);
1239 *value = vpd_read_value(vrs, len);
1240
1241 return (len);
1242 }
1243
1244 /* subtract all data following first byte from checksum of RV element */
1245 static void
vpd_fixup_cksum(struct vpd_readstate * vrs,char * rvstring,int len)1246 vpd_fixup_cksum(struct vpd_readstate *vrs, char *rvstring, int len)
1247 {
1248 int i;
1249 uint8_t fixup;
1250
1251 fixup = 0;
1252 for (i = 1; i < len; i++)
1253 fixup += rvstring[i];
1254 vrs->cksum -= fixup;
1255 }
1256
1257 /* fetch one read-only element and return size of heading + data */
1258 static int
next_vpd_ro_elem(struct vpd_readstate * vrs,int maxsize)1259 next_vpd_ro_elem(struct vpd_readstate *vrs, int maxsize)
1260 {
1261 struct pcicfg_vpd *vpd;
1262 pcicfgregs *cfg;
1263 struct vpd_readonly *vpd_ros;
1264 int len;
1265
1266 cfg = vrs->cfg;
1267 vpd = &cfg->vpd;
1268
1269 if (maxsize < 3)
1270 return (-1);
1271 vpd->vpd_ros = alloc_buffer(vpd->vpd_ros, sizeof(*vpd->vpd_ros), vpd->vpd_rocnt);
1272 vpd_ros = &vpd->vpd_ros[vpd->vpd_rocnt];
1273 maxsize -= 3;
1274 len = vpd_read_elem_data(vrs, vpd_ros->keyword, &vpd_ros->value, maxsize);
1275 if (vpd_ros->value == NULL)
1276 return (-1);
1277 vpd_ros->len = len;
1278 if (vpd_ros->keyword[0] == 'R' && vpd_ros->keyword[1] == 'V') {
1279 vpd_fixup_cksum(vrs, vpd_ros->value, len);
1280 if (vrs->cksum != 0) {
1281 pci_printf(cfg,
1282 "invalid VPD checksum %#hhx\n", vrs->cksum);
1283 return (-1);
1284 }
1285 }
1286 vpd->vpd_rocnt++;
1287
1288 return (len + 3);
1289 }
1290
1291 /* fetch one writable element and return size of heading + data */
1292 static int
next_vpd_rw_elem(struct vpd_readstate * vrs,int maxsize)1293 next_vpd_rw_elem(struct vpd_readstate *vrs, int maxsize)
1294 {
1295 struct pcicfg_vpd *vpd;
1296 pcicfgregs *cfg;
1297 struct vpd_write *vpd_w;
1298 int len;
1299
1300 cfg = vrs->cfg;
1301 vpd = &cfg->vpd;
1302
1303 if (maxsize < 3)
1304 return (-1);
1305 vpd->vpd_w = alloc_buffer(vpd->vpd_w, sizeof(*vpd->vpd_w), vpd->vpd_wcnt);
1306 if (vpd->vpd_w == NULL) {
1307 pci_printf(cfg, "out of memory");
1308 return (-1);
1309 }
1310 vpd_w = &vpd->vpd_w[vpd->vpd_wcnt];
1311 maxsize -= 3;
1312 vpd_w->start = vrs->off + 3 - vrs->bytesinval;
1313 len = vpd_read_elem_data(vrs, vpd_w->keyword, &vpd_w->value, maxsize);
1314 if (vpd_w->value == NULL)
1315 return (-1);
1316 vpd_w->len = len;
1317 vpd->vpd_wcnt++;
1318
1319 return (len + 3);
1320 }
1321
1322 /* free all memory allocated for VPD data */
1323 static void
vpd_free(struct pcicfg_vpd * vpd)1324 vpd_free(struct pcicfg_vpd *vpd)
1325 {
1326 int i;
1327
1328 free(vpd->vpd_ident, M_DEVBUF);
1329 for (i = 0; i < vpd->vpd_rocnt; i++)
1330 free(vpd->vpd_ros[i].value, M_DEVBUF);
1331 free(vpd->vpd_ros, M_DEVBUF);
1332 vpd->vpd_rocnt = 0;
1333 for (i = 0; i < vpd->vpd_wcnt; i++)
1334 free(vpd->vpd_w[i].value, M_DEVBUF);
1335 free(vpd->vpd_w, M_DEVBUF);
1336 vpd->vpd_wcnt = 0;
1337 }
1338
1339 #define VPD_TAG_END ((0x0f << 3) | 0) /* small tag, len == 0 */
1340 #define VPD_TAG_IDENT (0x02 | 0x80) /* large tag */
1341 #define VPD_TAG_RO (0x10 | 0x80) /* large tag */
1342 #define VPD_TAG_RW (0x11 | 0x80) /* large tag */
1343
1344 static int
pci_parse_vpd(device_t pcib,pcicfgregs * cfg)1345 pci_parse_vpd(device_t pcib, pcicfgregs *cfg)
1346 {
1347 struct vpd_readstate vrs;
1348 int cksumvalid;
1349 int size, elem_size;
1350
1351 /* init vpd reader */
1352 vrs.bytesinval = 0;
1353 vrs.off = 0;
1354 vrs.pcib = pcib;
1355 vrs.cfg = cfg;
1356 vrs.cksum = 0;
1357
1358 /* read VPD ident element - mandatory */
1359 size = vpd_read_tag_size(&vrs, VPD_TAG_IDENT);
1360 if (size <= 0) {
1361 pci_printf(cfg, "no VPD ident found\n");
1362 return (0);
1363 }
1364 cfg->vpd.vpd_ident = vpd_read_value(&vrs, size);
1365 if (cfg->vpd.vpd_ident == NULL) {
1366 pci_printf(cfg, "error accessing VPD ident data\n");
1367 return (0);
1368 }
1369
1370 /* read VPD RO elements - mandatory */
1371 size = vpd_read_tag_size(&vrs, VPD_TAG_RO);
1372 if (size <= 0) {
1373 pci_printf(cfg, "no read-only VPD data found\n");
1374 return (0);
1375 }
1376 while (size > 0) {
1377 elem_size = next_vpd_ro_elem(&vrs, size);
1378 if (elem_size < 0) {
1379 pci_printf(cfg, "error accessing read-only VPD data\n");
1380 return (-1);
1381 }
1382 size -= elem_size;
1383 }
1384 cksumvalid = (vrs.cksum == 0);
1385 if (!cksumvalid)
1386 return (-1);
1387
1388 /* read VPD RW elements - optional */
1389 size = vpd_read_tag_size(&vrs, VPD_TAG_RW);
1390 if (size == -2)
1391 return (-1);
1392 while (size > 0) {
1393 elem_size = next_vpd_rw_elem(&vrs, size);
1394 if (elem_size < 0) {
1395 pci_printf(cfg, "error accessing writeable VPD data\n");
1396 return (-1);
1397 }
1398 size -= elem_size;
1399 }
1400
1401 /* read empty END tag - mandatory */
1402 size = vpd_read_tag_size(&vrs, VPD_TAG_END);
1403 if (size != 0) {
1404 pci_printf(cfg, "No valid VPD end tag found\n");
1405 }
1406 return (0);
1407 }
1408
1409 static void
pci_read_vpd(device_t pcib,pcicfgregs * cfg)1410 pci_read_vpd(device_t pcib, pcicfgregs *cfg)
1411 {
1412 int status;
1413
1414 status = pci_parse_vpd(pcib, cfg);
1415 if (status < 0)
1416 vpd_free(&cfg->vpd);
1417 cfg->vpd.vpd_cached = 1;
1418 #undef REG
1419 #undef WREG
1420 }
1421
1422 int
pci_get_vpd_ident_method(device_t dev,device_t child,const char ** identptr)1423 pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr)
1424 {
1425 struct pci_devinfo *dinfo = device_get_ivars(child);
1426 pcicfgregs *cfg = &dinfo->cfg;
1427
1428 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1429 pci_read_vpd(device_get_parent(dev), cfg);
1430
1431 *identptr = cfg->vpd.vpd_ident;
1432
1433 if (*identptr == NULL)
1434 return (ENXIO);
1435
1436 return (0);
1437 }
1438
1439 int
pci_get_vpd_readonly_method(device_t dev,device_t child,const char * kw,const char ** vptr)1440 pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
1441 const char **vptr)
1442 {
1443 struct pci_devinfo *dinfo = device_get_ivars(child);
1444 pcicfgregs *cfg = &dinfo->cfg;
1445 int i;
1446
1447 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1448 pci_read_vpd(device_get_parent(dev), cfg);
1449
1450 for (i = 0; i < cfg->vpd.vpd_rocnt; i++)
1451 if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword,
1452 sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) {
1453 *vptr = cfg->vpd.vpd_ros[i].value;
1454 return (0);
1455 }
1456
1457 *vptr = NULL;
1458 return (ENXIO);
1459 }
1460
1461 struct pcicfg_vpd *
pci_fetch_vpd_list(device_t dev)1462 pci_fetch_vpd_list(device_t dev)
1463 {
1464 struct pci_devinfo *dinfo = device_get_ivars(dev);
1465 pcicfgregs *cfg = &dinfo->cfg;
1466
1467 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1468 pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg);
1469 return (&cfg->vpd);
1470 }
1471
1472 /*
1473 * Find the requested HyperTransport capability and return the offset
1474 * in configuration space via the pointer provided. The function
1475 * returns 0 on success and an error code otherwise.
1476 */
1477 int
pci_find_htcap_method(device_t dev,device_t child,int capability,int * capreg)1478 pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg)
1479 {
1480 int ptr, error;
1481 uint16_t val;
1482
1483 error = pci_find_cap(child, PCIY_HT, &ptr);
1484 if (error)
1485 return (error);
1486
1487 /*
1488 * Traverse the capabilities list checking each HT capability
1489 * to see if it matches the requested HT capability.
1490 */
1491 for (;;) {
1492 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1493 if (capability == PCIM_HTCAP_SLAVE ||
1494 capability == PCIM_HTCAP_HOST)
1495 val &= 0xe000;
1496 else
1497 val &= PCIM_HTCMD_CAP_MASK;
1498 if (val == capability) {
1499 if (capreg != NULL)
1500 *capreg = ptr;
1501 return (0);
1502 }
1503
1504 /* Skip to the next HT capability. */
1505 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1506 break;
1507 }
1508
1509 return (ENOENT);
1510 }
1511
1512 /*
1513 * Find the next requested HyperTransport capability after start and return
1514 * the offset in configuration space via the pointer provided. The function
1515 * returns 0 on success and an error code otherwise.
1516 */
1517 int
pci_find_next_htcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1518 pci_find_next_htcap_method(device_t dev, device_t child, int capability,
1519 int start, int *capreg)
1520 {
1521 int ptr;
1522 uint16_t val;
1523
1524 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT,
1525 ("start capability is not HyperTransport capability"));
1526 ptr = start;
1527
1528 /*
1529 * Traverse the capabilities list checking each HT capability
1530 * to see if it matches the requested HT capability.
1531 */
1532 for (;;) {
1533 /* Skip to the next HT capability. */
1534 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1535 break;
1536
1537 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1538 if (capability == PCIM_HTCAP_SLAVE ||
1539 capability == PCIM_HTCAP_HOST)
1540 val &= 0xe000;
1541 else
1542 val &= PCIM_HTCMD_CAP_MASK;
1543 if (val == capability) {
1544 if (capreg != NULL)
1545 *capreg = ptr;
1546 return (0);
1547 }
1548 }
1549
1550 return (ENOENT);
1551 }
1552
1553 /*
1554 * Find the requested capability and return the offset in
1555 * configuration space via the pointer provided. The function returns
1556 * 0 on success and an error code otherwise.
1557 */
1558 int
pci_find_cap_method(device_t dev,device_t child,int capability,int * capreg)1559 pci_find_cap_method(device_t dev, device_t child, int capability,
1560 int *capreg)
1561 {
1562 struct pci_devinfo *dinfo = device_get_ivars(child);
1563 pcicfgregs *cfg = &dinfo->cfg;
1564 uint32_t status;
1565 uint8_t ptr;
1566 int cnt;
1567
1568 /*
1569 * Check the CAP_LIST bit of the PCI status register first.
1570 */
1571 status = pci_read_config(child, PCIR_STATUS, 2);
1572 if (!(status & PCIM_STATUS_CAPPRESENT))
1573 return (ENXIO);
1574
1575 /*
1576 * Determine the start pointer of the capabilities list.
1577 */
1578 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1579 case PCIM_HDRTYPE_NORMAL:
1580 case PCIM_HDRTYPE_BRIDGE:
1581 ptr = PCIR_CAP_PTR;
1582 break;
1583 case PCIM_HDRTYPE_CARDBUS:
1584 ptr = PCIR_CAP_PTR_2;
1585 break;
1586 default:
1587 /* XXX: panic? */
1588 return (ENXIO); /* no extended capabilities support */
1589 }
1590 ptr = pci_read_config(child, ptr, 1);
1591
1592 /*
1593 * Traverse the capabilities list. Limit by total theoretical
1594 * maximum number of caps: capability needs at least id and
1595 * next registers, and any type X header cannot contain caps.
1596 */
1597 for (cnt = 0; ptr != 0 && cnt < (PCIE_REGMAX - 0x40) / 2; cnt++) {
1598 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1599 if (capreg != NULL)
1600 *capreg = ptr;
1601 return (0);
1602 }
1603 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1604 }
1605
1606 return (ENOENT);
1607 }
1608
1609 /*
1610 * Find the next requested capability after start and return the offset in
1611 * configuration space via the pointer provided. The function returns
1612 * 0 on success and an error code otherwise.
1613 */
1614 int
pci_find_next_cap_method(device_t dev,device_t child,int capability,int start,int * capreg)1615 pci_find_next_cap_method(device_t dev, device_t child, int capability,
1616 int start, int *capreg)
1617 {
1618 uint8_t ptr;
1619
1620 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability,
1621 ("start capability is not expected capability"));
1622
1623 ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1);
1624 while (ptr != 0) {
1625 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1626 if (capreg != NULL)
1627 *capreg = ptr;
1628 return (0);
1629 }
1630 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1631 }
1632
1633 return (ENOENT);
1634 }
1635
1636 /*
1637 * Find the requested extended capability and return the offset in
1638 * configuration space via the pointer provided. The function returns
1639 * 0 on success and an error code otherwise.
1640 */
1641 int
pci_find_extcap_method(device_t dev,device_t child,int capability,int * capreg)1642 pci_find_extcap_method(device_t dev, device_t child, int capability,
1643 int *capreg)
1644 {
1645 struct pci_devinfo *dinfo = device_get_ivars(child);
1646 pcicfgregs *cfg = &dinfo->cfg;
1647 uint32_t ecap;
1648 uint16_t ptr;
1649
1650 /* Only supported for PCI-express devices. */
1651 if (cfg->pcie.pcie_location == 0)
1652 return (ENXIO);
1653
1654 ptr = PCIR_EXTCAP;
1655 ecap = pci_read_config(child, ptr, 4);
1656 if (ecap == 0xffffffff || ecap == 0)
1657 return (ENOENT);
1658 for (;;) {
1659 if (PCI_EXTCAP_ID(ecap) == capability) {
1660 if (capreg != NULL)
1661 *capreg = ptr;
1662 return (0);
1663 }
1664 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1665 if (ptr == 0)
1666 break;
1667 ecap = pci_read_config(child, ptr, 4);
1668 }
1669
1670 return (ENOENT);
1671 }
1672
1673 /*
1674 * Find the next requested extended capability after start and return the
1675 * offset in configuration space via the pointer provided. The function
1676 * returns 0 on success and an error code otherwise.
1677 */
1678 int
pci_find_next_extcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1679 pci_find_next_extcap_method(device_t dev, device_t child, int capability,
1680 int start, int *capreg)
1681 {
1682 struct pci_devinfo *dinfo = device_get_ivars(child);
1683 pcicfgregs *cfg = &dinfo->cfg;
1684 uint32_t ecap;
1685 uint16_t ptr;
1686
1687 /* Only supported for PCI-express devices. */
1688 if (cfg->pcie.pcie_location == 0)
1689 return (ENXIO);
1690
1691 ecap = pci_read_config(child, start, 4);
1692 KASSERT(PCI_EXTCAP_ID(ecap) == capability,
1693 ("start extended capability is not expected capability"));
1694 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1695 while (ptr != 0) {
1696 ecap = pci_read_config(child, ptr, 4);
1697 if (PCI_EXTCAP_ID(ecap) == capability) {
1698 if (capreg != NULL)
1699 *capreg = ptr;
1700 return (0);
1701 }
1702 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1703 }
1704
1705 return (ENOENT);
1706 }
1707
1708 /*
1709 * Support for MSI-X message interrupts.
1710 */
1711 static void
pci_write_msix_entry(device_t dev,u_int index,uint64_t address,uint32_t data)1712 pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data)
1713 {
1714 struct pci_devinfo *dinfo = device_get_ivars(dev);
1715 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1716 uint32_t offset;
1717
1718 KASSERT(msix->msix_table_len > index, ("bogus index"));
1719 offset = msix->msix_table_offset + index * 16;
1720 bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
1721 bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
1722 bus_write_4(msix->msix_table_res, offset + 8, data);
1723 }
1724
1725 void
pci_enable_msix_method(device_t dev,device_t child,u_int index,uint64_t address,uint32_t data)1726 pci_enable_msix_method(device_t dev, device_t child, u_int index,
1727 uint64_t address, uint32_t data)
1728 {
1729
1730 if (pci_msix_rewrite_table) {
1731 struct pci_devinfo *dinfo = device_get_ivars(child);
1732 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1733
1734 /*
1735 * Some VM hosts require MSIX to be disabled in the
1736 * control register before updating the MSIX table
1737 * entries are allowed. It is not enough to only
1738 * disable MSIX while updating a single entry. MSIX
1739 * must be disabled while updating all entries in the
1740 * table.
1741 */
1742 pci_write_config(child,
1743 msix->msix_location + PCIR_MSIX_CTRL,
1744 msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
1745 pci_resume_msix(child);
1746 } else
1747 pci_write_msix_entry(child, index, address, data);
1748
1749 /* Enable MSI -> HT mapping. */
1750 pci_ht_map_msi(child, address);
1751 }
1752
1753 void
pci_mask_msix(device_t dev,u_int index)1754 pci_mask_msix(device_t dev, u_int index)
1755 {
1756 struct pci_devinfo *dinfo = device_get_ivars(dev);
1757 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1758 uint32_t offset, val;
1759
1760 KASSERT(PCI_MSIX_MSGNUM(msix->msix_ctrl) > index, ("bogus index"));
1761 offset = msix->msix_table_offset + index * 16 + 12;
1762 val = bus_read_4(msix->msix_table_res, offset);
1763 val |= PCIM_MSIX_VCTRL_MASK;
1764
1765 /*
1766 * Some devices (e.g. Samsung PM961) do not support reads of this
1767 * register, so always write the new value.
1768 */
1769 bus_write_4(msix->msix_table_res, offset, val);
1770 }
1771
1772 void
pci_unmask_msix(device_t dev,u_int index)1773 pci_unmask_msix(device_t dev, u_int index)
1774 {
1775 struct pci_devinfo *dinfo = device_get_ivars(dev);
1776 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1777 uint32_t offset, val;
1778
1779 KASSERT(PCI_MSIX_MSGNUM(msix->msix_ctrl) > index, ("bogus index"));
1780 offset = msix->msix_table_offset + index * 16 + 12;
1781 val = bus_read_4(msix->msix_table_res, offset);
1782 val &= ~PCIM_MSIX_VCTRL_MASK;
1783
1784 /*
1785 * Some devices (e.g. Samsung PM961) do not support reads of this
1786 * register, so always write the new value.
1787 */
1788 bus_write_4(msix->msix_table_res, offset, val);
1789 }
1790
1791 int
pci_pending_msix(device_t dev,u_int index)1792 pci_pending_msix(device_t dev, u_int index)
1793 {
1794 struct pci_devinfo *dinfo = device_get_ivars(dev);
1795 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1796 uint32_t offset, bit;
1797
1798 KASSERT(msix->msix_table_len > index, ("bogus index"));
1799 offset = msix->msix_pba_offset + (index / 32) * 4;
1800 bit = 1 << index % 32;
1801 return (bus_read_4(msix->msix_pba_res, offset) & bit);
1802 }
1803
1804 /*
1805 * Restore MSI-X registers and table during resume. If MSI-X is
1806 * enabled then walk the virtual table to restore the actual MSI-X
1807 * table.
1808 */
1809 static void
pci_resume_msix(device_t dev)1810 pci_resume_msix(device_t dev)
1811 {
1812 struct pci_devinfo *dinfo = device_get_ivars(dev);
1813 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1814 struct msix_table_entry *mte;
1815 struct msix_vector *mv;
1816 u_int i, msgnum;
1817
1818 if (msix->msix_alloc > 0) {
1819 msgnum = PCI_MSIX_MSGNUM(msix->msix_ctrl);
1820
1821 /* First, mask all vectors. */
1822 for (i = 0; i < msgnum; i++)
1823 pci_mask_msix(dev, i);
1824
1825 /* Second, program any messages with at least one handler. */
1826 for (i = 0; i < msix->msix_table_len; i++) {
1827 mte = &msix->msix_table[i];
1828 if (mte->mte_vector == 0 || mte->mte_handlers == 0)
1829 continue;
1830 mv = &msix->msix_vectors[mte->mte_vector - 1];
1831 pci_write_msix_entry(dev, i, mv->mv_address,
1832 mv->mv_data);
1833 pci_unmask_msix(dev, i);
1834 }
1835 }
1836 pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
1837 msix->msix_ctrl, 2);
1838 }
1839
1840 /*
1841 * Attempt to allocate *count MSI-X messages. The actual number allocated is
1842 * returned in *count. After this function returns, each message will be
1843 * available to the driver as SYS_RES_IRQ resources starting at rid 1.
1844 */
1845 int
pci_alloc_msix_method(device_t dev,device_t child,int * count)1846 pci_alloc_msix_method(device_t dev, device_t child, int *count)
1847 {
1848 struct pci_devinfo *dinfo = device_get_ivars(child);
1849 pcicfgregs *cfg = &dinfo->cfg;
1850 struct resource_list_entry *rle;
1851 u_int actual, i, max;
1852 int error, irq;
1853 uint16_t ctrl, msgnum;
1854
1855 /* Don't let count == 0 get us into trouble. */
1856 if (*count < 1)
1857 return (EINVAL);
1858
1859 /* If rid 0 is allocated, then fail. */
1860 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
1861 if (rle != NULL && rle->res != NULL)
1862 return (ENXIO);
1863
1864 /* Already have allocated messages? */
1865 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
1866 return (ENXIO);
1867
1868 /* If MSI-X is blacklisted for this system, fail. */
1869 if (pci_msix_blacklisted())
1870 return (ENXIO);
1871
1872 /* MSI-X capability present? */
1873 if (cfg->msix.msix_location == 0 || !pci_do_msix)
1874 return (ENODEV);
1875
1876 /* Make sure the appropriate BARs are mapped. */
1877 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1878 cfg->msix.msix_table_bar);
1879 if (rle == NULL || rle->res == NULL ||
1880 !(rman_get_flags(rle->res) & RF_ACTIVE))
1881 return (ENXIO);
1882 cfg->msix.msix_table_res = rle->res;
1883 if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
1884 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1885 cfg->msix.msix_pba_bar);
1886 if (rle == NULL || rle->res == NULL ||
1887 !(rman_get_flags(rle->res) & RF_ACTIVE))
1888 return (ENXIO);
1889 }
1890 cfg->msix.msix_pba_res = rle->res;
1891
1892 ctrl = pci_read_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1893 2);
1894 msgnum = PCI_MSIX_MSGNUM(ctrl);
1895 if (bootverbose)
1896 device_printf(child,
1897 "attempting to allocate %d MSI-X vectors (%d supported)\n",
1898 *count, msgnum);
1899 max = min(*count, msgnum);
1900 for (i = 0; i < max; i++) {
1901 /* Allocate a message. */
1902 error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
1903 if (error) {
1904 if (i == 0)
1905 return (error);
1906 break;
1907 }
1908 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
1909 irq, 1);
1910 }
1911 actual = i;
1912
1913 if (bootverbose) {
1914 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
1915 if (actual == 1)
1916 device_printf(child, "using IRQ %ju for MSI-X\n",
1917 rle->start);
1918 else {
1919 bool run;
1920
1921 /*
1922 * Be fancy and try to print contiguous runs of
1923 * IRQ values as ranges. 'irq' is the previous IRQ.
1924 * 'run' is true if we are in a range.
1925 */
1926 device_printf(child, "using IRQs %ju", rle->start);
1927 irq = rle->start;
1928 run = false;
1929 for (i = 1; i < actual; i++) {
1930 rle = resource_list_find(&dinfo->resources,
1931 SYS_RES_IRQ, i + 1);
1932
1933 /* Still in a run? */
1934 if (rle->start == irq + 1) {
1935 run = true;
1936 irq++;
1937 continue;
1938 }
1939
1940 /* Finish previous range. */
1941 if (run) {
1942 printf("-%d", irq);
1943 run = false;
1944 }
1945
1946 /* Start new range. */
1947 printf(",%ju", rle->start);
1948 irq = rle->start;
1949 }
1950
1951 /* Unfinished range? */
1952 if (run)
1953 printf("-%d", irq);
1954 printf(" for MSI-X\n");
1955 }
1956 }
1957
1958 /*
1959 * Mask all vectors. Note that the message index assertion in
1960 * pci_mask_msix requires msix_ctrl to be set.
1961 */
1962 cfg->msix.msix_ctrl = ctrl;
1963 for (i = 0; i < msgnum; i++)
1964 pci_mask_msix(child, i);
1965
1966 /* Allocate and initialize vector data and virtual table. */
1967 cfg->msix.msix_vectors = mallocarray(actual, sizeof(struct msix_vector),
1968 M_DEVBUF, M_WAITOK | M_ZERO);
1969 cfg->msix.msix_table = mallocarray(actual,
1970 sizeof(struct msix_table_entry), M_DEVBUF, M_WAITOK | M_ZERO);
1971 for (i = 0; i < actual; i++) {
1972 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
1973 cfg->msix.msix_vectors[i].mv_irq = rle->start;
1974 cfg->msix.msix_table[i].mte_vector = i + 1;
1975 }
1976
1977 /* Update control register to enable MSI-X. */
1978 ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1979 pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1980 ctrl, 2);
1981 cfg->msix.msix_ctrl = ctrl;
1982
1983 /* Update counts of alloc'd messages. */
1984 cfg->msix.msix_alloc = actual;
1985 cfg->msix.msix_table_len = actual;
1986 *count = actual;
1987 return (0);
1988 }
1989
1990 /*
1991 * By default, pci_alloc_msix() will assign the allocated IRQ
1992 * resources consecutively to the first N messages in the MSI-X table.
1993 * However, device drivers may want to use different layouts if they
1994 * either receive fewer messages than they asked for, or they wish to
1995 * populate the MSI-X table sparsely. This method allows the driver
1996 * to specify what layout it wants. It must be called after a
1997 * successful pci_alloc_msix() but before any of the associated
1998 * SYS_RES_IRQ resources are allocated via bus_alloc_resource().
1999 *
2000 * The 'vectors' array contains 'count' message vectors. The array
2001 * maps directly to the MSI-X table in that index 0 in the array
2002 * specifies the vector for the first message in the MSI-X table, etc.
2003 * The vector value in each array index can either be 0 to indicate
2004 * that no vector should be assigned to a message slot, or it can be a
2005 * number from 1 to N (where N is the count returned from a
2006 * succcessful call to pci_alloc_msix()) to indicate which message
2007 * vector (IRQ) to be used for the corresponding message.
2008 *
2009 * On successful return, each message with a non-zero vector will have
2010 * an associated SYS_RES_IRQ whose rid is equal to the array index +
2011 * 1. Additionally, if any of the IRQs allocated via the previous
2012 * call to pci_alloc_msix() are not used in the mapping, those IRQs
2013 * will be freed back to the system automatically.
2014 *
2015 * For example, suppose a driver has a MSI-X table with 6 messages and
2016 * asks for 6 messages, but pci_alloc_msix() only returns a count of
2017 * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and
2018 * C. After the call to pci_alloc_msix(), the device will be setup to
2019 * have an MSI-X table of ABC--- (where - means no vector assigned).
2020 * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 },
2021 * then the MSI-X table will look like A-AB-B, and the 'C' vector will
2022 * be freed back to the system. This device will also have valid
2023 * SYS_RES_IRQ rids of 1, 3, 4, and 6.
2024 *
2025 * In any case, the SYS_RES_IRQ rid X will always map to the message
2026 * at MSI-X table index X - 1 and will only be valid if a vector is
2027 * assigned to that table entry.
2028 */
2029 int
pci_remap_msix_method(device_t dev,device_t child,int count,const u_int * vectors)2030 pci_remap_msix_method(device_t dev, device_t child, int count,
2031 const u_int *vectors)
2032 {
2033 struct pci_devinfo *dinfo = device_get_ivars(child);
2034 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2035 struct resource_list_entry *rle;
2036 u_int i, irq, j;
2037 bool *used;
2038
2039 /*
2040 * Have to have at least one message in the table but the
2041 * table can't be bigger than the actual MSI-X table in the
2042 * device.
2043 */
2044 if (count < 1 || count > PCI_MSIX_MSGNUM(msix->msix_ctrl))
2045 return (EINVAL);
2046
2047 /* Sanity check the vectors. */
2048 for (i = 0; i < count; i++)
2049 if (vectors[i] > msix->msix_alloc)
2050 return (EINVAL);
2051
2052 /*
2053 * Make sure there aren't any holes in the vectors to be used.
2054 * It's a big pain to support it, and it doesn't really make
2055 * sense anyway. Also, at least one vector must be used.
2056 */
2057 used = mallocarray(msix->msix_alloc, sizeof(*used), M_DEVBUF, M_WAITOK |
2058 M_ZERO);
2059 for (i = 0; i < count; i++)
2060 if (vectors[i] != 0)
2061 used[vectors[i] - 1] = true;
2062 for (i = 0; i < msix->msix_alloc - 1; i++)
2063 if (!used[i] && used[i + 1]) {
2064 free(used, M_DEVBUF);
2065 return (EINVAL);
2066 }
2067 if (!used[0]) {
2068 free(used, M_DEVBUF);
2069 return (EINVAL);
2070 }
2071
2072 /* Make sure none of the resources are allocated. */
2073 for (i = 0; i < msix->msix_table_len; i++) {
2074 if (msix->msix_table[i].mte_vector == 0)
2075 continue;
2076 if (msix->msix_table[i].mte_handlers > 0) {
2077 free(used, M_DEVBUF);
2078 return (EBUSY);
2079 }
2080 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2081 KASSERT(rle != NULL, ("missing resource"));
2082 if (rle->res != NULL) {
2083 free(used, M_DEVBUF);
2084 return (EBUSY);
2085 }
2086 }
2087
2088 /* Free the existing resource list entries. */
2089 for (i = 0; i < msix->msix_table_len; i++) {
2090 if (msix->msix_table[i].mte_vector == 0)
2091 continue;
2092 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2093 }
2094
2095 /*
2096 * Build the new virtual table keeping track of which vectors are
2097 * used.
2098 */
2099 free(msix->msix_table, M_DEVBUF);
2100 msix->msix_table = mallocarray(count, sizeof(struct msix_table_entry),
2101 M_DEVBUF, M_WAITOK | M_ZERO);
2102 for (i = 0; i < count; i++)
2103 msix->msix_table[i].mte_vector = vectors[i];
2104 msix->msix_table_len = count;
2105
2106 /* Free any unused IRQs and resize the vectors array if necessary. */
2107 j = msix->msix_alloc - 1;
2108 if (!used[j]) {
2109 struct msix_vector *vec;
2110
2111 while (!used[j]) {
2112 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2113 msix->msix_vectors[j].mv_irq);
2114 j--;
2115 }
2116 vec = mallocarray(j + 1, sizeof(struct msix_vector), M_DEVBUF,
2117 M_WAITOK);
2118 bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) *
2119 (j + 1));
2120 free(msix->msix_vectors, M_DEVBUF);
2121 msix->msix_vectors = vec;
2122 msix->msix_alloc = j + 1;
2123 }
2124 free(used, M_DEVBUF);
2125
2126 /* Map the IRQs onto the rids. */
2127 for (i = 0; i < count; i++) {
2128 if (vectors[i] == 0)
2129 continue;
2130 irq = msix->msix_vectors[vectors[i] - 1].mv_irq;
2131 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
2132 irq, 1);
2133 }
2134
2135 if (bootverbose) {
2136 device_printf(child, "Remapped MSI-X IRQs as: ");
2137 for (i = 0; i < count; i++) {
2138 if (i != 0)
2139 printf(", ");
2140 if (vectors[i] == 0)
2141 printf("---");
2142 else
2143 printf("%d",
2144 msix->msix_vectors[vectors[i] - 1].mv_irq);
2145 }
2146 printf("\n");
2147 }
2148
2149 return (0);
2150 }
2151
2152 static int
pci_release_msix(device_t dev,device_t child)2153 pci_release_msix(device_t dev, device_t child)
2154 {
2155 struct pci_devinfo *dinfo = device_get_ivars(child);
2156 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2157 struct resource_list_entry *rle;
2158 u_int i;
2159
2160 /* Do we have any messages to release? */
2161 if (msix->msix_alloc == 0)
2162 return (ENODEV);
2163
2164 /* Make sure none of the resources are allocated. */
2165 for (i = 0; i < msix->msix_table_len; i++) {
2166 if (msix->msix_table[i].mte_vector == 0)
2167 continue;
2168 if (msix->msix_table[i].mte_handlers > 0)
2169 return (EBUSY);
2170 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2171 KASSERT(rle != NULL, ("missing resource"));
2172 if (rle->res != NULL)
2173 return (EBUSY);
2174 }
2175
2176 /* Update control register to disable MSI-X. */
2177 msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
2178 pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
2179 msix->msix_ctrl, 2);
2180
2181 /* Free the resource list entries. */
2182 for (i = 0; i < msix->msix_table_len; i++) {
2183 if (msix->msix_table[i].mte_vector == 0)
2184 continue;
2185 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2186 }
2187 free(msix->msix_table, M_DEVBUF);
2188 msix->msix_table_len = 0;
2189
2190 /* Release the IRQs. */
2191 for (i = 0; i < msix->msix_alloc; i++)
2192 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2193 msix->msix_vectors[i].mv_irq);
2194 free(msix->msix_vectors, M_DEVBUF);
2195 msix->msix_alloc = 0;
2196 return (0);
2197 }
2198
2199 /*
2200 * Return the max supported MSI-X messages this device supports.
2201 * Basically, assuming the MD code can alloc messages, this function
2202 * should return the maximum value that pci_alloc_msix() can return.
2203 * Thus, it is subject to the tunables, etc.
2204 */
2205 int
pci_msix_count_method(device_t dev,device_t child)2206 pci_msix_count_method(device_t dev, device_t child)
2207 {
2208 struct pci_devinfo *dinfo = device_get_ivars(child);
2209 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2210 uint16_t ctrl;
2211
2212 if (pci_do_msix && msix->msix_location != 0) {
2213 ctrl = pci_read_config(child, msix->msix_location +
2214 PCIR_MSI_CTRL, 2);
2215 return (PCI_MSIX_MSGNUM(ctrl));
2216 }
2217 return (0);
2218 }
2219
2220 int
pci_msix_pba_bar_method(device_t dev,device_t child)2221 pci_msix_pba_bar_method(device_t dev, device_t child)
2222 {
2223 struct pci_devinfo *dinfo = device_get_ivars(child);
2224 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2225
2226 if (pci_do_msix && msix->msix_location != 0)
2227 return (msix->msix_pba_bar);
2228 return (-1);
2229 }
2230
2231 int
pci_msix_table_bar_method(device_t dev,device_t child)2232 pci_msix_table_bar_method(device_t dev, device_t child)
2233 {
2234 struct pci_devinfo *dinfo = device_get_ivars(child);
2235 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2236
2237 if (pci_do_msix && msix->msix_location != 0)
2238 return (msix->msix_table_bar);
2239 return (-1);
2240 }
2241
2242 /*
2243 * HyperTransport MSI mapping control
2244 */
2245 void
pci_ht_map_msi(device_t dev,uint64_t addr)2246 pci_ht_map_msi(device_t dev, uint64_t addr)
2247 {
2248 struct pci_devinfo *dinfo = device_get_ivars(dev);
2249 struct pcicfg_ht *ht = &dinfo->cfg.ht;
2250
2251 if (!ht->ht_msimap)
2252 return;
2253
2254 if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) &&
2255 ht->ht_msiaddr >> 20 == addr >> 20) {
2256 /* Enable MSI -> HT mapping. */
2257 ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
2258 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2259 ht->ht_msictrl, 2);
2260 }
2261
2262 if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
2263 /* Disable MSI -> HT mapping. */
2264 ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
2265 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2266 ht->ht_msictrl, 2);
2267 }
2268 }
2269
2270 int
pci_get_relaxed_ordering_enabled(device_t dev)2271 pci_get_relaxed_ordering_enabled(device_t dev)
2272 {
2273 struct pci_devinfo *dinfo = device_get_ivars(dev);
2274 int cap;
2275 uint16_t val;
2276
2277 cap = dinfo->cfg.pcie.pcie_location;
2278 if (cap == 0)
2279 return (0);
2280 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2281 val &= PCIEM_CTL_RELAXED_ORD_ENABLE;
2282 return (val != 0);
2283 }
2284
2285 int
pci_get_max_payload(device_t dev)2286 pci_get_max_payload(device_t dev)
2287 {
2288 struct pci_devinfo *dinfo = device_get_ivars(dev);
2289 int cap;
2290 uint16_t val;
2291
2292 cap = dinfo->cfg.pcie.pcie_location;
2293 if (cap == 0)
2294 return (0);
2295 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2296 val &= PCIEM_CTL_MAX_PAYLOAD;
2297 val >>= 5;
2298 return (1 << (val + 7));
2299 }
2300
2301 int
pci_get_max_read_req(device_t dev)2302 pci_get_max_read_req(device_t dev)
2303 {
2304 struct pci_devinfo *dinfo = device_get_ivars(dev);
2305 int cap;
2306 uint16_t val;
2307
2308 cap = dinfo->cfg.pcie.pcie_location;
2309 if (cap == 0)
2310 return (0);
2311 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2312 val &= PCIEM_CTL_MAX_READ_REQUEST;
2313 val >>= 12;
2314 return (1 << (val + 7));
2315 }
2316
2317 int
pci_set_max_read_req(device_t dev,int size)2318 pci_set_max_read_req(device_t dev, int size)
2319 {
2320 struct pci_devinfo *dinfo = device_get_ivars(dev);
2321 int cap;
2322 uint16_t val;
2323
2324 cap = dinfo->cfg.pcie.pcie_location;
2325 if (cap == 0)
2326 return (0);
2327 if (size < 128)
2328 size = 128;
2329 if (size > 4096)
2330 size = 4096;
2331 size = (1 << (fls(size) - 1));
2332 val = (fls(size) - 8) << 12;
2333 pcie_adjust_config(dev, PCIER_DEVICE_CTL,
2334 PCIEM_CTL_MAX_READ_REQUEST, val, 2);
2335 return (size);
2336 }
2337
2338 uint32_t
pcie_read_config(device_t dev,int reg,int width)2339 pcie_read_config(device_t dev, int reg, int width)
2340 {
2341 struct pci_devinfo *dinfo = device_get_ivars(dev);
2342 int cap;
2343
2344 cap = dinfo->cfg.pcie.pcie_location;
2345 if (cap == 0) {
2346 if (width == 2)
2347 return (0xffff);
2348 return (0xffffffff);
2349 }
2350
2351 return (pci_read_config(dev, cap + reg, width));
2352 }
2353
2354 void
pcie_write_config(device_t dev,int reg,uint32_t value,int width)2355 pcie_write_config(device_t dev, int reg, uint32_t value, int width)
2356 {
2357 struct pci_devinfo *dinfo = device_get_ivars(dev);
2358 int cap;
2359
2360 cap = dinfo->cfg.pcie.pcie_location;
2361 if (cap == 0)
2362 return;
2363 pci_write_config(dev, cap + reg, value, width);
2364 }
2365
2366 /*
2367 * Adjusts a PCI-e capability register by clearing the bits in mask
2368 * and setting the bits in (value & mask). Bits not set in mask are
2369 * not adjusted.
2370 *
2371 * Returns the old value on success or all ones on failure.
2372 */
2373 uint32_t
pcie_adjust_config(device_t dev,int reg,uint32_t mask,uint32_t value,int width)2374 pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
2375 int width)
2376 {
2377 struct pci_devinfo *dinfo = device_get_ivars(dev);
2378 uint16_t *saved;
2379 uint32_t old, new;
2380 int cap;
2381
2382 cap = dinfo->cfg.pcie.pcie_location;
2383 if (cap == 0) {
2384 if (width == 2)
2385 return (0xffff);
2386 return (0xffffffff);
2387 }
2388
2389 old = pci_read_config(dev, cap + reg, width);
2390 new = old & ~mask;
2391 new |= (value & mask);
2392 pci_write_config(dev, cap + reg, new, width);
2393 /* Apply only the requested policy bits to the saved restore image. */
2394 if (width == 2) {
2395 saved = NULL;
2396 switch (reg) {
2397 case PCIER_DEVICE_CTL:
2398 saved = &dinfo->cfg.pcie.pcie_device_ctl;
2399 break;
2400 case PCIER_ROOT_CTL:
2401 saved = &dinfo->cfg.pcie.pcie_root_ctl;
2402 break;
2403 }
2404 if (saved != NULL) {
2405 *saved &= ~(uint16_t)mask;
2406 *saved |= (uint16_t)(value & mask);
2407 }
2408 }
2409 return (old);
2410 }
2411
2412 /*
2413 * Support for MSI message signalled interrupts.
2414 */
2415 void
pci_enable_msi_method(device_t dev,device_t child,uint64_t address,uint16_t data)2416 pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
2417 uint16_t data)
2418 {
2419 struct pci_devinfo *dinfo = device_get_ivars(child);
2420 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2421
2422 /* Write data and address values. */
2423 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
2424 address & 0xffffffff, 4);
2425 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2426 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
2427 address >> 32, 4);
2428 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
2429 data, 2);
2430 } else
2431 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
2432 2);
2433
2434 /* Enable MSI in the control register. */
2435 msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE;
2436 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2437 msi->msi_ctrl, 2);
2438
2439 /* Enable MSI -> HT mapping. */
2440 pci_ht_map_msi(child, address);
2441 }
2442
2443 void
pci_disable_msi_method(device_t dev,device_t child)2444 pci_disable_msi_method(device_t dev, device_t child)
2445 {
2446 struct pci_devinfo *dinfo = device_get_ivars(child);
2447 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2448
2449 /* Disable MSI -> HT mapping. */
2450 pci_ht_map_msi(child, 0);
2451
2452 /* Disable MSI in the control register. */
2453 msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
2454 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2455 msi->msi_ctrl, 2);
2456 }
2457
2458 /*
2459 * Restore MSI registers during resume. If MSI is enabled then
2460 * restore the data and address registers in addition to the control
2461 * register.
2462 */
2463 static void
pci_resume_msi(device_t dev)2464 pci_resume_msi(device_t dev)
2465 {
2466 struct pci_devinfo *dinfo = device_get_ivars(dev);
2467 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2468 uint64_t address;
2469 uint16_t data;
2470
2471 if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) {
2472 address = msi->msi_addr;
2473 data = msi->msi_data;
2474 pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
2475 address & 0xffffffff, 4);
2476 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2477 pci_write_config(dev, msi->msi_location +
2478 PCIR_MSI_ADDR_HIGH, address >> 32, 4);
2479 pci_write_config(dev, msi->msi_location +
2480 PCIR_MSI_DATA_64BIT, data, 2);
2481 } else
2482 pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA,
2483 data, 2);
2484 }
2485 pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
2486 2);
2487 }
2488
2489 static int
pci_remap_intr_method(device_t bus,device_t dev,u_int irq)2490 pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
2491 {
2492 struct pci_devinfo *dinfo = device_get_ivars(dev);
2493 pcicfgregs *cfg = &dinfo->cfg;
2494 struct resource_list_entry *rle;
2495 struct msix_table_entry *mte;
2496 struct msix_vector *mv;
2497 uint64_t addr;
2498 uint32_t data;
2499 u_int i, j;
2500 int error;
2501
2502 /*
2503 * Handle MSI first. We try to find this IRQ among our list
2504 * of MSI IRQs. If we find it, we request updated address and
2505 * data registers and apply the results.
2506 */
2507 if (cfg->msi.msi_alloc > 0) {
2508 /* If we don't have any active handlers, nothing to do. */
2509 if (cfg->msi.msi_handlers == 0)
2510 return (0);
2511 for (i = 0; i < cfg->msi.msi_alloc; i++) {
2512 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
2513 i + 1);
2514 if (rle->start == irq) {
2515 error = PCIB_MAP_MSI(device_get_parent(bus),
2516 dev, irq, &addr, &data);
2517 if (error)
2518 return (error);
2519 pci_disable_msi(dev);
2520 dinfo->cfg.msi.msi_addr = addr;
2521 dinfo->cfg.msi.msi_data = data;
2522 pci_enable_msi(dev, addr, data);
2523 return (0);
2524 }
2525 }
2526 return (ENOENT);
2527 }
2528
2529 /*
2530 * For MSI-X, we check to see if we have this IRQ. If we do,
2531 * we request the updated mapping info. If that works, we go
2532 * through all the slots that use this IRQ and update them.
2533 */
2534 if (cfg->msix.msix_alloc > 0) {
2535 bool found = false;
2536
2537 for (i = 0; i < cfg->msix.msix_alloc; i++) {
2538 mv = &cfg->msix.msix_vectors[i];
2539 if (mv->mv_irq == irq) {
2540 error = PCIB_MAP_MSI(device_get_parent(bus),
2541 dev, irq, &addr, &data);
2542 if (error)
2543 return (error);
2544 mv->mv_address = addr;
2545 mv->mv_data = data;
2546 for (j = 0; j < cfg->msix.msix_table_len; j++) {
2547 mte = &cfg->msix.msix_table[j];
2548 if (mte->mte_vector != i + 1)
2549 continue;
2550 if (mte->mte_handlers == 0)
2551 continue;
2552 pci_mask_msix(dev, j);
2553 pci_enable_msix(dev, j, addr, data);
2554 pci_unmask_msix(dev, j);
2555 }
2556 found = true;
2557 }
2558 }
2559 return (found ? 0 : ENOENT);
2560 }
2561
2562 return (ENOENT);
2563 }
2564
2565 /*
2566 * Returns true if the specified device is blacklisted because MSI
2567 * doesn't work.
2568 */
2569 int
pci_msi_device_blacklisted(device_t dev)2570 pci_msi_device_blacklisted(device_t dev)
2571 {
2572
2573 if (!pci_honor_msi_blacklist)
2574 return (0);
2575
2576 return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
2577 }
2578
2579 /*
2580 * Determine if MSI is blacklisted globally on this system. Currently,
2581 * we just check for blacklisted chipsets as represented by the
2582 * host-PCI bridge at device 0:0:0. In the future, it may become
2583 * necessary to check other system attributes, such as the kenv values
2584 * that give the motherboard manufacturer and model number.
2585 */
2586 static int
pci_msi_blacklisted(void)2587 pci_msi_blacklisted(void)
2588 {
2589 device_t dev;
2590
2591 if (!pci_honor_msi_blacklist)
2592 return (0);
2593
2594 /* Blacklist all non-PCI-express and non-PCI-X chipsets. */
2595 if (!(pcie_chipset || pcix_chipset)) {
2596 if (vm_guest != VM_GUEST_NO) {
2597 /*
2598 * Whitelist older chipsets in virtual
2599 * machines known to support MSI.
2600 */
2601 dev = pci_find_bsf(0, 0, 0);
2602 if (dev != NULL)
2603 return (!pci_has_quirk(pci_get_devid(dev),
2604 PCI_QUIRK_ENABLE_MSI_VM));
2605 }
2606 return (1);
2607 }
2608
2609 dev = pci_find_bsf(0, 0, 0);
2610 if (dev != NULL)
2611 return (pci_msi_device_blacklisted(dev));
2612 return (0);
2613 }
2614
2615 /*
2616 * Returns true if the specified device is blacklisted because MSI-X
2617 * doesn't work. Note that this assumes that if MSI doesn't work,
2618 * MSI-X doesn't either.
2619 */
2620 int
pci_msix_device_blacklisted(device_t dev)2621 pci_msix_device_blacklisted(device_t dev)
2622 {
2623
2624 if (!pci_honor_msi_blacklist)
2625 return (0);
2626
2627 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
2628 return (1);
2629
2630 return (pci_msi_device_blacklisted(dev));
2631 }
2632
2633 /*
2634 * Determine if MSI-X is blacklisted globally on this system. If MSI
2635 * is blacklisted, assume that MSI-X is as well. Check for additional
2636 * chipsets where MSI works but MSI-X does not.
2637 */
2638 static int
pci_msix_blacklisted(void)2639 pci_msix_blacklisted(void)
2640 {
2641 device_t dev;
2642
2643 if (!pci_honor_msi_blacklist)
2644 return (0);
2645
2646 dev = pci_find_bsf(0, 0, 0);
2647 if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
2648 PCI_QUIRK_DISABLE_MSIX))
2649 return (1);
2650
2651 return (pci_msi_blacklisted());
2652 }
2653
2654 /*
2655 * Attempt to allocate *count MSI messages. The actual number allocated is
2656 * returned in *count. After this function returns, each message will be
2657 * available to the driver as SYS_RES_IRQ resources starting at a rid 1.
2658 */
2659 int
pci_alloc_msi_method(device_t dev,device_t child,int * count)2660 pci_alloc_msi_method(device_t dev, device_t child, int *count)
2661 {
2662 struct pci_devinfo *dinfo = device_get_ivars(child);
2663 pcicfgregs *cfg = &dinfo->cfg;
2664 struct resource_list_entry *rle;
2665 u_int actual, i;
2666 int error, irqs[32];
2667 uint16_t ctrl, msgnum;
2668
2669 /* Don't let count == 0 get us into trouble. */
2670 if (*count < 1)
2671 return (EINVAL);
2672
2673 /* If rid 0 is allocated, then fail. */
2674 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
2675 if (rle != NULL && rle->res != NULL)
2676 return (ENXIO);
2677
2678 /* Already have allocated messages? */
2679 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
2680 return (ENXIO);
2681
2682 /* If MSI is blacklisted for this system, fail. */
2683 if (pci_msi_blacklisted())
2684 return (ENXIO);
2685
2686 /* MSI capability present? */
2687 if (cfg->msi.msi_location == 0 || !pci_do_msi)
2688 return (ENODEV);
2689
2690 ctrl = pci_read_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, 2);
2691 msgnum = PCI_MSI_MSGNUM(ctrl);
2692 if (bootverbose)
2693 device_printf(child,
2694 "attempting to allocate %d MSI vectors (%u supported)\n",
2695 *count, msgnum);
2696
2697 /* Don't ask for more than the device supports. */
2698 actual = min(*count, msgnum);
2699
2700 /* Don't ask for more than 32 messages. */
2701 actual = min(actual, 32);
2702
2703 /* MSI requires power of 2 number of messages. */
2704 if (!powerof2(actual))
2705 return (EINVAL);
2706
2707 for (;;) {
2708 /* Try to allocate N messages. */
2709 error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual,
2710 actual, irqs);
2711 if (error == 0)
2712 break;
2713 if (actual == 1)
2714 return (error);
2715
2716 /* Try N / 2. */
2717 actual >>= 1;
2718 }
2719
2720 /*
2721 * We now have N actual messages mapped onto SYS_RES_IRQ
2722 * resources in the irqs[] array, so add new resources
2723 * starting at rid 1.
2724 */
2725 for (i = 0; i < actual; i++)
2726 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1,
2727 irqs[i], irqs[i], 1);
2728
2729 if (bootverbose) {
2730 if (actual == 1)
2731 device_printf(child, "using IRQ %d for MSI\n", irqs[0]);
2732 else {
2733 bool run;
2734
2735 /*
2736 * Be fancy and try to print contiguous runs
2737 * of IRQ values as ranges. 'run' is true if
2738 * we are in a range.
2739 */
2740 device_printf(child, "using IRQs %d", irqs[0]);
2741 run = false;
2742 for (i = 1; i < actual; i++) {
2743 /* Still in a run? */
2744 if (irqs[i] == irqs[i - 1] + 1) {
2745 run = true;
2746 continue;
2747 }
2748
2749 /* Finish previous range. */
2750 if (run) {
2751 printf("-%d", irqs[i - 1]);
2752 run = false;
2753 }
2754
2755 /* Start new range. */
2756 printf(",%d", irqs[i]);
2757 }
2758
2759 /* Unfinished range? */
2760 if (run)
2761 printf("-%d", irqs[actual - 1]);
2762 printf(" for MSI\n");
2763 }
2764 }
2765
2766 /* Update control register with actual count. */
2767 ctrl &= ~PCIM_MSICTRL_MME_MASK;
2768 ctrl |= (ffs(actual) - 1) << 4;
2769 cfg->msi.msi_ctrl = ctrl;
2770 pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2);
2771
2772 /* Update counts of alloc'd messages. */
2773 cfg->msi.msi_alloc = actual;
2774 cfg->msi.msi_handlers = 0;
2775 *count = actual;
2776 return (0);
2777 }
2778
2779 /* Release the MSI messages associated with this device. */
2780 int
pci_release_msi_method(device_t dev,device_t child)2781 pci_release_msi_method(device_t dev, device_t child)
2782 {
2783 struct pci_devinfo *dinfo = device_get_ivars(child);
2784 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2785 struct resource_list_entry *rle;
2786 u_int i, irqs[32];
2787 int error;
2788
2789 /* Try MSI-X first. */
2790 error = pci_release_msix(dev, child);
2791 if (error != ENODEV)
2792 return (error);
2793
2794 /* Do we have any messages to release? */
2795 if (msi->msi_alloc == 0)
2796 return (ENODEV);
2797 KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));
2798
2799 /* Make sure none of the resources are allocated. */
2800 if (msi->msi_handlers > 0)
2801 return (EBUSY);
2802 for (i = 0; i < msi->msi_alloc; i++) {
2803 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2804 KASSERT(rle != NULL, ("missing MSI resource"));
2805 if (rle->res != NULL)
2806 return (EBUSY);
2807 irqs[i] = rle->start;
2808 }
2809
2810 /* Update control register with 0 count. */
2811 KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
2812 ("%s: MSI still enabled", __func__));
2813 msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
2814 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2815 msi->msi_ctrl, 2);
2816
2817 /* Release the messages. */
2818 PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs);
2819 for (i = 0; i < msi->msi_alloc; i++)
2820 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2821
2822 /* Update alloc count. */
2823 msi->msi_alloc = 0;
2824 msi->msi_addr = 0;
2825 msi->msi_data = 0;
2826 return (0);
2827 }
2828
2829 /*
2830 * Return the max supported MSI messages this device supports.
2831 * Basically, assuming the MD code can alloc messages, this function
2832 * should return the maximum value that pci_alloc_msi() can return.
2833 * Thus, it is subject to the tunables, etc.
2834 */
2835 int
pci_msi_count_method(device_t dev,device_t child)2836 pci_msi_count_method(device_t dev, device_t child)
2837 {
2838 struct pci_devinfo *dinfo = device_get_ivars(child);
2839 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2840 uint16_t ctrl;
2841
2842 if (pci_do_msi && msi->msi_location != 0) {
2843 ctrl = pci_read_config(child, msi->msi_location + PCIR_MSI_CTRL,
2844 2);
2845 return (PCI_MSI_MSGNUM(ctrl));
2846 }
2847 return (0);
2848 }
2849
2850 /* free pcicfgregs structure and all depending data structures */
2851
2852 int
pci_freecfg(struct pci_devinfo * dinfo)2853 pci_freecfg(struct pci_devinfo *dinfo)
2854 {
2855 struct devlist *devlist_head;
2856 struct pci_map *pm, *next;
2857
2858 devlist_head = &pci_devq;
2859
2860 if (dinfo->cfg.vpd.vpd_reg)
2861 vpd_free(&dinfo->cfg.vpd);
2862
2863 STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) {
2864 free(pm, M_DEVBUF);
2865 }
2866 STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links);
2867 free(dinfo, M_DEVBUF);
2868
2869 /* increment the generation count */
2870 pci_generation++;
2871
2872 /* we're losing one device */
2873 pci_numdevs--;
2874 return (0);
2875 }
2876
2877 /*
2878 * PCI power manangement
2879 */
2880 int
pci_set_powerstate_method(device_t dev,device_t child,int state)2881 pci_set_powerstate_method(device_t dev, device_t child, int state)
2882 {
2883 struct pci_devinfo *dinfo = device_get_ivars(child);
2884 pcicfgregs *cfg = &dinfo->cfg;
2885 uint16_t status;
2886 int oldstate, highest, delay;
2887
2888 if (cfg->pp.pp_location == 0)
2889 return (EOPNOTSUPP);
2890
2891 /*
2892 * Optimize a no state change request away. While it would be OK to
2893 * write to the hardware in theory, some devices have shown odd
2894 * behavior when going from D3 -> D3.
2895 */
2896 oldstate = pci_get_powerstate(child);
2897 if (oldstate == state)
2898 return (0);
2899
2900 /*
2901 * The PCI power management specification states that after a state
2902 * transition between PCI power states, system software must
2903 * guarantee a minimal delay before the function accesses the device.
2904 * Compute the worst case delay that we need to guarantee before we
2905 * access the device. Many devices will be responsive much more
2906 * quickly than this delay, but there are some that don't respond
2907 * instantly to state changes. Transitions to/from D3 state require
2908 * 10ms, while D2 requires 200us, and D0/1 require none. The delay
2909 * is done below with DELAY rather than a sleeper function because
2910 * this function can be called from contexts where we cannot sleep.
2911 */
2912 highest = (oldstate > state) ? oldstate : state;
2913 if (highest == PCI_POWERSTATE_D3)
2914 delay = 10000;
2915 else if (highest == PCI_POWERSTATE_D2)
2916 delay = 200;
2917 else
2918 delay = 0;
2919 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_location +
2920 PCIR_POWER_STATUS, 2) & ~PCIM_PSTAT_DMASK;
2921 switch (state) {
2922 case PCI_POWERSTATE_D0:
2923 status |= PCIM_PSTAT_D0;
2924 break;
2925 case PCI_POWERSTATE_D1:
2926 if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0)
2927 return (EOPNOTSUPP);
2928 status |= PCIM_PSTAT_D1;
2929 break;
2930 case PCI_POWERSTATE_D2:
2931 if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0)
2932 return (EOPNOTSUPP);
2933 status |= PCIM_PSTAT_D2;
2934 break;
2935 case PCI_POWERSTATE_D3:
2936 status |= PCIM_PSTAT_D3;
2937 break;
2938 default:
2939 return (EINVAL);
2940 }
2941
2942 if (bootverbose)
2943 pci_printf(cfg, "Transition from %s to %s\n",
2944 pci_powerstate_to_str(oldstate),
2945 pci_powerstate_to_str(state));
2946
2947 PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_location + PCIR_POWER_STATUS,
2948 status, 2);
2949 if (delay)
2950 DELAY(delay);
2951 return (0);
2952 }
2953
2954 int
pci_get_powerstate_method(device_t dev,device_t child)2955 pci_get_powerstate_method(device_t dev, device_t child)
2956 {
2957 struct pci_devinfo *dinfo = device_get_ivars(child);
2958 pcicfgregs *cfg = &dinfo->cfg;
2959 uint16_t status;
2960 int result;
2961
2962 if (cfg->pp.pp_location != 0) {
2963 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_location +
2964 PCIR_POWER_STATUS, 2);
2965 switch (status & PCIM_PSTAT_DMASK) {
2966 case PCIM_PSTAT_D0:
2967 result = PCI_POWERSTATE_D0;
2968 break;
2969 case PCIM_PSTAT_D1:
2970 result = PCI_POWERSTATE_D1;
2971 break;
2972 case PCIM_PSTAT_D2:
2973 result = PCI_POWERSTATE_D2;
2974 break;
2975 case PCIM_PSTAT_D3:
2976 result = PCI_POWERSTATE_D3;
2977 break;
2978 default:
2979 result = PCI_POWERSTATE_UNKNOWN;
2980 break;
2981 }
2982 } else {
2983 /* No support, device is always at D0 */
2984 result = PCI_POWERSTATE_D0;
2985 }
2986 return (result);
2987 }
2988
2989 /* Clear any active PME# and disable PME# generation. */
2990 void
pci_clear_pme(device_t dev)2991 pci_clear_pme(device_t dev)
2992 {
2993 struct pci_devinfo *dinfo = device_get_ivars(dev);
2994 pcicfgregs *cfg = &dinfo->cfg;
2995 uint16_t status;
2996
2997 if (cfg->pp.pp_location != 0) {
2998 status = pci_read_config(dev, dinfo->cfg.pp.pp_location +
2999 PCIR_POWER_STATUS, 2);
3000 status &= ~PCIM_PSTAT_PMEENABLE;
3001 status |= PCIM_PSTAT_PME;
3002 pci_write_config(dev, dinfo->cfg.pp.pp_location +
3003 PCIR_POWER_STATUS, status, 2);
3004 }
3005 }
3006
3007 /* Clear any active PME# and enable PME# generation. */
3008 void
pci_enable_pme(device_t dev)3009 pci_enable_pme(device_t dev)
3010 {
3011 struct pci_devinfo *dinfo = device_get_ivars(dev);
3012 pcicfgregs *cfg = &dinfo->cfg;
3013 uint16_t status;
3014
3015 if (cfg->pp.pp_location != 0) {
3016 status = pci_read_config(dev, dinfo->cfg.pp.pp_location +
3017 PCIR_POWER_STATUS, 2);
3018 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
3019 pci_write_config(dev, dinfo->cfg.pp.pp_location +
3020 PCIR_POWER_STATUS, status, 2);
3021 }
3022 }
3023
3024 bool
pci_has_pm(device_t dev)3025 pci_has_pm(device_t dev)
3026 {
3027 struct pci_devinfo *dinfo = device_get_ivars(dev);
3028 pcicfgregs *cfg = &dinfo->cfg;
3029
3030 return (cfg->pp.pp_location != 0);
3031 }
3032
3033 /*
3034 * Some convenience functions for PCI device drivers.
3035 */
3036
3037 static __inline void
pci_set_command_bit(device_t dev,device_t child,uint16_t bit)3038 pci_set_command_bit(device_t dev, device_t child, uint16_t bit)
3039 {
3040 uint16_t command;
3041
3042 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
3043 command |= bit;
3044 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
3045 }
3046
3047 static __inline void
pci_clear_command_bit(device_t dev,device_t child,uint16_t bit)3048 pci_clear_command_bit(device_t dev, device_t child, uint16_t bit)
3049 {
3050 uint16_t command;
3051
3052 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
3053 command &= ~bit;
3054 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
3055 }
3056
3057 int
pci_enable_busmaster_method(device_t dev,device_t child)3058 pci_enable_busmaster_method(device_t dev, device_t child)
3059 {
3060 pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
3061 return (0);
3062 }
3063
3064 int
pci_disable_busmaster_method(device_t dev,device_t child)3065 pci_disable_busmaster_method(device_t dev, device_t child)
3066 {
3067 pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
3068 return (0);
3069 }
3070
3071 int
pci_enable_io_method(device_t dev,device_t child,int space)3072 pci_enable_io_method(device_t dev, device_t child, int space)
3073 {
3074 uint16_t bit;
3075
3076 switch(space) {
3077 case SYS_RES_IOPORT:
3078 bit = PCIM_CMD_PORTEN;
3079 break;
3080 case SYS_RES_MEMORY:
3081 bit = PCIM_CMD_MEMEN;
3082 break;
3083 default:
3084 return (EINVAL);
3085 }
3086 pci_set_command_bit(dev, child, bit);
3087 return (0);
3088 }
3089
3090 int
pci_disable_io_method(device_t dev,device_t child,int space)3091 pci_disable_io_method(device_t dev, device_t child, int space)
3092 {
3093 uint16_t bit;
3094
3095 switch(space) {
3096 case SYS_RES_IOPORT:
3097 bit = PCIM_CMD_PORTEN;
3098 break;
3099 case SYS_RES_MEMORY:
3100 bit = PCIM_CMD_MEMEN;
3101 break;
3102 default:
3103 return (EINVAL);
3104 }
3105 pci_clear_command_bit(dev, child, bit);
3106 return (0);
3107 }
3108
3109 /*
3110 * New style pci driver. Parent device is either a pci-host-bridge or a
3111 * pci-pci-bridge. Both kinds are represented by instances of pcib.
3112 */
3113
3114 void
pci_print_verbose(struct pci_devinfo * dinfo)3115 pci_print_verbose(struct pci_devinfo *dinfo)
3116 {
3117
3118 if (bootverbose) {
3119 pcicfgregs *cfg = &dinfo->cfg;
3120
3121 printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n",
3122 cfg->vendor, cfg->device, cfg->revid);
3123 printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n",
3124 cfg->domain, cfg->bus, cfg->slot, cfg->func);
3125 printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n",
3126 cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype,
3127 cfg->mfdev);
3128 printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n",
3129 cfg->cmdreg, cfg->statreg, cfg->cachelnsz);
3130 printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n",
3131 cfg->lattimer, cfg->lattimer * 30, cfg->mingnt,
3132 cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250);
3133 if (cfg->intpin > 0)
3134 printf("\tintpin=%c, irq=%d\n",
3135 cfg->intpin +'a' -1, cfg->intline);
3136 if (cfg->pp.pp_location) {
3137 uint16_t status;
3138
3139 status = pci_read_config(cfg->dev, cfg->pp.pp_location +
3140 PCIR_POWER_STATUS, 2);
3141 printf("\tpowerspec %d supports D0%s%s D3 current D%d\n",
3142 cfg->pp.pp_cap & PCIM_PCAP_SPEC,
3143 cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "",
3144 cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "",
3145 status & PCIM_PSTAT_DMASK);
3146 }
3147 if (cfg->msi.msi_location) {
3148 uint16_t ctrl, msgnum;
3149
3150 ctrl = cfg->msi.msi_ctrl;
3151 msgnum = PCI_MSI_MSGNUM(ctrl);
3152 printf("\tMSI supports %d message%s%s%s\n",
3153 msgnum, (msgnum == 1) ? "" : "s",
3154 (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "",
3155 (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":"");
3156 }
3157 if (cfg->msix.msix_location) {
3158 uint16_t msgnum;
3159
3160 msgnum = PCI_MSIX_MSGNUM(cfg->msix.msix_ctrl);
3161 printf("\tMSI-X supports %d message%s ",
3162 msgnum, (msgnum == 1) ? "" : "s");
3163 if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar)
3164 printf("in map 0x%x\n",
3165 cfg->msix.msix_table_bar);
3166 else
3167 printf("in maps 0x%x and 0x%x\n",
3168 cfg->msix.msix_table_bar,
3169 cfg->msix.msix_pba_bar);
3170 }
3171 }
3172 }
3173
3174 static int
pci_porten(device_t dev)3175 pci_porten(device_t dev)
3176 {
3177 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0;
3178 }
3179
3180 static int
pci_memen(device_t dev)3181 pci_memen(device_t dev)
3182 {
3183 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0;
3184 }
3185
3186 void
pci_read_bar(device_t dev,int reg,pci_addr_t * mapp,pci_addr_t * testvalp,int * bar64)3187 pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp,
3188 int *bar64)
3189 {
3190 struct pci_devinfo *dinfo;
3191 pci_addr_t map, testval;
3192 int ln2range;
3193 uint16_t cmd;
3194
3195 /*
3196 * The device ROM BAR is special. It is always a 32-bit
3197 * memory BAR. Bit 0 is special and should not be set when
3198 * sizing the BAR.
3199 */
3200 dinfo = device_get_ivars(dev);
3201 if (PCIR_IS_BIOS(&dinfo->cfg, reg)) {
3202 map = pci_read_config(dev, reg, 4);
3203 pci_write_config(dev, reg, 0xfffffffe, 4);
3204 testval = pci_read_config(dev, reg, 4);
3205 pci_write_config(dev, reg, map, 4);
3206 *mapp = map;
3207 *testvalp = testval;
3208 if (bar64 != NULL)
3209 *bar64 = 0;
3210 return;
3211 }
3212
3213 map = pci_read_config(dev, reg, 4);
3214 ln2range = pci_maprange(map);
3215 if (ln2range == 64)
3216 map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3217
3218 /*
3219 * Disable decoding via the command register before
3220 * determining the BAR's length since we will be placing it in
3221 * a weird state.
3222 */
3223 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3224 pci_write_config(dev, PCIR_COMMAND,
3225 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
3226
3227 /*
3228 * Determine the BAR's length by writing all 1's. The bottom
3229 * log_2(size) bits of the BAR will stick as 0 when we read
3230 * the value back.
3231 *
3232 * NB: according to the PCI Local Bus Specification, rev. 3.0:
3233 * "Software writes 0FFFFFFFFh to both registers, reads them back,
3234 * and combines the result into a 64-bit value." (section 6.2.5.1)
3235 *
3236 * Writes to both registers must be performed before attempting to
3237 * read back the size value.
3238 */
3239 testval = 0;
3240 pci_write_config(dev, reg, 0xffffffff, 4);
3241 if (ln2range == 64) {
3242 pci_write_config(dev, reg + 4, 0xffffffff, 4);
3243 testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3244 }
3245 testval |= pci_read_config(dev, reg, 4);
3246
3247 /*
3248 * Restore the original value of the BAR. We may have reprogrammed
3249 * the BAR of the low-level console device and when booting verbose,
3250 * we need the console device addressable.
3251 */
3252 pci_write_config(dev, reg, map, 4);
3253 if (ln2range == 64)
3254 pci_write_config(dev, reg + 4, map >> 32, 4);
3255 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3256
3257 *mapp = map;
3258 *testvalp = testval;
3259 if (bar64 != NULL)
3260 *bar64 = (ln2range == 64);
3261 }
3262
3263 static void
pci_write_bar(device_t dev,struct pci_map * pm,pci_addr_t base)3264 pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base)
3265 {
3266 struct pci_devinfo *dinfo;
3267 int ln2range;
3268
3269 /* The device ROM BAR is always a 32-bit memory BAR. */
3270 dinfo = device_get_ivars(dev);
3271 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3272 ln2range = 32;
3273 else
3274 ln2range = pci_maprange(pm->pm_value);
3275 pci_write_config(dev, pm->pm_reg, base, 4);
3276 if (ln2range == 64)
3277 pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4);
3278 pm->pm_value = pci_read_config(dev, pm->pm_reg, 4);
3279 if (ln2range == 64)
3280 pm->pm_value |= (pci_addr_t)pci_read_config(dev,
3281 pm->pm_reg + 4, 4) << 32;
3282 }
3283
3284 struct pci_map *
pci_find_bar(device_t dev,int reg)3285 pci_find_bar(device_t dev, int reg)
3286 {
3287 struct pci_devinfo *dinfo;
3288 struct pci_map *pm;
3289
3290 dinfo = device_get_ivars(dev);
3291 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3292 if (pm->pm_reg == reg)
3293 return (pm);
3294 }
3295 return (NULL);
3296 }
3297
3298 struct pci_map *
pci_first_bar(device_t dev)3299 pci_first_bar(device_t dev)
3300 {
3301 struct pci_devinfo *dinfo;
3302
3303 dinfo = device_get_ivars(dev);
3304 return (STAILQ_FIRST(&dinfo->cfg.maps));
3305 }
3306
3307 struct pci_map *
pci_next_bar(struct pci_map * pm)3308 pci_next_bar(struct pci_map *pm)
3309 {
3310 return (STAILQ_NEXT(pm, pm_link));
3311 }
3312
3313 int
pci_bar_enabled(device_t dev,struct pci_map * pm)3314 pci_bar_enabled(device_t dev, struct pci_map *pm)
3315 {
3316 struct pci_devinfo *dinfo;
3317 uint16_t cmd;
3318
3319 dinfo = device_get_ivars(dev);
3320 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) &&
3321 !(pm->pm_value & PCIM_BIOS_ENABLE))
3322 return (0);
3323 #ifdef PCI_IOV
3324 if ((dinfo->cfg.flags & PCICFG_VF) != 0) {
3325 struct pcicfg_iov *iov;
3326
3327 iov = dinfo->cfg.iov;
3328 cmd = pci_read_config(iov->iov_pf,
3329 iov->iov_pos + PCIR_SRIOV_CTL, 2);
3330 return ((cmd & PCIM_SRIOV_VF_MSE) != 0);
3331 }
3332 #endif
3333 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3334 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value))
3335 return ((cmd & PCIM_CMD_MEMEN) != 0);
3336 else
3337 return ((cmd & PCIM_CMD_PORTEN) != 0);
3338 }
3339
3340 struct pci_map *
pci_add_bar(device_t dev,int reg,pci_addr_t value,pci_addr_t size)3341 pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
3342 {
3343 struct pci_devinfo *dinfo;
3344 struct pci_map *pm, *prev;
3345
3346 dinfo = device_get_ivars(dev);
3347 pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO);
3348 pm->pm_reg = reg;
3349 pm->pm_value = value;
3350 pm->pm_size = size;
3351 STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) {
3352 KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x",
3353 reg));
3354 if (STAILQ_NEXT(prev, pm_link) == NULL ||
3355 STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg)
3356 break;
3357 }
3358 if (prev != NULL)
3359 STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link);
3360 else
3361 STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link);
3362 return (pm);
3363 }
3364
3365 static void
pci_restore_bars(device_t dev)3366 pci_restore_bars(device_t dev)
3367 {
3368 struct pci_devinfo *dinfo;
3369 struct pci_map *pm;
3370 int ln2range;
3371
3372 dinfo = device_get_ivars(dev);
3373 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3374 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3375 ln2range = 32;
3376 else
3377 ln2range = pci_maprange(pm->pm_value);
3378 pci_write_config(dev, pm->pm_reg, pm->pm_value, 4);
3379 if (ln2range == 64)
3380 pci_write_config(dev, pm->pm_reg + 4,
3381 pm->pm_value >> 32, 4);
3382 }
3383 }
3384
3385 /*
3386 * Add a resource based on a pci map register. Return 1 if the map
3387 * register is a 32bit map register or 2 if it is a 64bit register.
3388 */
3389 static int
pci_add_map(device_t bus,device_t dev,int reg,struct resource_list * rl,int force,int prefetch)3390 pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
3391 int force, int prefetch)
3392 {
3393 struct pci_map *pm;
3394 pci_addr_t base, map, testval;
3395 pci_addr_t start, end, count;
3396 int barlen, basezero, flags, maprange, mapsize, type;
3397 uint16_t cmd;
3398 struct resource *res;
3399
3400 /*
3401 * The BAR may already exist if the device is a CardBus card
3402 * whose CIS is stored in this BAR.
3403 */
3404 pm = pci_find_bar(dev, reg);
3405 if (pm != NULL) {
3406 maprange = pci_maprange(pm->pm_value);
3407 barlen = maprange == 64 ? 2 : 1;
3408 return (barlen);
3409 }
3410
3411 pci_read_bar(dev, reg, &map, &testval, NULL);
3412 if (PCI_BAR_MEM(map)) {
3413 type = SYS_RES_MEMORY;
3414 if (map & PCIM_BAR_MEM_PREFETCH)
3415 prefetch = 1;
3416 } else
3417 type = SYS_RES_IOPORT;
3418 mapsize = pci_mapsize(testval);
3419 base = pci_mapbase(map);
3420 #ifdef __PCI_BAR_ZERO_VALID
3421 basezero = 0;
3422 #else
3423 basezero = base == 0;
3424 #endif
3425 maprange = pci_maprange(map);
3426 barlen = maprange == 64 ? 2 : 1;
3427
3428 /*
3429 * For I/O registers, if bottom bit is set, and the next bit up
3430 * isn't clear, we know we have a BAR that doesn't conform to the
3431 * spec, so ignore it. Also, sanity check the size of the data
3432 * areas to the type of memory involved. Memory must be at least
3433 * 16 bytes in size, while I/O ranges must be at least 4.
3434 */
3435 if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0)
3436 return (barlen);
3437 if ((type == SYS_RES_MEMORY && mapsize < 4) ||
3438 (type == SYS_RES_IOPORT && mapsize < 2))
3439 return (barlen);
3440
3441 /* Save a record of this BAR. */
3442 pm = pci_add_bar(dev, reg, map, mapsize);
3443 if (bootverbose) {
3444 printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d",
3445 reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize);
3446 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3447 printf(", port disabled\n");
3448 else if (type == SYS_RES_MEMORY && !pci_memen(dev))
3449 printf(", memory disabled\n");
3450 else
3451 printf(", enabled\n");
3452 }
3453
3454 /*
3455 * If base is 0, then we have problems if this architecture does
3456 * not allow that. It is best to ignore such entries for the
3457 * moment. These will be allocated later if the driver specifically
3458 * requests them. However, some removable buses look better when
3459 * all resources are allocated, so allow '0' to be overridden.
3460 *
3461 * Similarly treat maps whose values is the same as the test value
3462 * read back. These maps have had all f's written to them by the
3463 * BIOS in an attempt to disable the resources.
3464 */
3465 if (!force && (basezero || map == testval))
3466 return (barlen);
3467 if ((u_long)base != base) {
3468 device_printf(bus,
3469 "pci%d:%d:%d:%d bar %#x too many address bits",
3470 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
3471 pci_get_function(dev), reg);
3472 return (barlen);
3473 }
3474
3475 /*
3476 * This code theoretically does the right thing, but has
3477 * undesirable side effects in some cases where peripherals
3478 * respond oddly to having these bits enabled. Let the user
3479 * be able to turn them off (since pci_enable_io_modes is 1 by
3480 * default).
3481 */
3482 if (pci_enable_io_modes) {
3483 /* Turn on resources that have been left off by a lazy BIOS */
3484 if (type == SYS_RES_IOPORT && !pci_porten(dev)) {
3485 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3486 cmd |= PCIM_CMD_PORTEN;
3487 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3488 }
3489 if (type == SYS_RES_MEMORY && !pci_memen(dev)) {
3490 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3491 cmd |= PCIM_CMD_MEMEN;
3492 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3493 }
3494 } else {
3495 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3496 return (barlen);
3497 if (type == SYS_RES_MEMORY && !pci_memen(dev))
3498 return (barlen);
3499 }
3500
3501 count = (pci_addr_t)1 << mapsize;
3502 flags = RF_ALIGNMENT_LOG2(mapsize);
3503 if (prefetch)
3504 flags |= RF_PREFETCHABLE;
3505 if (basezero || base == pci_mapbase(testval) || pci_clear_bars) {
3506 start = 0; /* Let the parent decide. */
3507 end = ~0;
3508 } else {
3509 start = base;
3510 end = base + count - 1;
3511 }
3512 resource_list_add(rl, type, reg, start, end, count);
3513
3514 /*
3515 * Try to allocate the resource for this BAR from our parent
3516 * so that this resource range is already reserved. The
3517 * driver for this device will later inherit this resource in
3518 * pci_alloc_resource().
3519 */
3520 res = resource_list_reserve(rl, bus, dev, type, reg, start, end, count,
3521 flags);
3522 if ((pci_do_realloc_bars
3523 || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR))
3524 && res == NULL && (start != 0 || end != ~0)) {
3525 /*
3526 * If the allocation fails, try to allocate a resource for
3527 * this BAR using any available range. The firmware felt
3528 * it was important enough to assign a resource, so don't
3529 * disable decoding if we can help it.
3530 */
3531 resource_list_delete(rl, type, reg);
3532 resource_list_add(rl, type, reg, 0, ~0, count);
3533 res = resource_list_reserve(rl, bus, dev, type, reg, 0, ~0,
3534 count, flags);
3535 }
3536 if (res == NULL) {
3537 /*
3538 * If the allocation fails, delete the resource list entry
3539 * and disable decoding for this device.
3540 *
3541 * If the driver requests this resource in the future,
3542 * pci_reserve_map() will try to allocate a fresh
3543 * resource range.
3544 */
3545 resource_list_delete(rl, type, reg);
3546 pci_disable_io(dev, type);
3547 if (bootverbose)
3548 device_printf(bus,
3549 "pci%d:%d:%d:%d bar %#x failed to allocate\n",
3550 pci_get_domain(dev), pci_get_bus(dev),
3551 pci_get_slot(dev), pci_get_function(dev), reg);
3552 } else {
3553 start = rman_get_start(res);
3554 pci_write_bar(dev, pm, start);
3555 }
3556 return (barlen);
3557 }
3558
3559 /*
3560 * For ATA devices we need to decide early what addressing mode to use.
3561 * Legacy demands that the primary and secondary ATA ports sits on the
3562 * same addresses that old ISA hardware did. This dictates that we use
3563 * those addresses and ignore the BAR's if we cannot set PCI native
3564 * addressing mode.
3565 */
3566 static void
pci_ata_maps(device_t bus,device_t dev,struct resource_list * rl,int force,uint32_t prefetchmask)3567 pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
3568 uint32_t prefetchmask)
3569 {
3570 int rid, type, progif;
3571 #if 0
3572 /* if this device supports PCI native addressing use it */
3573 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3574 if ((progif & 0x8a) == 0x8a) {
3575 if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) &&
3576 pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) {
3577 printf("Trying ATA native PCI addressing mode\n");
3578 pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1);
3579 }
3580 }
3581 #endif
3582 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3583 type = SYS_RES_IOPORT;
3584 if (progif & PCIP_STORAGE_IDE_MODEPRIM) {
3585 pci_add_map(bus, dev, PCIR_BAR(0), rl, force,
3586 prefetchmask & (1 << 0));
3587 pci_add_map(bus, dev, PCIR_BAR(1), rl, force,
3588 prefetchmask & (1 << 1));
3589 } else {
3590 rid = PCIR_BAR(0);
3591 resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
3592 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x1f0,
3593 0x1f7, 8, 0);
3594 rid = PCIR_BAR(1);
3595 resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
3596 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x3f6,
3597 0x3f6, 1, 0);
3598 }
3599 if (progif & PCIP_STORAGE_IDE_MODESEC) {
3600 pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
3601 prefetchmask & (1 << 2));
3602 pci_add_map(bus, dev, PCIR_BAR(3), rl, force,
3603 prefetchmask & (1 << 3));
3604 } else {
3605 rid = PCIR_BAR(2);
3606 resource_list_add(rl, type, rid, 0x170, 0x177, 8);
3607 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x170,
3608 0x177, 8, 0);
3609 rid = PCIR_BAR(3);
3610 resource_list_add(rl, type, rid, 0x376, 0x376, 1);
3611 (void)resource_list_reserve(rl, bus, dev, type, rid, 0x376,
3612 0x376, 1, 0);
3613 }
3614 pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
3615 prefetchmask & (1 << 4));
3616 pci_add_map(bus, dev, PCIR_BAR(5), rl, force,
3617 prefetchmask & (1 << 5));
3618 }
3619
3620 static void
pci_assign_interrupt(device_t bus,device_t dev,int force_route)3621 pci_assign_interrupt(device_t bus, device_t dev, int force_route)
3622 {
3623 struct pci_devinfo *dinfo = device_get_ivars(dev);
3624 pcicfgregs *cfg = &dinfo->cfg;
3625 char tunable_name[64];
3626 int irq;
3627
3628 /* Has to have an intpin to have an interrupt. */
3629 if (cfg->intpin == 0)
3630 return;
3631
3632 /* Let the user override the IRQ with a tunable. */
3633 irq = PCI_INVALID_IRQ;
3634 snprintf(tunable_name, sizeof(tunable_name),
3635 "hw.pci%d.%d.%d.INT%c.irq",
3636 cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
3637 if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0))
3638 irq = PCI_INVALID_IRQ;
3639
3640 /*
3641 * If we didn't get an IRQ via the tunable, then we either use the
3642 * IRQ value in the intline register or we ask the bus to route an
3643 * interrupt for us. If force_route is true, then we only use the
3644 * value in the intline register if the bus was unable to assign an
3645 * IRQ.
3646 */
3647 if (!PCI_INTERRUPT_VALID(irq)) {
3648 if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route)
3649 irq = PCI_ASSIGN_INTERRUPT(bus, dev);
3650 if (!PCI_INTERRUPT_VALID(irq))
3651 irq = cfg->intline;
3652 }
3653
3654 /* If after all that we don't have an IRQ, just bail. */
3655 if (!PCI_INTERRUPT_VALID(irq))
3656 return;
3657
3658 /* Update the config register if it changed. */
3659 if (irq != cfg->intline) {
3660 cfg->intline = irq;
3661 pci_write_config(dev, PCIR_INTLINE, irq, 1);
3662 }
3663
3664 /* Add this IRQ as rid 0 interrupt resource. */
3665 resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1);
3666 }
3667
3668 /* Perform early OHCI takeover from SMM. */
3669 static void
ohci_early_takeover(device_t self)3670 ohci_early_takeover(device_t self)
3671 {
3672 struct resource *res;
3673 uint32_t ctl;
3674 int rid;
3675 int i;
3676
3677 rid = PCIR_BAR(0);
3678 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3679 if (res == NULL)
3680 return;
3681
3682 ctl = bus_read_4(res, OHCI_CONTROL);
3683 if (ctl & OHCI_IR) {
3684 if (bootverbose)
3685 printf("ohci early: "
3686 "SMM active, request owner change\n");
3687 bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR);
3688 for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) {
3689 DELAY(1000);
3690 ctl = bus_read_4(res, OHCI_CONTROL);
3691 }
3692 if (ctl & OHCI_IR) {
3693 if (bootverbose)
3694 printf("ohci early: "
3695 "SMM does not respond, resetting\n");
3696 bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET);
3697 }
3698 /* Disable interrupts */
3699 bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS);
3700 }
3701
3702 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3703 }
3704
3705 /* Perform early UHCI takeover from SMM. */
3706 static void
uhci_early_takeover(device_t self)3707 uhci_early_takeover(device_t self)
3708 {
3709 struct resource *res;
3710 int rid;
3711
3712 /*
3713 * Set the PIRQD enable bit and switch off all the others. We don't
3714 * want legacy support to interfere with us XXX Does this also mean
3715 * that the BIOS won't touch the keyboard anymore if it is connected
3716 * to the ports of the root hub?
3717 */
3718 pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);
3719
3720 /* Disable interrupts */
3721 rid = PCI_UHCI_BASE_REG;
3722 res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE);
3723 if (res != NULL) {
3724 bus_write_2(res, UHCI_INTR, 0);
3725 bus_release_resource(self, SYS_RES_IOPORT, rid, res);
3726 }
3727 }
3728
3729 /* Perform early EHCI takeover from SMM. */
3730 static void
ehci_early_takeover(device_t self)3731 ehci_early_takeover(device_t self)
3732 {
3733 struct resource *res;
3734 uint32_t cparams;
3735 uint32_t eec;
3736 uint8_t eecp;
3737 uint8_t bios_sem;
3738 uint8_t offs;
3739 int rid;
3740 int i;
3741
3742 rid = PCIR_BAR(0);
3743 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3744 if (res == NULL)
3745 return;
3746
3747 cparams = bus_read_4(res, EHCI_HCCPARAMS);
3748
3749 /* Synchronise with the BIOS if it owns the controller. */
3750 for (eecp = EHCI_HCC_EECP(cparams); eecp != 0;
3751 eecp = EHCI_EECP_NEXT(eec)) {
3752 eec = pci_read_config(self, eecp, 4);
3753 if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) {
3754 continue;
3755 }
3756 bios_sem = pci_read_config(self, eecp +
3757 EHCI_LEGSUP_BIOS_SEM, 1);
3758 if (bios_sem == 0) {
3759 continue;
3760 }
3761 if (bootverbose)
3762 printf("ehci early: "
3763 "SMM active, request owner change\n");
3764
3765 pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1);
3766
3767 for (i = 0; (i < 100) && (bios_sem != 0); i++) {
3768 DELAY(1000);
3769 bios_sem = pci_read_config(self, eecp +
3770 EHCI_LEGSUP_BIOS_SEM, 1);
3771 }
3772
3773 if (bios_sem != 0) {
3774 if (bootverbose)
3775 printf("ehci early: "
3776 "SMM does not respond\n");
3777 }
3778 /* Disable interrupts */
3779 offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION));
3780 bus_write_4(res, offs + EHCI_USBINTR, 0);
3781 }
3782 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3783 }
3784
3785 /* Perform early XHCI takeover from SMM. */
3786 static void
xhci_early_takeover(device_t self)3787 xhci_early_takeover(device_t self)
3788 {
3789 struct resource *res;
3790 uint32_t cparams;
3791 uint32_t eec;
3792 uint32_t eecp;
3793 uint8_t bios_sem;
3794 uint8_t offs;
3795 int rid;
3796 int i;
3797
3798 rid = PCIR_BAR(0);
3799 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3800 if (res == NULL)
3801 return;
3802
3803 cparams = bus_read_4(res, XHCI_HCCPARAMS1);
3804
3805 eec = -1;
3806
3807 /* Synchronise with the BIOS if it owns the controller. */
3808 for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec);
3809 eecp += XHCI_XECP_NEXT(eec) << 2) {
3810 eec = bus_read_4(res, eecp);
3811
3812 if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY)
3813 continue;
3814
3815 bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM);
3816 if (bios_sem == 0)
3817 continue;
3818
3819 if (bootverbose)
3820 printf("xhci early: "
3821 "SMM active, request owner change\n");
3822
3823 bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1);
3824
3825 /* wait a maximum of 5 second */
3826
3827 for (i = 0; (i < 5000) && (bios_sem != 0); i++) {
3828 DELAY(1000);
3829 bios_sem = bus_read_1(res, eecp +
3830 XHCI_XECP_BIOS_SEM);
3831 }
3832
3833 if (bios_sem != 0) {
3834 if (bootverbose)
3835 printf("xhci early: "
3836 "SMM does not respond\n");
3837 }
3838
3839 /* Disable interrupts */
3840 offs = bus_read_1(res, XHCI_CAPLENGTH);
3841 bus_write_4(res, offs + XHCI_USBCMD, 0);
3842 bus_read_4(res, offs + XHCI_USBSTS);
3843 }
3844 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3845 }
3846
3847 static void
pci_reserve_secbus(device_t bus,device_t dev,pcicfgregs * cfg,struct resource_list * rl)3848 pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg,
3849 struct resource_list *rl)
3850 {
3851 struct resource *res;
3852 char *cp;
3853 rman_res_t start, end, count;
3854 int sec_bus, sec_reg, sub_bus, sub_reg, sup_bus;
3855
3856 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3857 case PCIM_HDRTYPE_BRIDGE:
3858 sec_reg = PCIR_SECBUS_1;
3859 sub_reg = PCIR_SUBBUS_1;
3860 break;
3861 case PCIM_HDRTYPE_CARDBUS:
3862 sec_reg = PCIR_SECBUS_2;
3863 sub_reg = PCIR_SUBBUS_2;
3864 break;
3865 default:
3866 return;
3867 }
3868
3869 /*
3870 * If the existing bus range is valid, attempt to reserve it
3871 * from our parent. If this fails for any reason, clear the
3872 * secbus and subbus registers.
3873 *
3874 * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus?
3875 * This would at least preserve the existing sec_bus if it is
3876 * valid.
3877 */
3878 sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1);
3879 sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1);
3880
3881 /* Quirk handling. */
3882 switch (pci_get_devid(dev)) {
3883 case 0x12258086: /* Intel 82454KX/GX (Orion) */
3884 sup_bus = pci_read_config(dev, 0x41, 1);
3885 if (sup_bus != 0xff) {
3886 sec_bus = sup_bus + 1;
3887 sub_bus = sup_bus + 1;
3888 PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1);
3889 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3890 }
3891 break;
3892
3893 case 0x00dd10de:
3894 /* Compaq R3000 BIOS sets wrong subordinate bus number. */
3895 if ((cp = kern_getenv("smbios.planar.maker")) == NULL)
3896 break;
3897 if (strncmp(cp, "Compal", 6) != 0) {
3898 freeenv(cp);
3899 break;
3900 }
3901 freeenv(cp);
3902 if ((cp = kern_getenv("smbios.planar.product")) == NULL)
3903 break;
3904 if (strncmp(cp, "08A0", 4) != 0) {
3905 freeenv(cp);
3906 break;
3907 }
3908 freeenv(cp);
3909 if (sub_bus < 0xa) {
3910 sub_bus = 0xa;
3911 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3912 }
3913 break;
3914 }
3915
3916 if (bootverbose)
3917 printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus);
3918 if (sec_bus > 0 && sub_bus >= sec_bus) {
3919 start = sec_bus;
3920 end = sub_bus;
3921 count = end - start + 1;
3922
3923 resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count);
3924
3925 /*
3926 * If requested, clear secondary bus registers in
3927 * bridge devices to force a complete renumbering
3928 * rather than reserving the existing range. However,
3929 * preserve the existing size.
3930 */
3931 if (pci_clear_buses)
3932 goto clear;
3933
3934 res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, 0,
3935 start, end, count, 0);
3936 if (res != NULL)
3937 return;
3938
3939 if (bootverbose)
3940 device_printf(bus,
3941 "pci%d:%d:%d:%d secbus failed to allocate\n",
3942 pci_get_domain(dev), pci_get_bus(dev),
3943 pci_get_slot(dev), pci_get_function(dev));
3944 }
3945
3946 clear:
3947 PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1);
3948 PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1);
3949 }
3950
3951 static struct resource *
pci_alloc_secbus(device_t dev,device_t child,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)3952 pci_alloc_secbus(device_t dev, device_t child, int rid, rman_res_t start,
3953 rman_res_t end, rman_res_t count, u_int flags)
3954 {
3955 struct pci_devinfo *dinfo;
3956 pcicfgregs *cfg;
3957 struct resource_list *rl;
3958 struct resource *res;
3959 int sec_reg, sub_reg;
3960
3961 dinfo = device_get_ivars(child);
3962 cfg = &dinfo->cfg;
3963 rl = &dinfo->resources;
3964 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3965 case PCIM_HDRTYPE_BRIDGE:
3966 sec_reg = PCIR_SECBUS_1;
3967 sub_reg = PCIR_SUBBUS_1;
3968 break;
3969 case PCIM_HDRTYPE_CARDBUS:
3970 sec_reg = PCIR_SECBUS_2;
3971 sub_reg = PCIR_SUBBUS_2;
3972 break;
3973 default:
3974 return (NULL);
3975 }
3976
3977 if (rid != 0)
3978 return (NULL);
3979
3980 if (resource_list_find(rl, PCI_RES_BUS, rid) == NULL)
3981 resource_list_add(rl, PCI_RES_BUS, rid, start, end, count);
3982 if (!resource_list_reserved(rl, PCI_RES_BUS, rid)) {
3983 res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid,
3984 start, end, count, flags & ~RF_ACTIVE);
3985 if (res == NULL) {
3986 resource_list_delete(rl, PCI_RES_BUS, rid);
3987 device_printf(child, "allocating %ju bus%s failed\n",
3988 count, count == 1 ? "" : "es");
3989 return (NULL);
3990 }
3991 if (bootverbose)
3992 device_printf(child,
3993 "Lazy allocation of %ju bus%s at %ju\n", count,
3994 count == 1 ? "" : "es", rman_get_start(res));
3995 PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1);
3996 PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1);
3997 }
3998 return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start,
3999 end, count, flags));
4000 }
4001
4002 static int
pci_ea_bei_to_rid(device_t dev,int bei)4003 pci_ea_bei_to_rid(device_t dev, int bei)
4004 {
4005 #ifdef PCI_IOV
4006 struct pci_devinfo *dinfo;
4007 int iov_pos;
4008 struct pcicfg_iov *iov;
4009
4010 dinfo = device_get_ivars(dev);
4011 iov = dinfo->cfg.iov;
4012 if (iov != NULL)
4013 iov_pos = iov->iov_pos;
4014 else
4015 iov_pos = 0;
4016 #endif
4017
4018 /* Check if matches BAR */
4019 if ((bei >= PCIM_EA_BEI_BAR_0) &&
4020 (bei <= PCIM_EA_BEI_BAR_5))
4021 return (PCIR_BAR(bei));
4022
4023 /* Check ROM */
4024 if (bei == PCIM_EA_BEI_ROM)
4025 return (PCIR_BIOS);
4026
4027 #ifdef PCI_IOV
4028 /* Check if matches VF_BAR */
4029 if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) &&
4030 (bei <= PCIM_EA_BEI_VF_BAR_5))
4031 return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) +
4032 iov_pos);
4033 #endif
4034
4035 return (-1);
4036 }
4037
4038 int
pci_ea_is_enabled(device_t dev,int rid)4039 pci_ea_is_enabled(device_t dev, int rid)
4040 {
4041 struct pci_ea_entry *ea;
4042 struct pci_devinfo *dinfo;
4043
4044 dinfo = device_get_ivars(dev);
4045
4046 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
4047 if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid)
4048 return ((ea->eae_flags & PCIM_EA_ENABLE) > 0);
4049 }
4050
4051 return (0);
4052 }
4053
4054 void
pci_add_resources_ea(device_t bus,device_t dev,int alloc_iov)4055 pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
4056 {
4057 struct pci_ea_entry *ea;
4058 struct pci_devinfo *dinfo;
4059 pci_addr_t start, end, count;
4060 struct resource_list *rl;
4061 int type, flags, rid;
4062 struct resource *res;
4063 uint32_t tmp;
4064 #ifdef PCI_IOV
4065 struct pcicfg_iov *iov;
4066 #endif
4067
4068 dinfo = device_get_ivars(dev);
4069 rl = &dinfo->resources;
4070 flags = 0;
4071
4072 #ifdef PCI_IOV
4073 iov = dinfo->cfg.iov;
4074 #endif
4075
4076 if (dinfo->cfg.ea.ea_location == 0)
4077 return;
4078
4079 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
4080 /*
4081 * TODO: Ignore EA-BAR if is not enabled.
4082 * Currently the EA implementation supports
4083 * only situation, where EA structure contains
4084 * predefined entries. In case they are not enabled
4085 * leave them unallocated and proceed with
4086 * a legacy-BAR mechanism.
4087 */
4088 if ((ea->eae_flags & PCIM_EA_ENABLE) == 0)
4089 continue;
4090
4091 switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) {
4092 case PCIM_EA_P_MEM_PREFETCH:
4093 case PCIM_EA_P_VF_MEM_PREFETCH:
4094 flags = RF_PREFETCHABLE;
4095 /* FALLTHROUGH */
4096 case PCIM_EA_P_VF_MEM:
4097 case PCIM_EA_P_MEM:
4098 type = SYS_RES_MEMORY;
4099 break;
4100 case PCIM_EA_P_IO:
4101 type = SYS_RES_IOPORT;
4102 break;
4103 default:
4104 continue;
4105 }
4106
4107 if (alloc_iov != 0) {
4108 #ifdef PCI_IOV
4109 /* Allocating IOV, confirm BEI matches */
4110 if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) ||
4111 (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5))
4112 continue;
4113 #else
4114 continue;
4115 #endif
4116 } else {
4117 /* Allocating BAR, confirm BEI matches */
4118 if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) ||
4119 (ea->eae_bei > PCIM_EA_BEI_BAR_5)) &&
4120 (ea->eae_bei != PCIM_EA_BEI_ROM))
4121 continue;
4122 }
4123
4124 rid = pci_ea_bei_to_rid(dev, ea->eae_bei);
4125 if (rid < 0)
4126 continue;
4127
4128 /* Skip resources already allocated by EA */
4129 if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) ||
4130 (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL))
4131 continue;
4132
4133 start = ea->eae_base;
4134 count = ea->eae_max_offset + 1;
4135 #ifdef PCI_IOV
4136 if (iov != NULL)
4137 count = count * iov->iov_num_vfs;
4138 #endif
4139 end = start + count - 1;
4140 if (count == 0)
4141 continue;
4142
4143 resource_list_add(rl, type, rid, start, end, count);
4144 res = resource_list_reserve(rl, bus, dev, type, rid, start, end, count,
4145 flags);
4146 if (res == NULL) {
4147 resource_list_delete(rl, type, rid);
4148
4149 /*
4150 * Failed to allocate using EA, disable entry.
4151 * Another attempt to allocation will be performed
4152 * further, but this time using legacy BAR registers
4153 */
4154 tmp = pci_read_config(dev, ea->eae_cfg_offset, 4);
4155 tmp &= ~PCIM_EA_ENABLE;
4156 pci_write_config(dev, ea->eae_cfg_offset, tmp, 4);
4157
4158 /*
4159 * Disabling entry might fail in case it is hardwired.
4160 * Read flags again to match current status.
4161 */
4162 ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4);
4163
4164 continue;
4165 }
4166
4167 /* As per specification, fill BAR with zeros */
4168 pci_write_config(dev, rid, 0, 4);
4169 }
4170 }
4171
4172 void
pci_add_resources(device_t bus,device_t dev,int force,uint32_t prefetchmask)4173 pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
4174 {
4175 struct pci_devinfo *dinfo;
4176 pcicfgregs *cfg;
4177 struct resource_list *rl;
4178 const struct pci_quirk *q;
4179 uint32_t devid;
4180 int i;
4181
4182 dinfo = device_get_ivars(dev);
4183 cfg = &dinfo->cfg;
4184 rl = &dinfo->resources;
4185 devid = (cfg->device << 16) | cfg->vendor;
4186
4187 /* Allocate resources using Enhanced Allocation */
4188 pci_add_resources_ea(bus, dev, 0);
4189
4190 /* ATA devices needs special map treatment */
4191 if ((pci_get_class(dev) == PCIC_STORAGE) &&
4192 (pci_get_subclass(dev) == PCIS_STORAGE_IDE) &&
4193 ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) ||
4194 (!pci_read_config(dev, PCIR_BAR(0), 4) &&
4195 !pci_read_config(dev, PCIR_BAR(2), 4))) )
4196 pci_ata_maps(bus, dev, rl, force, prefetchmask);
4197 else
4198 for (i = 0; i < cfg->nummaps;) {
4199 /* Skip resources already managed by EA */
4200 if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) ||
4201 (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) ||
4202 pci_ea_is_enabled(dev, PCIR_BAR(i))) {
4203 i++;
4204 continue;
4205 }
4206
4207 /*
4208 * Skip quirked resources.
4209 */
4210 for (q = &pci_quirks[0]; q->devid != 0; q++)
4211 if (q->devid == devid &&
4212 q->type == PCI_QUIRK_UNMAP_REG &&
4213 q->arg1 == PCIR_BAR(i))
4214 break;
4215 if (q->devid != 0) {
4216 i++;
4217 continue;
4218 }
4219 i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force,
4220 prefetchmask & (1 << i));
4221 }
4222
4223 /*
4224 * Add additional, quirked resources.
4225 */
4226 for (q = &pci_quirks[0]; q->devid != 0; q++)
4227 if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG)
4228 pci_add_map(bus, dev, q->arg1, rl, force, 0);
4229
4230 if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline) &&
4231 pci_intx_reroute) {
4232 /*
4233 * Try to re-route interrupts. Sometimes the BIOS or
4234 * firmware may leave bogus values in these registers.
4235 * If the re-route fails, then just stick with what we
4236 * have.
4237 */
4238 pci_assign_interrupt(bus, dev, 1);
4239 }
4240
4241 if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS &&
4242 pci_get_subclass(dev) == PCIS_SERIALBUS_USB) {
4243 if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI)
4244 xhci_early_takeover(dev);
4245 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI)
4246 ehci_early_takeover(dev);
4247 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI)
4248 ohci_early_takeover(dev);
4249 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI)
4250 uhci_early_takeover(dev);
4251 }
4252
4253 /*
4254 * Reserve resources for secondary bus ranges behind bridge
4255 * devices.
4256 */
4257 pci_reserve_secbus(bus, dev, cfg, rl);
4258 }
4259
4260 static struct pci_devinfo *
pci_identify_function(device_t pcib,device_t dev,int domain,int busno,int slot,int func)4261 pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
4262 int slot, int func)
4263 {
4264 struct pci_devinfo *dinfo;
4265
4266 dinfo = pci_read_device(pcib, dev, domain, busno, slot, func);
4267 if (dinfo != NULL)
4268 pci_add_child(dev, dinfo);
4269
4270 return (dinfo);
4271 }
4272
4273 void
pci_add_children(device_t dev,int domain,int busno)4274 pci_add_children(device_t dev, int domain, int busno)
4275 {
4276 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4277 device_t pcib = device_get_parent(dev);
4278 struct pci_devinfo *dinfo;
4279 int maxslots;
4280 int s, f, pcifunchigh;
4281 uint8_t hdrtype;
4282 int first_func;
4283
4284 /*
4285 * Try to detect a device at slot 0, function 0. If it exists, try to
4286 * enable ARI. We must enable ARI before detecting the rest of the
4287 * functions on this bus as ARI changes the set of slots and functions
4288 * that are legal on this bus.
4289 */
4290 dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0);
4291 if (dinfo != NULL && pci_enable_ari)
4292 PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);
4293
4294 /*
4295 * Start looking for new devices on slot 0 at function 1 because we
4296 * just identified the device at slot 0, function 0.
4297 */
4298 first_func = 1;
4299
4300 maxslots = PCIB_MAXSLOTS(pcib);
4301 for (s = 0; s <= maxslots; s++, first_func = 0) {
4302 pcifunchigh = 0;
4303 f = 0;
4304 DELAY(1);
4305
4306 /* If function 0 is not present, skip to the next slot. */
4307 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4308 continue;
4309 hdrtype = REG(PCIR_HDRTYPE, 1);
4310 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4311 continue;
4312 if (hdrtype & PCIM_MFDEV)
4313 pcifunchigh = PCIB_MAXFUNCS(pcib);
4314 for (f = first_func; f <= pcifunchigh; f++)
4315 pci_identify_function(pcib, dev, domain, busno, s, f);
4316 }
4317 pcie_reconcile_link_mps(dev);
4318 #undef REG
4319 }
4320
4321 int
pci_rescan_method(device_t dev)4322 pci_rescan_method(device_t dev)
4323 {
4324 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4325 device_t pcib = device_get_parent(dev);
4326 device_t child, *devlist, *unchanged;
4327 int devcount, error, i, j, maxslots, oldcount;
4328 int busno, domain, s, f, pcifunchigh;
4329 uint8_t hdrtype;
4330
4331 /* No need to check for ARI on a rescan. */
4332 error = device_get_children(dev, &devlist, &devcount);
4333 if (error)
4334 return (error);
4335 if (devcount != 0) {
4336 unchanged = malloc(devcount * sizeof(device_t), M_TEMP,
4337 M_NOWAIT | M_ZERO);
4338 if (unchanged == NULL) {
4339 free(devlist, M_TEMP);
4340 return (ENOMEM);
4341 }
4342 } else
4343 unchanged = NULL;
4344
4345 domain = pcib_get_domain(dev);
4346 busno = pcib_get_bus(dev);
4347 maxslots = PCIB_MAXSLOTS(pcib);
4348 for (s = 0; s <= maxslots; s++) {
4349 /* If function 0 is not present, skip to the next slot. */
4350 f = 0;
4351 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4352 continue;
4353 pcifunchigh = 0;
4354 hdrtype = REG(PCIR_HDRTYPE, 1);
4355 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4356 continue;
4357 if (hdrtype & PCIM_MFDEV)
4358 pcifunchigh = PCIB_MAXFUNCS(pcib);
4359 for (f = 0; f <= pcifunchigh; f++) {
4360 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4361 continue;
4362
4363 /*
4364 * Found a valid function. Check if a
4365 * device_t for this device already exists.
4366 */
4367 for (i = 0; i < devcount; i++) {
4368 child = devlist[i];
4369 if (child == NULL)
4370 continue;
4371 if (pci_get_slot(child) == s &&
4372 pci_get_function(child) == f) {
4373 unchanged[i] = child;
4374 goto next_func;
4375 }
4376 }
4377
4378 pci_identify_function(pcib, dev, domain, busno, s, f);
4379 next_func:;
4380 }
4381 }
4382
4383 /* Remove devices that are no longer present. */
4384 for (i = 0; i < devcount; i++) {
4385 if (unchanged[i] != NULL)
4386 continue;
4387 device_delete_child(dev, devlist[i]);
4388 }
4389
4390 free(devlist, M_TEMP);
4391 oldcount = devcount;
4392
4393 /* Try to attach the devices just added. */
4394 error = device_get_children(dev, &devlist, &devcount);
4395 if (error) {
4396 free(unchanged, M_TEMP);
4397 return (error);
4398 }
4399
4400 for (i = 0; i < devcount; i++) {
4401 for (j = 0; j < oldcount; j++) {
4402 if (devlist[i] == unchanged[j])
4403 goto next_device;
4404 }
4405
4406 device_probe_and_attach(devlist[i]);
4407 next_device:;
4408 }
4409
4410 free(unchanged, M_TEMP);
4411 free(devlist, M_TEMP);
4412 return (0);
4413 #undef REG
4414 }
4415
4416 #ifdef PCI_IOV
4417 device_t
pci_add_iov_child(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4418 pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid,
4419 uint16_t did)
4420 {
4421 struct pci_devinfo *vf_dinfo;
4422 device_t pcib;
4423 int busno, slot, func;
4424
4425 pcib = device_get_parent(bus);
4426
4427 PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func);
4428
4429 vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno,
4430 slot, func, vid, did);
4431
4432 vf_dinfo->cfg.flags |= PCICFG_VF;
4433 pci_add_child(bus, vf_dinfo);
4434
4435 return (vf_dinfo->cfg.dev);
4436 }
4437
4438 device_t
pci_create_iov_child_method(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4439 pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid,
4440 uint16_t vid, uint16_t did)
4441 {
4442
4443 return (pci_add_iov_child(bus, pf, rid, vid, did));
4444 }
4445 #endif
4446
4447 static int
pcie_mps_bytes(uint16_t mps)4448 pcie_mps_bytes(uint16_t mps)
4449 {
4450
4451 return (128 << (mps >> 5));
4452 }
4453
4454 static bool
pcie_mps_limit_value(uint16_t * mps)4455 pcie_mps_limit_value(uint16_t *mps)
4456 {
4457
4458 if (pci_mps_limit == 0)
4459 return (false);
4460 if (pci_mps_limit < 128 || pci_mps_limit > 4096 ||
4461 !powerof2(pci_mps_limit)) {
4462 if (!pci_mps_limit_warned) {
4463 printf("pci: invalid hw.pci.mps_limit=%d; ignoring\n",
4464 pci_mps_limit);
4465 pci_mps_limit_warned = true;
4466 }
4467 return (false);
4468 }
4469 *mps = (fls(pci_mps_limit) - 8) << 5;
4470 return (true);
4471 }
4472
4473 /* Return the smallest configured MPS above dev, if the walk reaches a root. */
4474 static bool
pcie_path_mps(device_t dev,uint16_t * mpsp)4475 pcie_path_mps(device_t dev, uint16_t *mpsp)
4476 {
4477 struct pci_devinfo *dinfo;
4478 device_t bus, pcib, start;
4479 uint16_t mps;
4480 bool found;
4481
4482 start = dev;
4483 found = false;
4484 for (;;) {
4485 bus = device_get_parent(dev);
4486 if (bus == NULL)
4487 break;
4488 pcib = device_get_parent(bus);
4489 if (pcib == NULL || !is_pci_device(pcib))
4490 break;
4491 /*
4492 * A PCI function may expose a host bridge for a synthetic PCI
4493 * domain. Its Device Control belongs to the parent domain and
4494 * does not describe an upstream link in the synthetic hierarchy.
4495 */
4496 if (pci_get_domain(pcib) != pci_get_domain(dev))
4497 break;
4498 dinfo = device_get_ivars(pcib);
4499 if (dinfo->cfg.pcie.pcie_location != 0) {
4500 mps = pcie_read_config(pcib, PCIER_DEVICE_CTL, 2) &
4501 PCIEM_CTL_MAX_PAYLOAD;
4502 if (!found || mps < *mpsp)
4503 *mpsp = mps;
4504 found = true;
4505 if (dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
4506 return (true);
4507 }
4508 dev = pcib;
4509 }
4510 if (found && bootverbose)
4511 device_printf(start,
4512 "PCIe MPS path walk did not reach a Root Port\n");
4513 return (false);
4514 }
4515
4516 static bool
pcie_mps_first_warning(device_t dev)4517 pcie_mps_first_warning(device_t dev)
4518 {
4519 struct pci_devinfo *dinfo;
4520
4521 dinfo = device_get_ivars(dev);
4522 if ((dinfo->cfg.flags & PCICFG_MPS_WARNED) != 0)
4523 return (false);
4524 dinfo->cfg.flags |= PCICFG_MPS_WARNED;
4525 return (true);
4526 }
4527
4528 static void
pcie_mps_conflict(device_t dev,uint16_t path_mps,uint16_t max_mps)4529 pcie_mps_conflict(device_t dev, uint16_t path_mps, uint16_t max_mps)
4530 {
4531
4532 if (!pcie_mps_first_warning(dev))
4533 return;
4534 device_printf(dev,
4535 "maximum supported MPS %d is below configured path MPS %d; "
4536 "cannot safely retune the shared ancestor hierarchy\n",
4537 pcie_mps_bytes(max_mps), pcie_mps_bytes(path_mps));
4538 }
4539
4540 static bool
pcie_mps_is_bridge(struct pci_devinfo * dinfo)4541 pcie_mps_is_bridge(struct pci_devinfo *dinfo)
4542 {
4543 uint8_t hdrtype;
4544
4545 hdrtype = dinfo->cfg.hdrtype & PCIM_HDRTYPE;
4546 return (hdrtype == PCIM_HDRTYPE_BRIDGE ||
4547 hdrtype == PCIM_HDRTYPE_CARDBUS);
4548 }
4549
4550 static void
pcie_mps_active_conflict(device_t dev,uint16_t path_mps,uint16_t device_mps)4551 pcie_mps_active_conflict(device_t dev, uint16_t path_mps,
4552 uint16_t device_mps)
4553 {
4554 struct pci_devinfo *dinfo;
4555 const char *action;
4556
4557 if (!pcie_mps_first_warning(dev))
4558 return;
4559 dinfo = device_get_ivars(dev);
4560 if (pci_mps_enforce && !pcie_mps_is_bridge(dinfo))
4561 action = "disabling device";
4562 else
4563 action = "leaving device unchanged";
4564 device_printf(dev,
4565 "configured MPS %d does not match path MPS %d while bus "
4566 "mastering is enabled; %s\n", pcie_mps_bytes(device_mps),
4567 pcie_mps_bytes(path_mps), action);
4568 }
4569
4570 static void
pcie_mps_mark_unreconciled(device_t dev)4571 pcie_mps_mark_unreconciled(device_t dev)
4572 {
4573 struct pci_devinfo *dinfo;
4574 uint16_t cmd;
4575
4576 dinfo = device_get_ivars(dev);
4577 if ((dinfo->cfg.flags & PCICFG_MPS_UNRECONCILED) != 0)
4578 return;
4579 dinfo->cfg.flags |= PCICFG_MPS_UNRECONCILED;
4580 if (!pci_mps_enforce)
4581 return;
4582 if (pcie_mps_is_bridge(dinfo)) {
4583 device_printf(dev,
4584 "not disabled by hw.pci.mps_enforce because it is a bridge\n");
4585 return;
4586 }
4587 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
4588 cmd &= ~(PCIM_CMD_PORTEN | PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
4589 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
4590 dinfo->cfg.cmdreg = cmd;
4591 device_disable(dev);
4592 device_printf(dev,
4593 "disabled because its MPS cannot be safely configured\n");
4594 }
4595
4596 static void
pcie_mps_unreconciled(device_t dev,uint16_t path_mps,uint16_t max_mps)4597 pcie_mps_unreconciled(device_t dev, uint16_t path_mps, uint16_t max_mps)
4598 {
4599
4600 pcie_mps_conflict(dev, path_mps, max_mps);
4601 pcie_mps_mark_unreconciled(dev);
4602 }
4603
4604 static void
pcie_mps_active_unreconciled(device_t dev,uint16_t path_mps,uint16_t device_mps)4605 pcie_mps_active_unreconciled(device_t dev, uint16_t path_mps,
4606 uint16_t device_mps)
4607 {
4608
4609 pcie_mps_active_conflict(dev, path_mps, device_mps);
4610 pcie_mps_mark_unreconciled(dev);
4611 }
4612
4613 static void
pcie_mps_mark_link_unreconciled(device_t * devlist,int count,uint16_t path_mps,bool all)4614 pcie_mps_mark_link_unreconciled(device_t *devlist, int count,
4615 uint16_t path_mps,
4616 bool all)
4617 {
4618 struct pci_devinfo *dinfo;
4619 device_t child;
4620 uint16_t mmps;
4621 int i;
4622
4623 for (i = 0; i < count; i++) {
4624 child = devlist[i];
4625 dinfo = device_get_ivars(child);
4626 if ((dinfo->cfg.flags & (PCICFG_VF |
4627 PCICFG_MPS_UNRECONCILED)) != 0 ||
4628 dinfo->cfg.pcie.pcie_location == 0)
4629 continue;
4630 if (all) {
4631 pcie_mps_mark_unreconciled(child);
4632 continue;
4633 }
4634 mmps = (pcie_read_config(child, PCIER_DEVICE_CAP, 2) &
4635 PCIEM_CAP_MAX_PAYLOAD) << 5;
4636 if (mmps < path_mps)
4637 pcie_mps_unreconciled(child, path_mps, mmps);
4638 }
4639 }
4640
4641 /*
4642 * Tune a function discovered by rescan or hot-add against the established
4643 * path. Never change a shared upstream port here: doing so requires
4644 * quiescing every driver and draining all outstanding transactions in the
4645 * hierarchy. Cold enumeration is reconciled by pcie_reconcile_link_mps().
4646 */
4647 static void
pcie_setup_mps(device_t dev)4648 pcie_setup_mps(device_t dev)
4649 {
4650 struct pci_devinfo *dinfo;
4651 device_t bus;
4652 uint16_t mmps, mps, path_mps;
4653
4654 dinfo = device_get_ivars(dev);
4655 /*
4656 * PCIe r4.0, sec 9.3.5.4 defines the VF MPS and MRRS fields as
4657 * Reserved and Preserved, with the PF settings applying to the VF.
4658 * Do not use the VF's hardwired value to configure the shared path.
4659 */
4660 if ((dinfo->cfg.flags & PCICFG_VF) != 0)
4661 return;
4662 if (dinfo->cfg.pcie.pcie_location == 0)
4663 return;
4664
4665 /* Cold enumeration is reconciled one complete link at a time. */
4666 bus = device_get_parent(dev);
4667 if (!device_is_attached(bus))
4668 return;
4669 path_mps = 0;
4670 if (!pcie_path_mps(dev, &path_mps))
4671 return;
4672
4673 mmps = (pcie_read_config(dev, PCIER_DEVICE_CAP, 2) &
4674 PCIEM_CAP_MAX_PAYLOAD) << 5;
4675 if (path_mps > mmps) {
4676 pcie_mps_unreconciled(dev, path_mps, mmps);
4677 return;
4678 }
4679 mps = pcie_read_config(dev, PCIER_DEVICE_CTL, 2) &
4680 PCIEM_CTL_MAX_PAYLOAD;
4681 if (mps == path_mps)
4682 return;
4683 if ((pci_read_config(dev, PCIR_COMMAND, 2) &
4684 PCIM_CMD_BUSMASTEREN) != 0) {
4685 pcie_mps_active_unreconciled(dev, path_mps, mps);
4686 return;
4687 }
4688 pcie_adjust_config(dev, PCIER_DEVICE_CTL, PCIEM_CTL_MAX_PAYLOAD,
4689 path_mps, 2);
4690 }
4691
4692 /*
4693 * Reconcile a newly enumerated link before attaching any child drivers. A
4694 * Root Port may be lowered because its complete downstream hierarchy is still
4695 * idle. A late reduction below a switch is not propagated through ancestors,
4696 * since sibling subtrees may already be active.
4697 */
4698 void
pcie_reconcile_link_mps(device_t bus)4699 pcie_reconcile_link_mps(device_t bus)
4700 {
4701 struct pci_devinfo *dinfo, *upinfo;
4702 device_t child, limiting, pcib, *devlist;
4703 uint16_t cap_target, lmps, mmps, mps, target, up_mmps, up_mps;
4704 int count, error, i;
4705 bool limit_requested;
4706
4707 if (!pci_enable_mps_tune)
4708 return;
4709 /* Shared-path tuning is only safe before this bus attaches children. */
4710 if (device_is_attached(bus))
4711 return;
4712 pcib = device_get_parent(bus);
4713 if (!is_pci_device(pcib))
4714 return;
4715 /*
4716 * A PCI function may provide a host bridge into a separate domain,
4717 * as Intel VMD does. Do not treat the function's host-facing PCIe
4718 * Device Control as the upstream end of a link in the child domain.
4719 */
4720 if (pci_get_domain(pcib) != pcib_get_domain(bus))
4721 return;
4722 upinfo = device_get_ivars(pcib);
4723 if (upinfo->cfg.pcie.pcie_location == 0)
4724 return;
4725 error = device_get_children(bus, &devlist, &count);
4726 if (error != 0)
4727 return;
4728
4729 up_mps = pcie_read_config(pcib, PCIER_DEVICE_CTL, 2) &
4730 PCIEM_CTL_MAX_PAYLOAD;
4731 cap_target = up_mps;
4732 limiting = NULL;
4733 up_mmps = (pcie_read_config(pcib, PCIER_DEVICE_CAP, 2) &
4734 PCIEM_CAP_MAX_PAYLOAD) << 5;
4735 if (cap_target > up_mmps) {
4736 cap_target = up_mmps;
4737 limiting = pcib;
4738 }
4739 /*
4740 * Firmware may leave Bus Master Enable set after handoff. Since no
4741 * child driver has attached during this cold pass, it is not a proxy
4742 * for a live FreeBSD consumer.
4743 */
4744 for (i = 0; i < count; i++) {
4745 child = devlist[i];
4746 dinfo = device_get_ivars(child);
4747 if ((dinfo->cfg.flags & (PCICFG_VF |
4748 PCICFG_MPS_UNRECONCILED)) != 0 ||
4749 dinfo->cfg.pcie.pcie_location == 0)
4750 continue;
4751 mmps = (pcie_read_config(child, PCIER_DEVICE_CAP, 2) &
4752 PCIEM_CAP_MAX_PAYLOAD) << 5;
4753 if (cap_target > mmps) {
4754 cap_target = mmps;
4755 limiting = child;
4756 }
4757 }
4758 target = cap_target;
4759 limit_requested = pcie_mps_limit_value(&lmps) && up_mps > lmps;
4760 if (limit_requested && target > lmps)
4761 target = lmps;
4762
4763 /*
4764 * Do not lower one link below a switch without also reconciling every
4765 * ancestor and sibling subtree. Recursive newbus attachment may already
4766 * have made another subtree live, so leave the established path intact.
4767 */
4768 if (target < up_mps &&
4769 upinfo->cfg.pcie.pcie_type != PCIEM_TYPE_ROOT_PORT) {
4770 if (cap_target < up_mps)
4771 pcie_mps_conflict(limiting, up_mps, cap_target);
4772 if (limit_requested) {
4773 device_printf(pcib,
4774 "cannot apply hw.pci.mps_limit=%d below a switch "
4775 "without retuning the shared ancestor hierarchy; "
4776 "leaving path MPS %d unchanged\n",
4777 pci_mps_limit, pcie_mps_bytes(up_mps));
4778 }
4779 pcie_mps_mark_link_unreconciled(devlist, count, up_mps,
4780 up_mps > up_mmps);
4781 /* Keep compatible functions at the established path MPS. */
4782 target = up_mps;
4783 }
4784 /* Lower downstream producers before lowering the shared Root Port. */
4785 for (i = 0; i < count; i++) {
4786 child = devlist[i];
4787 dinfo = device_get_ivars(child);
4788 if ((dinfo->cfg.flags & (PCICFG_VF |
4789 PCICFG_MPS_UNRECONCILED)) != 0 ||
4790 dinfo->cfg.pcie.pcie_location == 0)
4791 continue;
4792 mps = pcie_read_config(child, PCIER_DEVICE_CTL, 2) &
4793 PCIEM_CTL_MAX_PAYLOAD;
4794 if (mps > target)
4795 pcie_adjust_config(child, PCIER_DEVICE_CTL,
4796 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4797 }
4798 if (up_mps > target)
4799 pcie_adjust_config(pcib, PCIER_DEVICE_CTL,
4800 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4801
4802 /* Raise idle children only after the upstream port is configured. */
4803 for (i = 0; i < count; i++) {
4804 child = devlist[i];
4805 dinfo = device_get_ivars(child);
4806 if ((dinfo->cfg.flags & (PCICFG_VF |
4807 PCICFG_MPS_UNRECONCILED)) != 0 ||
4808 dinfo->cfg.pcie.pcie_location == 0)
4809 continue;
4810 mps = pcie_read_config(child, PCIER_DEVICE_CTL, 2) &
4811 PCIEM_CTL_MAX_PAYLOAD;
4812 if (mps < target)
4813 pcie_adjust_config(child, PCIER_DEVICE_CTL,
4814 PCIEM_CTL_MAX_PAYLOAD, target, 2);
4815 }
4816 free(devlist, M_TEMP);
4817 }
4818
4819 static void
pci_add_child_clear_aer(device_t dev,struct pci_devinfo * dinfo)4820 pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo)
4821 {
4822 int aer;
4823 uint32_t r;
4824
4825 if (dinfo->cfg.pcie.pcie_location != 0 &&
4826 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) {
4827 r = PCIEM_ROOT_CTL_SERR_CORR |
4828 PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL;
4829 pcie_adjust_config(dev, PCIER_ROOT_CTL, r, 0, 2);
4830 }
4831 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
4832 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
4833 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
4834 if (r != 0 && bootverbose) {
4835 pci_printf(&dinfo->cfg,
4836 "clearing AER UC 0x%08x -> 0x%08x\n",
4837 r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS,
4838 4));
4839 }
4840
4841 r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4);
4842 r &= ~(PCIM_AER_UC_TRAINING_ERROR |
4843 PCIM_AER_UC_DL_PROTOCOL_ERROR |
4844 PCIM_AER_UC_SURPRISE_LINK_DOWN |
4845 PCIM_AER_UC_POISONED_TLP |
4846 PCIM_AER_UC_FC_PROTOCOL_ERROR |
4847 PCIM_AER_UC_COMPLETION_TIMEOUT |
4848 PCIM_AER_UC_COMPLETER_ABORT |
4849 PCIM_AER_UC_UNEXPECTED_COMPLETION |
4850 PCIM_AER_UC_RECEIVER_OVERFLOW |
4851 PCIM_AER_UC_MALFORMED_TLP |
4852 PCIM_AER_UC_ECRC_ERROR |
4853 PCIM_AER_UC_UNSUPPORTED_REQUEST |
4854 PCIM_AER_UC_ACS_VIOLATION |
4855 PCIM_AER_UC_INTERNAL_ERROR |
4856 PCIM_AER_UC_MC_BLOCKED_TLP |
4857 PCIM_AER_UC_ATOMIC_EGRESS_BLK |
4858 PCIM_AER_UC_TLP_PREFIX_BLOCKED);
4859 pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4);
4860
4861 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
4862 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
4863 if (r != 0 && bootverbose) {
4864 pci_printf(&dinfo->cfg,
4865 "clearing AER COR 0x%08x -> 0x%08x\n",
4866 r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS,
4867 4));
4868 }
4869
4870 r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4);
4871 r &= ~(PCIM_AER_COR_RECEIVER_ERROR |
4872 PCIM_AER_COR_BAD_TLP |
4873 PCIM_AER_COR_BAD_DLLP |
4874 PCIM_AER_COR_REPLAY_ROLLOVER |
4875 PCIM_AER_COR_REPLAY_TIMEOUT |
4876 PCIM_AER_COR_ADVISORY_NF_ERROR |
4877 PCIM_AER_COR_INTERNAL_ERROR |
4878 PCIM_AER_COR_HEADER_LOG_OVFLOW);
4879 pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4);
4880
4881 r = PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE |
4882 PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE;
4883 pcie_adjust_config(dev, PCIER_DEVICE_CTL, r, r, 2);
4884 }
4885 }
4886
4887 void
pci_add_child(device_t bus,struct pci_devinfo * dinfo)4888 pci_add_child(device_t bus, struct pci_devinfo *dinfo)
4889 {
4890 device_t dev;
4891
4892 dinfo->cfg.dev = dev = device_add_child(bus, NULL, DEVICE_UNIT_ANY);
4893 device_set_ivars(dev, dinfo);
4894 resource_list_init(&dinfo->resources);
4895 pci_cfg_save(dev, dinfo, 0);
4896 pci_cfg_restore(dev, dinfo);
4897 pci_clear_pme(dev);
4898 pci_print_verbose(dinfo);
4899 pci_add_resources(bus, dev, 0, 0);
4900 if (pci_enable_mps_tune)
4901 pcie_setup_mps(dev);
4902 pci_child_added(dinfo->cfg.dev);
4903
4904 if (pci_clear_aer_on_attach)
4905 pci_add_child_clear_aer(dev, dinfo);
4906
4907 EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
4908 }
4909
4910 void
pci_child_added_method(device_t dev,device_t child)4911 pci_child_added_method(device_t dev, device_t child)
4912 {
4913
4914 }
4915
4916 static int
pci_probe(device_t dev)4917 pci_probe(device_t dev)
4918 {
4919
4920 device_set_desc(dev, "PCI bus");
4921
4922 /* Allow other subclasses to override this driver. */
4923 return (BUS_PROBE_GENERIC);
4924 }
4925
4926 int
pci_attach_common(device_t dev)4927 pci_attach_common(device_t dev)
4928 {
4929 struct pci_softc *sc;
4930 int busno, domain;
4931 int rid;
4932
4933 sc = device_get_softc(dev);
4934 domain = pcib_get_domain(dev);
4935 busno = pcib_get_bus(dev);
4936 rid = 0;
4937 sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno,
4938 1, 0);
4939 if (sc->sc_bus == NULL) {
4940 device_printf(dev, "failed to allocate bus number\n");
4941 return (ENXIO);
4942 }
4943 if (bootverbose)
4944 device_printf(dev, "domain=%d, physical bus=%d\n",
4945 domain, busno);
4946 sc->sc_dma_tag = bus_get_dma_tag(dev);
4947 return (0);
4948 }
4949
4950 int
pci_attach(device_t dev)4951 pci_attach(device_t dev)
4952 {
4953 int busno, domain, error;
4954
4955 error = pci_attach_common(dev);
4956 if (error)
4957 return (error);
4958
4959 /*
4960 * Since there can be multiple independently numbered PCI
4961 * buses on systems with multiple PCI domains, we can't use
4962 * the unit number to decide which bus we are probing. We ask
4963 * the parent pcib what our domain and bus numbers are.
4964 */
4965 domain = pcib_get_domain(dev);
4966 busno = pcib_get_bus(dev);
4967 pci_add_children(dev, domain, busno);
4968 bus_attach_children(dev);
4969 return (0);
4970 }
4971
4972 int
pci_detach(device_t dev)4973 pci_detach(device_t dev)
4974 {
4975 struct pci_softc *sc;
4976 int error;
4977
4978 error = bus_generic_detach(dev);
4979 if (error)
4980 return (error);
4981 sc = device_get_softc(dev);
4982 error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus);
4983 return (error);
4984 }
4985
4986 static void
pci_hint_device_unit(device_t dev,device_t child,const char * name,int * unitp)4987 pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp)
4988 {
4989 int line, unit;
4990 const char *at;
4991 char me1[24], me2[32];
4992 uint8_t b, s, f;
4993 uint32_t d;
4994 device_location_cache_t *cache;
4995
4996 d = pci_get_domain(child);
4997 b = pci_get_bus(child);
4998 s = pci_get_slot(child);
4999 f = pci_get_function(child);
5000 snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f);
5001 snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f);
5002 line = 0;
5003 cache = dev_wired_cache_init();
5004 while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) {
5005 resource_string_value(name, unit, "at", &at);
5006 if (strcmp(at, me1) == 0 || strcmp(at, me2) == 0) {
5007 *unitp = unit;
5008 break;
5009 }
5010 if (dev_wired_cache_match(cache, child, at)) {
5011 *unitp = unit;
5012 break;
5013 }
5014 }
5015 dev_wired_cache_fini(cache);
5016 }
5017
5018 static void
pci_set_power_child(device_t dev,device_t child,int state)5019 pci_set_power_child(device_t dev, device_t child, int state)
5020 {
5021 device_t pcib;
5022 int dstate;
5023
5024 /*
5025 * Set the device to the given state. If the firmware suggests
5026 * a different power state, use it instead. If power management
5027 * is not present, the firmware is responsible for managing
5028 * device power. Skip children who aren't attached since they
5029 * are handled separately.
5030 */
5031 pcib = device_get_parent(dev);
5032 dstate = state;
5033 if (device_is_attached(child) &&
5034 PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0)
5035 pci_set_powerstate(child, dstate);
5036 }
5037
5038 int
pci_suspend_child(device_t dev,device_t child)5039 pci_suspend_child(device_t dev, device_t child)
5040 {
5041 struct pci_devinfo *dinfo;
5042 struct resource_list_entry *rle;
5043 int error;
5044
5045 dinfo = device_get_ivars(child);
5046
5047 /*
5048 * Save the PCI configuration space for the child and set the
5049 * device in the appropriate power state for this sleep state.
5050 */
5051 pci_cfg_save(child, dinfo, 0);
5052
5053 /* Suspend devices before potentially powering them down. */
5054 error = bus_generic_suspend_child(dev, child);
5055
5056 if (error)
5057 return (error);
5058
5059 if (pci_do_power_suspend) {
5060 /*
5061 * Make sure this device's interrupt handler is not invoked
5062 * in the case the device uses a shared interrupt that can
5063 * be raised by some other device.
5064 * This is applicable only to regular (legacy) PCI interrupts
5065 * as MSI/MSI-X interrupts are never shared.
5066 */
5067 rle = resource_list_find(&dinfo->resources,
5068 SYS_RES_IRQ, 0);
5069 if (rle != NULL && rle->res != NULL)
5070 (void)bus_suspend_intr(child, rle->res);
5071 pci_set_power_child(dev, child, PCI_POWERSTATE_D3);
5072 }
5073
5074 return (0);
5075 }
5076
5077 int
pci_resume_child(device_t dev,device_t child)5078 pci_resume_child(device_t dev, device_t child)
5079 {
5080 struct pci_devinfo *dinfo;
5081 struct resource_list_entry *rle;
5082
5083 if (pci_do_power_resume)
5084 pci_set_power_child(dev, child, PCI_POWERSTATE_D0);
5085
5086 dinfo = device_get_ivars(child);
5087 pci_cfg_restore(child, dinfo);
5088 pci_clear_pme(child);
5089 if (!device_is_attached(child))
5090 pci_cfg_save(child, dinfo, 1);
5091
5092 bus_generic_resume_child(dev, child);
5093
5094 /*
5095 * Allow interrupts only after fully resuming the driver and hardware.
5096 */
5097 if (pci_do_power_suspend) {
5098 /* See pci_suspend_child for details. */
5099 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
5100 if (rle != NULL && rle->res != NULL)
5101 (void)bus_resume_intr(child, rle->res);
5102 }
5103
5104 return (0);
5105 }
5106
5107 int
pci_resume(device_t dev)5108 pci_resume(device_t dev)
5109 {
5110 device_t child, *devlist;
5111 int error, i, numdevs;
5112
5113 if ((error = device_get_children(dev, &devlist, &numdevs)) != 0)
5114 return (error);
5115
5116 /*
5117 * Resume critical devices first, then everything else later.
5118 */
5119 for (i = 0; i < numdevs; i++) {
5120 child = devlist[i];
5121 switch (pci_get_class(child)) {
5122 case PCIC_DISPLAY:
5123 case PCIC_MEMORY:
5124 case PCIC_BRIDGE:
5125 case PCIC_BASEPERIPH:
5126 BUS_RESUME_CHILD(dev, child);
5127 break;
5128 }
5129 }
5130 for (i = 0; i < numdevs; i++) {
5131 child = devlist[i];
5132 switch (pci_get_class(child)) {
5133 case PCIC_DISPLAY:
5134 case PCIC_MEMORY:
5135 case PCIC_BRIDGE:
5136 case PCIC_BASEPERIPH:
5137 break;
5138 default:
5139 BUS_RESUME_CHILD(dev, child);
5140 }
5141 }
5142 free(devlist, M_TEMP);
5143 return (0);
5144 }
5145
5146 static void
pci_load_vendor_data(void)5147 pci_load_vendor_data(void)
5148 {
5149 caddr_t data;
5150 void *ptr;
5151 size_t sz;
5152
5153 data = preload_search_by_type("pci_vendor_data");
5154 if (data != NULL) {
5155 ptr = preload_fetch_addr(data);
5156 sz = preload_fetch_size(data);
5157 if (ptr != NULL && sz != 0) {
5158 pci_vendordata = ptr;
5159 pci_vendordata_size = sz;
5160 /* terminate the database */
5161 pci_vendordata[pci_vendordata_size] = '\n';
5162 }
5163 }
5164 }
5165
5166 void
pci_driver_added(device_t dev,driver_t * driver)5167 pci_driver_added(device_t dev, driver_t *driver)
5168 {
5169 int numdevs;
5170 device_t *devlist;
5171 device_t child;
5172 struct pci_devinfo *dinfo;
5173 int i;
5174
5175 if (bootverbose)
5176 device_printf(dev, "driver added\n");
5177 DEVICE_IDENTIFY(driver, dev);
5178 if (device_get_children(dev, &devlist, &numdevs) != 0)
5179 return;
5180 for (i = 0; i < numdevs; i++) {
5181 child = devlist[i];
5182 if (device_get_state(child) != DS_NOTPRESENT)
5183 continue;
5184 dinfo = device_get_ivars(child);
5185 pci_print_verbose(dinfo);
5186 if (bootverbose)
5187 pci_printf(&dinfo->cfg, "reprobing on driver added\n");
5188 pci_cfg_restore(child, dinfo);
5189 if (device_probe_and_attach(child) != 0)
5190 pci_child_detached(dev, child);
5191 }
5192 free(devlist, M_TEMP);
5193 }
5194
5195 int
pci_setup_intr(device_t dev,device_t child,struct resource * irq,int flags,driver_filter_t * filter,driver_intr_t * intr,void * arg,void ** cookiep)5196 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
5197 driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep)
5198 {
5199 struct pci_devinfo *dinfo;
5200 struct msix_table_entry *mte;
5201 struct msix_vector *mv;
5202 uint64_t addr;
5203 uint32_t data;
5204 void *cookie;
5205 int error, rid;
5206
5207 error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr,
5208 arg, &cookie);
5209 if (error)
5210 return (error);
5211
5212 /* If this is not a direct child, just bail out. */
5213 if (device_get_parent(child) != dev) {
5214 *cookiep = cookie;
5215 return(0);
5216 }
5217
5218 rid = rman_get_rid(irq);
5219 if (rid == 0) {
5220 /* Make sure that INTx is enabled */
5221 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
5222 } else {
5223 /*
5224 * Check to see if the interrupt is MSI or MSI-X.
5225 * Ask our parent to map the MSI and give
5226 * us the address and data register values.
5227 * If we fail for some reason, teardown the
5228 * interrupt handler.
5229 */
5230 dinfo = device_get_ivars(child);
5231 if (dinfo->cfg.msi.msi_alloc > 0) {
5232 if (dinfo->cfg.msi.msi_addr == 0) {
5233 KASSERT(dinfo->cfg.msi.msi_handlers == 0,
5234 ("MSI has handlers, but vectors not mapped"));
5235 error = PCIB_MAP_MSI(device_get_parent(dev),
5236 child, rman_get_start(irq), &addr, &data);
5237 if (error)
5238 goto bad;
5239 dinfo->cfg.msi.msi_addr = addr;
5240 dinfo->cfg.msi.msi_data = data;
5241 }
5242 if (dinfo->cfg.msi.msi_handlers == 0)
5243 pci_enable_msi(child, dinfo->cfg.msi.msi_addr,
5244 dinfo->cfg.msi.msi_data);
5245 dinfo->cfg.msi.msi_handlers++;
5246 } else {
5247 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
5248 ("No MSI or MSI-X interrupts allocated"));
5249 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
5250 ("MSI-X index too high"));
5251 mte = &dinfo->cfg.msix.msix_table[rid - 1];
5252 KASSERT(mte->mte_vector != 0, ("no message vector"));
5253 mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
5254 KASSERT(mv->mv_irq == rman_get_start(irq),
5255 ("IRQ mismatch"));
5256 if (mv->mv_address == 0) {
5257 KASSERT(mte->mte_handlers == 0,
5258 ("MSI-X table entry has handlers, but vector not mapped"));
5259 error = PCIB_MAP_MSI(device_get_parent(dev),
5260 child, rman_get_start(irq), &addr, &data);
5261 if (error)
5262 goto bad;
5263 mv->mv_address = addr;
5264 mv->mv_data = data;
5265 }
5266
5267 /*
5268 * The MSIX table entry must be made valid by
5269 * incrementing the mte_handlers before
5270 * calling pci_enable_msix() and
5271 * pci_resume_msix(). Else the MSIX rewrite
5272 * table quirk will not work as expected.
5273 */
5274 mte->mte_handlers++;
5275 if (mte->mte_handlers == 1) {
5276 pci_enable_msix(child, rid - 1, mv->mv_address,
5277 mv->mv_data);
5278 pci_unmask_msix(child, rid - 1);
5279 }
5280 }
5281
5282 /*
5283 * Make sure that INTx is disabled if we are using MSI/MSI-X,
5284 * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG,
5285 * in which case we "enable" INTx so MSI/MSI-X actually works.
5286 */
5287 if (!pci_has_quirk(pci_get_devid(child),
5288 PCI_QUIRK_MSI_INTX_BUG))
5289 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
5290 else
5291 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
5292 bad:
5293 if (error) {
5294 (void)bus_generic_teardown_intr(dev, child, irq,
5295 cookie);
5296 return (error);
5297 }
5298 }
5299 *cookiep = cookie;
5300 return (0);
5301 }
5302
5303 int
pci_teardown_intr(device_t dev,device_t child,struct resource * irq,void * cookie)5304 pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
5305 void *cookie)
5306 {
5307 struct msix_table_entry *mte;
5308 struct resource_list_entry *rle;
5309 struct pci_devinfo *dinfo;
5310 int error, rid;
5311
5312 if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
5313 return (EINVAL);
5314
5315 /* If this isn't a direct child, just bail out */
5316 if (device_get_parent(child) != dev)
5317 return(bus_generic_teardown_intr(dev, child, irq, cookie));
5318
5319 rid = rman_get_rid(irq);
5320 if (rid == 0) {
5321 /* Mask INTx */
5322 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
5323 } else {
5324 /*
5325 * Check to see if the interrupt is MSI or MSI-X. If so,
5326 * decrement the appropriate handlers count and mask the
5327 * MSI-X message, or disable MSI messages if the count
5328 * drops to 0.
5329 */
5330 dinfo = device_get_ivars(child);
5331 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
5332 if (rle->res != irq)
5333 return (EINVAL);
5334 if (dinfo->cfg.msi.msi_alloc > 0) {
5335 KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
5336 ("MSI-X index too high"));
5337 if (dinfo->cfg.msi.msi_handlers == 0)
5338 return (EINVAL);
5339 dinfo->cfg.msi.msi_handlers--;
5340 if (dinfo->cfg.msi.msi_handlers == 0)
5341 pci_disable_msi(child);
5342 } else {
5343 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
5344 ("No MSI or MSI-X interrupts allocated"));
5345 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
5346 ("MSI-X index too high"));
5347 mte = &dinfo->cfg.msix.msix_table[rid - 1];
5348 if (mte->mte_handlers == 0)
5349 return (EINVAL);
5350 mte->mte_handlers--;
5351 if (mte->mte_handlers == 0)
5352 pci_mask_msix(child, rid - 1);
5353 }
5354 }
5355 error = bus_generic_teardown_intr(dev, child, irq, cookie);
5356 if (rid > 0)
5357 KASSERT(error == 0,
5358 ("%s: generic teardown failed for MSI/MSI-X", __func__));
5359 return (error);
5360 }
5361
5362 int
pci_print_child(device_t dev,device_t child)5363 pci_print_child(device_t dev, device_t child)
5364 {
5365 struct pci_devinfo *dinfo;
5366 struct resource_list *rl;
5367 int retval = 0;
5368
5369 dinfo = device_get_ivars(child);
5370 rl = &dinfo->resources;
5371
5372 retval += bus_print_child_header(dev, child);
5373
5374 retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
5375 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
5376 retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
5377 if (device_get_flags(dev))
5378 retval += printf(" flags %#x", device_get_flags(dev));
5379
5380 retval += printf(" at device %d.%d", pci_get_slot(child),
5381 pci_get_function(child));
5382
5383 retval += bus_print_child_domain(dev, child);
5384 retval += bus_print_child_footer(dev, child);
5385
5386 return (retval);
5387 }
5388
5389 static const struct
5390 {
5391 int class;
5392 int subclass;
5393 int report; /* 0 = bootverbose, 1 = always */
5394 const char *desc;
5395 } pci_nomatch_tab[] = {
5396 {PCIC_OLD, -1, 1, "old"},
5397 {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"},
5398 {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"},
5399 {PCIC_STORAGE, -1, 1, "mass storage"},
5400 {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"},
5401 {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"},
5402 {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"},
5403 {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"},
5404 {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"},
5405 {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"},
5406 {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"},
5407 {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"},
5408 {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"},
5409 {PCIC_NETWORK, -1, 1, "network"},
5410 {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"},
5411 {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"},
5412 {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"},
5413 {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"},
5414 {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"},
5415 {PCIC_DISPLAY, -1, 1, "display"},
5416 {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"},
5417 {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"},
5418 {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"},
5419 {PCIC_MULTIMEDIA, -1, 1, "multimedia"},
5420 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"},
5421 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"},
5422 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"},
5423 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"},
5424 {PCIC_MEMORY, -1, 1, "memory"},
5425 {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"},
5426 {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"},
5427 {PCIC_BRIDGE, -1, 1, "bridge"},
5428 {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"},
5429 {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"},
5430 {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"},
5431 {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"},
5432 {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"},
5433 {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"},
5434 {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"},
5435 {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"},
5436 {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"},
5437 {PCIC_SIMPLECOMM, -1, 1, "simple comms"},
5438 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */
5439 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"},
5440 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"},
5441 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"},
5442 {PCIC_BASEPERIPH, -1, 0, "base peripheral"},
5443 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"},
5444 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"},
5445 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"},
5446 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"},
5447 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"},
5448 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"},
5449 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"},
5450 {PCIC_INPUTDEV, -1, 1, "input device"},
5451 {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"},
5452 {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"},
5453 {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"},
5454 {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"},
5455 {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"},
5456 {PCIC_DOCKING, -1, 1, "docking station"},
5457 {PCIC_PROCESSOR, -1, 1, "processor"},
5458 {PCIC_SERIALBUS, -1, 1, "serial bus"},
5459 {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"},
5460 {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"},
5461 {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"},
5462 {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"},
5463 {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"},
5464 {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"},
5465 {PCIC_WIRELESS, -1, 1, "wireless controller"},
5466 {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"},
5467 {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"},
5468 {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"},
5469 {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"},
5470 {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"},
5471 {PCIC_SATCOM, -1, 1, "satellite communication"},
5472 {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"},
5473 {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"},
5474 {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"},
5475 {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"},
5476 {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"},
5477 {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"},
5478 {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"},
5479 {PCIC_DASP, -1, 0, "dasp"},
5480 {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"},
5481 {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"},
5482 {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"},
5483 {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"},
5484 {PCIC_INSTRUMENT, -1, 0, "non-essential instrumentation"},
5485 {0, 0, 0, NULL}
5486 };
5487
5488 void
pci_probe_nomatch(device_t dev,device_t child)5489 pci_probe_nomatch(device_t dev, device_t child)
5490 {
5491 int i, report;
5492 const char *cp, *scp;
5493 char *device;
5494
5495 /*
5496 * Look for a listing for this device in a loaded device database.
5497 */
5498 report = 1;
5499 if ((device = pci_describe_device(child)) != NULL) {
5500 device_printf(dev, "<%s>", device);
5501 free(device, M_DEVBUF);
5502 } else {
5503 /*
5504 * Scan the class/subclass descriptions for a general
5505 * description.
5506 */
5507 cp = "unknown";
5508 scp = NULL;
5509 for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) {
5510 if (pci_nomatch_tab[i].class == pci_get_class(child)) {
5511 if (pci_nomatch_tab[i].subclass == -1) {
5512 cp = pci_nomatch_tab[i].desc;
5513 report = pci_nomatch_tab[i].report;
5514 } else if (pci_nomatch_tab[i].subclass ==
5515 pci_get_subclass(child)) {
5516 scp = pci_nomatch_tab[i].desc;
5517 report = pci_nomatch_tab[i].report;
5518 }
5519 }
5520 }
5521 if (report || bootverbose) {
5522 device_printf(dev, "<%s%s%s>",
5523 cp ? cp : "",
5524 ((cp != NULL) && (scp != NULL)) ? ", " : "",
5525 scp ? scp : "");
5526 }
5527 }
5528 if (report || bootverbose) {
5529 printf(" at device %d.%d (no driver attached)\n",
5530 pci_get_slot(child), pci_get_function(child));
5531 }
5532 pci_cfg_save(child, device_get_ivars(child), 1);
5533 }
5534
5535 void
pci_child_detached(device_t dev,device_t child)5536 pci_child_detached(device_t dev, device_t child)
5537 {
5538 struct pci_devinfo *dinfo;
5539 struct resource_list *rl;
5540
5541 dinfo = device_get_ivars(child);
5542 rl = &dinfo->resources;
5543
5544 /*
5545 * Have to deallocate IRQs before releasing any MSI messages and
5546 * have to release MSI messages before deallocating any memory
5547 * BARs.
5548 */
5549 if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0)
5550 pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n");
5551 if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) {
5552 if (dinfo->cfg.msi.msi_alloc != 0)
5553 pci_printf(&dinfo->cfg, "Device leaked %d MSI "
5554 "vectors\n", dinfo->cfg.msi.msi_alloc);
5555 else
5556 pci_printf(&dinfo->cfg, "Device leaked %d MSI-X "
5557 "vectors\n", dinfo->cfg.msix.msix_alloc);
5558 (void)pci_release_msi(child);
5559 }
5560 if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0)
5561 pci_printf(&dinfo->cfg, "Device leaked memory resources\n");
5562 if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0)
5563 pci_printf(&dinfo->cfg, "Device leaked I/O resources\n");
5564 if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0)
5565 pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n");
5566
5567 pci_cfg_save(child, dinfo, 1);
5568 }
5569
5570 /*
5571 * Parse the PCI device database, if loaded, and return a pointer to a
5572 * description of the device.
5573 *
5574 * The database is flat text formatted as follows:
5575 *
5576 * Any line not in a valid format is ignored.
5577 * Lines are terminated with newline '\n' characters.
5578 *
5579 * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then
5580 * the vendor name.
5581 *
5582 * A DEVICE line is entered immediately below the corresponding VENDOR ID.
5583 * - devices cannot be listed without a corresponding VENDOR line.
5584 * A DEVICE line consists of a TAB, the 4 digit (hex) device code,
5585 * another TAB, then the device name.
5586 */
5587
5588 /*
5589 * Assuming (ptr) points to the beginning of a line in the database,
5590 * return the vendor or device and description of the next entry.
5591 * The value of (vendor) or (device) inappropriate for the entry type
5592 * is set to -1. Returns nonzero at the end of the database.
5593 *
5594 * Note that this is slightly unrobust in the face of corrupt data;
5595 * we attempt to safeguard against this by spamming the end of the
5596 * database with a newline when we initialise.
5597 */
5598 static int
pci_describe_parse_line(char ** ptr,int * vendor,int * device,char ** desc)5599 pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc)
5600 {
5601 char *cp = *ptr;
5602 int left;
5603
5604 *device = -1;
5605 *vendor = -1;
5606 **desc = '\0';
5607 for (;;) {
5608 left = pci_vendordata_size - (cp - pci_vendordata);
5609 if (left <= 0) {
5610 *ptr = cp;
5611 return(1);
5612 }
5613
5614 /* vendor entry? */
5615 if (*cp != '\t' &&
5616 sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2)
5617 break;
5618 /* device entry? */
5619 if (*cp == '\t' &&
5620 sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2)
5621 break;
5622
5623 /* skip to next line */
5624 while (*cp != '\n' && left > 0) {
5625 cp++;
5626 left--;
5627 }
5628 if (*cp == '\n') {
5629 cp++;
5630 left--;
5631 }
5632 }
5633 /* skip to next line */
5634 while (*cp != '\n' && left > 0) {
5635 cp++;
5636 left--;
5637 }
5638 if (*cp == '\n' && left > 0)
5639 cp++;
5640 *ptr = cp;
5641 return(0);
5642 }
5643
5644 static char *
pci_describe_device(device_t dev)5645 pci_describe_device(device_t dev)
5646 {
5647 int vendor, device;
5648 char *desc, *vp, *dp, *line;
5649
5650 desc = vp = dp = NULL;
5651
5652 /*
5653 * If we have no vendor data, we can't do anything.
5654 */
5655 if (pci_vendordata == NULL)
5656 goto out;
5657
5658 /*
5659 * Scan the vendor data looking for this device
5660 */
5661 line = pci_vendordata;
5662 if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5663 goto out;
5664 for (;;) {
5665 if (pci_describe_parse_line(&line, &vendor, &device, &vp))
5666 goto out;
5667 if (vendor == pci_get_vendor(dev))
5668 break;
5669 }
5670 if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5671 goto out;
5672 for (;;) {
5673 if (pci_describe_parse_line(&line, &vendor, &device, &dp)) {
5674 *dp = 0;
5675 break;
5676 }
5677 if (vendor != -1) {
5678 *dp = 0;
5679 break;
5680 }
5681 if (device == pci_get_device(dev))
5682 break;
5683 }
5684 if (dp[0] == '\0')
5685 snprintf(dp, 80, "0x%x", pci_get_device(dev));
5686 if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) !=
5687 NULL)
5688 sprintf(desc, "%s, %s", vp, dp);
5689 out:
5690 if (vp != NULL)
5691 free(vp, M_DEVBUF);
5692 if (dp != NULL)
5693 free(dp, M_DEVBUF);
5694 return(desc);
5695 }
5696
5697 int
pci_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)5698 pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
5699 {
5700 struct pci_devinfo *dinfo;
5701 pcicfgregs *cfg;
5702
5703 dinfo = device_get_ivars(child);
5704 cfg = &dinfo->cfg;
5705
5706 switch (which) {
5707 case PCI_IVAR_ETHADDR:
5708 /*
5709 * The generic accessor doesn't deal with failure, so
5710 * we set the return value, then return an error.
5711 */
5712 *((uint8_t **) result) = NULL;
5713 return (EINVAL);
5714 case PCI_IVAR_SUBVENDOR:
5715 *result = cfg->subvendor;
5716 break;
5717 case PCI_IVAR_SUBDEVICE:
5718 *result = cfg->subdevice;
5719 break;
5720 case PCI_IVAR_VENDOR:
5721 *result = cfg->vendor;
5722 break;
5723 case PCI_IVAR_DEVICE:
5724 *result = cfg->device;
5725 break;
5726 case PCI_IVAR_DEVID:
5727 *result = (cfg->device << 16) | cfg->vendor;
5728 break;
5729 case PCI_IVAR_CLASS:
5730 *result = cfg->baseclass;
5731 break;
5732 case PCI_IVAR_SUBCLASS:
5733 *result = cfg->subclass;
5734 break;
5735 case PCI_IVAR_PROGIF:
5736 *result = cfg->progif;
5737 break;
5738 case PCI_IVAR_REVID:
5739 *result = cfg->revid;
5740 break;
5741 case PCI_IVAR_INTPIN:
5742 *result = cfg->intpin;
5743 break;
5744 case PCI_IVAR_IRQ:
5745 *result = cfg->intline;
5746 break;
5747 case PCI_IVAR_DOMAIN:
5748 *result = cfg->domain;
5749 break;
5750 case PCI_IVAR_BUS:
5751 *result = cfg->bus;
5752 break;
5753 case PCI_IVAR_SLOT:
5754 *result = cfg->slot;
5755 break;
5756 case PCI_IVAR_FUNCTION:
5757 *result = cfg->func;
5758 break;
5759 case PCI_IVAR_CMDREG:
5760 *result = cfg->cmdreg;
5761 break;
5762 case PCI_IVAR_CACHELNSZ:
5763 *result = cfg->cachelnsz;
5764 break;
5765 case PCI_IVAR_MINGNT:
5766 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5767 *result = -1;
5768 return (EINVAL);
5769 }
5770 *result = cfg->mingnt;
5771 break;
5772 case PCI_IVAR_MAXLAT:
5773 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5774 *result = -1;
5775 return (EINVAL);
5776 }
5777 *result = cfg->maxlat;
5778 break;
5779 case PCI_IVAR_LATTIMER:
5780 *result = cfg->lattimer;
5781 break;
5782 default:
5783 return (ENOENT);
5784 }
5785 return (0);
5786 }
5787
5788 int
pci_write_ivar(device_t dev,device_t child,int which,uintptr_t value)5789 pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
5790 {
5791 struct pci_devinfo *dinfo;
5792
5793 dinfo = device_get_ivars(child);
5794
5795 switch (which) {
5796 case PCI_IVAR_INTPIN:
5797 dinfo->cfg.intpin = value;
5798 return (0);
5799 case PCI_IVAR_ETHADDR:
5800 case PCI_IVAR_SUBVENDOR:
5801 case PCI_IVAR_SUBDEVICE:
5802 case PCI_IVAR_VENDOR:
5803 case PCI_IVAR_DEVICE:
5804 case PCI_IVAR_DEVID:
5805 case PCI_IVAR_CLASS:
5806 case PCI_IVAR_SUBCLASS:
5807 case PCI_IVAR_PROGIF:
5808 case PCI_IVAR_REVID:
5809 case PCI_IVAR_IRQ:
5810 case PCI_IVAR_DOMAIN:
5811 case PCI_IVAR_BUS:
5812 case PCI_IVAR_SLOT:
5813 case PCI_IVAR_FUNCTION:
5814 return (EINVAL); /* disallow for now */
5815
5816 default:
5817 return (ENOENT);
5818 }
5819 }
5820
5821 #include "opt_ddb.h"
5822 #ifdef DDB
5823 #include <ddb/ddb.h>
5824 #include <sys/cons.h>
5825
5826 /*
5827 * List resources based on pci map registers, used for within ddb
5828 */
5829
DB_SHOW_COMMAND_FLAGS(pciregs,db_pci_dump,DB_CMD_MEMSAFE)5830 DB_SHOW_COMMAND_FLAGS(pciregs, db_pci_dump, DB_CMD_MEMSAFE)
5831 {
5832 struct pci_devinfo *dinfo;
5833 struct devlist *devlist_head;
5834 struct pci_conf *p;
5835 const char *name;
5836 int i, error, none_count;
5837
5838 none_count = 0;
5839 /* get the head of the device queue */
5840 devlist_head = &pci_devq;
5841
5842 /*
5843 * Go through the list of devices and print out devices
5844 */
5845 for (error = 0, i = 0,
5846 dinfo = STAILQ_FIRST(devlist_head);
5847 (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit;
5848 dinfo = STAILQ_NEXT(dinfo, pci_links), i++) {
5849 /* Populate pd_name and pd_unit */
5850 name = NULL;
5851 if (dinfo->cfg.dev)
5852 name = device_get_name(dinfo->cfg.dev);
5853
5854 p = &dinfo->conf;
5855 db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x "
5856 "chip=0x%08x rev=0x%02x hdr=0x%02x\n",
5857 (name && *name) ? name : "none",
5858 (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) :
5859 none_count++,
5860 p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev,
5861 p->pc_sel.pc_func, (p->pc_class << 16) |
5862 (p->pc_subclass << 8) | p->pc_progif,
5863 (p->pc_subdevice << 16) | p->pc_subvendor,
5864 (p->pc_device << 16) | p->pc_vendor,
5865 p->pc_revid, p->pc_hdr);
5866 }
5867 }
5868 #endif /* DDB */
5869
5870 struct resource *
pci_reserve_map(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int num,u_int flags)5871 pci_reserve_map(device_t dev, device_t child, int type, int rid,
5872 rman_res_t start, rman_res_t end, rman_res_t count, u_int num,
5873 u_int flags)
5874 {
5875 struct pci_devinfo *dinfo = device_get_ivars(child);
5876 struct resource_list *rl = &dinfo->resources;
5877 struct resource *res;
5878 struct pci_map *pm;
5879 uint16_t cmd;
5880 pci_addr_t map, testval;
5881 int mapsize;
5882
5883 res = NULL;
5884
5885 /* If rid is managed by EA, ignore it */
5886 if (pci_ea_is_enabled(child, rid))
5887 goto out;
5888
5889 pm = pci_find_bar(child, rid);
5890 if (pm != NULL) {
5891 /* This is a BAR that we failed to allocate earlier. */
5892 mapsize = pm->pm_size;
5893 map = pm->pm_value;
5894 } else {
5895 /*
5896 * Weed out the bogons, and figure out how large the
5897 * BAR/map is. BARs that read back 0 here are bogus
5898 * and unimplemented. Note: atapci in legacy mode are
5899 * special and handled elsewhere in the code. If you
5900 * have a atapci device in legacy mode and it fails
5901 * here, that other code is broken.
5902 */
5903 pci_read_bar(child, rid, &map, &testval, NULL);
5904
5905 /*
5906 * Determine the size of the BAR and ignore BARs with a size
5907 * of 0. Device ROM BARs use a different mask value.
5908 */
5909 if (PCIR_IS_BIOS(&dinfo->cfg, rid))
5910 mapsize = pci_romsize(testval);
5911 else
5912 mapsize = pci_mapsize(testval);
5913 if (mapsize == 0)
5914 goto out;
5915 pm = pci_add_bar(child, rid, map, mapsize);
5916 }
5917
5918 if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, rid)) {
5919 if (type != SYS_RES_MEMORY) {
5920 if (bootverbose)
5921 device_printf(dev,
5922 "child %s requested type %d for rid %#x,"
5923 " but the BAR says it is an memio\n",
5924 device_get_nameunit(child), type, rid);
5925 goto out;
5926 }
5927 } else {
5928 if (type != SYS_RES_IOPORT) {
5929 if (bootverbose)
5930 device_printf(dev,
5931 "child %s requested type %d for rid %#x,"
5932 " but the BAR says it is an ioport\n",
5933 device_get_nameunit(child), type, rid);
5934 goto out;
5935 }
5936 }
5937
5938 /*
5939 * For real BARs, we need to override the size that
5940 * the driver requests, because that's what the BAR
5941 * actually uses and we would otherwise have a
5942 * situation where we might allocate the excess to
5943 * another driver, which won't work.
5944 */
5945 count = ((pci_addr_t)1 << mapsize) * num;
5946 if (RF_ALIGNMENT(flags) < mapsize)
5947 flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize);
5948 if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH))
5949 flags |= RF_PREFETCHABLE;
5950
5951 /*
5952 * Allocate enough resource, and then write back the
5953 * appropriate BAR for that resource.
5954 */
5955 resource_list_add(rl, type, rid, start, end, count);
5956 res = resource_list_reserve(rl, dev, child, type, rid, start, end,
5957 count, flags & ~RF_ACTIVE);
5958 if (res == NULL) {
5959 resource_list_delete(rl, type, rid);
5960 device_printf(child,
5961 "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n",
5962 count, rid, type, start, end);
5963 goto out;
5964 }
5965 if (bootverbose)
5966 device_printf(child,
5967 "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n",
5968 count, rid, type, rman_get_start(res));
5969
5970 /* Disable decoding via the CMD register before updating the BAR */
5971 cmd = pci_read_config(child, PCIR_COMMAND, 2);
5972 pci_write_config(child, PCIR_COMMAND,
5973 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
5974
5975 map = rman_get_start(res);
5976 pci_write_bar(child, pm, map);
5977
5978 /* Restore the original value of the CMD register */
5979 pci_write_config(child, PCIR_COMMAND, cmd, 2);
5980 out:
5981 return (res);
5982 }
5983
5984 struct resource *
pci_alloc_multi_resource(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_long num,u_int flags)5985 pci_alloc_multi_resource(device_t dev, device_t child, int type, int rid,
5986 rman_res_t start, rman_res_t end, rman_res_t count, u_long num,
5987 u_int flags)
5988 {
5989 struct pci_devinfo *dinfo;
5990 struct resource_list *rl;
5991 struct resource_list_entry *rle;
5992 struct resource *res;
5993 pcicfgregs *cfg;
5994
5995 /*
5996 * Perform lazy resource allocation
5997 */
5998 dinfo = device_get_ivars(child);
5999 rl = &dinfo->resources;
6000 cfg = &dinfo->cfg;
6001 switch (type) {
6002 case PCI_RES_BUS:
6003 return (pci_alloc_secbus(dev, child, rid, start, end, count,
6004 flags));
6005 case SYS_RES_IRQ:
6006 /*
6007 * Can't alloc legacy interrupt once MSI messages have
6008 * been allocated.
6009 */
6010 if (rid == 0 && (cfg->msi.msi_alloc > 0 ||
6011 cfg->msix.msix_alloc > 0))
6012 return (NULL);
6013
6014 /*
6015 * If the child device doesn't have an interrupt
6016 * routed and is deserving of an interrupt, try to
6017 * assign it one.
6018 */
6019 if (rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) &&
6020 (cfg->intpin != 0))
6021 pci_assign_interrupt(dev, child, 0);
6022 break;
6023 case SYS_RES_IOPORT:
6024 case SYS_RES_MEMORY:
6025 /*
6026 * PCI-PCI bridge I/O window resources are not BARs.
6027 * For those allocations just pass the request up the
6028 * tree.
6029 */
6030 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) {
6031 switch (rid) {
6032 case PCIR_IOBASEL_1:
6033 case PCIR_MEMBASE_1:
6034 case PCIR_PMBASEL_1:
6035 /*
6036 * XXX: Should we bother creating a resource
6037 * list entry?
6038 */
6039 return (bus_generic_alloc_resource(dev, child,
6040 type, rid, start, end, count, flags));
6041 }
6042 }
6043 /* Reserve resources for this BAR if needed. */
6044 rle = resource_list_find(rl, type, rid);
6045 if (rle == NULL) {
6046 res = pci_reserve_map(dev, child, type, rid, start, end,
6047 count, num, flags);
6048 if (res == NULL)
6049 return (NULL);
6050 }
6051 }
6052 return (resource_list_alloc(rl, dev, child, type, rid,
6053 start, end, count, flags));
6054 }
6055
6056 struct resource *
pci_alloc_resource(device_t dev,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)6057 pci_alloc_resource(device_t dev, device_t child, int type, int rid,
6058 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
6059 {
6060 #ifdef PCI_IOV
6061 struct pci_devinfo *dinfo;
6062 #endif
6063
6064 if (device_get_parent(child) != dev)
6065 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
6066 type, rid, start, end, count, flags));
6067
6068 #ifdef PCI_IOV
6069 dinfo = device_get_ivars(child);
6070 if (dinfo->cfg.flags & PCICFG_VF) {
6071 switch (type) {
6072 /* VFs can't have I/O BARs. */
6073 case SYS_RES_IOPORT:
6074 return (NULL);
6075 case SYS_RES_MEMORY:
6076 return (pci_vf_alloc_mem_resource(dev, child, rid,
6077 start, end, count, flags));
6078 }
6079
6080 /* Fall through for other types of resource allocations. */
6081 }
6082 #endif
6083
6084 return (pci_alloc_multi_resource(dev, child, type, rid, start, end,
6085 count, 1, flags));
6086 }
6087
6088 int
pci_release_resource(device_t dev,device_t child,struct resource * r)6089 pci_release_resource(device_t dev, device_t child, struct resource *r)
6090 {
6091 struct pci_devinfo *dinfo;
6092 struct resource_list *rl;
6093 pcicfgregs *cfg __unused;
6094
6095 if (device_get_parent(child) != dev)
6096 return (bus_generic_release_resource(dev, child, r));
6097
6098 dinfo = device_get_ivars(child);
6099 cfg = &dinfo->cfg;
6100
6101 #ifdef PCI_IOV
6102 if (cfg->flags & PCICFG_VF) {
6103 switch (rman_get_type(r)) {
6104 /* VFs can't have I/O BARs. */
6105 case SYS_RES_IOPORT:
6106 return (EDOOFUS);
6107 case SYS_RES_MEMORY:
6108 return (pci_vf_release_mem_resource(dev, child, r));
6109 }
6110
6111 /* Fall through for other types of resource allocations. */
6112 }
6113 #endif
6114
6115 /*
6116 * PCI-PCI bridge I/O window resources are not BARs. For
6117 * those allocations just pass the request up the tree.
6118 */
6119 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE &&
6120 (rman_get_type(r) == SYS_RES_IOPORT ||
6121 rman_get_type(r) == SYS_RES_MEMORY)) {
6122 switch (rman_get_rid(r)) {
6123 case PCIR_IOBASEL_1:
6124 case PCIR_MEMBASE_1:
6125 case PCIR_PMBASEL_1:
6126 return (bus_generic_release_resource(dev, child, r));
6127 }
6128 }
6129
6130 rl = &dinfo->resources;
6131 return (resource_list_release(rl, dev, child, r));
6132 }
6133
6134 int
pci_activate_resource(device_t dev,device_t child,struct resource * r)6135 pci_activate_resource(device_t dev, device_t child, struct resource *r)
6136 {
6137 struct pci_devinfo *dinfo;
6138 int error, rid, type;
6139
6140 if (device_get_parent(child) != dev)
6141 return (bus_generic_activate_resource(dev, child, r));
6142
6143 dinfo = device_get_ivars(child);
6144 #ifdef PCI_IOV
6145 if (dinfo->cfg.flags & PCICFG_VF) {
6146 switch (rman_get_type(r)) {
6147 /* VFs can't have I/O BARs. */
6148 case SYS_RES_IOPORT:
6149 error = EINVAL;
6150 break;
6151 case SYS_RES_MEMORY:
6152 error = pci_vf_activate_mem_resource(dev, child, r);
6153 break;
6154 default:
6155 error = bus_generic_activate_resource(dev, child, r);
6156 break;
6157 }
6158 } else
6159 #endif
6160 error = bus_generic_activate_resource(dev, child, r);
6161 if (error)
6162 return (error);
6163
6164 rid = rman_get_rid(r);
6165 type = rman_get_type(r);
6166
6167 /* Device ROMs need their decoding explicitly enabled. */
6168 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
6169 pci_write_bar(child, pci_find_bar(child, rid),
6170 rman_get_start(r) | PCIM_BIOS_ENABLE);
6171
6172 /* Enable decoding in the command register when activating BARs. */
6173 switch (type) {
6174 case SYS_RES_IOPORT:
6175 case SYS_RES_MEMORY:
6176 error = PCI_ENABLE_IO(dev, child, type);
6177 break;
6178 }
6179 return (error);
6180 }
6181
6182 int
pci_deactivate_resource(device_t dev,device_t child,struct resource * r)6183 pci_deactivate_resource(device_t dev, device_t child, struct resource *r)
6184 {
6185 struct pci_devinfo *dinfo;
6186 int error, rid, type;
6187
6188 if (device_get_parent(child) != dev)
6189 return (bus_generic_deactivate_resource(dev, child, r));
6190
6191 dinfo = device_get_ivars(child);
6192 #ifdef PCI_IOV
6193 if (dinfo->cfg.flags & PCICFG_VF) {
6194 switch (rman_get_type(r)) {
6195 /* VFs can't have I/O BARs. */
6196 case SYS_RES_IOPORT:
6197 error = EINVAL;
6198 break;
6199 case SYS_RES_MEMORY:
6200 error = pci_vf_deactivate_mem_resource(dev, child, r);
6201 break;
6202 default:
6203 error = bus_generic_deactivate_resource(dev, child, r);
6204 break;
6205 }
6206 } else
6207 #endif
6208 error = bus_generic_deactivate_resource(dev, child, r);
6209 if (error)
6210 return (error);
6211
6212 /* Disable decoding for device ROMs. */
6213 rid = rman_get_rid(r);
6214 type = rman_get_type(r);
6215 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
6216 pci_write_bar(child, pci_find_bar(child, rid),
6217 rman_get_start(r));
6218 return (0);
6219 }
6220
6221 int
pci_adjust_resource(device_t dev,device_t child,struct resource * r,rman_res_t start,rman_res_t end)6222 pci_adjust_resource(device_t dev, device_t child, struct resource *r,
6223 rman_res_t start, rman_res_t end)
6224 {
6225 #ifdef PCI_IOV
6226 struct pci_devinfo *dinfo;
6227
6228 if (device_get_parent(child) != dev)
6229 return (bus_generic_adjust_resource(dev, child, r, start,
6230 end));
6231
6232 dinfo = device_get_ivars(child);
6233 if (dinfo->cfg.flags & PCICFG_VF) {
6234 switch (rman_get_type(r)) {
6235 /* VFs can't have I/O BARs. */
6236 case SYS_RES_IOPORT:
6237 return (EINVAL);
6238 case SYS_RES_MEMORY:
6239 return (pci_vf_adjust_mem_resource(dev, child, r,
6240 start, end));
6241 }
6242
6243 /* Fall through for other types of resource allocations. */
6244 }
6245 #endif
6246
6247 return (bus_generic_adjust_resource(dev, child, r, start, end));
6248 }
6249
6250 int
pci_map_resource(device_t dev,device_t child,struct resource * r,struct resource_map_request * argsp,struct resource_map * map)6251 pci_map_resource(device_t dev, device_t child, struct resource *r,
6252 struct resource_map_request *argsp, struct resource_map *map)
6253 {
6254 #ifdef PCI_IOV
6255 struct pci_devinfo *dinfo;
6256
6257 if (device_get_parent(child) != dev)
6258 return (bus_generic_map_resource(dev, child, r, argsp,
6259 map));
6260
6261 dinfo = device_get_ivars(child);
6262 if (dinfo->cfg.flags & PCICFG_VF) {
6263 switch (rman_get_type(r)) {
6264 /* VFs can't have I/O BARs. */
6265 case SYS_RES_IOPORT:
6266 return (EINVAL);
6267 case SYS_RES_MEMORY:
6268 return (pci_vf_map_mem_resource(dev, child, r, argsp,
6269 map));
6270 }
6271
6272 /* Fall through for other types of resource allocations. */
6273 }
6274 #endif
6275
6276 return (bus_generic_map_resource(dev, child, r, argsp, map));
6277 }
6278
6279 int
pci_unmap_resource(device_t dev,device_t child,struct resource * r,struct resource_map * map)6280 pci_unmap_resource(device_t dev, device_t child, struct resource *r,
6281 struct resource_map *map)
6282 {
6283 #ifdef PCI_IOV
6284 struct pci_devinfo *dinfo;
6285
6286 if (device_get_parent(child) != dev)
6287 return (bus_generic_unmap_resource(dev, child, r, map));
6288
6289 dinfo = device_get_ivars(child);
6290 if (dinfo->cfg.flags & PCICFG_VF) {
6291 switch (rman_get_type(r)) {
6292 /* VFs can't have I/O BARs. */
6293 case SYS_RES_IOPORT:
6294 return (EINVAL);
6295 case SYS_RES_MEMORY:
6296 return (pci_vf_unmap_mem_resource(dev, child, r, map));
6297 }
6298
6299 /* Fall through for other types of resource allocations. */
6300 }
6301 #endif
6302
6303 return (bus_generic_unmap_resource(dev, child, r, map));
6304 }
6305
6306 void
pci_child_deleted(device_t dev,device_t child)6307 pci_child_deleted(device_t dev, device_t child)
6308 {
6309 struct resource_list_entry *rle;
6310 struct resource_list *rl;
6311 struct pci_devinfo *dinfo;
6312
6313 dinfo = device_get_ivars(child);
6314 rl = &dinfo->resources;
6315
6316 EVENTHANDLER_INVOKE(pci_delete_device, child);
6317
6318 /* Turn off access to resources we're about to free */
6319 if (bus_child_present(child) != 0) {
6320 pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
6321 PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
6322
6323 pci_disable_busmaster(child);
6324 }
6325
6326 /* Free all allocated resources */
6327 STAILQ_FOREACH(rle, rl, link) {
6328 if (rle->res) {
6329 if (rman_get_flags(rle->res) & RF_ACTIVE ||
6330 resource_list_busy(rl, rle->type, rle->rid)) {
6331 pci_printf(&dinfo->cfg,
6332 "Resource still owned, oops. "
6333 "(type=%d, rid=%d, addr=%lx)\n",
6334 rle->type, rle->rid,
6335 rman_get_start(rle->res));
6336 bus_release_resource(child, rle->type, rle->rid,
6337 rle->res);
6338 }
6339 resource_list_unreserve(rl, dev, child, rle->type,
6340 rle->rid);
6341 }
6342 }
6343 resource_list_free(rl);
6344
6345 pci_freecfg(dinfo);
6346 }
6347
6348 void
pci_delete_resource(device_t dev,device_t child,int type,int rid)6349 pci_delete_resource(device_t dev, device_t child, int type, int rid)
6350 {
6351 struct pci_devinfo *dinfo;
6352 struct resource_list *rl;
6353 struct resource_list_entry *rle;
6354
6355 if (device_get_parent(child) != dev)
6356 return;
6357
6358 dinfo = device_get_ivars(child);
6359 rl = &dinfo->resources;
6360 rle = resource_list_find(rl, type, rid);
6361 if (rle == NULL)
6362 return;
6363
6364 if (rle->res) {
6365 if (rman_get_flags(rle->res) & RF_ACTIVE ||
6366 resource_list_busy(rl, type, rid)) {
6367 device_printf(dev, "delete_resource: "
6368 "Resource still owned by child, oops. "
6369 "(type=%d, rid=%d, addr=%jx)\n",
6370 type, rid, rman_get_start(rle->res));
6371 return;
6372 }
6373 resource_list_unreserve(rl, dev, child, type, rid);
6374 }
6375 resource_list_delete(rl, type, rid);
6376 }
6377
6378 struct resource_list *
pci_get_resource_list(device_t dev,device_t child)6379 pci_get_resource_list (device_t dev, device_t child)
6380 {
6381 struct pci_devinfo *dinfo = device_get_ivars(child);
6382
6383 return (&dinfo->resources);
6384 }
6385
6386 #ifdef IOMMU
6387 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)6388 pci_get_dma_tag(device_t bus, device_t dev)
6389 {
6390 bus_dma_tag_t tag;
6391 struct pci_softc *sc;
6392
6393 if (device_get_parent(dev) == bus) {
6394 /* try iommu and return if it works */
6395 tag = iommu_get_dma_tag(bus, dev);
6396 } else
6397 tag = NULL;
6398 if (tag == NULL) {
6399 sc = device_get_softc(bus);
6400 tag = sc->sc_dma_tag;
6401 }
6402 return (tag);
6403 }
6404 #else
6405 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)6406 pci_get_dma_tag(device_t bus, device_t dev)
6407 {
6408 struct pci_softc *sc = device_get_softc(bus);
6409
6410 return (sc->sc_dma_tag);
6411 }
6412 #endif
6413
6414 uint32_t
pci_read_config_method(device_t dev,device_t child,int reg,int width)6415 pci_read_config_method(device_t dev, device_t child, int reg, int width)
6416 {
6417 struct pci_devinfo *dinfo = device_get_ivars(child);
6418 pcicfgregs *cfg = &dinfo->cfg;
6419
6420 #ifdef PCI_IOV
6421 /*
6422 * SR-IOV VFs don't implement the VID or DID registers, so we have to
6423 * emulate them here.
6424 */
6425 if (cfg->flags & PCICFG_VF) {
6426 if (reg == PCIR_VENDOR) {
6427 switch (width) {
6428 case 4:
6429 return (cfg->device << 16 | cfg->vendor);
6430 case 2:
6431 return (cfg->vendor);
6432 case 1:
6433 return (cfg->vendor & 0xff);
6434 default:
6435 return (0xffffffff);
6436 }
6437 } else if (reg == PCIR_DEVICE) {
6438 switch (width) {
6439 /* Note that an unaligned 4-byte read is an error. */
6440 case 2:
6441 return (cfg->device);
6442 case 1:
6443 return (cfg->device & 0xff);
6444 default:
6445 return (0xffffffff);
6446 }
6447 }
6448 }
6449 #endif
6450
6451 return (PCIB_READ_CONFIG(device_get_parent(dev),
6452 cfg->bus, cfg->slot, cfg->func, reg, width));
6453 }
6454
6455 void
pci_write_config_method(device_t dev,device_t child,int reg,uint32_t val,int width)6456 pci_write_config_method(device_t dev, device_t child, int reg,
6457 uint32_t val, int width)
6458 {
6459 struct pci_devinfo *dinfo = device_get_ivars(child);
6460 pcicfgregs *cfg = &dinfo->cfg;
6461
6462 PCIB_WRITE_CONFIG(device_get_parent(dev),
6463 cfg->bus, cfg->slot, cfg->func, reg, val, width);
6464 }
6465
6466 int
pci_child_location_method(device_t dev,device_t child,struct sbuf * sb)6467 pci_child_location_method(device_t dev, device_t child, struct sbuf *sb)
6468 {
6469
6470 sbuf_printf(sb, "slot=%d function=%d dbsf=pci%d:%d:%d:%d",
6471 pci_get_slot(child), pci_get_function(child), pci_get_domain(child),
6472 pci_get_bus(child), pci_get_slot(child), pci_get_function(child));
6473 return (0);
6474 }
6475
6476 int
pci_child_pnpinfo_method(device_t dev,device_t child,struct sbuf * sb)6477 pci_child_pnpinfo_method(device_t dev, device_t child, struct sbuf *sb)
6478 {
6479 struct pci_devinfo *dinfo;
6480 pcicfgregs *cfg;
6481
6482 dinfo = device_get_ivars(child);
6483 cfg = &dinfo->cfg;
6484 sbuf_printf(sb, "vendor=0x%04x device=0x%04x subvendor=0x%04x "
6485 "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device,
6486 cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass,
6487 cfg->progif);
6488 return (0);
6489 }
6490
6491 int
pci_get_device_path_method(device_t bus,device_t child,const char * locator,struct sbuf * sb)6492 pci_get_device_path_method(device_t bus, device_t child, const char *locator,
6493 struct sbuf *sb)
6494 {
6495 device_t parent = device_get_parent(bus);
6496 int rv;
6497
6498 if (strcmp(locator, BUS_LOCATOR_UEFI) == 0) {
6499 rv = bus_generic_get_device_path(parent, bus, locator, sb);
6500 if (rv == 0) {
6501 sbuf_printf(sb, "/Pci(0x%x,0x%x)", pci_get_slot(child),
6502 pci_get_function(child));
6503 }
6504 return (0);
6505 }
6506 return (bus_generic_get_device_path(bus, child, locator, sb));
6507 }
6508
6509 int
pci_assign_interrupt_method(device_t dev,device_t child)6510 pci_assign_interrupt_method(device_t dev, device_t child)
6511 {
6512 struct pci_devinfo *dinfo = device_get_ivars(child);
6513 pcicfgregs *cfg = &dinfo->cfg;
6514
6515 return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child,
6516 cfg->intpin));
6517 }
6518
6519 static void
pci_lookup(void * arg,const char * name,device_t * dev)6520 pci_lookup(void *arg, const char *name, device_t *dev)
6521 {
6522 long val;
6523 char *end;
6524 int domain, bus, slot, func;
6525
6526 if (*dev != NULL)
6527 return;
6528
6529 /*
6530 * Accept pciconf-style selectors of either pciD:B:S:F or
6531 * pciB:S:F. In the latter case, the domain is assumed to
6532 * be zero.
6533 */
6534 if (strncmp(name, "pci", 3) != 0)
6535 return;
6536 val = strtol(name + 3, &end, 10);
6537 if (val < 0 || val > INT_MAX || *end != ':')
6538 return;
6539 domain = val;
6540 val = strtol(end + 1, &end, 10);
6541 if (val < 0 || val > INT_MAX || *end != ':')
6542 return;
6543 bus = val;
6544 val = strtol(end + 1, &end, 10);
6545 if (val < 0 || val > INT_MAX)
6546 return;
6547 slot = val;
6548 if (*end == ':') {
6549 val = strtol(end + 1, &end, 10);
6550 if (val < 0 || val > INT_MAX || *end != '\0')
6551 return;
6552 func = val;
6553 } else if (*end == '\0') {
6554 func = slot;
6555 slot = bus;
6556 bus = domain;
6557 domain = 0;
6558 } else
6559 return;
6560
6561 if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX ||
6562 func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX))
6563 return;
6564
6565 *dev = pci_find_dbsf(domain, bus, slot, func);
6566 }
6567
6568 static int
pci_modevent(module_t mod,int what,void * arg)6569 pci_modevent(module_t mod, int what, void *arg)
6570 {
6571 static struct cdev *pci_cdev;
6572 static eventhandler_tag tag;
6573
6574 switch (what) {
6575 case MOD_LOAD:
6576 STAILQ_INIT(&pci_devq);
6577 pci_generation = 0;
6578 pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644,
6579 "pci");
6580 pci_load_vendor_data();
6581 tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL,
6582 1000);
6583 break;
6584
6585 case MOD_UNLOAD:
6586 if (tag != NULL)
6587 EVENTHANDLER_DEREGISTER(dev_lookup, tag);
6588 destroy_dev(pci_cdev);
6589 break;
6590 }
6591
6592 return (0);
6593 }
6594
6595 static void
pci_cfg_restore_pcie(device_t dev,struct pci_devinfo * dinfo)6596 pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo)
6597 {
6598 #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2)
6599 struct pcicfg_pcie *cfg;
6600 int version, pos;
6601
6602 cfg = &dinfo->cfg.pcie;
6603 pos = cfg->pcie_location;
6604
6605 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6606
6607 WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl);
6608
6609 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6610 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6611 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6612 WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl);
6613
6614 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6615 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6616 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6617 WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl);
6618
6619 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6620 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6621 WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl);
6622
6623 if (version > 1) {
6624 WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2);
6625 WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2);
6626 WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2);
6627 }
6628 #undef WREG
6629 }
6630
6631 static void
pci_cfg_restore_pcix(device_t dev,struct pci_devinfo * dinfo)6632 pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo)
6633 {
6634 pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND,
6635 dinfo->cfg.pcix.pcix_command, 2);
6636 }
6637
6638 void
pci_cfg_restore(device_t dev,struct pci_devinfo * dinfo)6639 pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
6640 {
6641
6642 /*
6643 * Restore the device to full power mode. We must do this
6644 * before we restore the registers because moving from D3 to
6645 * D0 will cause the chip's BARs and some other registers to
6646 * be reset to some unknown power on reset values. Cut down
6647 * the noise on boot by doing nothing if we are already in
6648 * state D0.
6649 */
6650 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0)
6651 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6652 pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1);
6653 pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1);
6654 pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1);
6655 pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1);
6656 pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
6657 pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
6658 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6659 case PCIM_HDRTYPE_NORMAL:
6660 pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1);
6661 pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1);
6662 break;
6663 case PCIM_HDRTYPE_BRIDGE:
6664 pci_write_config(dev, PCIR_SECLAT_1,
6665 dinfo->cfg.bridge.br_seclat, 1);
6666 pci_write_config(dev, PCIR_SUBBUS_1,
6667 dinfo->cfg.bridge.br_subbus, 1);
6668 pci_write_config(dev, PCIR_SECBUS_1,
6669 dinfo->cfg.bridge.br_secbus, 1);
6670 pci_write_config(dev, PCIR_PRIBUS_1,
6671 dinfo->cfg.bridge.br_pribus, 1);
6672 pci_write_config(dev, PCIR_BRIDGECTL_1,
6673 dinfo->cfg.bridge.br_control, 2);
6674 break;
6675 case PCIM_HDRTYPE_CARDBUS:
6676 pci_write_config(dev, PCIR_SECLAT_2,
6677 dinfo->cfg.bridge.br_seclat, 1);
6678 pci_write_config(dev, PCIR_SUBBUS_2,
6679 dinfo->cfg.bridge.br_subbus, 1);
6680 pci_write_config(dev, PCIR_SECBUS_2,
6681 dinfo->cfg.bridge.br_secbus, 1);
6682 pci_write_config(dev, PCIR_PRIBUS_2,
6683 dinfo->cfg.bridge.br_pribus, 1);
6684 pci_write_config(dev, PCIR_BRIDGECTL_2,
6685 dinfo->cfg.bridge.br_control, 2);
6686 break;
6687 }
6688 pci_restore_bars(dev);
6689
6690 if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE)
6691 pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2);
6692
6693 /*
6694 * Restore extended capabilities for PCI-Express and PCI-X
6695 */
6696 if (dinfo->cfg.pcie.pcie_location != 0)
6697 pci_cfg_restore_pcie(dev, dinfo);
6698 if (dinfo->cfg.pcix.pcix_location != 0)
6699 pci_cfg_restore_pcix(dev, dinfo);
6700
6701 /* Restore MSI and MSI-X configurations if they are present. */
6702 if (dinfo->cfg.msi.msi_location != 0)
6703 pci_resume_msi(dev);
6704 if (dinfo->cfg.msix.msix_location != 0)
6705 pci_resume_msix(dev);
6706
6707 #ifdef PCI_IOV
6708 /* The SR-IOV capability is implemented only by PFs. */
6709 if (dinfo->cfg.iov != NULL &&
6710 (dinfo->cfg.flags & PCICFG_VF) == 0)
6711 pci_iov_cfg_restore(dev, dinfo);
6712 #endif
6713 }
6714
6715 static void
pci_cfg_save_pcie(device_t dev,struct pci_devinfo * dinfo)6716 pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo)
6717 {
6718 #define RREG(n) pci_read_config(dev, pos + (n), 2)
6719 struct pcicfg_pcie *cfg;
6720 int version, pos;
6721
6722 cfg = &dinfo->cfg.pcie;
6723 pos = cfg->pcie_location;
6724
6725 cfg->pcie_flags = RREG(PCIER_FLAGS);
6726
6727 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6728
6729 cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL);
6730
6731 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6732 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6733 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6734 cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL);
6735
6736 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6737 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6738 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6739 cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL);
6740
6741 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6742 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6743 cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL);
6744
6745 if (version > 1) {
6746 cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2);
6747 cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2);
6748 cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2);
6749 }
6750 #undef RREG
6751 }
6752
6753 static void
pci_cfg_save_pcix(device_t dev,struct pci_devinfo * dinfo)6754 pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo)
6755 {
6756 dinfo->cfg.pcix.pcix_command = pci_read_config(dev,
6757 dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2);
6758 }
6759
6760 void
pci_cfg_save(device_t dev,struct pci_devinfo * dinfo,int setstate)6761 pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
6762 {
6763 uint32_t cls;
6764 int ps;
6765
6766 /*
6767 * Some drivers apparently write to these registers w/o updating our
6768 * cached copy. No harm happens if we update the copy, so do so here
6769 * so we can restore them. The COMMAND register is modified by the
6770 * bus w/o updating the cache. This should represent the normally
6771 * writable portion of the 'defined' part of type 0/1/2 headers.
6772 */
6773 dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2);
6774 dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2);
6775 dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2);
6776 dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1);
6777 dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1);
6778 dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
6779 dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
6780 dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1);
6781 dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1);
6782 dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1);
6783 dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1);
6784 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6785 case PCIM_HDRTYPE_NORMAL:
6786 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2);
6787 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2);
6788 dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1);
6789 dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1);
6790 break;
6791 case PCIM_HDRTYPE_BRIDGE:
6792 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6793 PCIR_SECLAT_1, 1);
6794 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6795 PCIR_SUBBUS_1, 1);
6796 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6797 PCIR_SECBUS_1, 1);
6798 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6799 PCIR_PRIBUS_1, 1);
6800 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6801 PCIR_BRIDGECTL_1, 2);
6802 break;
6803 case PCIM_HDRTYPE_CARDBUS:
6804 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6805 PCIR_SECLAT_2, 1);
6806 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6807 PCIR_SUBBUS_2, 1);
6808 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6809 PCIR_SECBUS_2, 1);
6810 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6811 PCIR_PRIBUS_2, 1);
6812 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6813 PCIR_BRIDGECTL_2, 2);
6814 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2);
6815 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2);
6816 break;
6817 }
6818
6819 if (dinfo->cfg.pcie.pcie_location != 0)
6820 pci_cfg_save_pcie(dev, dinfo);
6821
6822 if (dinfo->cfg.pcix.pcix_location != 0)
6823 pci_cfg_save_pcix(dev, dinfo);
6824
6825 #ifdef PCI_IOV
6826 /* The SR-IOV capability is implemented only by PFs. */
6827 if (dinfo->cfg.iov != NULL &&
6828 (dinfo->cfg.flags & PCICFG_VF) == 0)
6829 pci_iov_cfg_save(dev, dinfo);
6830 #endif
6831
6832 /*
6833 * don't set the state for display devices, base peripherals and
6834 * memory devices since bad things happen when they are powered down.
6835 * We should (a) have drivers that can easily detach and (b) use
6836 * generic drivers for these devices so that some device actually
6837 * attaches. We need to make sure that when we implement (a) we don't
6838 * power the device down on a reattach.
6839 */
6840 cls = pci_get_class(dev);
6841 if (!setstate)
6842 return;
6843 switch (pci_do_power_nodriver)
6844 {
6845 case 0: /* NO powerdown at all */
6846 return;
6847 case 1: /* Conservative about what to power down */
6848 if (cls == PCIC_STORAGE)
6849 return;
6850 /*FALLTHROUGH*/
6851 case 2: /* Aggressive about what to power down */
6852 if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY ||
6853 cls == PCIC_BASEPERIPH)
6854 return;
6855 /*FALLTHROUGH*/
6856 case 3: /* Power down everything */
6857 break;
6858 }
6859 /*
6860 * PCI spec says we can only go into D3 state from D0 state.
6861 * Transition from D[12] into D0 before going to D3 state.
6862 */
6863 ps = pci_get_powerstate(dev);
6864 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6865 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6866 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
6867 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6868 }
6869
6870 /* Wrapper APIs suitable for device driver use. */
6871 void
pci_save_state(device_t dev)6872 pci_save_state(device_t dev)
6873 {
6874 struct pci_devinfo *dinfo;
6875
6876 dinfo = device_get_ivars(dev);
6877 pci_cfg_save(dev, dinfo, 0);
6878 }
6879
6880 void
pci_restore_state(device_t dev)6881 pci_restore_state(device_t dev)
6882 {
6883 struct pci_devinfo *dinfo;
6884
6885 dinfo = device_get_ivars(dev);
6886 pci_cfg_restore(dev, dinfo);
6887 }
6888
6889 static int
pci_get_id_method(device_t dev,device_t child,enum pci_id_type type,uintptr_t * id)6890 pci_get_id_method(device_t dev, device_t child, enum pci_id_type type,
6891 uintptr_t *id)
6892 {
6893
6894 return (PCIB_GET_ID(device_get_parent(dev), child, type, id));
6895 }
6896
6897 /* Find the upstream port of a given PCI device in a root complex. */
6898 device_t
pci_find_pcie_root_port(device_t dev)6899 pci_find_pcie_root_port(device_t dev)
6900 {
6901 struct pci_devinfo *dinfo;
6902 device_t pcib, bus;
6903
6904 KASSERT(is_pci_device(dev),
6905 ("%s: non-pci device %s", __func__, device_get_nameunit(dev)));
6906
6907 /*
6908 * Walk the bridge hierarchy until we find a PCI-e root
6909 * port or a non-PCI device.
6910 */
6911 for (;;) {
6912 bus = device_get_parent(dev);
6913 KASSERT(bus != NULL, ("%s: null parent of %s", __func__,
6914 device_get_nameunit(dev)));
6915
6916 pcib = device_get_parent(bus);
6917 KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__,
6918 device_get_nameunit(bus)));
6919
6920 if (!is_pci_device(pcib))
6921 return (NULL);
6922
6923 dinfo = device_get_ivars(pcib);
6924 if (dinfo->cfg.pcie.pcie_location != 0 &&
6925 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
6926 return (pcib);
6927
6928 dev = pcib;
6929 }
6930 }
6931
6932 /*
6933 * Wait for pending transactions to complete on a PCI-express function.
6934 *
6935 * The maximum delay is specified in milliseconds in max_delay. Note
6936 * that this function may sleep.
6937 *
6938 * Returns true if the function is idle and false if the timeout is
6939 * exceeded. If dev is not a PCI-express function, this returns true.
6940 */
6941 bool
pcie_wait_for_pending_transactions(device_t dev,u_int max_delay)6942 pcie_wait_for_pending_transactions(device_t dev, u_int max_delay)
6943 {
6944 struct pci_devinfo *dinfo = device_get_ivars(dev);
6945 uint16_t sta;
6946 int cap;
6947
6948 cap = dinfo->cfg.pcie.pcie_location;
6949 if (cap == 0)
6950 return (true);
6951
6952 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6953 while (sta & PCIEM_STA_TRANSACTION_PND) {
6954 if (max_delay == 0)
6955 return (false);
6956
6957 /* Poll once every 100 milliseconds up to the timeout. */
6958 if (max_delay > 100) {
6959 pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK);
6960 max_delay -= 100;
6961 } else {
6962 pause_sbt("pcietp", max_delay * SBT_1MS, 0,
6963 C_HARDCLOCK);
6964 max_delay = 0;
6965 }
6966 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6967 }
6968
6969 return (true);
6970 }
6971
6972 /*
6973 * Determine the maximum Completion Timeout in microseconds.
6974 *
6975 * For non-PCI-express functions this returns 0.
6976 */
6977 int
pcie_get_max_completion_timeout(device_t dev)6978 pcie_get_max_completion_timeout(device_t dev)
6979 {
6980 struct pci_devinfo *dinfo = device_get_ivars(dev);
6981 int cap;
6982
6983 cap = dinfo->cfg.pcie.pcie_location;
6984 if (cap == 0)
6985 return (0);
6986
6987 /*
6988 * Functions using the 1.x spec use the default timeout range of
6989 * 50 microseconds to 50 milliseconds. Functions that do not
6990 * support programmable timeouts also use this range.
6991 */
6992 if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 ||
6993 (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) &
6994 PCIEM_CAP2_COMP_TIMO_RANGES) == 0)
6995 return (50 * 1000);
6996
6997 switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) &
6998 PCIEM_CTL2_COMP_TIMO_VAL) {
6999 case PCIEM_CTL2_COMP_TIMO_100US:
7000 return (100);
7001 case PCIEM_CTL2_COMP_TIMO_10MS:
7002 return (10 * 1000);
7003 case PCIEM_CTL2_COMP_TIMO_55MS:
7004 return (55 * 1000);
7005 case PCIEM_CTL2_COMP_TIMO_210MS:
7006 return (210 * 1000);
7007 case PCIEM_CTL2_COMP_TIMO_900MS:
7008 return (900 * 1000);
7009 case PCIEM_CTL2_COMP_TIMO_3500MS:
7010 return (3500 * 1000);
7011 case PCIEM_CTL2_COMP_TIMO_13S:
7012 return (13 * 1000 * 1000);
7013 case PCIEM_CTL2_COMP_TIMO_64S:
7014 return (64 * 1000 * 1000);
7015 default:
7016 return (50 * 1000);
7017 }
7018 }
7019
7020 void
pcie_apei_error(device_t dev,int sev,uint8_t * aerp)7021 pcie_apei_error(device_t dev, int sev, uint8_t *aerp)
7022 {
7023 struct pci_devinfo *dinfo = device_get_ivars(dev);
7024 const char *s;
7025 int aer;
7026 uint32_t r, r1;
7027 uint16_t rs;
7028
7029 if (sev == PCIEM_STA_CORRECTABLE_ERROR)
7030 s = "Correctable";
7031 else if (sev == PCIEM_STA_NON_FATAL_ERROR)
7032 s = "Uncorrectable (Non-Fatal)";
7033 else
7034 s = "Uncorrectable (Fatal)";
7035 device_printf(dev, "%s PCIe error reported by APEI\n", s);
7036 if (aerp) {
7037 if (sev == PCIEM_STA_CORRECTABLE_ERROR) {
7038 r = le32dec(aerp + PCIR_AER_COR_STATUS);
7039 r1 = le32dec(aerp + PCIR_AER_COR_MASK);
7040 } else {
7041 r = le32dec(aerp + PCIR_AER_UC_STATUS);
7042 r1 = le32dec(aerp + PCIR_AER_UC_MASK);
7043 }
7044 device_printf(dev, "status 0x%08x mask 0x%08x", r, r1);
7045 if (sev != PCIEM_STA_CORRECTABLE_ERROR) {
7046 r = le32dec(aerp + PCIR_AER_UC_SEVERITY);
7047 rs = le16dec(aerp + PCIR_AER_CAP_CONTROL);
7048 printf(" severity 0x%08x first %d\n",
7049 r, rs & 0x1f);
7050 } else
7051 printf("\n");
7052 }
7053
7054 /* As kind of recovery just report and clear the error statuses. */
7055 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
7056 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7057 if (r != 0) {
7058 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
7059 device_printf(dev, "Clearing UC AER errors 0x%08x\n", r);
7060 }
7061
7062 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7063 if (r != 0) {
7064 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
7065 device_printf(dev, "Clearing COR AER errors 0x%08x\n", r);
7066 }
7067 }
7068 if (dinfo->cfg.pcie.pcie_location != 0) {
7069 rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
7070 PCIER_DEVICE_STA, 2);
7071 if ((rs & (PCIEM_STA_CORRECTABLE_ERROR |
7072 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
7073 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
7074 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
7075 PCIER_DEVICE_STA, rs, 2);
7076 device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs);
7077 }
7078 }
7079 }
7080
7081 /*
7082 * Return true if the device supports FLR, taking both its advertised
7083 * capability and the PCI quirk policy into account.
7084 */
7085 bool
pcie_flr_supported(device_t dev)7086 pcie_flr_supported(device_t dev)
7087 {
7088 struct pci_devinfo *dinfo = device_get_ivars(dev);
7089 int cap;
7090
7091 cap = dinfo->cfg.pcie.pcie_location;
7092 if (cap == 0)
7093 return (false);
7094
7095 if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR) &&
7096 !pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_ENABLE_FLR))
7097 return (false);
7098 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_FLR))
7099 return (false);
7100 return (true);
7101 }
7102
7103 /*
7104 * Perform a Function Level Reset (FLR) on a device.
7105 *
7106 * This function first waits for any pending transactions to complete
7107 * within the timeout specified by max_delay. If transactions are
7108 * still pending, the function will return false without attempting a
7109 * reset.
7110 *
7111 * If dev is not a PCI-express function, or neither advertises FLR nor
7112 * has a quirk enabling FLR, this function returns false.
7113 *
7114 * Note that no registers are saved or restored. The caller is
7115 * responsible for saving and restoring any registers including
7116 * PCI-standard registers via pci_save_state() and
7117 * pci_restore_state().
7118 */
7119 bool
pcie_flr(device_t dev,u_int max_delay,bool force)7120 pcie_flr(device_t dev, u_int max_delay, bool force)
7121 {
7122 struct pci_devinfo *dinfo = device_get_ivars(dev);
7123 uint16_t cmd, ctl;
7124 int compl_delay;
7125 int cap;
7126
7127 if (!pcie_flr_supported(dev))
7128 return (false);
7129
7130 cap = dinfo->cfg.pcie.pcie_location;
7131
7132 /*
7133 * Disable busmastering to prevent generation of new
7134 * transactions while waiting for the device to go idle. If
7135 * the idle timeout fails, the command register is restored
7136 * which will re-enable busmastering.
7137 */
7138 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
7139 pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2);
7140 if (!pcie_wait_for_pending_transactions(dev, max_delay)) {
7141 if (!force) {
7142 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
7143 return (false);
7144 }
7145 pci_printf(&dinfo->cfg,
7146 "Resetting with transactions pending after %d ms\n",
7147 max_delay);
7148
7149 /*
7150 * Extend the post-FLR delay to cover the maximum
7151 * Completion Timeout delay of anything in flight
7152 * during the FLR delay. Enforce a minimum delay of
7153 * at least 10ms.
7154 */
7155 compl_delay = pcie_get_max_completion_timeout(dev) / 1000;
7156 if (compl_delay < 10)
7157 compl_delay = 10;
7158 } else
7159 compl_delay = 0;
7160
7161 /* Initiate the reset. */
7162 ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
7163 pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl |
7164 PCIEM_CTL_INITIATE_FLR, 2);
7165
7166 /* Wait for 100ms. */
7167 pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK);
7168
7169 if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) &
7170 PCIEM_STA_TRANSACTION_PND)
7171 pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
7172 return (true);
7173 }
7174
7175 /*
7176 * Attempt a power-management reset by cycling the device in/out of D3
7177 * state. PCI spec says we can only go into D3 state from D0 state.
7178 * Transition from D[12] into D0 before going to D3 state.
7179 */
7180 int
pci_power_reset(device_t dev)7181 pci_power_reset(device_t dev)
7182 {
7183 int ps;
7184
7185 ps = pci_get_powerstate(dev);
7186 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
7187 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
7188 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
7189 pci_set_powerstate(dev, ps);
7190 return (0);
7191 }
7192
7193 /*
7194 * Try link drop and retrain of the downstream port of upstream
7195 * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must
7196 * cause Conventional Hot reset of the device in the slot.
7197 * Alternative, for PCIe, could be the secondary bus reset initiatied
7198 * on the upstream switch PCIR_BRIDGECTL_1, bit 6.
7199 */
7200 int
pcie_link_reset(device_t port,int pcie_location)7201 pcie_link_reset(device_t port, int pcie_location)
7202 {
7203 uint16_t v;
7204
7205 v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2);
7206 v |= PCIEM_LINK_CTL_LINK_DIS;
7207 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
7208 pause_sbt("pcier1", mstosbt(20), 0, 0);
7209 v &= ~PCIEM_LINK_CTL_LINK_DIS;
7210 v |= PCIEM_LINK_CTL_RETRAIN_LINK;
7211 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
7212 pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */
7213 v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2);
7214 return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0);
7215 }
7216
7217 static int
pci_reset_post(device_t dev,device_t child)7218 pci_reset_post(device_t dev, device_t child)
7219 {
7220
7221 if (dev == device_get_parent(child))
7222 pci_restore_state(child);
7223 return (0);
7224 }
7225
7226 static int
pci_reset_prepare(device_t dev,device_t child)7227 pci_reset_prepare(device_t dev, device_t child)
7228 {
7229
7230 if (dev == device_get_parent(child))
7231 pci_save_state(child);
7232 return (0);
7233 }
7234
7235 static int
pci_reset_child(device_t dev,device_t child,int flags)7236 pci_reset_child(device_t dev, device_t child, int flags)
7237 {
7238 int error;
7239
7240 if (dev == NULL || device_get_parent(child) != dev)
7241 return (0);
7242 if ((flags & DEVF_RESET_DETACH) != 0) {
7243 error = device_get_state(child) == DS_ATTACHED ?
7244 device_detach(child) : 0;
7245 } else {
7246 error = BUS_SUSPEND_CHILD(dev, child);
7247 }
7248 if (error == 0) {
7249 if (!pcie_flr(child, 1000, false)) {
7250 error = BUS_RESET_PREPARE(dev, child);
7251 if (error == 0)
7252 pci_power_reset(child);
7253 BUS_RESET_POST(dev, child);
7254 }
7255 if ((flags & DEVF_RESET_DETACH) != 0)
7256 device_probe_and_attach(child);
7257 else
7258 BUS_RESUME_CHILD(dev, child);
7259 }
7260 return (error);
7261 }
7262
7263 const struct pci_device_table *
pci_match_device(device_t child,const struct pci_device_table * id,size_t nelt)7264 pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt)
7265 {
7266 bool match;
7267 uint16_t vendor, device, subvendor, subdevice, class, subclass, revid;
7268
7269 vendor = pci_get_vendor(child);
7270 device = pci_get_device(child);
7271 subvendor = pci_get_subvendor(child);
7272 subdevice = pci_get_subdevice(child);
7273 class = pci_get_class(child);
7274 subclass = pci_get_subclass(child);
7275 revid = pci_get_revid(child);
7276 while (nelt-- > 0) {
7277 match = true;
7278 if (id->match_flag_vendor)
7279 match &= vendor == id->vendor;
7280 if (id->match_flag_device)
7281 match &= device == id->device;
7282 if (id->match_flag_subvendor)
7283 match &= subvendor == id->subvendor;
7284 if (id->match_flag_subdevice)
7285 match &= subdevice == id->subdevice;
7286 if (id->match_flag_class)
7287 match &= class == id->class_id;
7288 if (id->match_flag_subclass)
7289 match &= subclass == id->subclass;
7290 if (id->match_flag_revid)
7291 match &= revid == id->revid;
7292 if (match)
7293 return (id);
7294 id++;
7295 }
7296 return (NULL);
7297 }
7298
7299 static void
pci_print_faulted_dev_name(const struct pci_devinfo * dinfo)7300 pci_print_faulted_dev_name(const struct pci_devinfo *dinfo)
7301 {
7302 const char *dev_name;
7303 device_t dev;
7304
7305 dev = dinfo->cfg.dev;
7306 printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus,
7307 dinfo->cfg.slot, dinfo->cfg.func);
7308 dev_name = device_get_name(dev);
7309 if (dev_name != NULL)
7310 printf(" (%s%d)", dev_name, device_get_unit(dev));
7311 }
7312
7313 void
pci_print_faulted_dev(void)7314 pci_print_faulted_dev(void)
7315 {
7316 struct pci_devinfo *dinfo;
7317 device_t dev;
7318 int aer, i;
7319 uint32_t r1, r2;
7320 uint16_t status;
7321
7322 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
7323 dev = dinfo->cfg.dev;
7324 status = pci_read_config(dev, PCIR_STATUS, 2);
7325 status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
7326 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
7327 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
7328 if (status != 0) {
7329 pci_print_faulted_dev_name(dinfo);
7330 printf(" error 0x%04x\n", status);
7331 }
7332 if (dinfo->cfg.pcie.pcie_location != 0) {
7333 status = pci_read_config(dev,
7334 dinfo->cfg.pcie.pcie_location +
7335 PCIER_DEVICE_STA, 2);
7336 if ((status & (PCIEM_STA_CORRECTABLE_ERROR |
7337 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
7338 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
7339 pci_print_faulted_dev_name(dinfo);
7340 printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n",
7341 pci_read_config(dev,
7342 dinfo->cfg.pcie.pcie_location +
7343 PCIER_DEVICE_CTL, 2),
7344 status);
7345 }
7346 }
7347 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
7348 r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7349 r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7350 if (r1 != 0 || r2 != 0) {
7351 pci_print_faulted_dev_name(dinfo);
7352 printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n"
7353 " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n",
7354 r1, pci_read_config(dev, aer +
7355 PCIR_AER_UC_MASK, 4),
7356 pci_read_config(dev, aer +
7357 PCIR_AER_UC_SEVERITY, 4),
7358 r2, pci_read_config(dev, aer +
7359 PCIR_AER_COR_MASK, 4),
7360 pci_read_config(dev, aer +
7361 PCIR_AER_CAP_CONTROL, 4));
7362 for (i = 0; i < 4; i++) {
7363 r1 = pci_read_config(dev, aer +
7364 PCIR_AER_HEADER_LOG + i * 4, 4);
7365 printf(" HL%d: 0x%08x\n", i, r1);
7366 }
7367 }
7368 }
7369 }
7370 }
7371
7372 bool
is_pci_device(device_t dev)7373 is_pci_device(device_t dev)
7374 {
7375 devclass_t pci_class;
7376
7377 if (device_get_parent(dev) == NULL)
7378 return (false);
7379 pci_class = devclass_find("pci");
7380 return (device_get_devclass(device_get_parent(dev)) == pci_class);
7381 }
7382
7383 #ifdef DDB
DB_SHOW_COMMAND_FLAGS(pcierr,pci_print_faulted_dev_db,DB_CMD_MEMSAFE)7384 DB_SHOW_COMMAND_FLAGS(pcierr, pci_print_faulted_dev_db, DB_CMD_MEMSAFE)
7385 {
7386
7387 pci_print_faulted_dev();
7388 }
7389
7390 static void
db_clear_pcie_errors(const struct pci_devinfo * dinfo)7391 db_clear_pcie_errors(const struct pci_devinfo *dinfo)
7392 {
7393 device_t dev;
7394 int aer;
7395 uint32_t r;
7396
7397 dev = dinfo->cfg.dev;
7398 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
7399 PCIER_DEVICE_STA, 2);
7400 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
7401 PCIER_DEVICE_STA, r, 2);
7402
7403 if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0)
7404 return;
7405 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
7406 if (r != 0)
7407 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
7408 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
7409 if (r != 0)
7410 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
7411 }
7412
DB_COMMAND_FLAGS(pci_clearerr,db_pci_clearerr,DB_CMD_MEMSAFE)7413 DB_COMMAND_FLAGS(pci_clearerr, db_pci_clearerr, DB_CMD_MEMSAFE)
7414 {
7415 struct pci_devinfo *dinfo;
7416 device_t dev;
7417 uint16_t status, status1;
7418
7419 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
7420 dev = dinfo->cfg.dev;
7421 status1 = status = pci_read_config(dev, PCIR_STATUS, 2);
7422 status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
7423 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
7424 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
7425 if (status1 != 0) {
7426 status &= ~status1;
7427 pci_write_config(dev, PCIR_STATUS, status, 2);
7428 }
7429 if (dinfo->cfg.pcie.pcie_location != 0)
7430 db_clear_pcie_errors(dinfo);
7431 }
7432 }
7433 #endif
7434