xref: /illumos-gate/usr/src/uts/common/sys/pcie_impl.h (revision ead3c390933666e6c332491643357cede94c341e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2019 Joyent, Inc.
24  * Copyright 2024 Oxide Computer Company
25  */
26 
27 #ifndef	_SYS_PCIE_IMPL_H
28 #define	_SYS_PCIE_IMPL_H
29 
30 #ifdef	__cplusplus
31 extern "C" {
32 #endif
33 
34 #include <sys/pcie.h>
35 #include <sys/pciev.h>
36 #include <sys/taskq_impl.h>
37 
38 #define	PCI_GET_BDF(dip)	\
39 	PCIE_DIP2BUS(dip)->bus_bdf
40 #define	PCI_GET_SEC_BUS(dip)	\
41 	PCIE_DIP2BUS(dip)->bus_bdg_secbus
42 #define	PCI_GET_PCIE2PCI_SECBUS(dip) \
43 	PCIE_DIP2BUS(dip)->bus_pcie2pci_secbus
44 
45 #define	DEVI_PORT_TYPE_PCI \
46 	((PCI_CLASS_BRIDGE << 16) | (PCI_BRIDGE_PCI << 8) | \
47 	PCI_BRIDGE_PCI_IF_PCI2PCI)
48 
49 #define	PCIE_DIP2BUS(dip) \
50 	(ndi_port_type(dip, B_TRUE, DEVI_PORT_TYPE_PCI) ? \
51 	PCIE_DIP2UPBUS(dip) : \
52 	ndi_port_type(dip, B_FALSE, DEVI_PORT_TYPE_PCI) ? \
53 	PCIE_DIP2DOWNBUS(dip) : NULL)
54 
55 #define	PCIE_DIP2UPBUS(dip) \
56 	((pcie_bus_t *)ndi_get_bus_private(dip, B_TRUE))
57 #define	PCIE_DIP2DOWNBUS(dip) \
58 	((pcie_bus_t *)ndi_get_bus_private(dip, B_FALSE))
59 #define	PCIE_DIP2PFD(dip) (PCIE_DIP2BUS(dip))->bus_pfd
60 #define	PCIE_PFD2BUS(pfd_p) pfd_p->pe_bus_p
61 #define	PCIE_PFD2DIP(pfd_p) PCIE_PFD2BUS(pfd_p)->bus_dip
62 #define	PCIE_BUS2DIP(bus_p) bus_p->bus_dip
63 #define	PCIE_BUS2PFD(bus_p) PCIE_DIP2PFD(PCIE_BUS2DIP(bus_p))
64 #define	PCIE_BUS2DOM(bus_p) bus_p->bus_dom
65 #define	PCIE_DIP2DOM(dip) PCIE_BUS2DOM(PCIE_DIP2BUS(dip))
66 
67 /*
68  * These macros depend on initialization of type related data in bus_p.
69  */
70 #define	PCIE_IS_PCIE(bus_p) (bus_p->bus_pcie_off)
71 #define	PCIE_IS_PCIX(bus_p) (bus_p->bus_pcix_off)
72 #define	PCIE_IS_PCI(bus_p) (!PCIE_IS_PCIE(bus_p))
73 #define	PCIE_HAS_AER(bus_p) (bus_p->bus_aer_off)
74 /* IS_ROOT = is RC or RP */
75 #define	PCIE_IS_ROOT(bus_p) (PCIE_IS_RC(bus_p) || PCIE_IS_RP(bus_p))
76 
77 #define	PCIE_IS_HOTPLUG_CAPABLE(dip) \
78 	(PCIE_DIP2BUS(dip)->bus_hp_sup_modes)
79 
80 #define	PCIE_IS_HOTPLUG_ENABLED(dip) \
81 	((PCIE_DIP2BUS(dip)->bus_hp_curr_mode == PCIE_PCI_HP_MODE) || \
82 	(PCIE_DIP2BUS(dip)->bus_hp_curr_mode == PCIE_NATIVE_HP_MODE))
83 
84 /*
85  * This is a pseudo pcie "device type", but it's needed to explain describe
86  * nodes such as PX and NPE, which aren't really PCI devices but do control or
87  * interaction with PCI error handling.
88  */
89 #define	PCIE_IS_RC(bus_p) \
90 	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO)
91 #define	PCIE_IS_RP(bus_p) \
92 	((bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_ROOT) && \
93 	    PCIE_IS_PCIE(bus_p))
94 #define	PCIE_IS_SWU(bus_p) \
95 	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_UP)
96 #define	PCIE_IS_SWD(bus_p) \
97 	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_DOWN)
98 #define	PCIE_IS_SW(bus_p) \
99 	(PCIE_IS_SWU(bus_p) || PCIE_IS_SWD(bus_p))
100 #define	PCIE_IS_BDG(bus_p)  (bus_p->bus_hdr_type == PCI_HEADER_ONE)
101 #define	PCIE_IS_PCI_BDG(bus_p) (PCIE_IS_PCI(bus_p) && PCIE_IS_BDG(bus_p))
102 #define	PCIE_IS_PCIE_BDG(bus_p) \
103 	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_PCIE2PCI)
104 #define	PCIE_IS_PCI2PCIE(bus_p) \
105 	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_PCI2PCIE)
106 #define	PCIE_IS_PCIE_SEC(bus_p) \
107 	(PCIE_IS_PCIE(bus_p) && PCIE_IS_BDG(bus_p) && !PCIE_IS_PCIE_BDG(bus_p))
108 #define	PCIX_ECC_VERSION_CHECK(bus_p) \
109 	((bus_p->bus_ecc_ver == PCI_PCIX_VER_1) || \
110 	    (bus_p->bus_ecc_ver == PCI_PCIX_VER_2))
111 
112 #define	PCIE_VENID(bus_p)	(bus_p->bus_dev_ven_id & 0xffff)
113 #define	PCIE_DEVID(bus_p)	((bus_p->bus_dev_ven_id >> 16) & 0xffff)
114 
115 /* PCIE Cap/AER shortcuts */
116 #define	PCIE_GET(sz, bus_p, off) \
117 	pci_config_get ## sz(bus_p->bus_cfg_hdl, off)
118 #define	PCIE_PUT(sz, bus_p, off, val) \
119 	pci_config_put ## sz(bus_p->bus_cfg_hdl, off, val)
120 #define	PCIE_CAP_GET(sz, bus_p, off) \
121 	PCI_CAP_GET ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_pcie_off, off)
122 #define	PCIE_CAP_PUT(sz, bus_p, off, val) \
123 	PCI_CAP_PUT ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_pcie_off, off, \
124 	    val)
125 #define	PCIE_AER_GET(sz, bus_p, off) \
126 	PCI_XCAP_GET ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_aer_off, off)
127 #define	PCIE_AER_PUT(sz, bus_p, off, val) \
128 	PCI_XCAP_PUT ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_aer_off, off, \
129 	    val)
130 #define	PCIX_CAP_GET(sz, bus_p, off) \
131 	PCI_CAP_GET ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_pcix_off, off)
132 #define	PCIX_CAP_PUT(sz, bus_p, off, val) \
133 	PCI_CAP_PUT ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_pcix_off, off, \
134 	    val)
135 
136 /* Translate PF error return values to DDI_FM values */
137 #define	PF_ERR2DDIFM_ERR(sts) \
138 	(sts & PF_ERR_FATAL_FLAGS ? DDI_FM_FATAL :	\
139 	(sts == PF_ERR_NO_ERROR ? DDI_FM_OK : DDI_FM_NONFATAL))
140 
141 /*
142  * The following flag is used for Broadcom 5714/5715 bridge prefetch issue.
143  * This flag will be used both by px and pcieb nexus drivers.
144  */
145 #define	PX_DMAI_FLAGS_MAP_BUFZONE	0x40000
146 
147 /*
148  * PCI(e/-X) structures used to to gather and report errors detected by
149  * PCI(e/-X) compliant devices.  These registers only contain "dynamic" data.
150  * Static data such as Capability Offsets and Version #s is saved in the parent
151  * private data.
152  */
153 #define	PCI_ERR_REG(pfd_p)	   pfd_p->pe_pci_regs
154 #define	PCI_BDG_ERR_REG(pfd_p)	   PCI_ERR_REG(pfd_p)->pci_bdg_regs
155 #define	PCIX_ERR_REG(pfd_p)	   pfd_p->pe_ext.pe_pcix_regs
156 #define	PCIX_ECC_REG(pfd_p)	   PCIX_ERR_REG(pfd_p)->pcix_ecc_regs
157 #define	PCIX_BDG_ERR_REG(pfd_p)	   pfd_p->pe_pcix_bdg_regs
158 #define	PCIX_BDG_ECC_REG(pfd_p, n) PCIX_BDG_ERR_REG(pfd_p)->pcix_bdg_ecc_regs[n]
159 #define	PCIE_ERR_REG(pfd_p)	   pfd_p->pe_ext.pe_pcie_regs
160 #define	PCIE_RP_REG(pfd_p)	   PCIE_ERR_REG(pfd_p)->pcie_rp_regs
161 #define	PCIE_ROOT_FAULT(pfd_p)	   pfd_p->pe_root_fault
162 #define	PCIE_ROOT_EH_SRC(pfd_p)    pfd_p->pe_root_eh_src
163 #define	PCIE_ADV_REG(pfd_p)	   PCIE_ERR_REG(pfd_p)->pcie_adv_regs
164 #define	PCIE_ADV_HDR(pfd_p, n)	   PCIE_ADV_REG(pfd_p)->pcie_ue_hdr[n]
165 #define	PCIE_ADV_BDG_REG(pfd_p) \
166 	PCIE_ADV_REG(pfd_p)->pcie_ext.pcie_adv_bdg_regs
167 #define	PCIE_ADV_BDG_HDR(pfd_p, n) PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_hdr[n]
168 #define	PCIE_ADV_RP_REG(pfd_p) \
169 	PCIE_ADV_REG(pfd_p)->pcie_ext.pcie_adv_rp_regs
170 #define	PCIE_SLOT_REG(pfd_p)		pfd_p->pe_pcie_slot_regs
171 #define	PFD_AFFECTED_DEV(pfd_p)	   pfd_p->pe_affected_dev
172 #define	PFD_SET_AFFECTED_FLAG(pfd_p, aff_flag) \
173 	PFD_AFFECTED_DEV(pfd_p)->pe_affected_flags = aff_flag
174 #define	PFD_SET_AFFECTED_BDF(pfd_p, bdf) \
175 	PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf = bdf
176 
177 #define	PFD_IS_ROOT(pfd_p)	   PCIE_IS_ROOT(PCIE_PFD2BUS(pfd_p))
178 #define	PFD_IS_RC(pfd_p)	   PCIE_IS_RC(PCIE_PFD2BUS(pfd_p))
179 #define	PFD_IS_RP(pfd_p)	   PCIE_IS_RP(PCIE_PFD2BUS(pfd_p))
180 
181 /* bus_hp_mode field */
182 typedef enum {
183 	PCIE_NONE_HP_MODE	= 0x0,
184 	PCIE_ACPI_HP_MODE	= 0x1,
185 	PCIE_PCI_HP_MODE	= 0x2,
186 	PCIE_NATIVE_HP_MODE	= 0x4
187 } pcie_hp_mode_t;
188 
189 typedef struct pf_pci_bdg_err_regs {
190 	uint16_t pci_bdg_sec_stat;	/* PCI secondary status reg */
191 	uint16_t pci_bdg_ctrl;		/* PCI bridge control reg */
192 } pf_pci_bdg_err_regs_t;
193 
194 typedef struct pf_pci_err_regs {
195 	uint16_t pci_err_status;	/* pci status register */
196 	uint16_t pci_cfg_comm;		/* pci command register */
197 	pf_pci_bdg_err_regs_t *pci_bdg_regs;
198 } pf_pci_err_regs_t;
199 
200 typedef struct pf_pcix_ecc_regs {
201 	uint32_t pcix_ecc_ctlstat;	/* pcix ecc control status reg */
202 	uint32_t pcix_ecc_fstaddr;	/* pcix ecc first address reg */
203 	uint32_t pcix_ecc_secaddr;	/* pcix ecc second address reg */
204 	uint32_t pcix_ecc_attr;		/* pcix ecc attributes reg */
205 } pf_pcix_ecc_regs_t;
206 
207 typedef struct pf_pcix_err_regs {
208 	uint16_t pcix_command;		/* pcix command register */
209 	uint32_t pcix_status;		/* pcix status register */
210 	pf_pcix_ecc_regs_t *pcix_ecc_regs;	/* pcix ecc registers */
211 } pf_pcix_err_regs_t;
212 
213 typedef struct pf_pcix_bdg_err_regs {
214 	uint16_t pcix_bdg_sec_stat;	/* pcix bridge secondary status reg */
215 	uint32_t pcix_bdg_stat;		/* pcix bridge status reg */
216 	pf_pcix_ecc_regs_t *pcix_bdg_ecc_regs[2];	/* pcix ecc registers */
217 } pf_pcix_bdg_err_regs_t;
218 
219 typedef struct pf_pcie_adv_bdg_err_regs {
220 	uint32_t pcie_sue_ctl;		/* pcie bridge secondary ue control */
221 	uint32_t pcie_sue_status;	/* pcie bridge secondary ue status */
222 	uint32_t pcie_sue_mask;		/* pcie bridge secondary ue mask */
223 	uint32_t pcie_sue_sev;		/* pcie bridge secondary ue severity */
224 	uint32_t pcie_sue_hdr[4];	/* pcie bridge secondary ue hdr log */
225 	uint32_t pcie_sue_tgt_trans;	/* Fault trans type from SAER Logs */
226 	uint64_t pcie_sue_tgt_addr;	/* Fault addr from SAER Logs */
227 	pcie_req_id_t pcie_sue_tgt_bdf;	/* Fault bdf from SAER Logs */
228 } pf_pcie_adv_bdg_err_regs_t;
229 
230 typedef struct pf_pcie_adv_rp_err_regs {
231 	uint32_t pcie_rp_err_status;	/* pcie root complex error status reg */
232 	uint32_t pcie_rp_err_cmd;	/* pcie root complex error cmd reg */
233 	uint16_t pcie_rp_ce_src_id;	/* pcie root complex ce sourpe id */
234 	uint16_t pcie_rp_ue_src_id;	/* pcie root complex ue sourpe id */
235 } pf_pcie_adv_rp_err_regs_t;
236 
237 typedef struct pf_pcie_adv_err_regs {
238 	uint32_t pcie_adv_ctl;		/* pcie advanced control reg */
239 	uint32_t pcie_ue_status;	/* pcie ue error status reg */
240 	uint32_t pcie_ue_mask;		/* pcie ue error mask reg */
241 	uint32_t pcie_ue_sev;		/* pcie ue error severity reg */
242 	uint32_t pcie_ue_hdr[4];	/* pcie ue header log */
243 	uint32_t pcie_ce_status;	/* pcie ce error status reg */
244 	uint32_t pcie_ce_mask;		/* pcie ce error mask reg */
245 	union {
246 		pf_pcie_adv_bdg_err_regs_t *pcie_adv_bdg_regs; /* bdg regs */
247 		pf_pcie_adv_rp_err_regs_t *pcie_adv_rp_regs;	 /* rp regs */
248 	} pcie_ext;
249 	uint32_t pcie_ue_tgt_trans;	/* Fault trans type from AER Logs */
250 	uint64_t pcie_ue_tgt_addr;	/* Fault addr from AER Logs */
251 	pcie_req_id_t pcie_ue_tgt_bdf;	/* Fault bdf from AER Logs */
252 } pf_pcie_adv_err_regs_t;
253 
254 typedef struct pf_pcie_rp_err_regs {
255 	uint32_t pcie_rp_status;	/* root complex status register */
256 	uint16_t pcie_rp_ctl;		/* root complex control register */
257 } pf_pcie_rp_err_regs_t;
258 
259 typedef struct pf_pcie_err_regs {
260 	uint16_t pcie_err_status;	/* pcie device status register */
261 	uint16_t pcie_err_ctl;		/* pcie error control register */
262 	uint32_t pcie_dev_cap;		/* pcie device capabilities register */
263 	pf_pcie_rp_err_regs_t *pcie_rp_regs;	 /* pcie root complex regs */
264 	pf_pcie_adv_err_regs_t *pcie_adv_regs; /* pcie aer regs */
265 } pf_pcie_err_regs_t;
266 
267 /*
268  * Slot register values for hotplug-capable Downstream Ports or Root Ports with
269  * the Slot Implemented capability bit set. We gather these to help determine
270  * whether the slot's child device is physically present.
271  */
272 typedef struct pf_pcie_slot_regs {
273 	boolean_t pcie_slot_regs_valid; /* true if register values are valid */
274 	uint32_t pcie_slot_cap;		/* pcie slot capabilities register */
275 	uint16_t pcie_slot_control;	/* pcie slot control register */
276 	uint16_t pcie_slot_status;	/* pcie slot status register */
277 } pf_pcie_slot_regs_t;
278 
279 typedef enum {
280 	PF_INTR_TYPE_NONE = 0,
281 	PF_INTR_TYPE_FABRIC = 1,	/* Fabric Message */
282 	PF_INTR_TYPE_DATA,		/* Data Access Failure, failed loads */
283 	PF_INTR_TYPE_AER,		/* Root Port AER MSI */
284 	PF_INTR_TYPE_INTERNAL		/* Chip specific internal errors */
285 } pf_intr_type_t;
286 
287 typedef struct pf_root_eh_src {
288 	pf_intr_type_t	intr_type;
289 	void		*intr_data;	/* Interrupt Data */
290 } pf_root_eh_src_t;
291 
292 typedef struct pf_root_fault {
293 	pcie_req_id_t	scan_bdf;	/* BDF from error logs */
294 	uint64_t	scan_addr;	/* Addr from error logs */
295 	boolean_t	full_scan;	/* Option to do a full scan */
296 } pf_root_fault_t;
297 
298 typedef struct pf_data pf_data_t;
299 
300 typedef enum pcie_link_width {
301 	PCIE_LINK_WIDTH_UNKNOWN,
302 	PCIE_LINK_WIDTH_X1,
303 	PCIE_LINK_WIDTH_X2,
304 	PCIE_LINK_WIDTH_X4,
305 	PCIE_LINK_WIDTH_X8,
306 	PCIE_LINK_WIDTH_X12,
307 	PCIE_LINK_WIDTH_X16,
308 	PCIE_LINK_WIDTH_X32
309 } pcie_link_width_t;
310 
311 /*
312  * Note, this member should always be treated as a bit field, as a device may
313  * support multiple speeds.
314  */
315 typedef enum pcie_link_speed {
316 	PCIE_LINK_SPEED_UNKNOWN = 0x00,
317 	PCIE_LINK_SPEED_2_5	= 1 << 0,
318 	PCIE_LINK_SPEED_5	= 1 << 1,
319 	PCIE_LINK_SPEED_8	= 1 << 2,
320 	PCIE_LINK_SPEED_16	= 1 << 3,
321 	PCIE_LINK_SPEED_32	= 1 << 4,
322 	PCIE_LINK_SPEED_64	= 1 << 5
323 } pcie_link_speed_t;
324 
325 #define	PCIE_NSPEEDS	6
326 
327 typedef enum pcie_link_flags {
328 	PCIE_LINK_F_ADMIN_TARGET	= 1 << 1
329 } pcie_link_flags_t;
330 
331 typedef enum {
332 	PCIE_LBW_S_ENABLED	= 1 << 0,
333 	PCIE_LBW_S_DISPATCHED	= 1 << 1,
334 	PCIE_LBW_S_RUNNING	= 1 << 2
335 } pcie_lbw_state_t;
336 
337 /*
338  * This structure is used to keep track of a given bus hierarchy and the set of
339  * PCIe tags that we have enabled on it.
340  */
341 typedef enum {
342 	PCIE_TAG_5B		= 0,
343 	PCIE_TAG_8B		= 1 << 0,
344 	PCIE_TAG_10B_COMP	= 1 << 1,
345 	PCIE_TAG_14B_COMP	= 1 << 2
346 } pcie_tag_t;
347 
348 #define	PCIE_TAG_ALL	(PCIE_TAG_8B | PCIE_TAG_10B_COMP | PCIE_TAG_14B_COMP)
349 
350 typedef enum {
351 	/*
352 	 * This flag is kept around for debugging and noticing that we're in the
353 	 * process of trying to perform a scan.
354 	 */
355 	PCIE_FABRIC_F_SCANNING	= 1 << 0,
356 	/*
357 	 * This is used to indicate that we have discovered a topology that is
358 	 * too complex for us to be able to set advanced settings on and
359 	 * therefore have to leave it at the bare minimum.
360 	 */
361 	PCIE_FABRIC_F_COMPLEX	= 1 << 1,
362 	/*
363 	 * Indicates that we found a hot-pluggable root port in the fabric.
364 	 */
365 	PCIE_FABRIC_F_RP_HP	= 1 << 2
366 } pcie_fabric_flags_t;
367 
368 /*
369  * This structure represents hierarchy wide settings that are used in a given
370  * PCIe fabric (what the spec calls a "hierarchy domain"). This keeps track of
371  * what we have found and enabled in the fabric as part of our initialization.
372  * For more information on this, please see the theory statement in
373  * uts/common/io/pciex/pcie.c.
374  */
375 typedef struct pice_fabric_data {
376 	pcie_fabric_flags_t	pfd_flags;
377 	uint16_t		pfd_mps_found;
378 	uint16_t		pfd_mps_act;
379 	pcie_tag_t		pfd_tag_found;
380 	pcie_tag_t		pfd_tag_act;
381 } pcie_fabric_data_t;
382 
383 /*
384  * For hot plugged device, these data are init'ed during during probe
385  * For non-hotplugged device, these data are init'ed in pci_autoconfig (on x86),
386  * or in px_attach()(on sparc).
387  *
388  * For root complex the fields are initialized in pcie_rc_init_bus();
389  * for others part of the fields are initialized in pcie_init_bus(),
390  * and part of fields initialized in pcie_post_init_bus(). See comments
391  * on top of respective functions for details.
392  */
393 typedef struct pcie_bus {
394 	/* Needed for PCI/PCIe fabric error handling */
395 	dev_info_t	*bus_dip;
396 	dev_info_t	*bus_rp_dip;
397 	ddi_acc_handle_t bus_cfg_hdl;		/* error handling acc hdle */
398 	uint_t		bus_fm_flags;
399 	uint_t		bus_soft_state;
400 
401 	/* Static PCI/PCIe information */
402 	pcie_req_id_t	bus_bdf;
403 	pcie_req_id_t	bus_rp_bdf;		/* BDF of device's Root Port */
404 	uint32_t	bus_dev_ven_id;		/* device/vendor ID */
405 	uint8_t		bus_rev_id;		/* revision ID */
406 	uint8_t		bus_hdr_type;		/* pci header type, see pci.h */
407 	uint16_t	bus_dev_type;		/* PCI-E dev type, see pcie.h */
408 	uint8_t		bus_bdg_secbus;		/* Bridge secondary bus num */
409 	uint8_t		bus_pcie_vers;		/* Version of the PCIe cap */
410 	uint16_t	bus_pcie_off;		/* PCIe Capability Offset */
411 	uint16_t	bus_aer_off;		/* PCIe Advanced Error Offset */
412 	uint16_t	bus_dev3_off;		/* PCIe Device 3 Capability */
413 	uint16_t	bus_pcix_off;		/* PCIx Capability Offset */
414 	uint16_t	bus_pci_hp_off;		/* PCI HP (SHPC) Cap Offset */
415 	uint16_t	bus_ecc_ver;		/* PCIX ecc version */
416 	pci_bus_range_t	bus_bus_range;		/* pci bus-range property */
417 	ppb_ranges_t	*bus_addr_ranges;	/* pci range property */
418 	int		bus_addr_entries;	/* number of range prop */
419 	pci_regspec_t	*bus_assigned_addr;	/* "assigned-address" prop */
420 	int		bus_assigned_entries;	/* number of prop entries */
421 
422 	/* Cache of last fault data */
423 	pf_data_t	*bus_pfd;
424 	pcie_domain_t	*bus_dom;
425 
426 	void		*bus_plat_private;	/* Platform specific */
427 	/* Hotplug specific fields */
428 	pcie_hp_mode_t	bus_hp_sup_modes;	/* HP modes supported */
429 	pcie_hp_mode_t	bus_hp_curr_mode;	/* HP mode used */
430 	void		*bus_hp_ctrl;		/* HP bus ctrl data */
431 	int		bus_ari;		/* ARI device */
432 
433 	/* workaround for PCI/PCI-X devs behind PCIe2PCI Bridge */
434 	pcie_req_id_t   bus_pcie2pci_secbus;
435 
436 	/*
437 	 * Link speed specific fields.
438 	 */
439 	kmutex_t		bus_speed_mutex;
440 	pcie_link_flags_t	bus_speed_flags;
441 	pcie_link_width_t	bus_max_width;
442 	pcie_link_width_t	bus_cur_width;
443 	pcie_link_speed_t	bus_sup_speed;
444 	pcie_link_speed_t	bus_max_speed;
445 	pcie_link_speed_t	bus_cur_speed;
446 	pcie_link_speed_t	bus_target_speed;
447 
448 	/*
449 	 * Link Bandwidth Monitoring
450 	 */
451 	kmutex_t		bus_lbw_mutex;
452 	kcondvar_t		bus_lbw_cv;
453 	pcie_lbw_state_t	bus_lbw_state;
454 	taskq_ent_t		bus_lbw_ent;
455 	uint64_t		bus_lbw_nevents;
456 	hrtime_t		bus_lbw_last_ts;
457 	char			*bus_lbw_pbuf;
458 	char			*bus_lbw_cbuf;
459 
460 	/*
461 	 * The following contains fabric wide settings and information that are
462 	 * used. This member is only valid on the root port. It is NULL on all
463 	 * other pcie_bus_t members who instead need to access this through the
464 	 * corresponding root port dip information.
465 	 */
466 	pcie_fabric_data_t	*bus_fab;
467 } pcie_bus_t;
468 
469 /*
470  * Data structure to log what devices are affected in relationship to the
471  * severity after all the errors bits have been analyzed.
472  */
473 #define	PF_AFFECTED_ROOT	(1 << 0) /* RP/RC is affected */
474 #define	PF_AFFECTED_SELF	(1 << 1) /* Reporting Device is affected */
475 #define	PF_AFFECTED_PARENT	(1 << 2) /* Parent device is affected */
476 #define	PF_AFFECTED_CHILDREN	(1 << 3) /* All children below are affected */
477 #define	PF_AFFECTED_BDF		(1 << 4) /* See affected_bdf */
478 #define	PF_AFFECTED_AER		(1 << 5) /* See AER Registers */
479 #define	PF_AFFECTED_SAER	(1 << 6) /* See SAER Registers */
480 #define	PF_AFFECTED_ADDR	(1 << 7) /* Device targeted by addr */
481 
482 #define	PF_MAX_AFFECTED_FLAG	PF_AFFECTED_ADDR
483 
484 typedef struct pf_affected_dev {
485 	uint16_t		pe_affected_flags;
486 	pcie_req_id_t		pe_affected_bdf;
487 } pf_affected_dev_t;
488 
489 struct pf_data {
490 	boolean_t		pe_lock;
491 	boolean_t		pe_valid;
492 	uint32_t		pe_severity_flags;	/* Severity of error */
493 	uint32_t		pe_severity_mask;
494 	uint32_t		pe_orig_severity_flags; /* Original severity */
495 	pf_affected_dev_t	*pe_affected_dev;
496 	pcie_bus_t		*pe_bus_p;
497 	pf_root_fault_t		*pe_root_fault; /* Only valid for RC and RP */
498 	pf_root_eh_src_t	*pe_root_eh_src; /* Only valid for RC and RP */
499 	pf_pci_err_regs_t	*pe_pci_regs;	/* PCI error reg */
500 	union {
501 		pf_pcix_err_regs_t	*pe_pcix_regs;	/* PCI-X error reg */
502 		pf_pcie_err_regs_t	*pe_pcie_regs;	/* PCIe error reg */
503 	} pe_ext;
504 	pf_pcix_bdg_err_regs_t *pe_pcix_bdg_regs; /* PCI-X bridge regs */
505 	pf_pcie_slot_regs_t	*pe_pcie_slot_regs; /* PCIe slot regs */
506 	pf_data_t		*pe_prev;	/* Next error in queue */
507 	pf_data_t		*pe_next;	/* Next error in queue */
508 	boolean_t		pe_rber_fatal;
509 };
510 
511 /* Information used while handling errors in the fabric. */
512 typedef struct pf_impl {
513 	ddi_fm_error_t	*pf_derr;
514 	pf_root_fault_t	*pf_fault;	/* captured fault bdf/addr to scan */
515 	pf_data_t	*pf_dq_head_p;	/* ptr to fault data queue */
516 	pf_data_t	*pf_dq_tail_p;	/* ptr pt last fault data q */
517 	uint32_t	pf_total;	/* total non RC pf_datas */
518 } pf_impl_t;
519 
520 /* bus_fm_flags field */
521 #define	PF_FM_READY		(1 << 0)	/* bus_fm_lock initialized */
522 #define	PF_FM_IS_NH		(1 << 1)	/* known as non-hardened */
523 #define	PF_FM_IS_PASSTHRU	(1 << 2)	/* device is controlled by VM */
524 
525 /*
526  * PCIe fabric handle lookup address flags.  Used to define what type of
527  * transaction the address is for.  These same value are defined again in
528  * fabric-xlate FM module.  Do not modify these variables, without modifying
529  * those.
530  */
531 #define	PF_ADDR_DMA		(1 << 0)
532 #define	PF_ADDR_PIO		(1 << 1)
533 #define	PF_ADDR_CFG		(1 << 2)
534 
535 /* PCIe fabric error scanning status flags */
536 #define	PF_SCAN_SUCCESS		(1 << 0)
537 #define	PF_SCAN_CB_FAILURE	(1 << 1) /* hardened device callback failure */
538 #define	PF_SCAN_NO_ERR_IN_CHILD	(1 << 2) /* no errors in bridge sec stat reg */
539 #define	PF_SCAN_IN_DQ		(1 << 3) /* already present in the faultq */
540 #define	PF_SCAN_DEADLOCK	(1 << 4) /* deadlock detected */
541 #define	PF_SCAN_BAD_RESPONSE	(1 << 5) /* Incorrect device response */
542 
543 /* PCIe fabric error handling severity return flags */
544 #define	PF_ERR_NO_ERROR		(1 << 0) /* No error seen */
545 #define	PF_ERR_CE		(1 << 1) /* Correctable Error */
546 #define	PF_ERR_NO_PANIC		(1 << 2) /* Error should not panic sys */
547 #define	PF_ERR_MATCHED_DEVICE	(1 << 3) /* Error Handled By Device */
548 #define	PF_ERR_MATCHED_RC	(1 << 4) /* Error Handled By RC */
549 #define	PF_ERR_MATCHED_PARENT	(1 << 5) /* Error Handled By Parent */
550 #define	PF_ERR_PANIC		(1 << 6) /* Error should panic system */
551 #define	PF_ERR_PANIC_DEADLOCK	(1 << 7) /* deadlock detected */
552 #define	PF_ERR_BAD_RESPONSE	(1 << 8) /* Device bad/no response */
553 #define	PF_ERR_MATCH_DOM	(1 << 9) /* Error Handled By IO domain */
554 
555 #define	PF_ERR_FATAL_FLAGS		(PF_ERR_PANIC | PF_ERR_PANIC_DEADLOCK)
556 
557 #define	PF_HDL_FOUND		1
558 #define	PF_HDL_NOTFOUND		2
559 
560 /*
561  * PCIe Capability Device Type Pseudo Definitions.
562  *
563  * PCI_PSEUDO is used on real PCI devices.  The Legacy PCI definition in the
564  * PCIe spec really refers to PCIe devices that *require* IO Space access.  IO
565  * Space access is usually frowned upon now in PCIe, but there for legacy
566  * purposes.
567  */
568 #define	PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO		0x100
569 #define	PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO	0x101
570 
571 #define	PCIE_INVALID_BDF	0xFFFF
572 #define	PCIE_CHECK_VALID_BDF(x)	(x != PCIE_INVALID_BDF)
573 
574 /*
575  * Default interrupt priority for all PCI and PCIe nexus drivers including
576  * hotplug interrupts.
577  */
578 #define	PCIE_INTR_PRI		(LOCK_LEVEL - 1)
579 
580 /*
581  * XXX - PCIE_IS_PCIE check is required in order not to invoke these macros
582  * for non-standard PCI or PCI Express Hotplug Controllers.
583  */
584 #define	PCIE_ENABLE_ERRORS(dip)	\
585 	if (PCIE_IS_PCIE(PCIE_DIP2BUS(dip))) {	\
586 		pcie_enable_errors(dip);	\
587 		(void) pcie_enable_ce(dip);	\
588 	}
589 
590 #define	PCIE_DISABLE_ERRORS(dip)		\
591 	if (PCIE_IS_PCIE(PCIE_DIP2BUS(dip))) {	\
592 		pcie_disable_errors(dip);	\
593 	}
594 
595 /*
596  * pcie_init_buspcie_fini_bus specific flags
597  */
598 #define	PCIE_BUS_INITIAL	0x0001
599 #define	PCIE_BUS_FINAL		0x0002
600 #define	PCIE_BUS_ALL		(PCIE_BUS_INITIAL | PCIE_BUS_FINAL)
601 
602 #ifdef	DEBUG
603 #define	PCIE_DBG pcie_dbg
604 /* Common Debugging shortcuts */
605 #define	PCIE_DBG_CFG(dip, bus_p, name, sz, off, org) \
606 	PCIE_DBG("%s:%d:(0x%x) %s(0x%x) 0x%x -> 0x%x\n", ddi_node_name(dip), \
607 	    ddi_get_instance(dip), bus_p->bus_bdf, name, off, org, \
608 	    PCIE_GET(sz, bus_p, off))
609 #define	PCIE_DBG_CAP(dip, bus_p, name, sz, off, org) \
610 	PCIE_DBG("%s:%d:(0x%x) %s(0x%x) 0x%x -> 0x%x\n", ddi_node_name(dip), \
611 	    ddi_get_instance(dip), bus_p->bus_bdf, name, off, org, \
612 	    PCIE_CAP_GET(sz, bus_p, off))
613 #define	PCIE_DBG_AER(dip, bus_p, name, sz, off, org) \
614 	PCIE_DBG("%s:%d:(0x%x) %s(0x%x) 0x%x -> 0x%x\n", ddi_node_name(dip), \
615 	    ddi_get_instance(dip), bus_p->bus_bdf, name, off, org, \
616 	    PCIE_AER_GET(sz, bus_p, off))
617 
618 #else	/* DEBUG */
619 
620 #define	PCIE_DBG_CFG(...)	(void)(0)
621 #define	PCIE_DBG(...)		(void)(0)
622 #define	PCIE_ARI_DBG(...)	(void)(0)
623 #define	PCIE_DBG_CAP(...)	(void)(0)
624 #define	PCIE_DBG_AER(...)	(void)(0)
625 
626 #endif	/* DEBUG */
627 
628 /* PCIe Friendly Functions */
629 extern int pcie_init(dev_info_t *dip, caddr_t arg);
630 extern int pcie_uninit(dev_info_t *dip);
631 extern int pcie_hpintr_enable(dev_info_t *dip);
632 extern int pcie_hpintr_disable(dev_info_t *dip);
633 extern int pcie_intr(dev_info_t *dip);
634 extern int pcie_open(dev_info_t *dip, dev_t *devp, int flags, int otyp,
635     cred_t *credp);
636 extern int pcie_close(dev_info_t *dip, dev_t dev, int flags, int otyp,
637     cred_t *credp);
638 extern int pcie_ioctl(dev_info_t *dip, dev_t dev, int cmd, intptr_t arg,
639     int mode, cred_t *credp, int *rvalp);
640 extern int pcie_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op,
641     int flags, char *name, caddr_t valuep, int *lengthp);
642 
643 extern void pcie_fabric_setup(dev_info_t *dip);
644 extern int pcie_initchild(dev_info_t *dip);
645 extern void pcie_uninitchild(dev_info_t *dip);
646 extern int pcie_init_cfghdl(dev_info_t *dip);
647 extern void pcie_fini_cfghdl(dev_info_t *dip);
648 extern void pcie_clear_errors(dev_info_t *dip);
649 extern int pcie_postattach_child(dev_info_t *dip);
650 extern void pcie_enable_errors(dev_info_t *dip);
651 extern void pcie_disable_errors(dev_info_t *dip);
652 extern int pcie_enable_ce(dev_info_t *dip);
653 extern boolean_t pcie_bridge_is_link_disabled(dev_info_t *);
654 extern boolean_t pcie_is_pci_device(dev_info_t *dip);
655 
656 extern pcie_bus_t *pcie_init_bus(dev_info_t *dip, pcie_req_id_t bdf,
657     uint8_t flags);
658 extern void pcie_fini_bus(dev_info_t *dip, uint8_t flags);
659 extern void pcie_fab_init_bus(dev_info_t *dip, uint8_t flags);
660 extern void pcie_fab_fini_bus(dev_info_t *dip, uint8_t flags);
661 extern void pcie_rc_init_bus(dev_info_t *dip);
662 extern void pcie_rc_fini_bus(dev_info_t *dip);
663 extern void pcie_rc_init_pfd(dev_info_t *dip, pf_data_t *pfd);
664 extern void pcie_rc_fini_pfd(pf_data_t *pfd);
665 extern boolean_t pcie_is_child(dev_info_t *dip, dev_info_t *rdip);
666 extern int pcie_get_bdf_from_dip(dev_info_t *dip, pcie_req_id_t *bdf);
667 extern dev_info_t *pcie_get_my_childs_dip(dev_info_t *dip, dev_info_t *rdip);
668 extern uint32_t pcie_get_bdf_for_dma_xfer(dev_info_t *dip, dev_info_t *rdip);
669 extern int pcie_dev(dev_info_t *dip);
670 extern int pcie_root_port(dev_info_t *dip);
671 extern void pcie_set_rber_fatal(dev_info_t *dip, boolean_t val);
672 extern boolean_t pcie_get_rber_fatal(dev_info_t *dip);
673 
674 extern uint32_t pcie_get_aer_uce_mask();
675 extern uint32_t pcie_get_aer_ce_mask();
676 extern uint32_t pcie_get_aer_suce_mask();
677 extern uint32_t pcie_get_serr_mask();
678 extern void pcie_set_aer_uce_mask(uint32_t mask);
679 extern void pcie_set_aer_ce_mask(uint32_t mask);
680 extern void pcie_set_aer_suce_mask(uint32_t mask);
681 extern void pcie_set_serr_mask(uint32_t mask);
682 extern void pcie_init_plat(dev_info_t *dip);
683 extern void pcie_fini_plat(dev_info_t *dip);
684 extern int pcie_read_only_probe(dev_info_t *, char *, dev_info_t **);
685 extern dev_info_t *pcie_func_to_dip(dev_info_t *dip, pcie_req_id_t function);
686 extern int pcie_ari_disable(dev_info_t *dip);
687 extern int pcie_ari_enable(dev_info_t *dip);
688 
689 #define	PCIE_ARI_FORW_NOT_SUPPORTED	0
690 #define	PCIE_ARI_FORW_SUPPORTED		1
691 
692 extern int pcie_ari_supported(dev_info_t *dip);
693 
694 #define	PCIE_ARI_FORW_DISABLED	0
695 #define	PCIE_ARI_FORW_ENABLED	1
696 
697 extern int pcie_ari_is_enabled(dev_info_t *dip);
698 
699 #define	PCIE_NOT_ARI_DEVICE		0
700 #define	PCIE_ARI_DEVICE			1
701 
702 extern int pcie_ari_device(dev_info_t *dip);
703 extern int pcie_ari_get_next_function(dev_info_t *dip, int *func);
704 
705 /* PCIe error handling functions */
706 extern void pf_eh_enter(pcie_bus_t *bus_p);
707 extern void pf_eh_exit(pcie_bus_t *bus_p);
708 extern int pf_scan_fabric(dev_info_t *rpdip, ddi_fm_error_t *derr,
709     pf_data_t *root_pfd_p);
710 extern void pf_set_passthru(dev_info_t *, boolean_t);
711 extern void pf_init(dev_info_t *, ddi_iblock_cookie_t, ddi_attach_cmd_t);
712 extern void pf_fini(dev_info_t *, ddi_detach_cmd_t);
713 extern int pf_hdl_lookup(dev_info_t *, uint64_t, uint32_t, uint64_t,
714     pcie_req_id_t);
715 extern int pf_tlp_decode(pcie_bus_t *, pf_pcie_adv_err_regs_t *);
716 extern void pcie_force_fullscan();
717 
718 #ifdef	DEBUG
719 extern uint_t pcie_debug_flags;
720 extern void pcie_dbg(char *fmt, ...);
721 #endif	/* DEBUG */
722 
723 /* PCIe IOV functions */
724 extern dev_info_t *pcie_find_dip_by_bdf(dev_info_t *rootp, pcie_req_id_t bdf);
725 
726 extern boolean_t pf_in_bus_range(pcie_bus_t *, pcie_req_id_t);
727 extern boolean_t pf_in_assigned_addr(pcie_bus_t *, uint64_t);
728 extern int pf_pci_decode(pf_data_t *, uint16_t *);
729 extern pcie_bus_t *pf_find_busp_by_bdf(pf_impl_t *, pcie_req_id_t);
730 extern pcie_bus_t *pf_find_busp_by_addr(pf_impl_t *, uint64_t);
731 extern pcie_bus_t *pf_find_busp_by_aer(pf_impl_t *, pf_data_t *);
732 extern pcie_bus_t *pf_find_busp_by_saer(pf_impl_t *, pf_data_t *);
733 
734 extern int pciev_eh(pf_data_t *, pf_impl_t *);
735 extern pcie_bus_t *pciev_get_affected_dev(pf_impl_t *, pf_data_t *,
736     uint16_t, uint16_t);
737 extern void pciev_eh_exit(pf_data_t *, uint_t);
738 extern boolean_t pcie_in_domain(pcie_bus_t *, uint_t);
739 
740 /* Link Bandwidth Monitoring */
741 extern boolean_t pcie_link_bw_supported(dev_info_t *);
742 extern int pcie_link_bw_enable(dev_info_t *);
743 extern int pcie_link_bw_disable(dev_info_t *);
744 
745 /* Link Management */
746 extern int pcie_link_set_target(dev_info_t *, pcie_link_speed_t);
747 extern int pcie_link_retrain(dev_info_t *);
748 
749 #define	PCIE_ZALLOC(data) kmem_zalloc(sizeof (data), KM_SLEEP)
750 
751 
752 #ifdef	__cplusplus
753 }
754 #endif
755 
756 #endif	/* _SYS_PCIE_IMPL_H */
757