xref: /linux/drivers/pci/controller/dwc/pcie-tegra194.c (revision e04e2b760ddbe3d7b283a05898c3a029085cd8cd)
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * PCIe host controller driver for the following SoCs
4  * Tegra194
5  * Tegra234
6  *
7  * Copyright (C) 2019-2022 NVIDIA Corporation.
8  *
9  * Author: Vidya Sagar <vidyas@nvidia.com>
10  */
11 
12 #include <linux/bitfield.h>
13 #include <linux/clk.h>
14 #include <linux/debugfs.h>
15 #include <linux/delay.h>
16 #include <linux/gpio/consumer.h>
17 #include <linux/interconnect.h>
18 #include <linux/interrupt.h>
19 #include <linux/iopoll.h>
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/of.h>
23 #include <linux/of_pci.h>
24 #include <linux/pci.h>
25 #include <linux/phy/phy.h>
26 #include <linux/pinctrl/consumer.h>
27 #include <linux/platform_device.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/random.h>
30 #include <linux/reset.h>
31 #include <linux/resource.h>
32 #include <linux/types.h>
33 #include "pcie-designware.h"
34 #include <soc/tegra/bpmp.h>
35 #include <soc/tegra/bpmp-abi.h>
36 #include "../../pci.h"
37 
38 #define TEGRA194_DWC_IP_VER			0x490A
39 #define TEGRA234_DWC_IP_VER			0x562A
40 
41 #define APPL_PINMUX				0x0
42 #define APPL_PINMUX_PEX_RST			BIT(0)
43 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN		BIT(2)
44 #define APPL_PINMUX_CLKREQ_OVERRIDE		BIT(3)
45 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN	BIT(4)
46 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE	BIT(5)
47 
48 #define APPL_CTRL				0x4
49 #define APPL_CTRL_SYS_PRE_DET_STATE		BIT(6)
50 #define APPL_CTRL_LTSSM_EN			BIT(7)
51 #define APPL_CTRL_HW_HOT_RST_EN			BIT(20)
52 #define APPL_CTRL_HW_HOT_RST_MODE_MASK		GENMASK(1, 0)
53 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT		22
54 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST	0x1
55 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN	0x2
56 
57 #define APPL_INTR_EN_L0_0			0x8
58 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN	BIT(0)
59 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN	BIT(4)
60 #define APPL_INTR_EN_L0_0_INT_INT_EN		BIT(8)
61 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN	BIT(15)
62 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN	BIT(19)
63 #define APPL_INTR_EN_L0_0_SYS_INTR_EN		BIT(30)
64 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN	BIT(31)
65 
66 #define APPL_INTR_STATUS_L0			0xC
67 #define APPL_INTR_STATUS_L0_LINK_STATE_INT	BIT(0)
68 #define APPL_INTR_STATUS_L0_INT_INT		BIT(8)
69 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT	BIT(15)
70 #define APPL_INTR_STATUS_L0_PEX_RST_INT		BIT(16)
71 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT	BIT(18)
72 
73 #define APPL_INTR_EN_L1_0_0				0x1C
74 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN	BIT(1)
75 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN		BIT(3)
76 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN	BIT(30)
77 
78 #define APPL_INTR_STATUS_L1_0_0				0x20
79 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED	BIT(1)
80 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED	BIT(3)
81 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE		BIT(30)
82 
83 #define APPL_INTR_STATUS_L1_1			0x2C
84 #define APPL_INTR_STATUS_L1_2			0x30
85 #define APPL_INTR_STATUS_L1_3			0x34
86 #define APPL_INTR_STATUS_L1_6			0x3C
87 #define APPL_INTR_STATUS_L1_7			0x40
88 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED	BIT(1)
89 
90 #define APPL_INTR_EN_L1_8_0			0x44
91 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN		BIT(2)
92 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN	BIT(3)
93 #define APPL_INTR_EN_L1_8_INTX_EN		BIT(11)
94 #define APPL_INTR_EN_L1_8_AER_INT_EN		BIT(15)
95 
96 #define APPL_INTR_STATUS_L1_8_0			0x4C
97 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK	GENMASK(11, 6)
98 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS	BIT(2)
99 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS	BIT(3)
100 
101 #define APPL_INTR_STATUS_L1_9			0x54
102 #define APPL_INTR_STATUS_L1_10			0x58
103 #define APPL_INTR_STATUS_L1_11			0x64
104 #define APPL_INTR_STATUS_L1_13			0x74
105 #define APPL_INTR_STATUS_L1_14			0x78
106 #define APPL_INTR_STATUS_L1_15			0x7C
107 #define APPL_INTR_STATUS_L1_17			0x88
108 
109 #define APPL_INTR_EN_L1_18				0x90
110 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT		BIT(2)
111 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR		BIT(1)
112 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
113 
114 #define APPL_INTR_STATUS_L1_18				0x94
115 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT	BIT(2)
116 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR	BIT(1)
117 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
118 
119 #define APPL_MSI_CTRL_1				0xAC
120 
121 #define APPL_MSI_CTRL_2				0xB0
122 
123 #define APPL_LEGACY_INTX			0xB8
124 
125 #define APPL_LTR_MSG_1				0xC4
126 #define LTR_MSG_REQ				BIT(15)
127 #define LTR_NOSNOOP_MSG_REQ			BIT(31)
128 
129 #define APPL_LTR_MSG_2				0xC8
130 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE	BIT(3)
131 
132 #define APPL_LINK_STATUS			0xCC
133 #define APPL_LINK_STATUS_RDLH_LINK_UP		BIT(0)
134 
135 #define APPL_DEBUG				0xD0
136 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT		BIT(21)
137 #define APPL_DEBUG_PM_LINKST_IN_L0		0x11
138 #define APPL_DEBUG_LTSSM_STATE_MASK		GENMASK(8, 3)
139 #define APPL_DEBUG_LTSSM_STATE_SHIFT		3
140 #define LTSSM_STATE_PRE_DETECT			5
141 
142 #define APPL_RADM_STATUS			0xE4
143 #define APPL_PM_XMT_TURNOFF_STATE		BIT(0)
144 
145 #define APPL_DM_TYPE				0x100
146 #define APPL_DM_TYPE_MASK			GENMASK(3, 0)
147 #define APPL_DM_TYPE_RP				0x4
148 #define APPL_DM_TYPE_EP				0x0
149 
150 #define APPL_CFG_BASE_ADDR			0x104
151 #define APPL_CFG_BASE_ADDR_MASK			GENMASK(31, 12)
152 
153 #define APPL_CFG_IATU_DMA_BASE_ADDR		0x108
154 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK	GENMASK(31, 18)
155 
156 #define APPL_CFG_MISC				0x110
157 #define APPL_CFG_MISC_SLV_EP_MODE		BIT(14)
158 #define APPL_CFG_MISC_ARCACHE_MASK		GENMASK(13, 10)
159 #define APPL_CFG_MISC_ARCACHE_SHIFT		10
160 #define APPL_CFG_MISC_ARCACHE_VAL		3
161 
162 #define APPL_CFG_SLCG_OVERRIDE			0x114
163 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER	BIT(0)
164 
165 #define APPL_CAR_RESET_OVRD				0x12C
166 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N	BIT(0)
167 
168 #define IO_BASE_IO_DECODE				BIT(0)
169 #define IO_BASE_IO_DECODE_BIT8				BIT(8)
170 
171 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE		BIT(0)
172 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE	BIT(16)
173 
174 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF	0x718
175 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT	(19)
176 
177 #define N_FTS_VAL					52
178 #define FTS_VAL						52
179 
180 #define GEN3_EQ_CONTROL_OFF			0x8a8
181 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT	8
182 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK	GENMASK(23, 8)
183 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK	GENMASK(3, 0)
184 
185 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT	0x8D0
186 #define AMBA_ERROR_RESPONSE_CRS_SHIFT		3
187 #define AMBA_ERROR_RESPONSE_CRS_MASK		GENMASK(1, 0)
188 #define AMBA_ERROR_RESPONSE_CRS_OKAY		0
189 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF	1
190 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001	2
191 
192 #define MSIX_ADDR_MATCH_LOW_OFF			0x940
193 #define MSIX_ADDR_MATCH_LOW_OFF_EN		BIT(0)
194 #define MSIX_ADDR_MATCH_LOW_OFF_MASK		GENMASK(31, 2)
195 
196 #define MSIX_ADDR_MATCH_HIGH_OFF		0x944
197 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK		GENMASK(31, 0)
198 
199 #define PORT_LOGIC_MSIX_DOORBELL			0x948
200 
201 #define CAP_SPCIE_CAP_OFF			0x154
202 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK	GENMASK(3, 0)
203 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK	GENMASK(11, 8)
204 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT	8
205 
206 #define PME_ACK_TIMEOUT 10000
207 
208 #define LTSSM_TIMEOUT 50000	/* 50ms */
209 
210 #define GEN3_GEN4_EQ_PRESET_INIT	5
211 
212 #define GEN1_CORE_CLK_FREQ	62500000
213 #define GEN2_CORE_CLK_FREQ	125000000
214 #define GEN3_CORE_CLK_FREQ	250000000
215 #define GEN4_CORE_CLK_FREQ	500000000
216 
217 #define LTR_MSG_TIMEOUT		(100 * 1000)
218 
219 #define PERST_DEBOUNCE_TIME	(5 * 1000)
220 
221 #define EP_STATE_DISABLED	0
222 #define EP_STATE_ENABLED	1
223 
224 static const unsigned int pcie_gen_freq[] = {
225 	GEN1_CORE_CLK_FREQ,	/* PCI_EXP_LNKSTA_CLS == 0; undefined */
226 	GEN1_CORE_CLK_FREQ,
227 	GEN2_CORE_CLK_FREQ,
228 	GEN3_CORE_CLK_FREQ,
229 	GEN4_CORE_CLK_FREQ
230 };
231 
232 struct tegra_pcie_dw_of_data {
233 	u32 version;
234 	enum dw_pcie_device_mode mode;
235 	bool has_msix_doorbell_access_fix;
236 	bool has_sbr_reset_fix;
237 	bool has_l1ss_exit_fix;
238 	bool has_ltr_req_fix;
239 	u32 cdm_chk_int_en_bit;
240 	u32 gen4_preset_vec;
241 	u8 n_fts[2];
242 };
243 
244 struct tegra_pcie_dw {
245 	struct device *dev;
246 	struct resource *appl_res;
247 	struct resource *dbi_res;
248 	struct resource *atu_dma_res;
249 	void __iomem *appl_base;
250 	struct clk *core_clk;
251 	struct reset_control *core_apb_rst;
252 	struct reset_control *core_rst;
253 	struct dw_pcie pci;
254 	struct tegra_bpmp *bpmp;
255 
256 	struct tegra_pcie_dw_of_data *of_data;
257 
258 	bool supports_clkreq;
259 	bool enable_cdm_check;
260 	bool enable_srns;
261 	bool link_state;
262 	bool update_fc_fixup;
263 	bool enable_ext_refclk;
264 	u8 init_link_width;
265 	u32 msi_ctrl_int;
266 	u32 num_lanes;
267 	u32 cid;
268 	u32 cfg_link_cap_l1sub;
269 	u32 ras_des_cap;
270 	u32 pcie_cap_base;
271 	u32 aspm_cmrt;
272 	u32 aspm_pwr_on_t;
273 	u32 aspm_l0s_enter_lat;
274 
275 	struct regulator *pex_ctl_supply;
276 	struct regulator *slot_ctl_3v3;
277 	struct regulator *slot_ctl_12v;
278 
279 	unsigned int phy_count;
280 	struct phy **phys;
281 
282 	struct dentry *debugfs;
283 
284 	/* Endpoint mode specific */
285 	struct gpio_desc *pex_rst_gpiod;
286 	struct gpio_desc *pex_refclk_sel_gpiod;
287 	unsigned int pex_rst_irq;
288 	int ep_state;
289 	long link_status;
290 	struct icc_path *icc_path;
291 };
292 
293 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
294 {
295 	return container_of(pci, struct tegra_pcie_dw, pci);
296 }
297 
298 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
299 			       const u32 reg)
300 {
301 	writel_relaxed(value, pcie->appl_base + reg);
302 }
303 
304 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
305 {
306 	return readl_relaxed(pcie->appl_base + reg);
307 }
308 
309 static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie)
310 {
311 	struct dw_pcie *pci = &pcie->pci;
312 	u32 val, speed, width;
313 
314 	val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
315 
316 	speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val);
317 	width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
318 
319 	val = width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]);
320 
321 	if (icc_set_bw(pcie->icc_path, Mbps_to_icc(val), 0))
322 		dev_err(pcie->dev, "can't set bw[%u]\n", val);
323 
324 	if (speed >= ARRAY_SIZE(pcie_gen_freq))
325 		speed = 0;
326 
327 	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]);
328 }
329 
330 static void apply_bad_link_workaround(struct dw_pcie_rp *pp)
331 {
332 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
333 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
334 	u32 current_link_width;
335 	u16 val;
336 
337 	/*
338 	 * NOTE:- Since this scenario is uncommon and link as such is not
339 	 * stable anyway, not waiting to confirm if link is really
340 	 * transitioning to Gen-2 speed
341 	 */
342 	val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
343 	if (val & PCI_EXP_LNKSTA_LBMS) {
344 		current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
345 		if (pcie->init_link_width > current_link_width) {
346 			dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
347 			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
348 						PCI_EXP_LNKCTL2);
349 			val &= ~PCI_EXP_LNKCTL2_TLS;
350 			val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
351 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
352 					   PCI_EXP_LNKCTL2, val);
353 
354 			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
355 						PCI_EXP_LNKCTL);
356 			val |= PCI_EXP_LNKCTL_RL;
357 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
358 					   PCI_EXP_LNKCTL, val);
359 		}
360 	}
361 }
362 
363 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
364 {
365 	struct tegra_pcie_dw *pcie = arg;
366 	struct dw_pcie *pci = &pcie->pci;
367 	struct dw_pcie_rp *pp = &pci->pp;
368 	u32 val, status_l0, status_l1;
369 	u16 val_w;
370 
371 	status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
372 	if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
373 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
374 		appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
375 		if (!pcie->of_data->has_sbr_reset_fix &&
376 		    status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
377 			/* SBR & Surprise Link Down WAR */
378 			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
379 			val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
380 			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
381 			udelay(1);
382 			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
383 			val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
384 			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
385 
386 			val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
387 			val |= PORT_LOGIC_SPEED_CHANGE;
388 			dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
389 		}
390 	}
391 
392 	if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) {
393 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
394 		if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
395 			appl_writel(pcie,
396 				    APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
397 				    APPL_INTR_STATUS_L1_8_0);
398 			apply_bad_link_workaround(pp);
399 		}
400 		if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
401 			val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
402 						  PCI_EXP_LNKSTA);
403 			val_w |= PCI_EXP_LNKSTA_LBMS;
404 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
405 					   PCI_EXP_LNKSTA, val_w);
406 
407 			appl_writel(pcie,
408 				    APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
409 				    APPL_INTR_STATUS_L1_8_0);
410 
411 			val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
412 						  PCI_EXP_LNKSTA);
413 			dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
414 				PCI_EXP_LNKSTA_CLS);
415 		}
416 	}
417 
418 	if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
419 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
420 		val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
421 		if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
422 			dev_info(pci->dev, "CDM check complete\n");
423 			val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
424 		}
425 		if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
426 			dev_err(pci->dev, "CDM comparison mismatch\n");
427 			val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
428 		}
429 		if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
430 			dev_err(pci->dev, "CDM Logic error\n");
431 			val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
432 		}
433 		dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);
434 		val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
435 		dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val);
436 	}
437 
438 	return IRQ_HANDLED;
439 }
440 
441 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
442 {
443 	u32 val;
444 
445 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
446 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
447 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
448 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
449 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
450 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
451 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
452 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
453 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
454 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
455 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
456 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
457 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
458 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
459 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
460 	appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
461 
462 	val = appl_readl(pcie, APPL_CTRL);
463 	val |= APPL_CTRL_LTSSM_EN;
464 	appl_writel(pcie, val, APPL_CTRL);
465 }
466 
467 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
468 {
469 	struct tegra_pcie_dw *pcie = arg;
470 	struct dw_pcie_ep *ep = &pcie->pci.ep;
471 	struct dw_pcie *pci = &pcie->pci;
472 	u32 val;
473 
474 	if (test_and_clear_bit(0, &pcie->link_status))
475 		dw_pcie_ep_linkup(ep);
476 
477 	tegra_pcie_icc_set(pcie);
478 
479 	if (pcie->of_data->has_ltr_req_fix)
480 		return IRQ_HANDLED;
481 
482 	/* If EP doesn't advertise L1SS, just return */
483 	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
484 	if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
485 		return IRQ_HANDLED;
486 
487 	/* Check if BME is set to '1' */
488 	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
489 	if (val & PCI_COMMAND_MASTER) {
490 		ktime_t timeout;
491 
492 		/* 110us for both snoop and no-snoop */
493 		val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) |
494 		      FIELD_PREP(PCI_LTR_SCALE_MASK, 2) |
495 		      LTR_MSG_REQ |
496 		      FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) |
497 		      FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) |
498 		      LTR_NOSNOOP_MSG_REQ;
499 		appl_writel(pcie, val, APPL_LTR_MSG_1);
500 
501 		/* Send LTR upstream */
502 		val = appl_readl(pcie, APPL_LTR_MSG_2);
503 		val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
504 		appl_writel(pcie, val, APPL_LTR_MSG_2);
505 
506 		timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
507 		for (;;) {
508 			val = appl_readl(pcie, APPL_LTR_MSG_2);
509 			if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
510 				break;
511 			if (ktime_after(ktime_get(), timeout))
512 				break;
513 			usleep_range(1000, 1100);
514 		}
515 		if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
516 			dev_err(pcie->dev, "Failed to send LTR message\n");
517 	}
518 
519 	return IRQ_HANDLED;
520 }
521 
522 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
523 {
524 	struct tegra_pcie_dw *pcie = arg;
525 	int spurious = 1;
526 	u32 status_l0, status_l1, link_status;
527 
528 	status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
529 	if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
530 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
531 		appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
532 
533 		if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
534 			pex_ep_event_hot_rst_done(pcie);
535 
536 		if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
537 			link_status = appl_readl(pcie, APPL_LINK_STATUS);
538 			if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
539 				dev_dbg(pcie->dev, "Link is up with Host\n");
540 				set_bit(0, &pcie->link_status);
541 				return IRQ_WAKE_THREAD;
542 			}
543 		}
544 
545 		spurious = 0;
546 	}
547 
548 	if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
549 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
550 		appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);
551 
552 		if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
553 			return IRQ_WAKE_THREAD;
554 
555 		spurious = 0;
556 	}
557 
558 	if (spurious) {
559 		dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
560 			 status_l0);
561 		appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);
562 	}
563 
564 	return IRQ_HANDLED;
565 }
566 
567 static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
568 				     int size, u32 *val)
569 {
570 	struct dw_pcie_rp *pp = bus->sysdata;
571 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
572 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
573 
574 	/*
575 	 * This is an endpoint mode specific register happen to appear even
576 	 * when controller is operating in root port mode and system hangs
577 	 * when it is accessed with link being in ASPM-L1 state.
578 	 * So skip accessing it altogether
579 	 */
580 	if (!pcie->of_data->has_msix_doorbell_access_fix &&
581 	    !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
582 		*val = 0x00000000;
583 		return PCIBIOS_SUCCESSFUL;
584 	}
585 
586 	return pci_generic_config_read(bus, devfn, where, size, val);
587 }
588 
589 static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
590 				     int size, u32 val)
591 {
592 	struct dw_pcie_rp *pp = bus->sysdata;
593 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
594 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
595 
596 	/*
597 	 * This is an endpoint mode specific register happen to appear even
598 	 * when controller is operating in root port mode and system hangs
599 	 * when it is accessed with link being in ASPM-L1 state.
600 	 * So skip accessing it altogether
601 	 */
602 	if (!pcie->of_data->has_msix_doorbell_access_fix &&
603 	    !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
604 		return PCIBIOS_SUCCESSFUL;
605 
606 	return pci_generic_config_write(bus, devfn, where, size, val);
607 }
608 
609 static struct pci_ops tegra_pci_ops = {
610 	.map_bus = dw_pcie_own_conf_map_bus,
611 	.read = tegra_pcie_dw_rd_own_conf,
612 	.write = tegra_pcie_dw_wr_own_conf,
613 };
614 
615 #if defined(CONFIG_PCIEASPM)
616 static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
617 {
618 	u32 val;
619 
620 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
621 	val &= ~PCI_L1SS_CAP_ASPM_L1_1;
622 	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
623 }
624 
625 static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
626 {
627 	u32 val;
628 
629 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
630 	val &= ~PCI_L1SS_CAP_ASPM_L1_2;
631 	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
632 }
633 
634 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
635 {
636 	u32 val;
637 
638 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
639 				PCIE_RAS_DES_EVENT_COUNTER_CONTROL);
640 	val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
641 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
642 	val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
643 	val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
644 	dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
645 			   PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
646 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
647 				PCIE_RAS_DES_EVENT_COUNTER_DATA);
648 
649 	return val;
650 }
651 
652 static int aspm_state_cnt(struct seq_file *s, void *data)
653 {
654 	struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
655 				     dev_get_drvdata(s->private);
656 	u32 val;
657 
658 	seq_printf(s, "Tx L0s entry count : %u\n",
659 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
660 
661 	seq_printf(s, "Rx L0s entry count : %u\n",
662 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
663 
664 	seq_printf(s, "Link L1 entry count : %u\n",
665 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
666 
667 	seq_printf(s, "Link L1.1 entry count : %u\n",
668 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
669 
670 	seq_printf(s, "Link L1.2 entry count : %u\n",
671 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
672 
673 	/* Clear all counters */
674 	dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
675 			   PCIE_RAS_DES_EVENT_COUNTER_CONTROL,
676 			   EVENT_COUNTER_ALL_CLEAR);
677 
678 	/* Re-enable counting */
679 	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
680 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
681 	dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
682 			   PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
683 
684 	return 0;
685 }
686 
687 static void init_host_aspm(struct tegra_pcie_dw *pcie)
688 {
689 	struct dw_pcie *pci = &pcie->pci;
690 	u32 val;
691 
692 	val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
693 	pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
694 
695 	pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci,
696 							PCI_EXT_CAP_ID_VNDR);
697 
698 	/* Enable ASPM counters */
699 	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
700 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
701 	dw_pcie_writel_dbi(pci, pcie->ras_des_cap +
702 			   PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
703 
704 	/* Program T_cmrt and T_pwr_on values */
705 	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
706 	val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
707 	val |= (pcie->aspm_cmrt << 8);
708 	val |= (pcie->aspm_pwr_on_t << 19);
709 	dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
710 
711 	/* Program L0s and L1 entrance latencies */
712 	val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
713 	val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
714 	val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
715 	val |= PORT_AFR_ENTER_ASPM;
716 	dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
717 }
718 
719 static void init_debugfs(struct tegra_pcie_dw *pcie)
720 {
721 	debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
722 				    aspm_state_cnt);
723 }
724 #else
725 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
726 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
727 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
728 static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
729 #endif
730 
731 static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp)
732 {
733 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
734 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
735 	u32 val;
736 	u16 val_w;
737 
738 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
739 	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
740 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
741 
742 	if (!pcie->of_data->has_sbr_reset_fix) {
743 		val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
744 		val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
745 		appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
746 	}
747 
748 	if (pcie->enable_cdm_check) {
749 		val = appl_readl(pcie, APPL_INTR_EN_L0_0);
750 		val |= pcie->of_data->cdm_chk_int_en_bit;
751 		appl_writel(pcie, val, APPL_INTR_EN_L0_0);
752 
753 		val = appl_readl(pcie, APPL_INTR_EN_L1_18);
754 		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
755 		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
756 		appl_writel(pcie, val, APPL_INTR_EN_L1_18);
757 	}
758 
759 	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
760 				  PCI_EXP_LNKSTA);
761 	pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w);
762 
763 	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
764 				  PCI_EXP_LNKCTL);
765 	val_w |= PCI_EXP_LNKCTL_LBMIE;
766 	dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
767 			   val_w);
768 }
769 
770 static void tegra_pcie_enable_intx_interrupts(struct dw_pcie_rp *pp)
771 {
772 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
773 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
774 	u32 val;
775 
776 	/* Enable INTX interrupt generation */
777 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
778 	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
779 	val |= APPL_INTR_EN_L0_0_INT_INT_EN;
780 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
781 
782 	val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
783 	val |= APPL_INTR_EN_L1_8_INTX_EN;
784 	val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
785 	val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
786 	if (IS_ENABLED(CONFIG_PCIEAER))
787 		val |= APPL_INTR_EN_L1_8_AER_INT_EN;
788 	appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
789 }
790 
791 static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp)
792 {
793 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
794 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
795 	u32 val;
796 
797 	/* Enable MSI interrupt generation */
798 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
799 	val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
800 	val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
801 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
802 }
803 
804 static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp)
805 {
806 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
807 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
808 
809 	/* Clear interrupt statuses before enabling interrupts */
810 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
811 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
812 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
813 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
814 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
815 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
816 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
817 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
818 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
819 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
820 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
821 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
822 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
823 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
824 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
825 
826 	tegra_pcie_enable_system_interrupts(pp);
827 	tegra_pcie_enable_intx_interrupts(pp);
828 	if (IS_ENABLED(CONFIG_PCI_MSI))
829 		tegra_pcie_enable_msi_interrupts(pp);
830 }
831 
832 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
833 {
834 	struct dw_pcie *pci = &pcie->pci;
835 	u32 val, offset, i;
836 
837 	/* Program init preset */
838 	for (i = 0; i < pcie->num_lanes; i++) {
839 		val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
840 		val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
841 		val |= GEN3_GEN4_EQ_PRESET_INIT;
842 		val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
843 		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
844 			   CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
845 		dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
846 
847 		offset = dw_pcie_find_ext_capability(pci,
848 						     PCI_EXT_CAP_ID_PL_16GT) +
849 				PCI_PL_16GT_LE_CTRL;
850 		val = dw_pcie_readb_dbi(pci, offset + i);
851 		val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
852 		val |= GEN3_GEN4_EQ_PRESET_INIT;
853 		val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
854 		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
855 			PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
856 		dw_pcie_writeb_dbi(pci, offset + i, val);
857 	}
858 
859 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
860 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
861 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
862 
863 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
864 	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
865 	val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
866 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
867 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
868 
869 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
870 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
871 	val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
872 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
873 
874 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
875 	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
876 	val |= (pcie->of_data->gen4_preset_vec <<
877 		GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
878 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
879 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
880 
881 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
882 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
883 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
884 }
885 
886 static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp)
887 {
888 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
889 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
890 	u32 val;
891 	u16 val_16;
892 
893 	pp->bridge->ops = &tegra_pci_ops;
894 
895 	if (!pcie->pcie_cap_base)
896 		pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
897 							      PCI_CAP_ID_EXP);
898 
899 	val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
900 	val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
901 	dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
902 
903 	val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
904 	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
905 	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
906 	dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
907 
908 	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
909 
910 	/* Enable as 0xFFFF0001 response for CRS */
911 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
912 	val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
913 	val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
914 		AMBA_ERROR_RESPONSE_CRS_SHIFT);
915 	dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
916 
917 	/* Clear Slot Clock Configuration bit if SRNS configuration */
918 	if (pcie->enable_srns) {
919 		val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
920 					   PCI_EXP_LNKSTA);
921 		val_16 &= ~PCI_EXP_LNKSTA_SLC;
922 		dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
923 				   val_16);
924 	}
925 
926 	config_gen3_gen4_eq_presets(pcie);
927 
928 	init_host_aspm(pcie);
929 
930 	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
931 	if (!pcie->supports_clkreq) {
932 		disable_aspm_l11(pcie);
933 		disable_aspm_l12(pcie);
934 	}
935 
936 	if (!pcie->of_data->has_l1ss_exit_fix) {
937 		val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
938 		val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
939 		dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
940 	}
941 
942 	if (pcie->update_fc_fixup) {
943 		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
944 		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
945 		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
946 	}
947 
948 	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
949 
950 	return 0;
951 }
952 
953 static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
954 {
955 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
956 	struct dw_pcie_rp *pp = &pci->pp;
957 	u32 val, offset, tmp;
958 	bool retry = true;
959 
960 	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
961 		enable_irq(pcie->pex_rst_irq);
962 		return 0;
963 	}
964 
965 retry_link:
966 	/* Assert RST */
967 	val = appl_readl(pcie, APPL_PINMUX);
968 	val &= ~APPL_PINMUX_PEX_RST;
969 	appl_writel(pcie, val, APPL_PINMUX);
970 
971 	usleep_range(100, 200);
972 
973 	/* Enable LTSSM */
974 	val = appl_readl(pcie, APPL_CTRL);
975 	val |= APPL_CTRL_LTSSM_EN;
976 	appl_writel(pcie, val, APPL_CTRL);
977 
978 	/* De-assert RST */
979 	val = appl_readl(pcie, APPL_PINMUX);
980 	val |= APPL_PINMUX_PEX_RST;
981 	appl_writel(pcie, val, APPL_PINMUX);
982 
983 	msleep(100);
984 
985 	if (dw_pcie_wait_for_link(pci)) {
986 		if (!retry)
987 			return 0;
988 		/*
989 		 * There are some endpoints which can't get the link up if
990 		 * root port has Data Link Feature (DLF) enabled.
991 		 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
992 		 * on Scaled Flow Control and DLF.
993 		 * So, need to confirm that is indeed the case here and attempt
994 		 * link up once again with DLF disabled.
995 		 */
996 		val = appl_readl(pcie, APPL_DEBUG);
997 		val &= APPL_DEBUG_LTSSM_STATE_MASK;
998 		val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
999 		tmp = appl_readl(pcie, APPL_LINK_STATUS);
1000 		tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
1001 		if (!(val == 0x11 && !tmp)) {
1002 			/* Link is down for all good reasons */
1003 			return 0;
1004 		}
1005 
1006 		dev_info(pci->dev, "Link is down in DLL");
1007 		dev_info(pci->dev, "Trying again with DLFE disabled\n");
1008 		/* Disable LTSSM */
1009 		val = appl_readl(pcie, APPL_CTRL);
1010 		val &= ~APPL_CTRL_LTSSM_EN;
1011 		appl_writel(pcie, val, APPL_CTRL);
1012 
1013 		reset_control_assert(pcie->core_rst);
1014 		reset_control_deassert(pcie->core_rst);
1015 
1016 		offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1017 		val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1018 		val &= ~PCI_DLF_EXCHANGE_ENABLE;
1019 		dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val);
1020 
1021 		tegra_pcie_dw_host_init(pp);
1022 		dw_pcie_setup_rc(pp);
1023 
1024 		retry = false;
1025 		goto retry_link;
1026 	}
1027 
1028 	tegra_pcie_icc_set(pcie);
1029 
1030 	tegra_pcie_enable_interrupts(pp);
1031 
1032 	return 0;
1033 }
1034 
1035 static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1036 {
1037 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1038 	u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1039 
1040 	return !!(val & PCI_EXP_LNKSTA_DLLLA);
1041 }
1042 
1043 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1044 {
1045 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1046 
1047 	disable_irq(pcie->pex_rst_irq);
1048 }
1049 
1050 static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1051 	.link_up = tegra_pcie_dw_link_up,
1052 	.start_link = tegra_pcie_dw_start_link,
1053 	.stop_link = tegra_pcie_dw_stop_link,
1054 };
1055 
1056 static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1057 	.init = tegra_pcie_dw_host_init,
1058 };
1059 
1060 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1061 {
1062 	unsigned int phy_count = pcie->phy_count;
1063 
1064 	while (phy_count--) {
1065 		phy_power_off(pcie->phys[phy_count]);
1066 		phy_exit(pcie->phys[phy_count]);
1067 	}
1068 }
1069 
1070 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1071 {
1072 	unsigned int i;
1073 	int ret;
1074 
1075 	for (i = 0; i < pcie->phy_count; i++) {
1076 		ret = phy_init(pcie->phys[i]);
1077 		if (ret < 0)
1078 			goto phy_power_off;
1079 
1080 		ret = phy_power_on(pcie->phys[i]);
1081 		if (ret < 0)
1082 			goto phy_exit;
1083 	}
1084 
1085 	return 0;
1086 
1087 phy_power_off:
1088 	while (i--) {
1089 		phy_power_off(pcie->phys[i]);
1090 phy_exit:
1091 		phy_exit(pcie->phys[i]);
1092 	}
1093 
1094 	return ret;
1095 }
1096 
1097 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1098 {
1099 	struct platform_device *pdev = to_platform_device(pcie->dev);
1100 	struct device_node *np = pcie->dev->of_node;
1101 	int ret;
1102 
1103 	pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1104 	if (!pcie->dbi_res) {
1105 		dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1106 		return -ENODEV;
1107 	}
1108 
1109 	ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1110 	if (ret < 0) {
1111 		dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1112 		return ret;
1113 	}
1114 
1115 	ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1116 				   &pcie->aspm_pwr_on_t);
1117 	if (ret < 0)
1118 		dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1119 			 ret);
1120 
1121 	ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1122 				   &pcie->aspm_l0s_enter_lat);
1123 	if (ret < 0)
1124 		dev_info(pcie->dev,
1125 			 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1126 
1127 	ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1128 	if (ret < 0) {
1129 		dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1130 		return ret;
1131 	}
1132 
1133 	ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1134 	if (ret) {
1135 		dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1136 		return ret;
1137 	}
1138 
1139 	ret = of_property_count_strings(np, "phy-names");
1140 	if (ret < 0) {
1141 		dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1142 			ret);
1143 		return ret;
1144 	}
1145 	pcie->phy_count = ret;
1146 
1147 	if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1148 		pcie->update_fc_fixup = true;
1149 
1150 	/* RP using an external REFCLK is supported only in Tegra234 */
1151 	if (pcie->of_data->version == TEGRA194_DWC_IP_VER) {
1152 		if (pcie->of_data->mode == DW_PCIE_EP_TYPE)
1153 			pcie->enable_ext_refclk = true;
1154 	} else {
1155 		pcie->enable_ext_refclk =
1156 			of_property_read_bool(pcie->dev->of_node,
1157 					      "nvidia,enable-ext-refclk");
1158 	}
1159 
1160 	pcie->supports_clkreq =
1161 		of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1162 
1163 	pcie->enable_cdm_check =
1164 		of_property_read_bool(np, "snps,enable-cdm-check");
1165 
1166 	if (pcie->of_data->version == TEGRA234_DWC_IP_VER)
1167 		pcie->enable_srns =
1168 			of_property_read_bool(np, "nvidia,enable-srns");
1169 
1170 	if (pcie->of_data->mode == DW_PCIE_RC_TYPE)
1171 		return 0;
1172 
1173 	/* Endpoint mode specific DT entries */
1174 	pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1175 	if (IS_ERR(pcie->pex_rst_gpiod)) {
1176 		int err = PTR_ERR(pcie->pex_rst_gpiod);
1177 		const char *level = KERN_ERR;
1178 
1179 		if (err == -EPROBE_DEFER)
1180 			level = KERN_DEBUG;
1181 
1182 		dev_printk(level, pcie->dev,
1183 			   dev_fmt("Failed to get PERST GPIO: %d\n"),
1184 			   err);
1185 		return err;
1186 	}
1187 
1188 	pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1189 						    "nvidia,refclk-select",
1190 						    GPIOD_OUT_HIGH);
1191 	if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1192 		int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1193 		const char *level = KERN_ERR;
1194 
1195 		if (err == -EPROBE_DEFER)
1196 			level = KERN_DEBUG;
1197 
1198 		dev_printk(level, pcie->dev,
1199 			   dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1200 			   err);
1201 		pcie->pex_refclk_sel_gpiod = NULL;
1202 	}
1203 
1204 	return 0;
1205 }
1206 
1207 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1208 					  bool enable)
1209 {
1210 	struct mrq_uphy_response resp;
1211 	struct tegra_bpmp_message msg;
1212 	struct mrq_uphy_request req;
1213 
1214 	/*
1215 	 * Controller-5 doesn't need to have its state set by BPMP-FW in
1216 	 * Tegra194
1217 	 */
1218 	if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5)
1219 		return 0;
1220 
1221 	memset(&req, 0, sizeof(req));
1222 	memset(&resp, 0, sizeof(resp));
1223 
1224 	req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1225 	req.controller_state.pcie_controller = pcie->cid;
1226 	req.controller_state.enable = enable;
1227 
1228 	memset(&msg, 0, sizeof(msg));
1229 	msg.mrq = MRQ_UPHY;
1230 	msg.tx.data = &req;
1231 	msg.tx.size = sizeof(req);
1232 	msg.rx.data = &resp;
1233 	msg.rx.size = sizeof(resp);
1234 
1235 	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1236 }
1237 
1238 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1239 					 bool enable)
1240 {
1241 	struct mrq_uphy_response resp;
1242 	struct tegra_bpmp_message msg;
1243 	struct mrq_uphy_request req;
1244 
1245 	memset(&req, 0, sizeof(req));
1246 	memset(&resp, 0, sizeof(resp));
1247 
1248 	if (enable) {
1249 		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1250 		req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1251 	} else {
1252 		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1253 		req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1254 	}
1255 
1256 	memset(&msg, 0, sizeof(msg));
1257 	msg.mrq = MRQ_UPHY;
1258 	msg.tx.data = &req;
1259 	msg.tx.size = sizeof(req);
1260 	msg.rx.data = &resp;
1261 	msg.rx.size = sizeof(resp);
1262 
1263 	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1264 }
1265 
1266 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1267 {
1268 	struct dw_pcie_rp *pp = &pcie->pci.pp;
1269 	struct pci_bus *child, *root_bus = NULL;
1270 	struct pci_dev *pdev;
1271 
1272 	/*
1273 	 * link doesn't go into L2 state with some of the endpoints with Tegra
1274 	 * if they are not in D0 state. So, need to make sure that immediate
1275 	 * downstream devices are in D0 state before sending PME_TurnOff to put
1276 	 * link into L2 state.
1277 	 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1278 	 * 5.2 Link State Power Management (Page #428).
1279 	 */
1280 
1281 	list_for_each_entry(child, &pp->bridge->bus->children, node) {
1282 		/* Bring downstream devices to D0 if they are not already in */
1283 		if (child->parent == pp->bridge->bus) {
1284 			root_bus = child;
1285 			break;
1286 		}
1287 	}
1288 
1289 	if (!root_bus) {
1290 		dev_err(pcie->dev, "Failed to find downstream devices\n");
1291 		return;
1292 	}
1293 
1294 	list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1295 		if (PCI_SLOT(pdev->devfn) == 0) {
1296 			if (pci_set_power_state(pdev, PCI_D0))
1297 				dev_err(pcie->dev,
1298 					"Failed to transition %s to D0 state\n",
1299 					dev_name(&pdev->dev));
1300 		}
1301 	}
1302 }
1303 
1304 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1305 {
1306 	pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1307 	if (IS_ERR(pcie->slot_ctl_3v3)) {
1308 		if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1309 			return PTR_ERR(pcie->slot_ctl_3v3);
1310 
1311 		pcie->slot_ctl_3v3 = NULL;
1312 	}
1313 
1314 	pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1315 	if (IS_ERR(pcie->slot_ctl_12v)) {
1316 		if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1317 			return PTR_ERR(pcie->slot_ctl_12v);
1318 
1319 		pcie->slot_ctl_12v = NULL;
1320 	}
1321 
1322 	return 0;
1323 }
1324 
1325 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1326 {
1327 	int ret;
1328 
1329 	if (pcie->slot_ctl_3v3) {
1330 		ret = regulator_enable(pcie->slot_ctl_3v3);
1331 		if (ret < 0) {
1332 			dev_err(pcie->dev,
1333 				"Failed to enable 3.3V slot supply: %d\n", ret);
1334 			return ret;
1335 		}
1336 	}
1337 
1338 	if (pcie->slot_ctl_12v) {
1339 		ret = regulator_enable(pcie->slot_ctl_12v);
1340 		if (ret < 0) {
1341 			dev_err(pcie->dev,
1342 				"Failed to enable 12V slot supply: %d\n", ret);
1343 			goto fail_12v_enable;
1344 		}
1345 	}
1346 
1347 	/*
1348 	 * According to PCI Express Card Electromechanical Specification
1349 	 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1350 	 * should be a minimum of 100ms.
1351 	 */
1352 	if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1353 		msleep(100);
1354 
1355 	return 0;
1356 
1357 fail_12v_enable:
1358 	if (pcie->slot_ctl_3v3)
1359 		regulator_disable(pcie->slot_ctl_3v3);
1360 	return ret;
1361 }
1362 
1363 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1364 {
1365 	if (pcie->slot_ctl_12v)
1366 		regulator_disable(pcie->slot_ctl_12v);
1367 	if (pcie->slot_ctl_3v3)
1368 		regulator_disable(pcie->slot_ctl_3v3);
1369 }
1370 
1371 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1372 					bool en_hw_hot_rst)
1373 {
1374 	int ret;
1375 	u32 val;
1376 
1377 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1378 	if (ret) {
1379 		dev_err(pcie->dev,
1380 			"Failed to enable controller %u: %d\n", pcie->cid, ret);
1381 		return ret;
1382 	}
1383 
1384 	if (pcie->enable_ext_refclk) {
1385 		ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1386 		if (ret) {
1387 			dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret);
1388 			goto fail_pll_init;
1389 		}
1390 	}
1391 
1392 	ret = tegra_pcie_enable_slot_regulators(pcie);
1393 	if (ret < 0)
1394 		goto fail_slot_reg_en;
1395 
1396 	ret = regulator_enable(pcie->pex_ctl_supply);
1397 	if (ret < 0) {
1398 		dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1399 		goto fail_reg_en;
1400 	}
1401 
1402 	ret = clk_prepare_enable(pcie->core_clk);
1403 	if (ret) {
1404 		dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1405 		goto fail_core_clk;
1406 	}
1407 
1408 	ret = reset_control_deassert(pcie->core_apb_rst);
1409 	if (ret) {
1410 		dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1411 			ret);
1412 		goto fail_core_apb_rst;
1413 	}
1414 
1415 	if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) {
1416 		/* Enable HW_HOT_RST mode */
1417 		val = appl_readl(pcie, APPL_CTRL);
1418 		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1419 			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1420 		val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN <<
1421 			APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1422 		val |= APPL_CTRL_HW_HOT_RST_EN;
1423 		appl_writel(pcie, val, APPL_CTRL);
1424 	}
1425 
1426 	ret = tegra_pcie_enable_phy(pcie);
1427 	if (ret) {
1428 		dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1429 		goto fail_phy;
1430 	}
1431 
1432 	/* Update CFG base address */
1433 	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1434 		    APPL_CFG_BASE_ADDR);
1435 
1436 	/* Configure this core for RP mode operation */
1437 	appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1438 
1439 	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1440 
1441 	val = appl_readl(pcie, APPL_CTRL);
1442 	appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1443 
1444 	val = appl_readl(pcie, APPL_CFG_MISC);
1445 	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1446 	appl_writel(pcie, val, APPL_CFG_MISC);
1447 
1448 	if (pcie->enable_srns || pcie->enable_ext_refclk) {
1449 		/*
1450 		 * When Tegra PCIe RP is using external clock, it cannot supply
1451 		 * same clock to its downstream hierarchy. Hence, gate PCIe RP
1452 		 * REFCLK out pads when RP & EP are using separate clocks or RP
1453 		 * is using an external REFCLK.
1454 		 */
1455 		val = appl_readl(pcie, APPL_PINMUX);
1456 		val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1457 		val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1458 		appl_writel(pcie, val, APPL_PINMUX);
1459 	}
1460 
1461 	if (!pcie->supports_clkreq) {
1462 		val = appl_readl(pcie, APPL_PINMUX);
1463 		val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1464 		val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1465 		appl_writel(pcie, val, APPL_PINMUX);
1466 	}
1467 
1468 	/* Update iATU_DMA base address */
1469 	appl_writel(pcie,
1470 		    pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1471 		    APPL_CFG_IATU_DMA_BASE_ADDR);
1472 
1473 	reset_control_deassert(pcie->core_rst);
1474 
1475 	return ret;
1476 
1477 fail_phy:
1478 	reset_control_assert(pcie->core_apb_rst);
1479 fail_core_apb_rst:
1480 	clk_disable_unprepare(pcie->core_clk);
1481 fail_core_clk:
1482 	regulator_disable(pcie->pex_ctl_supply);
1483 fail_reg_en:
1484 	tegra_pcie_disable_slot_regulators(pcie);
1485 fail_slot_reg_en:
1486 	if (pcie->enable_ext_refclk)
1487 		tegra_pcie_bpmp_set_pll_state(pcie, false);
1488 fail_pll_init:
1489 	tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1490 
1491 	return ret;
1492 }
1493 
1494 static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1495 {
1496 	int ret;
1497 
1498 	ret = reset_control_assert(pcie->core_rst);
1499 	if (ret)
1500 		dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1501 
1502 	tegra_pcie_disable_phy(pcie);
1503 
1504 	ret = reset_control_assert(pcie->core_apb_rst);
1505 	if (ret)
1506 		dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1507 
1508 	clk_disable_unprepare(pcie->core_clk);
1509 
1510 	ret = regulator_disable(pcie->pex_ctl_supply);
1511 	if (ret)
1512 		dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1513 
1514 	tegra_pcie_disable_slot_regulators(pcie);
1515 
1516 	if (pcie->enable_ext_refclk) {
1517 		ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1518 		if (ret)
1519 			dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret);
1520 	}
1521 
1522 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1523 	if (ret)
1524 		dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1525 			pcie->cid, ret);
1526 }
1527 
1528 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1529 {
1530 	struct dw_pcie *pci = &pcie->pci;
1531 	struct dw_pcie_rp *pp = &pci->pp;
1532 	int ret;
1533 
1534 	ret = tegra_pcie_config_controller(pcie, false);
1535 	if (ret < 0)
1536 		return ret;
1537 
1538 	pp->ops = &tegra_pcie_dw_host_ops;
1539 
1540 	ret = dw_pcie_host_init(pp);
1541 	if (ret < 0) {
1542 		dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1543 		goto fail_host_init;
1544 	}
1545 
1546 	return 0;
1547 
1548 fail_host_init:
1549 	tegra_pcie_unconfig_controller(pcie);
1550 	return ret;
1551 }
1552 
1553 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1554 {
1555 	u32 val;
1556 
1557 	if (!tegra_pcie_dw_link_up(&pcie->pci))
1558 		return 0;
1559 
1560 	val = appl_readl(pcie, APPL_RADM_STATUS);
1561 	val |= APPL_PM_XMT_TURNOFF_STATE;
1562 	appl_writel(pcie, val, APPL_RADM_STATUS);
1563 
1564 	return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1565 				 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1566 				 1, PME_ACK_TIMEOUT);
1567 }
1568 
1569 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1570 {
1571 	u32 data;
1572 	int err;
1573 
1574 	if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1575 		dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1576 		return;
1577 	}
1578 
1579 	/*
1580 	 * PCIe controller exits from L2 only if reset is applied, so
1581 	 * controller doesn't handle interrupts. But in cases where
1582 	 * L2 entry fails, PERST# is asserted which can trigger surprise
1583 	 * link down AER. However this function call happens in
1584 	 * suspend_noirq(), so AER interrupt will not be processed.
1585 	 * Disable all interrupts to avoid such a scenario.
1586 	 */
1587 	appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0);
1588 
1589 	if (tegra_pcie_try_link_l2(pcie)) {
1590 		dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1591 		/*
1592 		 * TX lane clock freq will reset to Gen1 only if link is in L2
1593 		 * or detect state.
1594 		 * So apply pex_rst to end point to force RP to go into detect
1595 		 * state
1596 		 */
1597 		data = appl_readl(pcie, APPL_PINMUX);
1598 		data &= ~APPL_PINMUX_PEX_RST;
1599 		appl_writel(pcie, data, APPL_PINMUX);
1600 
1601 		/*
1602 		 * Some cards do not go to detect state even after de-asserting
1603 		 * PERST#. So, de-assert LTSSM to bring link to detect state.
1604 		 */
1605 		data = readl(pcie->appl_base + APPL_CTRL);
1606 		data &= ~APPL_CTRL_LTSSM_EN;
1607 		writel(data, pcie->appl_base + APPL_CTRL);
1608 
1609 		err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1610 						data,
1611 						((data &
1612 						APPL_DEBUG_LTSSM_STATE_MASK) >>
1613 						APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1614 						LTSSM_STATE_PRE_DETECT,
1615 						1, LTSSM_TIMEOUT);
1616 		if (err)
1617 			dev_info(pcie->dev, "Link didn't go to detect state\n");
1618 	}
1619 	/*
1620 	 * DBI registers may not be accessible after this as PLL-E would be
1621 	 * down depending on how CLKREQ is pulled by end point
1622 	 */
1623 	data = appl_readl(pcie, APPL_PINMUX);
1624 	data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1625 	/* Cut REFCLK to slot */
1626 	data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1627 	data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1628 	appl_writel(pcie, data, APPL_PINMUX);
1629 }
1630 
1631 static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1632 {
1633 	tegra_pcie_downstream_dev_to_D0(pcie);
1634 	dw_pcie_host_deinit(&pcie->pci.pp);
1635 	tegra_pcie_dw_pme_turnoff(pcie);
1636 	tegra_pcie_unconfig_controller(pcie);
1637 }
1638 
1639 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1640 {
1641 	struct device *dev = pcie->dev;
1642 	char *name;
1643 	int ret;
1644 
1645 	pm_runtime_enable(dev);
1646 
1647 	ret = pm_runtime_get_sync(dev);
1648 	if (ret < 0) {
1649 		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1650 			ret);
1651 		goto fail_pm_get_sync;
1652 	}
1653 
1654 	ret = pinctrl_pm_select_default_state(dev);
1655 	if (ret < 0) {
1656 		dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1657 		goto fail_pm_get_sync;
1658 	}
1659 
1660 	ret = tegra_pcie_init_controller(pcie);
1661 	if (ret < 0) {
1662 		dev_err(dev, "Failed to initialize controller: %d\n", ret);
1663 		goto fail_pm_get_sync;
1664 	}
1665 
1666 	pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1667 	if (!pcie->link_state) {
1668 		ret = -ENOMEDIUM;
1669 		goto fail_host_init;
1670 	}
1671 
1672 	name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1673 	if (!name) {
1674 		ret = -ENOMEM;
1675 		goto fail_host_init;
1676 	}
1677 
1678 	pcie->debugfs = debugfs_create_dir(name, NULL);
1679 	init_debugfs(pcie);
1680 
1681 	return ret;
1682 
1683 fail_host_init:
1684 	tegra_pcie_deinit_controller(pcie);
1685 fail_pm_get_sync:
1686 	pm_runtime_put_sync(dev);
1687 	pm_runtime_disable(dev);
1688 	return ret;
1689 }
1690 
1691 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1692 {
1693 	u32 val;
1694 	int ret;
1695 
1696 	if (pcie->ep_state == EP_STATE_DISABLED)
1697 		return;
1698 
1699 	/* Disable LTSSM */
1700 	val = appl_readl(pcie, APPL_CTRL);
1701 	val &= ~APPL_CTRL_LTSSM_EN;
1702 	appl_writel(pcie, val, APPL_CTRL);
1703 
1704 	ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1705 				 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1706 				 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1707 				 LTSSM_STATE_PRE_DETECT,
1708 				 1, LTSSM_TIMEOUT);
1709 	if (ret)
1710 		dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1711 
1712 	pci_epc_deinit_notify(pcie->pci.ep.epc);
1713 	dw_pcie_ep_cleanup(&pcie->pci.ep);
1714 
1715 	reset_control_assert(pcie->core_rst);
1716 
1717 	tegra_pcie_disable_phy(pcie);
1718 
1719 	reset_control_assert(pcie->core_apb_rst);
1720 
1721 	clk_disable_unprepare(pcie->core_clk);
1722 
1723 	pm_runtime_put_sync(pcie->dev);
1724 
1725 	if (pcie->enable_ext_refclk) {
1726 		ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1727 		if (ret)
1728 			dev_err(pcie->dev, "Failed to turn off UPHY: %d\n",
1729 				ret);
1730 	}
1731 
1732 	ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1733 	if (ret)
1734 		dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1735 
1736 	pcie->ep_state = EP_STATE_DISABLED;
1737 	dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1738 }
1739 
1740 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1741 {
1742 	struct dw_pcie *pci = &pcie->pci;
1743 	struct dw_pcie_ep *ep = &pci->ep;
1744 	struct device *dev = pcie->dev;
1745 	u32 val;
1746 	int ret;
1747 	u16 val_16;
1748 
1749 	if (pcie->ep_state == EP_STATE_ENABLED)
1750 		return;
1751 
1752 	ret = pm_runtime_resume_and_get(dev);
1753 	if (ret < 0) {
1754 		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1755 			ret);
1756 		return;
1757 	}
1758 
1759 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1760 	if (ret) {
1761 		dev_err(pcie->dev, "Failed to enable controller %u: %d\n",
1762 			pcie->cid, ret);
1763 		goto fail_set_ctrl_state;
1764 	}
1765 
1766 	if (pcie->enable_ext_refclk) {
1767 		ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1768 		if (ret) {
1769 			dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n",
1770 				ret);
1771 			goto fail_pll_init;
1772 		}
1773 	}
1774 
1775 	ret = clk_prepare_enable(pcie->core_clk);
1776 	if (ret) {
1777 		dev_err(dev, "Failed to enable core clock: %d\n", ret);
1778 		goto fail_core_clk_enable;
1779 	}
1780 
1781 	ret = reset_control_deassert(pcie->core_apb_rst);
1782 	if (ret) {
1783 		dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1784 		goto fail_core_apb_rst;
1785 	}
1786 
1787 	ret = tegra_pcie_enable_phy(pcie);
1788 	if (ret) {
1789 		dev_err(dev, "Failed to enable PHY: %d\n", ret);
1790 		goto fail_phy;
1791 	}
1792 
1793 	/* Clear any stale interrupt statuses */
1794 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1795 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1796 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1797 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1798 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1799 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1800 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1801 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1802 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1803 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1804 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1805 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1806 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1807 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1808 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1809 
1810 	/* configure this core for EP mode operation */
1811 	val = appl_readl(pcie, APPL_DM_TYPE);
1812 	val &= ~APPL_DM_TYPE_MASK;
1813 	val |= APPL_DM_TYPE_EP;
1814 	appl_writel(pcie, val, APPL_DM_TYPE);
1815 
1816 	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1817 
1818 	val = appl_readl(pcie, APPL_CTRL);
1819 	val |= APPL_CTRL_SYS_PRE_DET_STATE;
1820 	val |= APPL_CTRL_HW_HOT_RST_EN;
1821 	appl_writel(pcie, val, APPL_CTRL);
1822 
1823 	val = appl_readl(pcie, APPL_CFG_MISC);
1824 	val |= APPL_CFG_MISC_SLV_EP_MODE;
1825 	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1826 	appl_writel(pcie, val, APPL_CFG_MISC);
1827 
1828 	val = appl_readl(pcie, APPL_PINMUX);
1829 	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1830 	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1831 	appl_writel(pcie, val, APPL_PINMUX);
1832 
1833 	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1834 		    APPL_CFG_BASE_ADDR);
1835 
1836 	appl_writel(pcie, pcie->atu_dma_res->start &
1837 		    APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1838 		    APPL_CFG_IATU_DMA_BASE_ADDR);
1839 
1840 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1841 	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1842 	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1843 	val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1844 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1845 
1846 	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1847 	val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1848 	val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1849 	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1850 
1851 	reset_control_deassert(pcie->core_rst);
1852 
1853 	if (pcie->update_fc_fixup) {
1854 		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1855 		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1856 		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1857 	}
1858 
1859 	config_gen3_gen4_eq_presets(pcie);
1860 
1861 	init_host_aspm(pcie);
1862 
1863 	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1864 	if (!pcie->supports_clkreq) {
1865 		disable_aspm_l11(pcie);
1866 		disable_aspm_l12(pcie);
1867 	}
1868 
1869 	if (!pcie->of_data->has_l1ss_exit_fix) {
1870 		val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1871 		val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1872 		dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1873 	}
1874 
1875 	pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1876 						      PCI_CAP_ID_EXP);
1877 
1878 	/* Clear Slot Clock Configuration bit if SRNS configuration */
1879 	if (pcie->enable_srns) {
1880 		val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
1881 					   PCI_EXP_LNKSTA);
1882 		val_16 &= ~PCI_EXP_LNKSTA_SLC;
1883 		dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
1884 				   val_16);
1885 	}
1886 
1887 	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1888 
1889 	val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1890 	val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1891 	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1892 	val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1893 	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1894 
1895 	ret = dw_pcie_ep_init_registers(ep);
1896 	if (ret) {
1897 		dev_err(dev, "Failed to complete initialization: %d\n", ret);
1898 		goto fail_init_complete;
1899 	}
1900 
1901 	pci_epc_init_notify(ep->epc);
1902 
1903 	/* Program the private control to allow sending LTR upstream */
1904 	if (pcie->of_data->has_ltr_req_fix) {
1905 		val = appl_readl(pcie, APPL_LTR_MSG_2);
1906 		val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
1907 		appl_writel(pcie, val, APPL_LTR_MSG_2);
1908 	}
1909 
1910 	/* Enable LTSSM */
1911 	val = appl_readl(pcie, APPL_CTRL);
1912 	val |= APPL_CTRL_LTSSM_EN;
1913 	appl_writel(pcie, val, APPL_CTRL);
1914 
1915 	pcie->ep_state = EP_STATE_ENABLED;
1916 	dev_dbg(dev, "Initialization of endpoint is completed\n");
1917 
1918 	return;
1919 
1920 fail_init_complete:
1921 	reset_control_assert(pcie->core_rst);
1922 	tegra_pcie_disable_phy(pcie);
1923 fail_phy:
1924 	reset_control_assert(pcie->core_apb_rst);
1925 fail_core_apb_rst:
1926 	clk_disable_unprepare(pcie->core_clk);
1927 fail_core_clk_enable:
1928 	tegra_pcie_bpmp_set_pll_state(pcie, false);
1929 fail_pll_init:
1930 	tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1931 fail_set_ctrl_state:
1932 	pm_runtime_put_sync(dev);
1933 }
1934 
1935 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1936 {
1937 	struct tegra_pcie_dw *pcie = arg;
1938 
1939 	if (gpiod_get_value(pcie->pex_rst_gpiod))
1940 		pex_ep_event_pex_rst_assert(pcie);
1941 	else
1942 		pex_ep_event_pex_rst_deassert(pcie);
1943 
1944 	return IRQ_HANDLED;
1945 }
1946 
1947 static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq)
1948 {
1949 	/* Tegra194 supports only INTA */
1950 	if (irq > 1)
1951 		return -EINVAL;
1952 
1953 	appl_writel(pcie, 1, APPL_LEGACY_INTX);
1954 	usleep_range(1000, 2000);
1955 	appl_writel(pcie, 0, APPL_LEGACY_INTX);
1956 	return 0;
1957 }
1958 
1959 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1960 {
1961 	if (unlikely(irq > 31))
1962 		return -EINVAL;
1963 
1964 	appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1965 
1966 	return 0;
1967 }
1968 
1969 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1970 {
1971 	struct dw_pcie_ep *ep = &pcie->pci.ep;
1972 
1973 	writel(irq, ep->msi_mem);
1974 
1975 	return 0;
1976 }
1977 
1978 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1979 				   unsigned int type, u16 interrupt_num)
1980 {
1981 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1982 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1983 
1984 	switch (type) {
1985 	case PCI_IRQ_INTX:
1986 		return tegra_pcie_ep_raise_intx_irq(pcie, interrupt_num);
1987 
1988 	case PCI_IRQ_MSI:
1989 		return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1990 
1991 	case PCI_IRQ_MSIX:
1992 		return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1993 
1994 	default:
1995 		dev_err(pci->dev, "Unknown IRQ type\n");
1996 		return -EPERM;
1997 	}
1998 
1999 	return 0;
2000 }
2001 
2002 static const struct pci_epc_features tegra_pcie_epc_features = {
2003 	.linkup_notifier = true,
2004 	.msi_capable = false,
2005 	.msix_capable = false,
2006 	.bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M,
2007 			.only_64bit = true, },
2008 	.bar[BAR_1] = { .type = BAR_RESERVED, },
2009 	.bar[BAR_2] = { .type = BAR_RESERVED, },
2010 	.bar[BAR_3] = { .type = BAR_RESERVED, },
2011 	.bar[BAR_4] = { .type = BAR_RESERVED, },
2012 	.bar[BAR_5] = { .type = BAR_RESERVED, },
2013 	.align = SZ_64K,
2014 };
2015 
2016 static const struct pci_epc_features*
2017 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
2018 {
2019 	return &tegra_pcie_epc_features;
2020 }
2021 
2022 static const struct dw_pcie_ep_ops pcie_ep_ops = {
2023 	.raise_irq = tegra_pcie_ep_raise_irq,
2024 	.get_features = tegra_pcie_ep_get_features,
2025 };
2026 
2027 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
2028 				struct platform_device *pdev)
2029 {
2030 	struct dw_pcie *pci = &pcie->pci;
2031 	struct device *dev = pcie->dev;
2032 	struct dw_pcie_ep *ep;
2033 	char *name;
2034 	int ret;
2035 
2036 	ep = &pci->ep;
2037 	ep->ops = &pcie_ep_ops;
2038 
2039 	ep->page_size = SZ_64K;
2040 
2041 	ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
2042 	if (ret < 0) {
2043 		dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
2044 			ret);
2045 		return ret;
2046 	}
2047 
2048 	ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2049 	if (ret < 0) {
2050 		dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2051 		return ret;
2052 	}
2053 	pcie->pex_rst_irq = (unsigned int)ret;
2054 
2055 	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2056 			      pcie->cid);
2057 	if (!name) {
2058 		dev_err(dev, "Failed to create PERST IRQ string\n");
2059 		return -ENOMEM;
2060 	}
2061 
2062 	irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2063 
2064 	pcie->ep_state = EP_STATE_DISABLED;
2065 
2066 	ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2067 					tegra_pcie_ep_pex_rst_irq,
2068 					IRQF_TRIGGER_RISING |
2069 					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2070 					name, (void *)pcie);
2071 	if (ret < 0) {
2072 		dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2073 		return ret;
2074 	}
2075 
2076 	pm_runtime_enable(dev);
2077 
2078 	ret = dw_pcie_ep_init(ep);
2079 	if (ret) {
2080 		dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2081 			ret);
2082 		pm_runtime_disable(dev);
2083 		return ret;
2084 	}
2085 
2086 	return 0;
2087 }
2088 
2089 static int tegra_pcie_dw_probe(struct platform_device *pdev)
2090 {
2091 	const struct tegra_pcie_dw_of_data *data;
2092 	struct device *dev = &pdev->dev;
2093 	struct resource *atu_dma_res;
2094 	struct tegra_pcie_dw *pcie;
2095 	struct dw_pcie_rp *pp;
2096 	struct dw_pcie *pci;
2097 	struct phy **phys;
2098 	char *name;
2099 	int ret;
2100 	u32 i;
2101 
2102 	data = of_device_get_match_data(dev);
2103 
2104 	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2105 	if (!pcie)
2106 		return -ENOMEM;
2107 
2108 	pci = &pcie->pci;
2109 	pci->dev = &pdev->dev;
2110 	pci->ops = &tegra_dw_pcie_ops;
2111 	pcie->dev = &pdev->dev;
2112 	pcie->of_data = (struct tegra_pcie_dw_of_data *)data;
2113 	pci->n_fts[0] = pcie->of_data->n_fts[0];
2114 	pci->n_fts[1] = pcie->of_data->n_fts[1];
2115 	pp = &pci->pp;
2116 	pp->num_vectors = MAX_MSI_IRQS;
2117 
2118 	ret = tegra_pcie_dw_parse_dt(pcie);
2119 	if (ret < 0) {
2120 		const char *level = KERN_ERR;
2121 
2122 		if (ret == -EPROBE_DEFER)
2123 			level = KERN_DEBUG;
2124 
2125 		dev_printk(level, dev,
2126 			   dev_fmt("Failed to parse device tree: %d\n"),
2127 			   ret);
2128 		return ret;
2129 	}
2130 
2131 	ret = tegra_pcie_get_slot_regulators(pcie);
2132 	if (ret < 0) {
2133 		const char *level = KERN_ERR;
2134 
2135 		if (ret == -EPROBE_DEFER)
2136 			level = KERN_DEBUG;
2137 
2138 		dev_printk(level, dev,
2139 			   dev_fmt("Failed to get slot regulators: %d\n"),
2140 			   ret);
2141 		return ret;
2142 	}
2143 
2144 	if (pcie->pex_refclk_sel_gpiod)
2145 		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2146 
2147 	pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2148 	if (IS_ERR(pcie->pex_ctl_supply)) {
2149 		ret = PTR_ERR(pcie->pex_ctl_supply);
2150 		if (ret != -EPROBE_DEFER)
2151 			dev_err(dev, "Failed to get regulator: %ld\n",
2152 				PTR_ERR(pcie->pex_ctl_supply));
2153 		return ret;
2154 	}
2155 
2156 	pcie->core_clk = devm_clk_get(dev, "core");
2157 	if (IS_ERR(pcie->core_clk)) {
2158 		dev_err(dev, "Failed to get core clock: %ld\n",
2159 			PTR_ERR(pcie->core_clk));
2160 		return PTR_ERR(pcie->core_clk);
2161 	}
2162 
2163 	pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2164 						      "appl");
2165 	if (!pcie->appl_res) {
2166 		dev_err(dev, "Failed to find \"appl\" region\n");
2167 		return -ENODEV;
2168 	}
2169 
2170 	pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2171 	if (IS_ERR(pcie->appl_base))
2172 		return PTR_ERR(pcie->appl_base);
2173 
2174 	pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2175 	if (IS_ERR(pcie->core_apb_rst)) {
2176 		dev_err(dev, "Failed to get APB reset: %ld\n",
2177 			PTR_ERR(pcie->core_apb_rst));
2178 		return PTR_ERR(pcie->core_apb_rst);
2179 	}
2180 
2181 	phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2182 	if (!phys)
2183 		return -ENOMEM;
2184 
2185 	for (i = 0; i < pcie->phy_count; i++) {
2186 		name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2187 		if (!name) {
2188 			dev_err(dev, "Failed to create P2U string\n");
2189 			return -ENOMEM;
2190 		}
2191 		phys[i] = devm_phy_get(dev, name);
2192 		kfree(name);
2193 		if (IS_ERR(phys[i])) {
2194 			ret = PTR_ERR(phys[i]);
2195 			if (ret != -EPROBE_DEFER)
2196 				dev_err(dev, "Failed to get PHY: %d\n", ret);
2197 			return ret;
2198 		}
2199 	}
2200 
2201 	pcie->phys = phys;
2202 
2203 	atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2204 						   "atu_dma");
2205 	if (!atu_dma_res) {
2206 		dev_err(dev, "Failed to find \"atu_dma\" region\n");
2207 		return -ENODEV;
2208 	}
2209 	pcie->atu_dma_res = atu_dma_res;
2210 
2211 	pci->atu_size = resource_size(atu_dma_res);
2212 	pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2213 	if (IS_ERR(pci->atu_base))
2214 		return PTR_ERR(pci->atu_base);
2215 
2216 	pcie->core_rst = devm_reset_control_get(dev, "core");
2217 	if (IS_ERR(pcie->core_rst)) {
2218 		dev_err(dev, "Failed to get core reset: %ld\n",
2219 			PTR_ERR(pcie->core_rst));
2220 		return PTR_ERR(pcie->core_rst);
2221 	}
2222 
2223 	pp->irq = platform_get_irq_byname(pdev, "intr");
2224 	if (pp->irq < 0)
2225 		return pp->irq;
2226 
2227 	pcie->bpmp = tegra_bpmp_get(dev);
2228 	if (IS_ERR(pcie->bpmp))
2229 		return PTR_ERR(pcie->bpmp);
2230 
2231 	platform_set_drvdata(pdev, pcie);
2232 
2233 	pcie->icc_path = devm_of_icc_get(&pdev->dev, "write");
2234 	ret = PTR_ERR_OR_ZERO(pcie->icc_path);
2235 	if (ret) {
2236 		tegra_bpmp_put(pcie->bpmp);
2237 		dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n");
2238 		return ret;
2239 	}
2240 
2241 	switch (pcie->of_data->mode) {
2242 	case DW_PCIE_RC_TYPE:
2243 		ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2244 				       IRQF_SHARED, "tegra-pcie-intr", pcie);
2245 		if (ret) {
2246 			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2247 				ret);
2248 			goto fail;
2249 		}
2250 
2251 		ret = tegra_pcie_config_rp(pcie);
2252 		if (ret && ret != -ENOMEDIUM)
2253 			goto fail;
2254 		else
2255 			return 0;
2256 		break;
2257 
2258 	case DW_PCIE_EP_TYPE:
2259 		ret = devm_request_threaded_irq(dev, pp->irq,
2260 						tegra_pcie_ep_hard_irq,
2261 						tegra_pcie_ep_irq_thread,
2262 						IRQF_SHARED | IRQF_ONESHOT,
2263 						"tegra-pcie-ep-intr", pcie);
2264 		if (ret) {
2265 			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2266 				ret);
2267 			goto fail;
2268 		}
2269 
2270 		ret = tegra_pcie_config_ep(pcie, pdev);
2271 		if (ret < 0)
2272 			goto fail;
2273 		else
2274 			return 0;
2275 		break;
2276 
2277 	default:
2278 		dev_err(dev, "Invalid PCIe device type %d\n",
2279 			pcie->of_data->mode);
2280 		ret = -EINVAL;
2281 	}
2282 
2283 fail:
2284 	tegra_bpmp_put(pcie->bpmp);
2285 	return ret;
2286 }
2287 
2288 static void tegra_pcie_dw_remove(struct platform_device *pdev)
2289 {
2290 	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2291 
2292 	if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2293 		if (!pcie->link_state)
2294 			return;
2295 
2296 		debugfs_remove_recursive(pcie->debugfs);
2297 		tegra_pcie_deinit_controller(pcie);
2298 		pm_runtime_put_sync(pcie->dev);
2299 	} else {
2300 		disable_irq(pcie->pex_rst_irq);
2301 		pex_ep_event_pex_rst_assert(pcie);
2302 	}
2303 
2304 	pm_runtime_disable(pcie->dev);
2305 	tegra_bpmp_put(pcie->bpmp);
2306 	if (pcie->pex_refclk_sel_gpiod)
2307 		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2308 }
2309 
2310 static int tegra_pcie_dw_suspend_late(struct device *dev)
2311 {
2312 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2313 	u32 val;
2314 
2315 	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2316 		dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n");
2317 		return -EPERM;
2318 	}
2319 
2320 	if (!pcie->link_state)
2321 		return 0;
2322 
2323 	/* Enable HW_HOT_RST mode */
2324 	if (!pcie->of_data->has_sbr_reset_fix) {
2325 		val = appl_readl(pcie, APPL_CTRL);
2326 		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2327 			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2328 		val |= APPL_CTRL_HW_HOT_RST_EN;
2329 		appl_writel(pcie, val, APPL_CTRL);
2330 	}
2331 
2332 	return 0;
2333 }
2334 
2335 static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2336 {
2337 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2338 
2339 	if (!pcie->link_state)
2340 		return 0;
2341 
2342 	tegra_pcie_downstream_dev_to_D0(pcie);
2343 	tegra_pcie_dw_pme_turnoff(pcie);
2344 	tegra_pcie_unconfig_controller(pcie);
2345 
2346 	return 0;
2347 }
2348 
2349 static int tegra_pcie_dw_resume_noirq(struct device *dev)
2350 {
2351 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2352 	int ret;
2353 
2354 	if (!pcie->link_state)
2355 		return 0;
2356 
2357 	ret = tegra_pcie_config_controller(pcie, true);
2358 	if (ret < 0)
2359 		return ret;
2360 
2361 	ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2362 	if (ret < 0) {
2363 		dev_err(dev, "Failed to init host: %d\n", ret);
2364 		goto fail_host_init;
2365 	}
2366 
2367 	dw_pcie_setup_rc(&pcie->pci.pp);
2368 
2369 	ret = tegra_pcie_dw_start_link(&pcie->pci);
2370 	if (ret < 0)
2371 		goto fail_host_init;
2372 
2373 	return 0;
2374 
2375 fail_host_init:
2376 	tegra_pcie_unconfig_controller(pcie);
2377 	return ret;
2378 }
2379 
2380 static int tegra_pcie_dw_resume_early(struct device *dev)
2381 {
2382 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2383 	u32 val;
2384 
2385 	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2386 		dev_err(dev, "Suspend is not supported in EP mode");
2387 		return -ENOTSUPP;
2388 	}
2389 
2390 	if (!pcie->link_state)
2391 		return 0;
2392 
2393 	/* Disable HW_HOT_RST mode */
2394 	if (!pcie->of_data->has_sbr_reset_fix) {
2395 		val = appl_readl(pcie, APPL_CTRL);
2396 		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2397 			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2398 		val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2399 		       APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2400 		val &= ~APPL_CTRL_HW_HOT_RST_EN;
2401 		appl_writel(pcie, val, APPL_CTRL);
2402 	}
2403 
2404 	return 0;
2405 }
2406 
2407 static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2408 {
2409 	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2410 
2411 	if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2412 		if (!pcie->link_state)
2413 			return;
2414 
2415 		debugfs_remove_recursive(pcie->debugfs);
2416 		tegra_pcie_downstream_dev_to_D0(pcie);
2417 
2418 		disable_irq(pcie->pci.pp.irq);
2419 		if (IS_ENABLED(CONFIG_PCI_MSI))
2420 			disable_irq(pcie->pci.pp.msi_irq[0]);
2421 
2422 		tegra_pcie_dw_pme_turnoff(pcie);
2423 		tegra_pcie_unconfig_controller(pcie);
2424 		pm_runtime_put_sync(pcie->dev);
2425 	} else {
2426 		disable_irq(pcie->pex_rst_irq);
2427 		pex_ep_event_pex_rst_assert(pcie);
2428 	}
2429 }
2430 
2431 static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = {
2432 	.version = TEGRA194_DWC_IP_VER,
2433 	.mode = DW_PCIE_RC_TYPE,
2434 	.cdm_chk_int_en_bit = BIT(19),
2435 	/* Gen4 - 5, 6, 8 and 9 presets enabled */
2436 	.gen4_preset_vec = 0x360,
2437 	.n_fts = { 52, 52 },
2438 };
2439 
2440 static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = {
2441 	.version = TEGRA194_DWC_IP_VER,
2442 	.mode = DW_PCIE_EP_TYPE,
2443 	.cdm_chk_int_en_bit = BIT(19),
2444 	/* Gen4 - 5, 6, 8 and 9 presets enabled */
2445 	.gen4_preset_vec = 0x360,
2446 	.n_fts = { 52, 52 },
2447 };
2448 
2449 static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = {
2450 	.version = TEGRA234_DWC_IP_VER,
2451 	.mode = DW_PCIE_RC_TYPE,
2452 	.has_msix_doorbell_access_fix = true,
2453 	.has_sbr_reset_fix = true,
2454 	.has_l1ss_exit_fix = true,
2455 	.cdm_chk_int_en_bit = BIT(18),
2456 	/* Gen4 - 6, 8 and 9 presets enabled */
2457 	.gen4_preset_vec = 0x340,
2458 	.n_fts = { 52, 80 },
2459 };
2460 
2461 static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = {
2462 	.version = TEGRA234_DWC_IP_VER,
2463 	.mode = DW_PCIE_EP_TYPE,
2464 	.has_l1ss_exit_fix = true,
2465 	.has_ltr_req_fix = true,
2466 	.cdm_chk_int_en_bit = BIT(18),
2467 	/* Gen4 - 6, 8 and 9 presets enabled */
2468 	.gen4_preset_vec = 0x340,
2469 	.n_fts = { 52, 80 },
2470 };
2471 
2472 static const struct of_device_id tegra_pcie_dw_of_match[] = {
2473 	{
2474 		.compatible = "nvidia,tegra194-pcie",
2475 		.data = &tegra194_pcie_dw_rc_of_data,
2476 	},
2477 	{
2478 		.compatible = "nvidia,tegra194-pcie-ep",
2479 		.data = &tegra194_pcie_dw_ep_of_data,
2480 	},
2481 	{
2482 		.compatible = "nvidia,tegra234-pcie",
2483 		.data = &tegra234_pcie_dw_rc_of_data,
2484 	},
2485 	{
2486 		.compatible = "nvidia,tegra234-pcie-ep",
2487 		.data = &tegra234_pcie_dw_ep_of_data,
2488 	},
2489 	{}
2490 };
2491 
2492 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2493 	.suspend_late = tegra_pcie_dw_suspend_late,
2494 	.suspend_noirq = tegra_pcie_dw_suspend_noirq,
2495 	.resume_noirq = tegra_pcie_dw_resume_noirq,
2496 	.resume_early = tegra_pcie_dw_resume_early,
2497 };
2498 
2499 static struct platform_driver tegra_pcie_dw_driver = {
2500 	.probe = tegra_pcie_dw_probe,
2501 	.remove_new = tegra_pcie_dw_remove,
2502 	.shutdown = tegra_pcie_dw_shutdown,
2503 	.driver = {
2504 		.name	= "tegra194-pcie",
2505 		.pm = &tegra_pcie_dw_pm_ops,
2506 		.of_match_table = tegra_pcie_dw_of_match,
2507 	},
2508 };
2509 module_platform_driver(tegra_pcie_dw_driver);
2510 
2511 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2512 
2513 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2514 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2515 MODULE_LICENSE("GPL v2");
2516