1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PCIe host controller driver for the following SoCs 4 * Tegra194 5 * Tegra234 6 * 7 * Copyright (C) 2019-2022 NVIDIA Corporation. 8 * 9 * Author: Vidya Sagar <vidyas@nvidia.com> 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/clk.h> 14 #include <linux/debugfs.h> 15 #include <linux/delay.h> 16 #include <linux/gpio/consumer.h> 17 #include <linux/interconnect.h> 18 #include <linux/interrupt.h> 19 #include <linux/iopoll.h> 20 #include <linux/kernel.h> 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_pci.h> 24 #include <linux/pci.h> 25 #include <linux/phy/phy.h> 26 #include <linux/pinctrl/consumer.h> 27 #include <linux/platform_device.h> 28 #include <linux/pm_runtime.h> 29 #include <linux/random.h> 30 #include <linux/reset.h> 31 #include <linux/resource.h> 32 #include <linux/types.h> 33 #include "pcie-designware.h" 34 #include <soc/tegra/bpmp.h> 35 #include <soc/tegra/bpmp-abi.h> 36 #include "../../pci.h" 37 38 #define TEGRA194_DWC_IP_VER 0x490A 39 #define TEGRA234_DWC_IP_VER 0x562A 40 41 #define APPL_PINMUX 0x0 42 #define APPL_PINMUX_PEX_RST BIT(0) 43 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) 44 #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3) 45 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4) 46 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5) 47 48 #define APPL_CTRL 0x4 49 #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6) 50 #define APPL_CTRL_LTSSM_EN BIT(7) 51 #define APPL_CTRL_HW_HOT_RST_EN BIT(20) 52 #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) 53 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 54 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 55 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x2 56 57 #define APPL_INTR_EN_L0_0 0x8 58 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) 59 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4) 60 #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8) 61 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15) 62 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19) 63 #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30) 64 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31) 65 66 #define APPL_INTR_STATUS_L0 0xC 67 #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0) 68 #define APPL_INTR_STATUS_L0_INT_INT BIT(8) 69 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15) 70 #define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16) 71 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18) 72 73 #define APPL_INTR_EN_L1_0_0 0x1C 74 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1) 75 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3) 76 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30) 77 78 #define APPL_INTR_STATUS_L1_0_0 0x20 79 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1) 80 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3) 81 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30) 82 83 #define APPL_INTR_STATUS_L1_1 0x2C 84 #define APPL_INTR_STATUS_L1_2 0x30 85 #define APPL_INTR_STATUS_L1_3 0x34 86 #define APPL_INTR_STATUS_L1_6 0x3C 87 #define APPL_INTR_STATUS_L1_7 0x40 88 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1) 89 90 #define APPL_INTR_EN_L1_8_0 0x44 91 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2) 92 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3) 93 #define APPL_INTR_EN_L1_8_INTX_EN BIT(11) 94 #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15) 95 96 #define APPL_INTR_STATUS_L1_8_0 0x4C 97 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6) 98 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2) 99 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3) 100 101 #define APPL_INTR_STATUS_L1_9 0x54 102 #define APPL_INTR_STATUS_L1_10 0x58 103 #define APPL_INTR_STATUS_L1_11 0x64 104 #define APPL_INTR_STATUS_L1_13 0x74 105 #define APPL_INTR_STATUS_L1_14 0x78 106 #define APPL_INTR_STATUS_L1_15 0x7C 107 #define APPL_INTR_STATUS_L1_17 0x88 108 109 #define APPL_INTR_EN_L1_18 0x90 110 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2) 111 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) 112 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) 113 114 #define APPL_INTR_STATUS_L1_18 0x94 115 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2) 116 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) 117 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) 118 119 #define APPL_MSI_CTRL_1 0xAC 120 121 #define APPL_MSI_CTRL_2 0xB0 122 123 #define APPL_LEGACY_INTX 0xB8 124 125 #define APPL_LTR_MSG_1 0xC4 126 #define LTR_MSG_REQ BIT(15) 127 #define LTR_NOSNOOP_MSG_REQ BIT(31) 128 129 #define APPL_LTR_MSG_2 0xC8 130 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) 131 132 #define APPL_LINK_STATUS 0xCC 133 #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0) 134 135 #define APPL_DEBUG 0xD0 136 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21) 137 #define APPL_DEBUG_PM_LINKST_IN_L0 0x11 138 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) 139 #define APPL_DEBUG_LTSSM_STATE_SHIFT 3 140 #define LTSSM_STATE_PRE_DETECT 5 141 142 #define APPL_RADM_STATUS 0xE4 143 #define APPL_PM_XMT_TURNOFF_STATE BIT(0) 144 145 #define APPL_DM_TYPE 0x100 146 #define APPL_DM_TYPE_MASK GENMASK(3, 0) 147 #define APPL_DM_TYPE_RP 0x4 148 #define APPL_DM_TYPE_EP 0x0 149 150 #define APPL_CFG_BASE_ADDR 0x104 151 #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12) 152 153 #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108 154 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18) 155 156 #define APPL_CFG_MISC 0x110 157 #define APPL_CFG_MISC_SLV_EP_MODE BIT(14) 158 #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10) 159 #define APPL_CFG_MISC_ARCACHE_SHIFT 10 160 #define APPL_CFG_MISC_ARCACHE_VAL 3 161 162 #define APPL_CFG_SLCG_OVERRIDE 0x114 163 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0) 164 165 #define APPL_CAR_RESET_OVRD 0x12C 166 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0) 167 168 #define IO_BASE_IO_DECODE BIT(0) 169 #define IO_BASE_IO_DECODE_BIT8 BIT(8) 170 171 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0) 172 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16) 173 174 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 175 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) 176 177 #define N_FTS_VAL 52 178 #define FTS_VAL 52 179 180 #define GEN3_EQ_CONTROL_OFF 0x8a8 181 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 182 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) 183 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) 184 185 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 186 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 187 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) 188 #define AMBA_ERROR_RESPONSE_CRS_OKAY 0 189 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 190 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 191 192 #define MSIX_ADDR_MATCH_LOW_OFF 0x940 193 #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0) 194 #define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2) 195 196 #define MSIX_ADDR_MATCH_HIGH_OFF 0x944 197 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0) 198 199 #define PORT_LOGIC_MSIX_DOORBELL 0x948 200 201 #define CAP_SPCIE_CAP_OFF 0x154 202 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0) 203 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) 204 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 205 206 #define PME_ACK_TIMEOUT 10000 207 208 #define LTSSM_TIMEOUT 50000 /* 50ms */ 209 210 #define GEN3_GEN4_EQ_PRESET_INIT 5 211 212 #define GEN1_CORE_CLK_FREQ 62500000 213 #define GEN2_CORE_CLK_FREQ 125000000 214 #define GEN3_CORE_CLK_FREQ 250000000 215 #define GEN4_CORE_CLK_FREQ 500000000 216 217 #define LTR_MSG_TIMEOUT (100 * 1000) 218 219 #define PERST_DEBOUNCE_TIME (5 * 1000) 220 221 #define EP_STATE_DISABLED 0 222 #define EP_STATE_ENABLED 1 223 224 static const unsigned int pcie_gen_freq[] = { 225 GEN1_CORE_CLK_FREQ, /* PCI_EXP_LNKSTA_CLS == 0; undefined */ 226 GEN1_CORE_CLK_FREQ, 227 GEN2_CORE_CLK_FREQ, 228 GEN3_CORE_CLK_FREQ, 229 GEN4_CORE_CLK_FREQ 230 }; 231 232 struct tegra_pcie_dw_of_data { 233 u32 version; 234 enum dw_pcie_device_mode mode; 235 bool has_msix_doorbell_access_fix; 236 bool has_sbr_reset_fix; 237 bool has_l1ss_exit_fix; 238 bool has_ltr_req_fix; 239 u32 cdm_chk_int_en_bit; 240 u32 gen4_preset_vec; 241 u8 n_fts[2]; 242 }; 243 244 struct tegra_pcie_dw { 245 struct device *dev; 246 struct resource *appl_res; 247 struct resource *dbi_res; 248 struct resource *atu_dma_res; 249 void __iomem *appl_base; 250 struct clk *core_clk; 251 struct reset_control *core_apb_rst; 252 struct reset_control *core_rst; 253 struct dw_pcie pci; 254 struct tegra_bpmp *bpmp; 255 256 struct tegra_pcie_dw_of_data *of_data; 257 258 bool supports_clkreq; 259 bool enable_cdm_check; 260 bool enable_srns; 261 bool link_state; 262 bool update_fc_fixup; 263 bool enable_ext_refclk; 264 u8 init_link_width; 265 u32 msi_ctrl_int; 266 u32 num_lanes; 267 u32 cid; 268 u32 cfg_link_cap_l1sub; 269 u32 ras_des_cap; 270 u32 pcie_cap_base; 271 u32 aspm_cmrt; 272 u32 aspm_pwr_on_t; 273 u32 aspm_l0s_enter_lat; 274 275 struct regulator *pex_ctl_supply; 276 struct regulator *slot_ctl_3v3; 277 struct regulator *slot_ctl_12v; 278 279 unsigned int phy_count; 280 struct phy **phys; 281 282 struct dentry *debugfs; 283 284 /* Endpoint mode specific */ 285 struct gpio_desc *pex_rst_gpiod; 286 struct gpio_desc *pex_refclk_sel_gpiod; 287 unsigned int pex_rst_irq; 288 int ep_state; 289 long link_status; 290 struct icc_path *icc_path; 291 }; 292 293 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) 294 { 295 return container_of(pci, struct tegra_pcie_dw, pci); 296 } 297 298 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value, 299 const u32 reg) 300 { 301 writel_relaxed(value, pcie->appl_base + reg); 302 } 303 304 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) 305 { 306 return readl_relaxed(pcie->appl_base + reg); 307 } 308 309 static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie) 310 { 311 struct dw_pcie *pci = &pcie->pci; 312 u32 val, speed, width; 313 314 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 315 316 speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val); 317 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val); 318 319 val = width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]); 320 321 if (icc_set_bw(pcie->icc_path, Mbps_to_icc(val), 0)) 322 dev_err(pcie->dev, "can't set bw[%u]\n", val); 323 324 if (speed >= ARRAY_SIZE(pcie_gen_freq)) 325 speed = 0; 326 327 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]); 328 } 329 330 static void apply_bad_link_workaround(struct dw_pcie_rp *pp) 331 { 332 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 333 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 334 u32 current_link_width; 335 u16 val; 336 337 /* 338 * NOTE:- Since this scenario is uncommon and link as such is not 339 * stable anyway, not waiting to confirm if link is really 340 * transitioning to Gen-2 speed 341 */ 342 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 343 if (val & PCI_EXP_LNKSTA_LBMS) { 344 current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val); 345 if (pcie->init_link_width > current_link_width) { 346 dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); 347 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 348 PCI_EXP_LNKCTL2); 349 val &= ~PCI_EXP_LNKCTL2_TLS; 350 val |= PCI_EXP_LNKCTL2_TLS_2_5GT; 351 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 352 PCI_EXP_LNKCTL2, val); 353 354 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 355 PCI_EXP_LNKCTL); 356 val |= PCI_EXP_LNKCTL_RL; 357 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 358 PCI_EXP_LNKCTL, val); 359 } 360 } 361 } 362 363 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) 364 { 365 struct tegra_pcie_dw *pcie = arg; 366 struct dw_pcie *pci = &pcie->pci; 367 struct dw_pcie_rp *pp = &pci->pp; 368 u32 val, status_l0, status_l1; 369 u16 val_w; 370 371 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0); 372 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) { 373 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); 374 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0); 375 if (!pcie->of_data->has_sbr_reset_fix && 376 status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { 377 /* SBR & Surprise Link Down WAR */ 378 val = appl_readl(pcie, APPL_CAR_RESET_OVRD); 379 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; 380 appl_writel(pcie, val, APPL_CAR_RESET_OVRD); 381 udelay(1); 382 val = appl_readl(pcie, APPL_CAR_RESET_OVRD); 383 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; 384 appl_writel(pcie, val, APPL_CAR_RESET_OVRD); 385 386 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); 387 val |= PORT_LOGIC_SPEED_CHANGE; 388 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); 389 } 390 } 391 392 if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) { 393 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); 394 if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { 395 appl_writel(pcie, 396 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, 397 APPL_INTR_STATUS_L1_8_0); 398 apply_bad_link_workaround(pp); 399 } 400 if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { 401 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 402 PCI_EXP_LNKSTA); 403 val_w |= PCI_EXP_LNKSTA_LBMS; 404 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 405 PCI_EXP_LNKSTA, val_w); 406 407 appl_writel(pcie, 408 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, 409 APPL_INTR_STATUS_L1_8_0); 410 411 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 412 PCI_EXP_LNKSTA); 413 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w & 414 PCI_EXP_LNKSTA_CLS); 415 } 416 } 417 418 if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { 419 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18); 420 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); 421 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { 422 dev_info(pci->dev, "CDM check complete\n"); 423 val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; 424 } 425 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { 426 dev_err(pci->dev, "CDM comparison mismatch\n"); 427 val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; 428 } 429 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { 430 dev_err(pci->dev, "CDM Logic error\n"); 431 val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; 432 } 433 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val); 434 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); 435 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val); 436 } 437 438 return IRQ_HANDLED; 439 } 440 441 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie) 442 { 443 u32 val; 444 445 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 446 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 447 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 448 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 449 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 450 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 451 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 452 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 453 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 454 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 455 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 456 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 457 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 458 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 459 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 460 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2); 461 462 val = appl_readl(pcie, APPL_CTRL); 463 val |= APPL_CTRL_LTSSM_EN; 464 appl_writel(pcie, val, APPL_CTRL); 465 } 466 467 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) 468 { 469 struct tegra_pcie_dw *pcie = arg; 470 struct dw_pcie_ep *ep = &pcie->pci.ep; 471 struct dw_pcie *pci = &pcie->pci; 472 u32 val; 473 474 if (test_and_clear_bit(0, &pcie->link_status)) 475 dw_pcie_ep_linkup(ep); 476 477 tegra_pcie_icc_set(pcie); 478 479 if (pcie->of_data->has_ltr_req_fix) 480 return IRQ_HANDLED; 481 482 /* If EP doesn't advertise L1SS, just return */ 483 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); 484 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2))) 485 return IRQ_HANDLED; 486 487 /* Check if BME is set to '1' */ 488 val = dw_pcie_readl_dbi(pci, PCI_COMMAND); 489 if (val & PCI_COMMAND_MASTER) { 490 ktime_t timeout; 491 492 /* 110us for both snoop and no-snoop */ 493 val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) | 494 FIELD_PREP(PCI_LTR_SCALE_MASK, 2) | 495 LTR_MSG_REQ | 496 FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) | 497 FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) | 498 LTR_NOSNOOP_MSG_REQ; 499 appl_writel(pcie, val, APPL_LTR_MSG_1); 500 501 /* Send LTR upstream */ 502 val = appl_readl(pcie, APPL_LTR_MSG_2); 503 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; 504 appl_writel(pcie, val, APPL_LTR_MSG_2); 505 506 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT); 507 for (;;) { 508 val = appl_readl(pcie, APPL_LTR_MSG_2); 509 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)) 510 break; 511 if (ktime_after(ktime_get(), timeout)) 512 break; 513 usleep_range(1000, 1100); 514 } 515 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE) 516 dev_err(pcie->dev, "Failed to send LTR message\n"); 517 } 518 519 return IRQ_HANDLED; 520 } 521 522 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) 523 { 524 struct tegra_pcie_dw *pcie = arg; 525 int spurious = 1; 526 u32 status_l0, status_l1, link_status; 527 528 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0); 529 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) { 530 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); 531 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0); 532 533 if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) 534 pex_ep_event_hot_rst_done(pcie); 535 536 if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { 537 link_status = appl_readl(pcie, APPL_LINK_STATUS); 538 if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) { 539 dev_dbg(pcie->dev, "Link is up with Host\n"); 540 set_bit(0, &pcie->link_status); 541 return IRQ_WAKE_THREAD; 542 } 543 } 544 545 spurious = 0; 546 } 547 548 if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { 549 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15); 550 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15); 551 552 if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) 553 return IRQ_WAKE_THREAD; 554 555 spurious = 0; 556 } 557 558 if (spurious) { 559 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n", 560 status_l0); 561 appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0); 562 } 563 564 return IRQ_HANDLED; 565 } 566 567 static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where, 568 int size, u32 *val) 569 { 570 struct dw_pcie_rp *pp = bus->sysdata; 571 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 572 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 573 574 /* 575 * This is an endpoint mode specific register happen to appear even 576 * when controller is operating in root port mode and system hangs 577 * when it is accessed with link being in ASPM-L1 state. 578 * So skip accessing it altogether 579 */ 580 if (!pcie->of_data->has_msix_doorbell_access_fix && 581 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) { 582 *val = 0x00000000; 583 return PCIBIOS_SUCCESSFUL; 584 } 585 586 return pci_generic_config_read(bus, devfn, where, size, val); 587 } 588 589 static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where, 590 int size, u32 val) 591 { 592 struct dw_pcie_rp *pp = bus->sysdata; 593 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 594 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 595 596 /* 597 * This is an endpoint mode specific register happen to appear even 598 * when controller is operating in root port mode and system hangs 599 * when it is accessed with link being in ASPM-L1 state. 600 * So skip accessing it altogether 601 */ 602 if (!pcie->of_data->has_msix_doorbell_access_fix && 603 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) 604 return PCIBIOS_SUCCESSFUL; 605 606 return pci_generic_config_write(bus, devfn, where, size, val); 607 } 608 609 static struct pci_ops tegra_pci_ops = { 610 .map_bus = dw_pcie_own_conf_map_bus, 611 .read = tegra_pcie_dw_rd_own_conf, 612 .write = tegra_pcie_dw_wr_own_conf, 613 }; 614 615 #if defined(CONFIG_PCIEASPM) 616 static void disable_aspm_l11(struct tegra_pcie_dw *pcie) 617 { 618 u32 val; 619 620 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); 621 val &= ~PCI_L1SS_CAP_ASPM_L1_1; 622 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); 623 } 624 625 static void disable_aspm_l12(struct tegra_pcie_dw *pcie) 626 { 627 u32 val; 628 629 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); 630 val &= ~PCI_L1SS_CAP_ASPM_L1_2; 631 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); 632 } 633 634 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) 635 { 636 u32 val; 637 638 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap + 639 PCIE_RAS_DES_EVENT_COUNTER_CONTROL); 640 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); 641 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 642 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; 643 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 644 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + 645 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); 646 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap + 647 PCIE_RAS_DES_EVENT_COUNTER_DATA); 648 649 return val; 650 } 651 652 static int aspm_state_cnt(struct seq_file *s, void *data) 653 { 654 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *) 655 dev_get_drvdata(s->private); 656 u32 val; 657 658 seq_printf(s, "Tx L0s entry count : %u\n", 659 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S)); 660 661 seq_printf(s, "Rx L0s entry count : %u\n", 662 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S)); 663 664 seq_printf(s, "Link L1 entry count : %u\n", 665 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1)); 666 667 seq_printf(s, "Link L1.1 entry count : %u\n", 668 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1)); 669 670 seq_printf(s, "Link L1.2 entry count : %u\n", 671 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); 672 673 /* Clear all counters */ 674 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + 675 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, 676 EVENT_COUNTER_ALL_CLEAR); 677 678 /* Re-enable counting */ 679 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 680 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 681 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + 682 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); 683 684 return 0; 685 } 686 687 static void init_host_aspm(struct tegra_pcie_dw *pcie) 688 { 689 struct dw_pcie *pci = &pcie->pci; 690 u32 val; 691 692 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); 693 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP; 694 695 pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci, 696 PCI_EXT_CAP_ID_VNDR); 697 698 /* Enable ASPM counters */ 699 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 700 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 701 dw_pcie_writel_dbi(pci, pcie->ras_des_cap + 702 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); 703 704 /* Program T_cmrt and T_pwr_on values */ 705 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); 706 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); 707 val |= (pcie->aspm_cmrt << 8); 708 val |= (pcie->aspm_pwr_on_t << 19); 709 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); 710 711 /* Program L0s and L1 entrance latencies */ 712 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); 713 val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK; 714 val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT); 715 val |= PORT_AFR_ENTER_ASPM; 716 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); 717 } 718 719 static void init_debugfs(struct tegra_pcie_dw *pcie) 720 { 721 debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs, 722 aspm_state_cnt); 723 } 724 #else 725 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } 726 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } 727 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } 728 static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; } 729 #endif 730 731 static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp) 732 { 733 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 734 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 735 u32 val; 736 u16 val_w; 737 738 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 739 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; 740 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 741 742 if (!pcie->of_data->has_sbr_reset_fix) { 743 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); 744 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; 745 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); 746 } 747 748 if (pcie->enable_cdm_check) { 749 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 750 val |= pcie->of_data->cdm_chk_int_en_bit; 751 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 752 753 val = appl_readl(pcie, APPL_INTR_EN_L1_18); 754 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR; 755 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR; 756 appl_writel(pcie, val, APPL_INTR_EN_L1_18); 757 } 758 759 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + 760 PCI_EXP_LNKSTA); 761 pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w); 762 763 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + 764 PCI_EXP_LNKCTL); 765 val_w |= PCI_EXP_LNKCTL_LBMIE; 766 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, 767 val_w); 768 } 769 770 static void tegra_pcie_enable_intx_interrupts(struct dw_pcie_rp *pp) 771 { 772 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 773 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 774 u32 val; 775 776 /* Enable INTX interrupt generation */ 777 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 778 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; 779 val |= APPL_INTR_EN_L0_0_INT_INT_EN; 780 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 781 782 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); 783 val |= APPL_INTR_EN_L1_8_INTX_EN; 784 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN; 785 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN; 786 if (IS_ENABLED(CONFIG_PCIEAER)) 787 val |= APPL_INTR_EN_L1_8_AER_INT_EN; 788 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); 789 } 790 791 static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp) 792 { 793 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 794 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 795 u32 val; 796 797 /* Enable MSI interrupt generation */ 798 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 799 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; 800 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN; 801 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 802 } 803 804 static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp) 805 { 806 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 807 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 808 809 /* Clear interrupt statuses before enabling interrupts */ 810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 812 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 813 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 814 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 815 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 816 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 817 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 818 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 819 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 820 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 821 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 822 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 823 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 824 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 825 826 tegra_pcie_enable_system_interrupts(pp); 827 tegra_pcie_enable_intx_interrupts(pp); 828 if (IS_ENABLED(CONFIG_PCI_MSI)) 829 tegra_pcie_enable_msi_interrupts(pp); 830 } 831 832 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) 833 { 834 struct dw_pcie *pci = &pcie->pci; 835 u32 val, offset, i; 836 837 /* Program init preset */ 838 for (i = 0; i < pcie->num_lanes; i++) { 839 val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2)); 840 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; 841 val |= GEN3_GEN4_EQ_PRESET_INIT; 842 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; 843 val |= (GEN3_GEN4_EQ_PRESET_INIT << 844 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); 845 dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val); 846 847 offset = dw_pcie_find_ext_capability(pci, 848 PCI_EXT_CAP_ID_PL_16GT) + 849 PCI_PL_16GT_LE_CTRL; 850 val = dw_pcie_readb_dbi(pci, offset + i); 851 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; 852 val |= GEN3_GEN4_EQ_PRESET_INIT; 853 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; 854 val |= (GEN3_GEN4_EQ_PRESET_INIT << 855 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); 856 dw_pcie_writeb_dbi(pci, offset + i, val); 857 } 858 859 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 860 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 861 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 862 863 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); 864 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; 865 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); 866 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; 867 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); 868 869 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 870 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 871 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT); 872 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 873 874 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); 875 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; 876 val |= (pcie->of_data->gen4_preset_vec << 877 GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); 878 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; 879 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); 880 881 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 882 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 883 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 884 } 885 886 static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp) 887 { 888 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 889 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 890 u32 val; 891 u16 val_16; 892 893 pp->bridge->ops = &tegra_pci_ops; 894 895 if (!pcie->pcie_cap_base) 896 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, 897 PCI_CAP_ID_EXP); 898 899 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); 900 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); 901 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val); 902 903 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE); 904 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE; 905 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE; 906 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val); 907 908 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); 909 910 /* Enable as 0xFFFF0001 response for CRS */ 911 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); 912 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); 913 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << 914 AMBA_ERROR_RESPONSE_CRS_SHIFT); 915 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); 916 917 /* Clear Slot Clock Configuration bit if SRNS configuration */ 918 if (pcie->enable_srns) { 919 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 920 PCI_EXP_LNKSTA); 921 val_16 &= ~PCI_EXP_LNKSTA_SLC; 922 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA, 923 val_16); 924 } 925 926 config_gen3_gen4_eq_presets(pcie); 927 928 init_host_aspm(pcie); 929 930 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */ 931 if (!pcie->supports_clkreq) { 932 disable_aspm_l11(pcie); 933 disable_aspm_l12(pcie); 934 } 935 936 if (!pcie->of_data->has_l1ss_exit_fix) { 937 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 938 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; 939 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 940 } 941 942 if (pcie->update_fc_fixup) { 943 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); 944 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; 945 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); 946 } 947 948 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); 949 950 return 0; 951 } 952 953 static int tegra_pcie_dw_start_link(struct dw_pcie *pci) 954 { 955 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 956 struct dw_pcie_rp *pp = &pci->pp; 957 u32 val, offset, tmp; 958 bool retry = true; 959 960 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { 961 enable_irq(pcie->pex_rst_irq); 962 return 0; 963 } 964 965 retry_link: 966 /* Assert RST */ 967 val = appl_readl(pcie, APPL_PINMUX); 968 val &= ~APPL_PINMUX_PEX_RST; 969 appl_writel(pcie, val, APPL_PINMUX); 970 971 usleep_range(100, 200); 972 973 /* Enable LTSSM */ 974 val = appl_readl(pcie, APPL_CTRL); 975 val |= APPL_CTRL_LTSSM_EN; 976 appl_writel(pcie, val, APPL_CTRL); 977 978 /* De-assert RST */ 979 val = appl_readl(pcie, APPL_PINMUX); 980 val |= APPL_PINMUX_PEX_RST; 981 appl_writel(pcie, val, APPL_PINMUX); 982 983 msleep(100); 984 985 if (dw_pcie_wait_for_link(pci)) { 986 if (!retry) 987 return 0; 988 /* 989 * There are some endpoints which can't get the link up if 990 * root port has Data Link Feature (DLF) enabled. 991 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info 992 * on Scaled Flow Control and DLF. 993 * So, need to confirm that is indeed the case here and attempt 994 * link up once again with DLF disabled. 995 */ 996 val = appl_readl(pcie, APPL_DEBUG); 997 val &= APPL_DEBUG_LTSSM_STATE_MASK; 998 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT; 999 tmp = appl_readl(pcie, APPL_LINK_STATUS); 1000 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP; 1001 if (!(val == 0x11 && !tmp)) { 1002 /* Link is down for all good reasons */ 1003 return 0; 1004 } 1005 1006 dev_info(pci->dev, "Link is down in DLL"); 1007 dev_info(pci->dev, "Trying again with DLFE disabled\n"); 1008 /* Disable LTSSM */ 1009 val = appl_readl(pcie, APPL_CTRL); 1010 val &= ~APPL_CTRL_LTSSM_EN; 1011 appl_writel(pcie, val, APPL_CTRL); 1012 1013 reset_control_assert(pcie->core_rst); 1014 reset_control_deassert(pcie->core_rst); 1015 1016 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); 1017 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); 1018 val &= ~PCI_DLF_EXCHANGE_ENABLE; 1019 dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val); 1020 1021 tegra_pcie_dw_host_init(pp); 1022 dw_pcie_setup_rc(pp); 1023 1024 retry = false; 1025 goto retry_link; 1026 } 1027 1028 tegra_pcie_icc_set(pcie); 1029 1030 tegra_pcie_enable_interrupts(pp); 1031 1032 return 0; 1033 } 1034 1035 static int tegra_pcie_dw_link_up(struct dw_pcie *pci) 1036 { 1037 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1038 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 1039 1040 return !!(val & PCI_EXP_LNKSTA_DLLLA); 1041 } 1042 1043 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci) 1044 { 1045 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1046 1047 disable_irq(pcie->pex_rst_irq); 1048 } 1049 1050 static const struct dw_pcie_ops tegra_dw_pcie_ops = { 1051 .link_up = tegra_pcie_dw_link_up, 1052 .start_link = tegra_pcie_dw_start_link, 1053 .stop_link = tegra_pcie_dw_stop_link, 1054 }; 1055 1056 static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { 1057 .init = tegra_pcie_dw_host_init, 1058 }; 1059 1060 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) 1061 { 1062 unsigned int phy_count = pcie->phy_count; 1063 1064 while (phy_count--) { 1065 phy_power_off(pcie->phys[phy_count]); 1066 phy_exit(pcie->phys[phy_count]); 1067 } 1068 } 1069 1070 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie) 1071 { 1072 unsigned int i; 1073 int ret; 1074 1075 for (i = 0; i < pcie->phy_count; i++) { 1076 ret = phy_init(pcie->phys[i]); 1077 if (ret < 0) 1078 goto phy_power_off; 1079 1080 ret = phy_power_on(pcie->phys[i]); 1081 if (ret < 0) 1082 goto phy_exit; 1083 } 1084 1085 return 0; 1086 1087 phy_power_off: 1088 while (i--) { 1089 phy_power_off(pcie->phys[i]); 1090 phy_exit: 1091 phy_exit(pcie->phys[i]); 1092 } 1093 1094 return ret; 1095 } 1096 1097 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) 1098 { 1099 struct platform_device *pdev = to_platform_device(pcie->dev); 1100 struct device_node *np = pcie->dev->of_node; 1101 int ret; 1102 1103 pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); 1104 if (!pcie->dbi_res) { 1105 dev_err(pcie->dev, "Failed to find \"dbi\" region\n"); 1106 return -ENODEV; 1107 } 1108 1109 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); 1110 if (ret < 0) { 1111 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); 1112 return ret; 1113 } 1114 1115 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us", 1116 &pcie->aspm_pwr_on_t); 1117 if (ret < 0) 1118 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n", 1119 ret); 1120 1121 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us", 1122 &pcie->aspm_l0s_enter_lat); 1123 if (ret < 0) 1124 dev_info(pcie->dev, 1125 "Failed to read ASPM L0s Entrance latency: %d\n", ret); 1126 1127 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes); 1128 if (ret < 0) { 1129 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret); 1130 return ret; 1131 } 1132 1133 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); 1134 if (ret) { 1135 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); 1136 return ret; 1137 } 1138 1139 ret = of_property_count_strings(np, "phy-names"); 1140 if (ret < 0) { 1141 dev_err(pcie->dev, "Failed to find PHY entries: %d\n", 1142 ret); 1143 return ret; 1144 } 1145 pcie->phy_count = ret; 1146 1147 if (of_property_read_bool(np, "nvidia,update-fc-fixup")) 1148 pcie->update_fc_fixup = true; 1149 1150 /* RP using an external REFCLK is supported only in Tegra234 */ 1151 if (pcie->of_data->version == TEGRA194_DWC_IP_VER) { 1152 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) 1153 pcie->enable_ext_refclk = true; 1154 } else { 1155 pcie->enable_ext_refclk = 1156 of_property_read_bool(pcie->dev->of_node, 1157 "nvidia,enable-ext-refclk"); 1158 } 1159 1160 pcie->supports_clkreq = 1161 of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); 1162 1163 pcie->enable_cdm_check = 1164 of_property_read_bool(np, "snps,enable-cdm-check"); 1165 1166 if (pcie->of_data->version == TEGRA234_DWC_IP_VER) 1167 pcie->enable_srns = 1168 of_property_read_bool(np, "nvidia,enable-srns"); 1169 1170 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) 1171 return 0; 1172 1173 /* Endpoint mode specific DT entries */ 1174 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN); 1175 if (IS_ERR(pcie->pex_rst_gpiod)) { 1176 int err = PTR_ERR(pcie->pex_rst_gpiod); 1177 const char *level = KERN_ERR; 1178 1179 if (err == -EPROBE_DEFER) 1180 level = KERN_DEBUG; 1181 1182 dev_printk(level, pcie->dev, 1183 dev_fmt("Failed to get PERST GPIO: %d\n"), 1184 err); 1185 return err; 1186 } 1187 1188 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev, 1189 "nvidia,refclk-select", 1190 GPIOD_OUT_HIGH); 1191 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) { 1192 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod); 1193 const char *level = KERN_ERR; 1194 1195 if (err == -EPROBE_DEFER) 1196 level = KERN_DEBUG; 1197 1198 dev_printk(level, pcie->dev, 1199 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"), 1200 err); 1201 pcie->pex_refclk_sel_gpiod = NULL; 1202 } 1203 1204 return 0; 1205 } 1206 1207 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, 1208 bool enable) 1209 { 1210 struct mrq_uphy_response resp; 1211 struct tegra_bpmp_message msg; 1212 struct mrq_uphy_request req; 1213 1214 /* 1215 * Controller-5 doesn't need to have its state set by BPMP-FW in 1216 * Tegra194 1217 */ 1218 if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5) 1219 return 0; 1220 1221 memset(&req, 0, sizeof(req)); 1222 memset(&resp, 0, sizeof(resp)); 1223 1224 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE; 1225 req.controller_state.pcie_controller = pcie->cid; 1226 req.controller_state.enable = enable; 1227 1228 memset(&msg, 0, sizeof(msg)); 1229 msg.mrq = MRQ_UPHY; 1230 msg.tx.data = &req; 1231 msg.tx.size = sizeof(req); 1232 msg.rx.data = &resp; 1233 msg.rx.size = sizeof(resp); 1234 1235 return tegra_bpmp_transfer(pcie->bpmp, &msg); 1236 } 1237 1238 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, 1239 bool enable) 1240 { 1241 struct mrq_uphy_response resp; 1242 struct tegra_bpmp_message msg; 1243 struct mrq_uphy_request req; 1244 1245 memset(&req, 0, sizeof(req)); 1246 memset(&resp, 0, sizeof(resp)); 1247 1248 if (enable) { 1249 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT; 1250 req.ep_ctrlr_pll_init.ep_controller = pcie->cid; 1251 } else { 1252 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF; 1253 req.ep_ctrlr_pll_off.ep_controller = pcie->cid; 1254 } 1255 1256 memset(&msg, 0, sizeof(msg)); 1257 msg.mrq = MRQ_UPHY; 1258 msg.tx.data = &req; 1259 msg.tx.size = sizeof(req); 1260 msg.rx.data = &resp; 1261 msg.rx.size = sizeof(resp); 1262 1263 return tegra_bpmp_transfer(pcie->bpmp, &msg); 1264 } 1265 1266 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) 1267 { 1268 struct dw_pcie_rp *pp = &pcie->pci.pp; 1269 struct pci_bus *child, *root_bus = NULL; 1270 struct pci_dev *pdev; 1271 1272 /* 1273 * link doesn't go into L2 state with some of the endpoints with Tegra 1274 * if they are not in D0 state. So, need to make sure that immediate 1275 * downstream devices are in D0 state before sending PME_TurnOff to put 1276 * link into L2 state. 1277 * This is as per PCI Express Base r4.0 v1.0 September 27-2017, 1278 * 5.2 Link State Power Management (Page #428). 1279 */ 1280 1281 list_for_each_entry(child, &pp->bridge->bus->children, node) { 1282 /* Bring downstream devices to D0 if they are not already in */ 1283 if (child->parent == pp->bridge->bus) { 1284 root_bus = child; 1285 break; 1286 } 1287 } 1288 1289 if (!root_bus) { 1290 dev_err(pcie->dev, "Failed to find downstream devices\n"); 1291 return; 1292 } 1293 1294 list_for_each_entry(pdev, &root_bus->devices, bus_list) { 1295 if (PCI_SLOT(pdev->devfn) == 0) { 1296 if (pci_set_power_state(pdev, PCI_D0)) 1297 dev_err(pcie->dev, 1298 "Failed to transition %s to D0 state\n", 1299 dev_name(&pdev->dev)); 1300 } 1301 } 1302 } 1303 1304 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) 1305 { 1306 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); 1307 if (IS_ERR(pcie->slot_ctl_3v3)) { 1308 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV) 1309 return PTR_ERR(pcie->slot_ctl_3v3); 1310 1311 pcie->slot_ctl_3v3 = NULL; 1312 } 1313 1314 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v"); 1315 if (IS_ERR(pcie->slot_ctl_12v)) { 1316 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV) 1317 return PTR_ERR(pcie->slot_ctl_12v); 1318 1319 pcie->slot_ctl_12v = NULL; 1320 } 1321 1322 return 0; 1323 } 1324 1325 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie) 1326 { 1327 int ret; 1328 1329 if (pcie->slot_ctl_3v3) { 1330 ret = regulator_enable(pcie->slot_ctl_3v3); 1331 if (ret < 0) { 1332 dev_err(pcie->dev, 1333 "Failed to enable 3.3V slot supply: %d\n", ret); 1334 return ret; 1335 } 1336 } 1337 1338 if (pcie->slot_ctl_12v) { 1339 ret = regulator_enable(pcie->slot_ctl_12v); 1340 if (ret < 0) { 1341 dev_err(pcie->dev, 1342 "Failed to enable 12V slot supply: %d\n", ret); 1343 goto fail_12v_enable; 1344 } 1345 } 1346 1347 /* 1348 * According to PCI Express Card Electromechanical Specification 1349 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive) 1350 * should be a minimum of 100ms. 1351 */ 1352 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v) 1353 msleep(100); 1354 1355 return 0; 1356 1357 fail_12v_enable: 1358 if (pcie->slot_ctl_3v3) 1359 regulator_disable(pcie->slot_ctl_3v3); 1360 return ret; 1361 } 1362 1363 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie) 1364 { 1365 if (pcie->slot_ctl_12v) 1366 regulator_disable(pcie->slot_ctl_12v); 1367 if (pcie->slot_ctl_3v3) 1368 regulator_disable(pcie->slot_ctl_3v3); 1369 } 1370 1371 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, 1372 bool en_hw_hot_rst) 1373 { 1374 int ret; 1375 u32 val; 1376 1377 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); 1378 if (ret) { 1379 dev_err(pcie->dev, 1380 "Failed to enable controller %u: %d\n", pcie->cid, ret); 1381 return ret; 1382 } 1383 1384 if (pcie->enable_ext_refclk) { 1385 ret = tegra_pcie_bpmp_set_pll_state(pcie, true); 1386 if (ret) { 1387 dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret); 1388 goto fail_pll_init; 1389 } 1390 } 1391 1392 ret = tegra_pcie_enable_slot_regulators(pcie); 1393 if (ret < 0) 1394 goto fail_slot_reg_en; 1395 1396 ret = regulator_enable(pcie->pex_ctl_supply); 1397 if (ret < 0) { 1398 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret); 1399 goto fail_reg_en; 1400 } 1401 1402 ret = clk_prepare_enable(pcie->core_clk); 1403 if (ret) { 1404 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret); 1405 goto fail_core_clk; 1406 } 1407 1408 ret = reset_control_deassert(pcie->core_apb_rst); 1409 if (ret) { 1410 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n", 1411 ret); 1412 goto fail_core_apb_rst; 1413 } 1414 1415 if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) { 1416 /* Enable HW_HOT_RST mode */ 1417 val = appl_readl(pcie, APPL_CTRL); 1418 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 1419 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 1420 val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN << 1421 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 1422 val |= APPL_CTRL_HW_HOT_RST_EN; 1423 appl_writel(pcie, val, APPL_CTRL); 1424 } 1425 1426 ret = tegra_pcie_enable_phy(pcie); 1427 if (ret) { 1428 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret); 1429 goto fail_phy; 1430 } 1431 1432 /* Update CFG base address */ 1433 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, 1434 APPL_CFG_BASE_ADDR); 1435 1436 /* Configure this core for RP mode operation */ 1437 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE); 1438 1439 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); 1440 1441 val = appl_readl(pcie, APPL_CTRL); 1442 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL); 1443 1444 val = appl_readl(pcie, APPL_CFG_MISC); 1445 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); 1446 appl_writel(pcie, val, APPL_CFG_MISC); 1447 1448 if (pcie->enable_srns || pcie->enable_ext_refclk) { 1449 /* 1450 * When Tegra PCIe RP is using external clock, it cannot supply 1451 * same clock to its downstream hierarchy. Hence, gate PCIe RP 1452 * REFCLK out pads when RP & EP are using separate clocks or RP 1453 * is using an external REFCLK. 1454 */ 1455 val = appl_readl(pcie, APPL_PINMUX); 1456 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1457 val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1458 appl_writel(pcie, val, APPL_PINMUX); 1459 } 1460 1461 if (!pcie->supports_clkreq) { 1462 val = appl_readl(pcie, APPL_PINMUX); 1463 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN; 1464 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE; 1465 appl_writel(pcie, val, APPL_PINMUX); 1466 } 1467 1468 /* Update iATU_DMA base address */ 1469 appl_writel(pcie, 1470 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK, 1471 APPL_CFG_IATU_DMA_BASE_ADDR); 1472 1473 reset_control_deassert(pcie->core_rst); 1474 1475 return ret; 1476 1477 fail_phy: 1478 reset_control_assert(pcie->core_apb_rst); 1479 fail_core_apb_rst: 1480 clk_disable_unprepare(pcie->core_clk); 1481 fail_core_clk: 1482 regulator_disable(pcie->pex_ctl_supply); 1483 fail_reg_en: 1484 tegra_pcie_disable_slot_regulators(pcie); 1485 fail_slot_reg_en: 1486 if (pcie->enable_ext_refclk) 1487 tegra_pcie_bpmp_set_pll_state(pcie, false); 1488 fail_pll_init: 1489 tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1490 1491 return ret; 1492 } 1493 1494 static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie) 1495 { 1496 int ret; 1497 1498 ret = reset_control_assert(pcie->core_rst); 1499 if (ret) 1500 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret); 1501 1502 tegra_pcie_disable_phy(pcie); 1503 1504 ret = reset_control_assert(pcie->core_apb_rst); 1505 if (ret) 1506 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); 1507 1508 clk_disable_unprepare(pcie->core_clk); 1509 1510 ret = regulator_disable(pcie->pex_ctl_supply); 1511 if (ret) 1512 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); 1513 1514 tegra_pcie_disable_slot_regulators(pcie); 1515 1516 if (pcie->enable_ext_refclk) { 1517 ret = tegra_pcie_bpmp_set_pll_state(pcie, false); 1518 if (ret) 1519 dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret); 1520 } 1521 1522 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1523 if (ret) 1524 dev_err(pcie->dev, "Failed to disable controller %d: %d\n", 1525 pcie->cid, ret); 1526 } 1527 1528 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) 1529 { 1530 struct dw_pcie *pci = &pcie->pci; 1531 struct dw_pcie_rp *pp = &pci->pp; 1532 int ret; 1533 1534 ret = tegra_pcie_config_controller(pcie, false); 1535 if (ret < 0) 1536 return ret; 1537 1538 pp->ops = &tegra_pcie_dw_host_ops; 1539 1540 ret = dw_pcie_host_init(pp); 1541 if (ret < 0) { 1542 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret); 1543 goto fail_host_init; 1544 } 1545 1546 return 0; 1547 1548 fail_host_init: 1549 tegra_pcie_unconfig_controller(pcie); 1550 return ret; 1551 } 1552 1553 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) 1554 { 1555 u32 val; 1556 1557 if (!tegra_pcie_dw_link_up(&pcie->pci)) 1558 return 0; 1559 1560 val = appl_readl(pcie, APPL_RADM_STATUS); 1561 val |= APPL_PM_XMT_TURNOFF_STATE; 1562 appl_writel(pcie, val, APPL_RADM_STATUS); 1563 1564 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val, 1565 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, 1566 1, PME_ACK_TIMEOUT); 1567 } 1568 1569 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) 1570 { 1571 u32 data; 1572 int err; 1573 1574 if (!tegra_pcie_dw_link_up(&pcie->pci)) { 1575 dev_dbg(pcie->dev, "PCIe link is not up...!\n"); 1576 return; 1577 } 1578 1579 /* 1580 * PCIe controller exits from L2 only if reset is applied, so 1581 * controller doesn't handle interrupts. But in cases where 1582 * L2 entry fails, PERST# is asserted which can trigger surprise 1583 * link down AER. However this function call happens in 1584 * suspend_noirq(), so AER interrupt will not be processed. 1585 * Disable all interrupts to avoid such a scenario. 1586 */ 1587 appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0); 1588 1589 if (tegra_pcie_try_link_l2(pcie)) { 1590 dev_info(pcie->dev, "Link didn't transition to L2 state\n"); 1591 /* 1592 * TX lane clock freq will reset to Gen1 only if link is in L2 1593 * or detect state. 1594 * So apply pex_rst to end point to force RP to go into detect 1595 * state 1596 */ 1597 data = appl_readl(pcie, APPL_PINMUX); 1598 data &= ~APPL_PINMUX_PEX_RST; 1599 appl_writel(pcie, data, APPL_PINMUX); 1600 1601 /* 1602 * Some cards do not go to detect state even after de-asserting 1603 * PERST#. So, de-assert LTSSM to bring link to detect state. 1604 */ 1605 data = readl(pcie->appl_base + APPL_CTRL); 1606 data &= ~APPL_CTRL_LTSSM_EN; 1607 writel(data, pcie->appl_base + APPL_CTRL); 1608 1609 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, 1610 data, 1611 ((data & 1612 APPL_DEBUG_LTSSM_STATE_MASK) >> 1613 APPL_DEBUG_LTSSM_STATE_SHIFT) == 1614 LTSSM_STATE_PRE_DETECT, 1615 1, LTSSM_TIMEOUT); 1616 if (err) 1617 dev_info(pcie->dev, "Link didn't go to detect state\n"); 1618 } 1619 /* 1620 * DBI registers may not be accessible after this as PLL-E would be 1621 * down depending on how CLKREQ is pulled by end point 1622 */ 1623 data = appl_readl(pcie, APPL_PINMUX); 1624 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE); 1625 /* Cut REFCLK to slot */ 1626 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1627 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1628 appl_writel(pcie, data, APPL_PINMUX); 1629 } 1630 1631 static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) 1632 { 1633 tegra_pcie_downstream_dev_to_D0(pcie); 1634 dw_pcie_host_deinit(&pcie->pci.pp); 1635 tegra_pcie_dw_pme_turnoff(pcie); 1636 tegra_pcie_unconfig_controller(pcie); 1637 } 1638 1639 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) 1640 { 1641 struct device *dev = pcie->dev; 1642 char *name; 1643 int ret; 1644 1645 pm_runtime_enable(dev); 1646 1647 ret = pm_runtime_get_sync(dev); 1648 if (ret < 0) { 1649 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", 1650 ret); 1651 goto fail_pm_get_sync; 1652 } 1653 1654 ret = pinctrl_pm_select_default_state(dev); 1655 if (ret < 0) { 1656 dev_err(dev, "Failed to configure sideband pins: %d\n", ret); 1657 goto fail_pm_get_sync; 1658 } 1659 1660 ret = tegra_pcie_init_controller(pcie); 1661 if (ret < 0) { 1662 dev_err(dev, "Failed to initialize controller: %d\n", ret); 1663 goto fail_pm_get_sync; 1664 } 1665 1666 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); 1667 if (!pcie->link_state) { 1668 ret = -ENOMEDIUM; 1669 goto fail_host_init; 1670 } 1671 1672 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); 1673 if (!name) { 1674 ret = -ENOMEM; 1675 goto fail_host_init; 1676 } 1677 1678 pcie->debugfs = debugfs_create_dir(name, NULL); 1679 init_debugfs(pcie); 1680 1681 return ret; 1682 1683 fail_host_init: 1684 tegra_pcie_deinit_controller(pcie); 1685 fail_pm_get_sync: 1686 pm_runtime_put_sync(dev); 1687 pm_runtime_disable(dev); 1688 return ret; 1689 } 1690 1691 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) 1692 { 1693 u32 val; 1694 int ret; 1695 1696 if (pcie->ep_state == EP_STATE_DISABLED) 1697 return; 1698 1699 /* Disable LTSSM */ 1700 val = appl_readl(pcie, APPL_CTRL); 1701 val &= ~APPL_CTRL_LTSSM_EN; 1702 appl_writel(pcie, val, APPL_CTRL); 1703 1704 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, 1705 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >> 1706 APPL_DEBUG_LTSSM_STATE_SHIFT) == 1707 LTSSM_STATE_PRE_DETECT, 1708 1, LTSSM_TIMEOUT); 1709 if (ret) 1710 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); 1711 1712 pci_epc_deinit_notify(pcie->pci.ep.epc); 1713 dw_pcie_ep_cleanup(&pcie->pci.ep); 1714 1715 reset_control_assert(pcie->core_rst); 1716 1717 tegra_pcie_disable_phy(pcie); 1718 1719 reset_control_assert(pcie->core_apb_rst); 1720 1721 clk_disable_unprepare(pcie->core_clk); 1722 1723 pm_runtime_put_sync(pcie->dev); 1724 1725 if (pcie->enable_ext_refclk) { 1726 ret = tegra_pcie_bpmp_set_pll_state(pcie, false); 1727 if (ret) 1728 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", 1729 ret); 1730 } 1731 1732 ret = tegra_pcie_bpmp_set_pll_state(pcie, false); 1733 if (ret) 1734 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret); 1735 1736 pcie->ep_state = EP_STATE_DISABLED; 1737 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n"); 1738 } 1739 1740 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) 1741 { 1742 struct dw_pcie *pci = &pcie->pci; 1743 struct dw_pcie_ep *ep = &pci->ep; 1744 struct device *dev = pcie->dev; 1745 u32 val; 1746 int ret; 1747 u16 val_16; 1748 1749 if (pcie->ep_state == EP_STATE_ENABLED) 1750 return; 1751 1752 ret = pm_runtime_resume_and_get(dev); 1753 if (ret < 0) { 1754 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", 1755 ret); 1756 return; 1757 } 1758 1759 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); 1760 if (ret) { 1761 dev_err(pcie->dev, "Failed to enable controller %u: %d\n", 1762 pcie->cid, ret); 1763 goto fail_set_ctrl_state; 1764 } 1765 1766 if (pcie->enable_ext_refclk) { 1767 ret = tegra_pcie_bpmp_set_pll_state(pcie, true); 1768 if (ret) { 1769 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", 1770 ret); 1771 goto fail_pll_init; 1772 } 1773 } 1774 1775 ret = clk_prepare_enable(pcie->core_clk); 1776 if (ret) { 1777 dev_err(dev, "Failed to enable core clock: %d\n", ret); 1778 goto fail_core_clk_enable; 1779 } 1780 1781 ret = reset_control_deassert(pcie->core_apb_rst); 1782 if (ret) { 1783 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret); 1784 goto fail_core_apb_rst; 1785 } 1786 1787 ret = tegra_pcie_enable_phy(pcie); 1788 if (ret) { 1789 dev_err(dev, "Failed to enable PHY: %d\n", ret); 1790 goto fail_phy; 1791 } 1792 1793 /* Clear any stale interrupt statuses */ 1794 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 1795 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 1796 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 1797 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 1798 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 1799 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 1800 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 1801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 1802 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 1803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 1804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 1805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 1806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 1807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 1808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 1809 1810 /* configure this core for EP mode operation */ 1811 val = appl_readl(pcie, APPL_DM_TYPE); 1812 val &= ~APPL_DM_TYPE_MASK; 1813 val |= APPL_DM_TYPE_EP; 1814 appl_writel(pcie, val, APPL_DM_TYPE); 1815 1816 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); 1817 1818 val = appl_readl(pcie, APPL_CTRL); 1819 val |= APPL_CTRL_SYS_PRE_DET_STATE; 1820 val |= APPL_CTRL_HW_HOT_RST_EN; 1821 appl_writel(pcie, val, APPL_CTRL); 1822 1823 val = appl_readl(pcie, APPL_CFG_MISC); 1824 val |= APPL_CFG_MISC_SLV_EP_MODE; 1825 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); 1826 appl_writel(pcie, val, APPL_CFG_MISC); 1827 1828 val = appl_readl(pcie, APPL_PINMUX); 1829 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1830 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1831 appl_writel(pcie, val, APPL_PINMUX); 1832 1833 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, 1834 APPL_CFG_BASE_ADDR); 1835 1836 appl_writel(pcie, pcie->atu_dma_res->start & 1837 APPL_CFG_IATU_DMA_BASE_ADDR_MASK, 1838 APPL_CFG_IATU_DMA_BASE_ADDR); 1839 1840 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 1841 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; 1842 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; 1843 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN; 1844 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 1845 1846 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); 1847 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN; 1848 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN; 1849 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); 1850 1851 reset_control_deassert(pcie->core_rst); 1852 1853 if (pcie->update_fc_fixup) { 1854 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); 1855 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; 1856 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); 1857 } 1858 1859 config_gen3_gen4_eq_presets(pcie); 1860 1861 init_host_aspm(pcie); 1862 1863 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */ 1864 if (!pcie->supports_clkreq) { 1865 disable_aspm_l11(pcie); 1866 disable_aspm_l12(pcie); 1867 } 1868 1869 if (!pcie->of_data->has_l1ss_exit_fix) { 1870 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 1871 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; 1872 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 1873 } 1874 1875 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, 1876 PCI_CAP_ID_EXP); 1877 1878 /* Clear Slot Clock Configuration bit if SRNS configuration */ 1879 if (pcie->enable_srns) { 1880 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 1881 PCI_EXP_LNKSTA); 1882 val_16 &= ~PCI_EXP_LNKSTA_SLC; 1883 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA, 1884 val_16); 1885 } 1886 1887 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); 1888 1889 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK); 1890 val |= MSIX_ADDR_MATCH_LOW_OFF_EN; 1891 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val); 1892 val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); 1893 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); 1894 1895 ret = dw_pcie_ep_init_registers(ep); 1896 if (ret) { 1897 dev_err(dev, "Failed to complete initialization: %d\n", ret); 1898 goto fail_init_complete; 1899 } 1900 1901 pci_epc_init_notify(ep->epc); 1902 1903 /* Program the private control to allow sending LTR upstream */ 1904 if (pcie->of_data->has_ltr_req_fix) { 1905 val = appl_readl(pcie, APPL_LTR_MSG_2); 1906 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; 1907 appl_writel(pcie, val, APPL_LTR_MSG_2); 1908 } 1909 1910 /* Enable LTSSM */ 1911 val = appl_readl(pcie, APPL_CTRL); 1912 val |= APPL_CTRL_LTSSM_EN; 1913 appl_writel(pcie, val, APPL_CTRL); 1914 1915 pcie->ep_state = EP_STATE_ENABLED; 1916 dev_dbg(dev, "Initialization of endpoint is completed\n"); 1917 1918 return; 1919 1920 fail_init_complete: 1921 reset_control_assert(pcie->core_rst); 1922 tegra_pcie_disable_phy(pcie); 1923 fail_phy: 1924 reset_control_assert(pcie->core_apb_rst); 1925 fail_core_apb_rst: 1926 clk_disable_unprepare(pcie->core_clk); 1927 fail_core_clk_enable: 1928 tegra_pcie_bpmp_set_pll_state(pcie, false); 1929 fail_pll_init: 1930 tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1931 fail_set_ctrl_state: 1932 pm_runtime_put_sync(dev); 1933 } 1934 1935 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg) 1936 { 1937 struct tegra_pcie_dw *pcie = arg; 1938 1939 if (gpiod_get_value(pcie->pex_rst_gpiod)) 1940 pex_ep_event_pex_rst_assert(pcie); 1941 else 1942 pex_ep_event_pex_rst_deassert(pcie); 1943 1944 return IRQ_HANDLED; 1945 } 1946 1947 static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq) 1948 { 1949 /* Tegra194 supports only INTA */ 1950 if (irq > 1) 1951 return -EINVAL; 1952 1953 appl_writel(pcie, 1, APPL_LEGACY_INTX); 1954 usleep_range(1000, 2000); 1955 appl_writel(pcie, 0, APPL_LEGACY_INTX); 1956 return 0; 1957 } 1958 1959 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) 1960 { 1961 if (unlikely(irq > 31)) 1962 return -EINVAL; 1963 1964 appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1); 1965 1966 return 0; 1967 } 1968 1969 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq) 1970 { 1971 struct dw_pcie_ep *ep = &pcie->pci.ep; 1972 1973 writel(irq, ep->msi_mem); 1974 1975 return 0; 1976 } 1977 1978 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, 1979 unsigned int type, u16 interrupt_num) 1980 { 1981 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1982 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1983 1984 switch (type) { 1985 case PCI_IRQ_INTX: 1986 return tegra_pcie_ep_raise_intx_irq(pcie, interrupt_num); 1987 1988 case PCI_IRQ_MSI: 1989 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num); 1990 1991 case PCI_IRQ_MSIX: 1992 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num); 1993 1994 default: 1995 dev_err(pci->dev, "Unknown IRQ type\n"); 1996 return -EPERM; 1997 } 1998 1999 return 0; 2000 } 2001 2002 static const struct pci_epc_features tegra_pcie_epc_features = { 2003 .linkup_notifier = true, 2004 .msi_capable = false, 2005 .msix_capable = false, 2006 .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M, 2007 .only_64bit = true, }, 2008 .bar[BAR_1] = { .type = BAR_RESERVED, }, 2009 .bar[BAR_2] = { .type = BAR_RESERVED, }, 2010 .bar[BAR_3] = { .type = BAR_RESERVED, }, 2011 .bar[BAR_4] = { .type = BAR_RESERVED, }, 2012 .bar[BAR_5] = { .type = BAR_RESERVED, }, 2013 .align = SZ_64K, 2014 }; 2015 2016 static const struct pci_epc_features* 2017 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep) 2018 { 2019 return &tegra_pcie_epc_features; 2020 } 2021 2022 static const struct dw_pcie_ep_ops pcie_ep_ops = { 2023 .raise_irq = tegra_pcie_ep_raise_irq, 2024 .get_features = tegra_pcie_ep_get_features, 2025 }; 2026 2027 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, 2028 struct platform_device *pdev) 2029 { 2030 struct dw_pcie *pci = &pcie->pci; 2031 struct device *dev = pcie->dev; 2032 struct dw_pcie_ep *ep; 2033 char *name; 2034 int ret; 2035 2036 ep = &pci->ep; 2037 ep->ops = &pcie_ep_ops; 2038 2039 ep->page_size = SZ_64K; 2040 2041 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME); 2042 if (ret < 0) { 2043 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n", 2044 ret); 2045 return ret; 2046 } 2047 2048 ret = gpiod_to_irq(pcie->pex_rst_gpiod); 2049 if (ret < 0) { 2050 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret); 2051 return ret; 2052 } 2053 pcie->pex_rst_irq = (unsigned int)ret; 2054 2055 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq", 2056 pcie->cid); 2057 if (!name) { 2058 dev_err(dev, "Failed to create PERST IRQ string\n"); 2059 return -ENOMEM; 2060 } 2061 2062 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN); 2063 2064 pcie->ep_state = EP_STATE_DISABLED; 2065 2066 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL, 2067 tegra_pcie_ep_pex_rst_irq, 2068 IRQF_TRIGGER_RISING | 2069 IRQF_TRIGGER_FALLING | IRQF_ONESHOT, 2070 name, (void *)pcie); 2071 if (ret < 0) { 2072 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret); 2073 return ret; 2074 } 2075 2076 pm_runtime_enable(dev); 2077 2078 ret = dw_pcie_ep_init(ep); 2079 if (ret) { 2080 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n", 2081 ret); 2082 pm_runtime_disable(dev); 2083 return ret; 2084 } 2085 2086 return 0; 2087 } 2088 2089 static int tegra_pcie_dw_probe(struct platform_device *pdev) 2090 { 2091 const struct tegra_pcie_dw_of_data *data; 2092 struct device *dev = &pdev->dev; 2093 struct resource *atu_dma_res; 2094 struct tegra_pcie_dw *pcie; 2095 struct dw_pcie_rp *pp; 2096 struct dw_pcie *pci; 2097 struct phy **phys; 2098 char *name; 2099 int ret; 2100 u32 i; 2101 2102 data = of_device_get_match_data(dev); 2103 2104 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); 2105 if (!pcie) 2106 return -ENOMEM; 2107 2108 pci = &pcie->pci; 2109 pci->dev = &pdev->dev; 2110 pci->ops = &tegra_dw_pcie_ops; 2111 pcie->dev = &pdev->dev; 2112 pcie->of_data = (struct tegra_pcie_dw_of_data *)data; 2113 pci->n_fts[0] = pcie->of_data->n_fts[0]; 2114 pci->n_fts[1] = pcie->of_data->n_fts[1]; 2115 pp = &pci->pp; 2116 pp->num_vectors = MAX_MSI_IRQS; 2117 2118 ret = tegra_pcie_dw_parse_dt(pcie); 2119 if (ret < 0) { 2120 const char *level = KERN_ERR; 2121 2122 if (ret == -EPROBE_DEFER) 2123 level = KERN_DEBUG; 2124 2125 dev_printk(level, dev, 2126 dev_fmt("Failed to parse device tree: %d\n"), 2127 ret); 2128 return ret; 2129 } 2130 2131 ret = tegra_pcie_get_slot_regulators(pcie); 2132 if (ret < 0) { 2133 const char *level = KERN_ERR; 2134 2135 if (ret == -EPROBE_DEFER) 2136 level = KERN_DEBUG; 2137 2138 dev_printk(level, dev, 2139 dev_fmt("Failed to get slot regulators: %d\n"), 2140 ret); 2141 return ret; 2142 } 2143 2144 if (pcie->pex_refclk_sel_gpiod) 2145 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1); 2146 2147 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl"); 2148 if (IS_ERR(pcie->pex_ctl_supply)) { 2149 ret = PTR_ERR(pcie->pex_ctl_supply); 2150 if (ret != -EPROBE_DEFER) 2151 dev_err(dev, "Failed to get regulator: %ld\n", 2152 PTR_ERR(pcie->pex_ctl_supply)); 2153 return ret; 2154 } 2155 2156 pcie->core_clk = devm_clk_get(dev, "core"); 2157 if (IS_ERR(pcie->core_clk)) { 2158 dev_err(dev, "Failed to get core clock: %ld\n", 2159 PTR_ERR(pcie->core_clk)); 2160 return PTR_ERR(pcie->core_clk); 2161 } 2162 2163 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 2164 "appl"); 2165 if (!pcie->appl_res) { 2166 dev_err(dev, "Failed to find \"appl\" region\n"); 2167 return -ENODEV; 2168 } 2169 2170 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res); 2171 if (IS_ERR(pcie->appl_base)) 2172 return PTR_ERR(pcie->appl_base); 2173 2174 pcie->core_apb_rst = devm_reset_control_get(dev, "apb"); 2175 if (IS_ERR(pcie->core_apb_rst)) { 2176 dev_err(dev, "Failed to get APB reset: %ld\n", 2177 PTR_ERR(pcie->core_apb_rst)); 2178 return PTR_ERR(pcie->core_apb_rst); 2179 } 2180 2181 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL); 2182 if (!phys) 2183 return -ENOMEM; 2184 2185 for (i = 0; i < pcie->phy_count; i++) { 2186 name = kasprintf(GFP_KERNEL, "p2u-%u", i); 2187 if (!name) { 2188 dev_err(dev, "Failed to create P2U string\n"); 2189 return -ENOMEM; 2190 } 2191 phys[i] = devm_phy_get(dev, name); 2192 kfree(name); 2193 if (IS_ERR(phys[i])) { 2194 ret = PTR_ERR(phys[i]); 2195 if (ret != -EPROBE_DEFER) 2196 dev_err(dev, "Failed to get PHY: %d\n", ret); 2197 return ret; 2198 } 2199 } 2200 2201 pcie->phys = phys; 2202 2203 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 2204 "atu_dma"); 2205 if (!atu_dma_res) { 2206 dev_err(dev, "Failed to find \"atu_dma\" region\n"); 2207 return -ENODEV; 2208 } 2209 pcie->atu_dma_res = atu_dma_res; 2210 2211 pci->atu_size = resource_size(atu_dma_res); 2212 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); 2213 if (IS_ERR(pci->atu_base)) 2214 return PTR_ERR(pci->atu_base); 2215 2216 pcie->core_rst = devm_reset_control_get(dev, "core"); 2217 if (IS_ERR(pcie->core_rst)) { 2218 dev_err(dev, "Failed to get core reset: %ld\n", 2219 PTR_ERR(pcie->core_rst)); 2220 return PTR_ERR(pcie->core_rst); 2221 } 2222 2223 pp->irq = platform_get_irq_byname(pdev, "intr"); 2224 if (pp->irq < 0) 2225 return pp->irq; 2226 2227 pcie->bpmp = tegra_bpmp_get(dev); 2228 if (IS_ERR(pcie->bpmp)) 2229 return PTR_ERR(pcie->bpmp); 2230 2231 platform_set_drvdata(pdev, pcie); 2232 2233 pcie->icc_path = devm_of_icc_get(&pdev->dev, "write"); 2234 ret = PTR_ERR_OR_ZERO(pcie->icc_path); 2235 if (ret) { 2236 tegra_bpmp_put(pcie->bpmp); 2237 dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n"); 2238 return ret; 2239 } 2240 2241 switch (pcie->of_data->mode) { 2242 case DW_PCIE_RC_TYPE: 2243 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler, 2244 IRQF_SHARED, "tegra-pcie-intr", pcie); 2245 if (ret) { 2246 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, 2247 ret); 2248 goto fail; 2249 } 2250 2251 ret = tegra_pcie_config_rp(pcie); 2252 if (ret && ret != -ENOMEDIUM) 2253 goto fail; 2254 else 2255 return 0; 2256 break; 2257 2258 case DW_PCIE_EP_TYPE: 2259 ret = devm_request_threaded_irq(dev, pp->irq, 2260 tegra_pcie_ep_hard_irq, 2261 tegra_pcie_ep_irq_thread, 2262 IRQF_SHARED | IRQF_ONESHOT, 2263 "tegra-pcie-ep-intr", pcie); 2264 if (ret) { 2265 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, 2266 ret); 2267 goto fail; 2268 } 2269 2270 ret = tegra_pcie_config_ep(pcie, pdev); 2271 if (ret < 0) 2272 goto fail; 2273 else 2274 return 0; 2275 break; 2276 2277 default: 2278 dev_err(dev, "Invalid PCIe device type %d\n", 2279 pcie->of_data->mode); 2280 ret = -EINVAL; 2281 } 2282 2283 fail: 2284 tegra_bpmp_put(pcie->bpmp); 2285 return ret; 2286 } 2287 2288 static void tegra_pcie_dw_remove(struct platform_device *pdev) 2289 { 2290 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); 2291 2292 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) { 2293 if (!pcie->link_state) 2294 return; 2295 2296 debugfs_remove_recursive(pcie->debugfs); 2297 tegra_pcie_deinit_controller(pcie); 2298 pm_runtime_put_sync(pcie->dev); 2299 } else { 2300 disable_irq(pcie->pex_rst_irq); 2301 pex_ep_event_pex_rst_assert(pcie); 2302 } 2303 2304 pm_runtime_disable(pcie->dev); 2305 tegra_bpmp_put(pcie->bpmp); 2306 if (pcie->pex_refclk_sel_gpiod) 2307 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0); 2308 } 2309 2310 static int tegra_pcie_dw_suspend_late(struct device *dev) 2311 { 2312 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2313 u32 val; 2314 2315 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { 2316 dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n"); 2317 return -EPERM; 2318 } 2319 2320 if (!pcie->link_state) 2321 return 0; 2322 2323 /* Enable HW_HOT_RST mode */ 2324 if (!pcie->of_data->has_sbr_reset_fix) { 2325 val = appl_readl(pcie, APPL_CTRL); 2326 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 2327 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 2328 val |= APPL_CTRL_HW_HOT_RST_EN; 2329 appl_writel(pcie, val, APPL_CTRL); 2330 } 2331 2332 return 0; 2333 } 2334 2335 static int tegra_pcie_dw_suspend_noirq(struct device *dev) 2336 { 2337 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2338 2339 if (!pcie->link_state) 2340 return 0; 2341 2342 tegra_pcie_downstream_dev_to_D0(pcie); 2343 tegra_pcie_dw_pme_turnoff(pcie); 2344 tegra_pcie_unconfig_controller(pcie); 2345 2346 return 0; 2347 } 2348 2349 static int tegra_pcie_dw_resume_noirq(struct device *dev) 2350 { 2351 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2352 int ret; 2353 2354 if (!pcie->link_state) 2355 return 0; 2356 2357 ret = tegra_pcie_config_controller(pcie, true); 2358 if (ret < 0) 2359 return ret; 2360 2361 ret = tegra_pcie_dw_host_init(&pcie->pci.pp); 2362 if (ret < 0) { 2363 dev_err(dev, "Failed to init host: %d\n", ret); 2364 goto fail_host_init; 2365 } 2366 2367 dw_pcie_setup_rc(&pcie->pci.pp); 2368 2369 ret = tegra_pcie_dw_start_link(&pcie->pci); 2370 if (ret < 0) 2371 goto fail_host_init; 2372 2373 return 0; 2374 2375 fail_host_init: 2376 tegra_pcie_unconfig_controller(pcie); 2377 return ret; 2378 } 2379 2380 static int tegra_pcie_dw_resume_early(struct device *dev) 2381 { 2382 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2383 u32 val; 2384 2385 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { 2386 dev_err(dev, "Suspend is not supported in EP mode"); 2387 return -ENOTSUPP; 2388 } 2389 2390 if (!pcie->link_state) 2391 return 0; 2392 2393 /* Disable HW_HOT_RST mode */ 2394 if (!pcie->of_data->has_sbr_reset_fix) { 2395 val = appl_readl(pcie, APPL_CTRL); 2396 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 2397 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 2398 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << 2399 APPL_CTRL_HW_HOT_RST_MODE_SHIFT; 2400 val &= ~APPL_CTRL_HW_HOT_RST_EN; 2401 appl_writel(pcie, val, APPL_CTRL); 2402 } 2403 2404 return 0; 2405 } 2406 2407 static void tegra_pcie_dw_shutdown(struct platform_device *pdev) 2408 { 2409 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); 2410 2411 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) { 2412 if (!pcie->link_state) 2413 return; 2414 2415 debugfs_remove_recursive(pcie->debugfs); 2416 tegra_pcie_downstream_dev_to_D0(pcie); 2417 2418 disable_irq(pcie->pci.pp.irq); 2419 if (IS_ENABLED(CONFIG_PCI_MSI)) 2420 disable_irq(pcie->pci.pp.msi_irq[0]); 2421 2422 tegra_pcie_dw_pme_turnoff(pcie); 2423 tegra_pcie_unconfig_controller(pcie); 2424 pm_runtime_put_sync(pcie->dev); 2425 } else { 2426 disable_irq(pcie->pex_rst_irq); 2427 pex_ep_event_pex_rst_assert(pcie); 2428 } 2429 } 2430 2431 static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = { 2432 .version = TEGRA194_DWC_IP_VER, 2433 .mode = DW_PCIE_RC_TYPE, 2434 .cdm_chk_int_en_bit = BIT(19), 2435 /* Gen4 - 5, 6, 8 and 9 presets enabled */ 2436 .gen4_preset_vec = 0x360, 2437 .n_fts = { 52, 52 }, 2438 }; 2439 2440 static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = { 2441 .version = TEGRA194_DWC_IP_VER, 2442 .mode = DW_PCIE_EP_TYPE, 2443 .cdm_chk_int_en_bit = BIT(19), 2444 /* Gen4 - 5, 6, 8 and 9 presets enabled */ 2445 .gen4_preset_vec = 0x360, 2446 .n_fts = { 52, 52 }, 2447 }; 2448 2449 static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = { 2450 .version = TEGRA234_DWC_IP_VER, 2451 .mode = DW_PCIE_RC_TYPE, 2452 .has_msix_doorbell_access_fix = true, 2453 .has_sbr_reset_fix = true, 2454 .has_l1ss_exit_fix = true, 2455 .cdm_chk_int_en_bit = BIT(18), 2456 /* Gen4 - 6, 8 and 9 presets enabled */ 2457 .gen4_preset_vec = 0x340, 2458 .n_fts = { 52, 80 }, 2459 }; 2460 2461 static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = { 2462 .version = TEGRA234_DWC_IP_VER, 2463 .mode = DW_PCIE_EP_TYPE, 2464 .has_l1ss_exit_fix = true, 2465 .has_ltr_req_fix = true, 2466 .cdm_chk_int_en_bit = BIT(18), 2467 /* Gen4 - 6, 8 and 9 presets enabled */ 2468 .gen4_preset_vec = 0x340, 2469 .n_fts = { 52, 80 }, 2470 }; 2471 2472 static const struct of_device_id tegra_pcie_dw_of_match[] = { 2473 { 2474 .compatible = "nvidia,tegra194-pcie", 2475 .data = &tegra194_pcie_dw_rc_of_data, 2476 }, 2477 { 2478 .compatible = "nvidia,tegra194-pcie-ep", 2479 .data = &tegra194_pcie_dw_ep_of_data, 2480 }, 2481 { 2482 .compatible = "nvidia,tegra234-pcie", 2483 .data = &tegra234_pcie_dw_rc_of_data, 2484 }, 2485 { 2486 .compatible = "nvidia,tegra234-pcie-ep", 2487 .data = &tegra234_pcie_dw_ep_of_data, 2488 }, 2489 {} 2490 }; 2491 2492 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { 2493 .suspend_late = tegra_pcie_dw_suspend_late, 2494 .suspend_noirq = tegra_pcie_dw_suspend_noirq, 2495 .resume_noirq = tegra_pcie_dw_resume_noirq, 2496 .resume_early = tegra_pcie_dw_resume_early, 2497 }; 2498 2499 static struct platform_driver tegra_pcie_dw_driver = { 2500 .probe = tegra_pcie_dw_probe, 2501 .remove_new = tegra_pcie_dw_remove, 2502 .shutdown = tegra_pcie_dw_shutdown, 2503 .driver = { 2504 .name = "tegra194-pcie", 2505 .pm = &tegra_pcie_dw_pm_ops, 2506 .of_match_table = tegra_pcie_dw_of_match, 2507 }, 2508 }; 2509 module_platform_driver(tegra_pcie_dw_driver); 2510 2511 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match); 2512 2513 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>"); 2514 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver"); 2515 MODULE_LICENSE("GPL v2"); 2516