1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Qualcomm PCIe root complex driver 4 * 5 * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved. 6 * Copyright 2015 Linaro Limited. 7 * 8 * Author: Stanimir Varbanov <svarbanov@mm-sol.com> 9 */ 10 11 #include <linux/clk.h> 12 #include <linux/crc8.h> 13 #include <linux/debugfs.h> 14 #include <linux/delay.h> 15 #include <linux/gpio/consumer.h> 16 #include <linux/interconnect.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/iopoll.h> 20 #include <linux/kernel.h> 21 #include <linux/limits.h> 22 #include <linux/init.h> 23 #include <linux/of.h> 24 #include <linux/of_pci.h> 25 #include <linux/pci.h> 26 #include <linux/pci-ecam.h> 27 #include <linux/pci-pwrctrl.h> 28 #include <linux/pm_opp.h> 29 #include <linux/pm_runtime.h> 30 #include <linux/platform_device.h> 31 #include <linux/phy/pcie.h> 32 #include <linux/phy/phy.h> 33 #include <linux/regulator/consumer.h> 34 #include <linux/reset.h> 35 #include <linux/slab.h> 36 #include <linux/types.h> 37 #include <linux/units.h> 38 39 #include "../../pci.h" 40 #include "../pci-host-common.h" 41 #include "pcie-designware.h" 42 #include "pcie-qcom-common.h" 43 44 /* PARF registers */ 45 #define PARF_SYS_CTRL 0x00 46 #define PARF_PM_CTRL 0x20 47 #define PARF_PCS_DEEMPH 0x34 48 #define PARF_PCS_SWING 0x38 49 #define PARF_PHY_CTRL 0x40 50 #define PARF_PHY_REFCLK 0x4c 51 #define PARF_CONFIG_BITS 0x50 52 #define PARF_DBI_BASE_ADDR 0x168 53 #define PARF_SLV_ADDR_SPACE_SIZE 0x16c 54 #define PARF_MHI_CLOCK_RESET_CTRL 0x174 55 #define PARF_AXI_MSTR_WR_ADDR_HALT 0x178 56 #define PARF_AXI_MSTR_WR_ADDR_HALT_V2 0x1a8 57 #define PARF_Q2A_FLUSH 0x1ac 58 #define PARF_LTSSM 0x1b0 59 #define PARF_SID_OFFSET 0x234 60 #define PARF_BDF_TRANSLATE_CFG 0x24c 61 #define PARF_DBI_BASE_ADDR_V2 0x350 62 #define PARF_DBI_BASE_ADDR_V2_HI 0x354 63 #define PARF_SLV_ADDR_SPACE_SIZE_V2 0x358 64 #define PARF_SLV_ADDR_SPACE_SIZE_V2_HI 0x35c 65 #define PARF_NO_SNOOP_OVERRIDE 0x3d4 66 #define PARF_ATU_BASE_ADDR 0x634 67 #define PARF_ATU_BASE_ADDR_HI 0x638 68 #define PARF_DEVICE_TYPE 0x1000 69 #define PARF_BDF_TO_SID_TABLE_N 0x2000 70 #define PARF_BDF_TO_SID_CFG 0x2c00 71 72 /* ELBI registers */ 73 #define ELBI_SYS_CTRL 0x04 74 #define ELBI_SYS_STTS 0x08 75 76 /* DBI registers */ 77 #define AXI_MSTR_RESP_COMP_CTRL0 0x818 78 #define AXI_MSTR_RESP_COMP_CTRL1 0x81c 79 80 /* MHI registers */ 81 #define PARF_DEBUG_CNT_PM_LINKST_IN_L2 0xc04 82 #define PARF_DEBUG_CNT_PM_LINKST_IN_L1 0xc0c 83 #define PARF_DEBUG_CNT_PM_LINKST_IN_L0S 0xc10 84 #define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1 0xc84 85 #define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2 0xc88 86 87 /* PARF_SYS_CTRL register fields */ 88 #define MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN BIT(29) 89 #define MST_WAKEUP_EN BIT(13) 90 #define SLV_WAKEUP_EN BIT(12) 91 #define MSTR_ACLK_CGC_DIS BIT(10) 92 #define SLV_ACLK_CGC_DIS BIT(9) 93 #define CORE_CLK_CGC_DIS BIT(6) 94 #define AUX_PWR_DET BIT(4) 95 #define L23_CLK_RMV_DIS BIT(2) 96 #define L1_CLK_RMV_DIS BIT(1) 97 98 /* PARF_PM_CTRL register fields */ 99 #define REQ_NOT_ENTR_L1 BIT(5) 100 101 /* PARF_PCS_DEEMPH register fields */ 102 #define PCS_DEEMPH_TX_DEEMPH_GEN1(x) FIELD_PREP(GENMASK(21, 16), x) 103 #define PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(x) FIELD_PREP(GENMASK(13, 8), x) 104 #define PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(x) FIELD_PREP(GENMASK(5, 0), x) 105 106 /* PARF_PCS_SWING register fields */ 107 #define PCS_SWING_TX_SWING_FULL(x) FIELD_PREP(GENMASK(14, 8), x) 108 #define PCS_SWING_TX_SWING_LOW(x) FIELD_PREP(GENMASK(6, 0), x) 109 110 /* PARF_PHY_CTRL register fields */ 111 #define PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK GENMASK(20, 16) 112 #define PHY_CTRL_PHY_TX0_TERM_OFFSET(x) FIELD_PREP(PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK, x) 113 #define PHY_TEST_PWR_DOWN BIT(0) 114 115 /* PARF_PHY_REFCLK register fields */ 116 #define PHY_REFCLK_SSP_EN BIT(16) 117 #define PHY_REFCLK_USE_PAD BIT(12) 118 119 /* PARF_CONFIG_BITS register fields */ 120 #define PHY_RX0_EQ(x) FIELD_PREP(GENMASK(26, 24), x) 121 122 /* PARF_SLV_ADDR_SPACE_SIZE register value */ 123 #define SLV_ADDR_SPACE_SZ 0x80000000 124 125 /* PARF_MHI_CLOCK_RESET_CTRL register fields */ 126 #define AHB_CLK_EN BIT(0) 127 #define MSTR_AXI_CLK_EN BIT(1) 128 #define BYPASS BIT(4) 129 130 /* PARF_AXI_MSTR_WR_ADDR_HALT register fields */ 131 #define EN BIT(31) 132 133 /* PARF_LTSSM register fields */ 134 #define LTSSM_EN BIT(8) 135 #define PARF_LTSSM_STATE_MASK GENMASK(5, 0) 136 137 /* PARF_NO_SNOOP_OVERRIDE register fields */ 138 #define WR_NO_SNOOP_OVERRIDE_EN BIT(1) 139 #define RD_NO_SNOOP_OVERRIDE_EN BIT(3) 140 141 /* PARF_DEVICE_TYPE register fields */ 142 #define DEVICE_TYPE_RC 0x4 143 144 /* PARF_BDF_TO_SID_CFG fields */ 145 #define BDF_TO_SID_BYPASS BIT(0) 146 147 /* ELBI_SYS_CTRL register fields */ 148 #define ELBI_SYS_CTRL_LT_ENABLE BIT(0) 149 #define ELBI_SYS_CTRL_PME_TURNOFF_MSG BIT(4) 150 151 /* ELBI_SYS_STTS register fields */ 152 #define ELBI_SYS_STTS_LTSSM_STATE_MASK GENMASK(17, 12) 153 154 /* AXI_MSTR_RESP_COMP_CTRL0 register fields */ 155 #define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K 0x4 156 #define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_4K 0x5 157 158 /* AXI_MSTR_RESP_COMP_CTRL1 register fields */ 159 #define CFG_BRIDGE_SB_INIT BIT(0) 160 161 /* PCI_EXP_SLTCAP register fields */ 162 #define PCIE_CAP_SLOT_POWER_LIMIT_VAL FIELD_PREP(PCI_EXP_SLTCAP_SPLV, 250) 163 #define PCIE_CAP_SLOT_POWER_LIMIT_SCALE FIELD_PREP(PCI_EXP_SLTCAP_SPLS, 1) 164 #define PCIE_CAP_SLOT_VAL (PCI_EXP_SLTCAP_ABP | \ 165 PCI_EXP_SLTCAP_PCP | \ 166 PCI_EXP_SLTCAP_MRLSP | \ 167 PCI_EXP_SLTCAP_AIP | \ 168 PCI_EXP_SLTCAP_PIP | \ 169 PCI_EXP_SLTCAP_HPS | \ 170 PCI_EXP_SLTCAP_EIP | \ 171 PCIE_CAP_SLOT_POWER_LIMIT_VAL | \ 172 PCIE_CAP_SLOT_POWER_LIMIT_SCALE) 173 174 #define PERST_DELAY_US 1000 175 176 #define QCOM_PCIE_CRC8_POLYNOMIAL (BIT(2) | BIT(1) | BIT(0)) 177 178 #define QCOM_PCIE_LINK_SPEED_TO_BW(speed) \ 179 Mbps_to_icc(PCIE_SPEED2MBS_ENC(pcie_get_link_speed(speed))) 180 181 struct qcom_pcie_resources_1_0_0 { 182 struct clk_bulk_data *clks; 183 int num_clks; 184 struct reset_control *core; 185 struct regulator *vdda; 186 }; 187 188 #define QCOM_PCIE_2_1_0_MAX_RESETS 6 189 #define QCOM_PCIE_2_1_0_MAX_SUPPLY 3 190 struct qcom_pcie_resources_2_1_0 { 191 struct clk_bulk_data *clks; 192 int num_clks; 193 struct reset_control_bulk_data resets[QCOM_PCIE_2_1_0_MAX_RESETS]; 194 int num_resets; 195 struct regulator_bulk_data supplies[QCOM_PCIE_2_1_0_MAX_SUPPLY]; 196 }; 197 198 #define QCOM_PCIE_2_3_2_MAX_SUPPLY 2 199 struct qcom_pcie_resources_2_3_2 { 200 struct clk_bulk_data *clks; 201 int num_clks; 202 struct regulator_bulk_data supplies[QCOM_PCIE_2_3_2_MAX_SUPPLY]; 203 }; 204 205 #define QCOM_PCIE_2_3_3_MAX_RESETS 7 206 struct qcom_pcie_resources_2_3_3 { 207 struct clk_bulk_data *clks; 208 int num_clks; 209 struct reset_control_bulk_data rst[QCOM_PCIE_2_3_3_MAX_RESETS]; 210 }; 211 212 #define QCOM_PCIE_2_4_0_MAX_RESETS 12 213 struct qcom_pcie_resources_2_4_0 { 214 struct clk_bulk_data *clks; 215 int num_clks; 216 struct reset_control_bulk_data resets[QCOM_PCIE_2_4_0_MAX_RESETS]; 217 int num_resets; 218 }; 219 220 #define QCOM_PCIE_2_7_0_MAX_SUPPLIES 2 221 struct qcom_pcie_resources_2_7_0 { 222 struct clk_bulk_data *clks; 223 int num_clks; 224 struct regulator_bulk_data supplies[QCOM_PCIE_2_7_0_MAX_SUPPLIES]; 225 struct reset_control *rst; 226 }; 227 228 struct qcom_pcie_resources_2_9_0 { 229 struct clk_bulk_data *clks; 230 int num_clks; 231 struct reset_control *rst; 232 }; 233 234 union qcom_pcie_resources { 235 struct qcom_pcie_resources_1_0_0 v1_0_0; 236 struct qcom_pcie_resources_2_1_0 v2_1_0; 237 struct qcom_pcie_resources_2_3_2 v2_3_2; 238 struct qcom_pcie_resources_2_3_3 v2_3_3; 239 struct qcom_pcie_resources_2_4_0 v2_4_0; 240 struct qcom_pcie_resources_2_7_0 v2_7_0; 241 struct qcom_pcie_resources_2_9_0 v2_9_0; 242 }; 243 244 struct qcom_pcie; 245 246 struct qcom_pcie_ops { 247 int (*get_resources)(struct qcom_pcie *pcie); 248 int (*init)(struct qcom_pcie *pcie); 249 int (*post_init)(struct qcom_pcie *pcie); 250 void (*host_post_init)(struct qcom_pcie *pcie); 251 void (*deinit)(struct qcom_pcie *pcie); 252 void (*ltssm_enable)(struct qcom_pcie *pcie); 253 int (*config_sid)(struct qcom_pcie *pcie); 254 enum dw_pcie_ltssm (*get_ltssm)(struct qcom_pcie *pcie); 255 }; 256 257 /** 258 * struct qcom_pcie_cfg - Per SoC config struct 259 * @ops: qcom PCIe ops structure 260 * @override_no_snoop: Override NO_SNOOP attribute in TLP to enable cache 261 * snooping 262 * @firmware_managed: Set if the Root Complex is firmware managed 263 */ 264 struct qcom_pcie_cfg { 265 const struct qcom_pcie_ops *ops; 266 bool override_no_snoop; 267 bool firmware_managed; 268 bool no_l0s; 269 }; 270 271 struct qcom_pcie_perst { 272 struct list_head list; 273 struct gpio_desc *desc; 274 }; 275 276 struct qcom_pcie_port { 277 struct list_head list; 278 struct phy *phy; 279 u32 l1ss_t_power_on; 280 struct list_head perst; 281 }; 282 283 struct qcom_pcie { 284 struct dw_pcie *pci; 285 void __iomem *parf; /* DT parf */ 286 void __iomem *mhi; 287 union qcom_pcie_resources res; 288 struct icc_path *icc_mem; 289 struct icc_path *icc_cpu; 290 const struct qcom_pcie_cfg *cfg; 291 struct dentry *debugfs; 292 struct list_head ports; 293 struct gpio_desc *reset; 294 bool use_pm_opp; 295 }; 296 297 #define to_qcom_pcie(x) dev_get_drvdata((x)->dev) 298 299 static void __qcom_pcie_perst_assert(struct qcom_pcie *pcie, bool assert) 300 { 301 struct qcom_pcie_perst *perst; 302 struct qcom_pcie_port *port; 303 int val = assert ? 1 : 0; 304 305 list_for_each_entry(port, &pcie->ports, list) { 306 list_for_each_entry(perst, &port->perst, list) 307 gpiod_set_value_cansleep(perst->desc, val); 308 } 309 310 usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500); 311 } 312 313 static void qcom_pcie_perst_assert(struct qcom_pcie *pcie) 314 { 315 __qcom_pcie_perst_assert(pcie, true); 316 } 317 318 static void qcom_pcie_perst_deassert(struct qcom_pcie *pcie) 319 { 320 /* Ensure that PERST# has been asserted for at least 100 ms */ 321 msleep(PCIE_T_PVPERL_MS); 322 __qcom_pcie_perst_assert(pcie, false); 323 } 324 325 static int qcom_pcie_start_link(struct dw_pcie *pci) 326 { 327 struct qcom_pcie *pcie = to_qcom_pcie(pci); 328 329 qcom_pcie_common_set_equalization(pci); 330 331 if (pcie_get_link_speed(pci->max_link_speed) == PCIE_SPEED_16_0GT) 332 qcom_pcie_common_set_16gt_lane_margining(pci); 333 334 /* Enable Link Training state machine */ 335 if (pcie->cfg->ops->ltssm_enable) 336 pcie->cfg->ops->ltssm_enable(pcie); 337 338 return 0; 339 } 340 341 static void qcom_pcie_clear_aspm_l0s(struct dw_pcie *pci) 342 { 343 struct qcom_pcie *pcie = to_qcom_pcie(pci); 344 u16 offset; 345 u32 val; 346 347 if (!pcie->cfg->no_l0s) 348 return; 349 350 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 351 352 dw_pcie_dbi_ro_wr_en(pci); 353 354 val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); 355 val &= ~PCI_EXP_LNKCAP_ASPM_L0S; 356 writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); 357 358 dw_pcie_dbi_ro_wr_dis(pci); 359 } 360 361 static void qcom_pcie_set_slot_nccs(struct dw_pcie *pci) 362 { 363 u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 364 u32 val; 365 366 dw_pcie_dbi_ro_wr_en(pci); 367 368 /* 369 * Qcom PCIe Root Ports do not support generating command completion 370 * notifications for the Hot-Plug commands. So set the NCCS field to 371 * avoid waiting for the completions. 372 */ 373 val = readl(pci->dbi_base + offset + PCI_EXP_SLTCAP); 374 val |= PCI_EXP_SLTCAP_NCCS; 375 writel(val, pci->dbi_base + offset + PCI_EXP_SLTCAP); 376 377 dw_pcie_dbi_ro_wr_dis(pci); 378 } 379 380 static void qcom_pcie_configure_dbi_base(struct qcom_pcie *pcie) 381 { 382 struct dw_pcie *pci = pcie->pci; 383 384 if (pci->dbi_phys_addr) { 385 /* 386 * PARF_DBI_BASE_ADDR register is in CPU domain and require to 387 * be programmed with CPU physical address. 388 */ 389 writel(lower_32_bits(pci->dbi_phys_addr), pcie->parf + 390 PARF_DBI_BASE_ADDR); 391 writel(SLV_ADDR_SPACE_SZ, pcie->parf + 392 PARF_SLV_ADDR_SPACE_SIZE); 393 } 394 } 395 396 static void qcom_pcie_configure_dbi_atu_base(struct qcom_pcie *pcie) 397 { 398 struct dw_pcie *pci = pcie->pci; 399 400 if (pci->dbi_phys_addr) { 401 /* 402 * PARF_DBI_BASE_ADDR_V2 and PARF_ATU_BASE_ADDR registers are 403 * in CPU domain and require to be programmed with CPU 404 * physical addresses. 405 */ 406 writel(lower_32_bits(pci->dbi_phys_addr), pcie->parf + 407 PARF_DBI_BASE_ADDR_V2); 408 writel(upper_32_bits(pci->dbi_phys_addr), pcie->parf + 409 PARF_DBI_BASE_ADDR_V2_HI); 410 411 if (pci->atu_phys_addr) { 412 writel(lower_32_bits(pci->atu_phys_addr), pcie->parf + 413 PARF_ATU_BASE_ADDR); 414 writel(upper_32_bits(pci->atu_phys_addr), pcie->parf + 415 PARF_ATU_BASE_ADDR_HI); 416 } 417 418 writel(0x0, pcie->parf + PARF_SLV_ADDR_SPACE_SIZE_V2); 419 writel(SLV_ADDR_SPACE_SZ, pcie->parf + 420 PARF_SLV_ADDR_SPACE_SIZE_V2_HI); 421 } 422 } 423 424 static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie) 425 { 426 struct dw_pcie *pci = pcie->pci; 427 u32 val; 428 429 if (!pci->elbi_base) { 430 dev_err(pci->dev, "ELBI is not present\n"); 431 return; 432 } 433 /* enable link training */ 434 val = readl(pci->elbi_base + ELBI_SYS_CTRL); 435 val |= ELBI_SYS_CTRL_LT_ENABLE; 436 writel(val, pci->elbi_base + ELBI_SYS_CTRL); 437 } 438 439 static enum dw_pcie_ltssm qcom_pcie_2_1_0_get_ltssm(struct qcom_pcie *pcie) 440 { 441 struct dw_pcie *pci = pcie->pci; 442 u32 val; 443 444 val = readl(pci->elbi_base + ELBI_SYS_STTS); 445 return (enum dw_pcie_ltssm)FIELD_GET(ELBI_SYS_STTS_LTSSM_STATE_MASK, val); 446 } 447 448 static int qcom_pcie_get_resources_2_1_0(struct qcom_pcie *pcie) 449 { 450 struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0; 451 struct dw_pcie *pci = pcie->pci; 452 struct device *dev = pci->dev; 453 bool is_apq = of_device_is_compatible(dev->of_node, "qcom,pcie-apq8064"); 454 int ret; 455 456 res->supplies[0].supply = "vdda"; 457 res->supplies[1].supply = "vdda_phy"; 458 res->supplies[2].supply = "vdda_refclk"; 459 ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies), 460 res->supplies); 461 if (ret) 462 return ret; 463 464 res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); 465 if (res->num_clks < 0) { 466 dev_err(dev, "Failed to get clocks\n"); 467 return res->num_clks; 468 } 469 470 res->resets[0].id = "pci"; 471 res->resets[1].id = "axi"; 472 res->resets[2].id = "ahb"; 473 res->resets[3].id = "por"; 474 res->resets[4].id = "phy"; 475 res->resets[5].id = "ext"; 476 477 /* ext is optional on APQ8016 */ 478 res->num_resets = is_apq ? 5 : 6; 479 ret = devm_reset_control_bulk_get_exclusive(dev, res->num_resets, res->resets); 480 if (ret < 0) 481 return ret; 482 483 return 0; 484 } 485 486 static void qcom_pcie_deinit_2_1_0(struct qcom_pcie *pcie) 487 { 488 struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0; 489 490 clk_bulk_disable_unprepare(res->num_clks, res->clks); 491 reset_control_bulk_assert(res->num_resets, res->resets); 492 493 writel(1, pcie->parf + PARF_PHY_CTRL); 494 495 regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); 496 } 497 498 static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie) 499 { 500 struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0; 501 struct dw_pcie *pci = pcie->pci; 502 struct device *dev = pci->dev; 503 int ret; 504 505 /* reset the PCIe interface as uboot can leave it undefined state */ 506 ret = reset_control_bulk_assert(res->num_resets, res->resets); 507 if (ret < 0) { 508 dev_err(dev, "cannot assert resets\n"); 509 return ret; 510 } 511 512 ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies); 513 if (ret < 0) { 514 dev_err(dev, "cannot enable regulators\n"); 515 return ret; 516 } 517 518 ret = reset_control_bulk_deassert(res->num_resets, res->resets); 519 if (ret < 0) { 520 dev_err(dev, "cannot deassert resets\n"); 521 regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); 522 return ret; 523 } 524 525 return 0; 526 } 527 528 static int qcom_pcie_post_init_2_1_0(struct qcom_pcie *pcie) 529 { 530 struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0; 531 struct dw_pcie *pci = pcie->pci; 532 struct device *dev = pci->dev; 533 struct device_node *node = dev->of_node; 534 u32 val; 535 int ret; 536 537 /* Force PHY out of lowest power state */ 538 val = readl(pcie->parf + PARF_PHY_CTRL); 539 val &= ~PHY_TEST_PWR_DOWN; 540 writel(val, pcie->parf + PARF_PHY_CTRL); 541 542 ret = clk_bulk_prepare_enable(res->num_clks, res->clks); 543 if (ret) 544 return ret; 545 546 if (of_device_is_compatible(node, "qcom,pcie-ipq8064") || 547 of_device_is_compatible(node, "qcom,pcie-ipq8064-v2")) { 548 writel(PCS_DEEMPH_TX_DEEMPH_GEN1(24) | 549 PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(24) | 550 PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(34), 551 pcie->parf + PARF_PCS_DEEMPH); 552 writel(PCS_SWING_TX_SWING_FULL(120) | 553 PCS_SWING_TX_SWING_LOW(120), 554 pcie->parf + PARF_PCS_SWING); 555 writel(PHY_RX0_EQ(4), pcie->parf + PARF_CONFIG_BITS); 556 } 557 558 if (of_device_is_compatible(node, "qcom,pcie-ipq8064")) { 559 /* set TX termination offset */ 560 val = readl(pcie->parf + PARF_PHY_CTRL); 561 val &= ~PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK; 562 val |= PHY_CTRL_PHY_TX0_TERM_OFFSET(7); 563 writel(val, pcie->parf + PARF_PHY_CTRL); 564 } 565 566 /* enable external reference clock */ 567 val = readl(pcie->parf + PARF_PHY_REFCLK); 568 /* USE_PAD is required only for ipq806x */ 569 if (!of_device_is_compatible(node, "qcom,pcie-apq8064")) 570 val &= ~PHY_REFCLK_USE_PAD; 571 val |= PHY_REFCLK_SSP_EN; 572 writel(val, pcie->parf + PARF_PHY_REFCLK); 573 574 /* wait for clock acquisition */ 575 usleep_range(1000, 1500); 576 577 /* Set the Max TLP size to 2K, instead of using default of 4K */ 578 writel(CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K, 579 pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL0); 580 writel(CFG_BRIDGE_SB_INIT, 581 pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL1); 582 583 qcom_pcie_set_slot_nccs(pcie->pci); 584 585 return 0; 586 } 587 588 static int qcom_pcie_get_resources_1_0_0(struct qcom_pcie *pcie) 589 { 590 struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0; 591 struct dw_pcie *pci = pcie->pci; 592 struct device *dev = pci->dev; 593 594 res->vdda = devm_regulator_get(dev, "vdda"); 595 if (IS_ERR(res->vdda)) 596 return PTR_ERR(res->vdda); 597 598 res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); 599 if (res->num_clks < 0) { 600 dev_err(dev, "Failed to get clocks\n"); 601 return res->num_clks; 602 } 603 604 res->core = devm_reset_control_get_exclusive(dev, "core"); 605 return PTR_ERR_OR_ZERO(res->core); 606 } 607 608 static void qcom_pcie_deinit_1_0_0(struct qcom_pcie *pcie) 609 { 610 struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0; 611 612 reset_control_assert(res->core); 613 clk_bulk_disable_unprepare(res->num_clks, res->clks); 614 regulator_disable(res->vdda); 615 } 616 617 static int qcom_pcie_init_1_0_0(struct qcom_pcie *pcie) 618 { 619 struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0; 620 struct dw_pcie *pci = pcie->pci; 621 struct device *dev = pci->dev; 622 int ret; 623 624 ret = reset_control_deassert(res->core); 625 if (ret) { 626 dev_err(dev, "cannot deassert core reset\n"); 627 return ret; 628 } 629 630 ret = clk_bulk_prepare_enable(res->num_clks, res->clks); 631 if (ret) { 632 dev_err(dev, "cannot prepare/enable clocks\n"); 633 goto err_assert_reset; 634 } 635 636 ret = regulator_enable(res->vdda); 637 if (ret) { 638 dev_err(dev, "cannot enable vdda regulator\n"); 639 goto err_disable_clks; 640 } 641 642 return 0; 643 644 err_disable_clks: 645 clk_bulk_disable_unprepare(res->num_clks, res->clks); 646 err_assert_reset: 647 reset_control_assert(res->core); 648 649 return ret; 650 } 651 652 static int qcom_pcie_post_init_1_0_0(struct qcom_pcie *pcie) 653 { 654 qcom_pcie_configure_dbi_base(pcie); 655 656 if (IS_ENABLED(CONFIG_PCI_MSI)) { 657 u32 val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT); 658 659 val |= EN; 660 writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT); 661 } 662 663 qcom_pcie_set_slot_nccs(pcie->pci); 664 665 return 0; 666 } 667 668 static void qcom_pcie_2_3_2_ltssm_enable(struct qcom_pcie *pcie) 669 { 670 u32 val; 671 672 /* enable link training */ 673 val = readl(pcie->parf + PARF_LTSSM); 674 val |= LTSSM_EN; 675 writel(val, pcie->parf + PARF_LTSSM); 676 } 677 678 static int qcom_pcie_get_resources_2_3_2(struct qcom_pcie *pcie) 679 { 680 struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2; 681 struct dw_pcie *pci = pcie->pci; 682 struct device *dev = pci->dev; 683 int ret; 684 685 res->supplies[0].supply = "vdda"; 686 res->supplies[1].supply = "vddpe-3v3"; 687 ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies), 688 res->supplies); 689 if (ret) 690 return ret; 691 692 res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); 693 if (res->num_clks < 0) { 694 dev_err(dev, "Failed to get clocks\n"); 695 return res->num_clks; 696 } 697 698 return 0; 699 } 700 701 static void qcom_pcie_deinit_2_3_2(struct qcom_pcie *pcie) 702 { 703 struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2; 704 u32 val; 705 706 /* Force PHY to lowest power state*/ 707 val = readl(pcie->parf + PARF_PHY_CTRL); 708 val |= PHY_TEST_PWR_DOWN; 709 writel(val, pcie->parf + PARF_PHY_CTRL); 710 711 clk_bulk_disable_unprepare(res->num_clks, res->clks); 712 regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); 713 } 714 715 static int qcom_pcie_init_2_3_2(struct qcom_pcie *pcie) 716 { 717 struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2; 718 struct dw_pcie *pci = pcie->pci; 719 struct device *dev = pci->dev; 720 int ret; 721 722 ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies); 723 if (ret < 0) { 724 dev_err(dev, "cannot enable regulators\n"); 725 return ret; 726 } 727 728 ret = clk_bulk_prepare_enable(res->num_clks, res->clks); 729 if (ret) { 730 dev_err(dev, "cannot prepare/enable clocks\n"); 731 regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); 732 return ret; 733 } 734 735 return 0; 736 } 737 738 static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie) 739 { 740 u32 val; 741 742 /* Force PHY out of lowest power state */ 743 val = readl(pcie->parf + PARF_PHY_CTRL); 744 val &= ~PHY_TEST_PWR_DOWN; 745 writel(val, pcie->parf + PARF_PHY_CTRL); 746 747 qcom_pcie_configure_dbi_base(pcie); 748 749 /* MAC PHY_POWERDOWN MUX DISABLE */ 750 val = readl(pcie->parf + PARF_SYS_CTRL); 751 val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN; 752 writel(val, pcie->parf + PARF_SYS_CTRL); 753 754 val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); 755 val |= BYPASS; 756 writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); 757 758 val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); 759 val |= EN; 760 writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); 761 762 qcom_pcie_set_slot_nccs(pcie->pci); 763 764 return 0; 765 } 766 767 static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie) 768 { 769 struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0; 770 struct dw_pcie *pci = pcie->pci; 771 struct device *dev = pci->dev; 772 bool is_ipq = of_device_is_compatible(dev->of_node, "qcom,pcie-ipq4019"); 773 int ret; 774 775 res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); 776 if (res->num_clks < 0) { 777 dev_err(dev, "Failed to get clocks\n"); 778 return res->num_clks; 779 } 780 781 res->resets[0].id = "axi_m"; 782 res->resets[1].id = "axi_s"; 783 res->resets[2].id = "axi_m_sticky"; 784 res->resets[3].id = "pipe_sticky"; 785 res->resets[4].id = "pwr"; 786 res->resets[5].id = "ahb"; 787 res->resets[6].id = "pipe"; 788 res->resets[7].id = "axi_m_vmid"; 789 res->resets[8].id = "axi_s_xpu"; 790 res->resets[9].id = "parf"; 791 res->resets[10].id = "phy"; 792 res->resets[11].id = "phy_ahb"; 793 794 res->num_resets = is_ipq ? 12 : 6; 795 796 ret = devm_reset_control_bulk_get_exclusive(dev, res->num_resets, res->resets); 797 if (ret < 0) 798 return ret; 799 800 return 0; 801 } 802 803 static void qcom_pcie_deinit_2_4_0(struct qcom_pcie *pcie) 804 { 805 struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0; 806 u32 val; 807 808 /* Force PHY to lowest power state*/ 809 val = readl(pcie->parf + PARF_PHY_CTRL); 810 val |= PHY_TEST_PWR_DOWN; 811 writel(val, pcie->parf + PARF_PHY_CTRL); 812 813 reset_control_bulk_assert(res->num_resets, res->resets); 814 clk_bulk_disable_unprepare(res->num_clks, res->clks); 815 } 816 817 static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie) 818 { 819 struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0; 820 struct dw_pcie *pci = pcie->pci; 821 struct device *dev = pci->dev; 822 int ret; 823 824 ret = reset_control_bulk_assert(res->num_resets, res->resets); 825 if (ret < 0) { 826 dev_err(dev, "cannot assert resets\n"); 827 return ret; 828 } 829 830 usleep_range(10000, 12000); 831 832 ret = reset_control_bulk_deassert(res->num_resets, res->resets); 833 if (ret < 0) { 834 dev_err(dev, "cannot deassert resets\n"); 835 return ret; 836 } 837 838 usleep_range(10000, 12000); 839 840 ret = clk_bulk_prepare_enable(res->num_clks, res->clks); 841 if (ret) { 842 reset_control_bulk_assert(res->num_resets, res->resets); 843 return ret; 844 } 845 846 return 0; 847 } 848 849 static int qcom_pcie_get_resources_2_3_3(struct qcom_pcie *pcie) 850 { 851 struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3; 852 struct dw_pcie *pci = pcie->pci; 853 struct device *dev = pci->dev; 854 int ret; 855 856 res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); 857 if (res->num_clks < 0) { 858 dev_err(dev, "Failed to get clocks\n"); 859 return res->num_clks; 860 } 861 862 res->rst[0].id = "axi_m"; 863 res->rst[1].id = "axi_s"; 864 res->rst[2].id = "pipe"; 865 res->rst[3].id = "axi_m_sticky"; 866 res->rst[4].id = "sticky"; 867 res->rst[5].id = "ahb"; 868 res->rst[6].id = "sleep"; 869 870 ret = devm_reset_control_bulk_get_exclusive(dev, ARRAY_SIZE(res->rst), res->rst); 871 if (ret < 0) 872 return ret; 873 874 return 0; 875 } 876 877 static void qcom_pcie_deinit_2_3_3(struct qcom_pcie *pcie) 878 { 879 struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3; 880 u32 val; 881 882 /* Force PHY to lowest power state */ 883 val = readl(pcie->parf + PARF_PHY_CTRL); 884 val |= PHY_TEST_PWR_DOWN; 885 writel(val, pcie->parf + PARF_PHY_CTRL); 886 887 clk_bulk_disable_unprepare(res->num_clks, res->clks); 888 } 889 890 static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie) 891 { 892 struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3; 893 struct dw_pcie *pci = pcie->pci; 894 struct device *dev = pci->dev; 895 int ret; 896 897 ret = reset_control_bulk_assert(ARRAY_SIZE(res->rst), res->rst); 898 if (ret < 0) { 899 dev_err(dev, "cannot assert resets\n"); 900 return ret; 901 } 902 903 usleep_range(2000, 2500); 904 905 ret = reset_control_bulk_deassert(ARRAY_SIZE(res->rst), res->rst); 906 if (ret < 0) { 907 dev_err(dev, "cannot deassert resets\n"); 908 return ret; 909 } 910 911 /* 912 * Don't have a way to see if the reset has completed. 913 * Wait for some time. 914 */ 915 usleep_range(2000, 2500); 916 917 ret = clk_bulk_prepare_enable(res->num_clks, res->clks); 918 if (ret) { 919 dev_err(dev, "cannot prepare/enable clocks\n"); 920 goto err_assert_resets; 921 } 922 923 return 0; 924 925 err_assert_resets: 926 /* 927 * Not checking for failure, will anyway return 928 * the original failure in 'ret'. 929 */ 930 reset_control_bulk_assert(ARRAY_SIZE(res->rst), res->rst); 931 932 return ret; 933 } 934 935 static int qcom_pcie_post_init_2_3_3(struct qcom_pcie *pcie) 936 { 937 struct dw_pcie *pci = pcie->pci; 938 u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 939 u32 val; 940 941 /* Force PHY out of lowest power state */ 942 val = readl(pcie->parf + PARF_PHY_CTRL); 943 val &= ~PHY_TEST_PWR_DOWN; 944 writel(val, pcie->parf + PARF_PHY_CTRL); 945 946 qcom_pcie_configure_dbi_atu_base(pcie); 947 948 writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS 949 | SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS | 950 AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS, 951 pcie->parf + PARF_SYS_CTRL); 952 writel(0, pcie->parf + PARF_Q2A_FLUSH); 953 954 writel(PCI_COMMAND_MASTER, pci->dbi_base + PCI_COMMAND); 955 956 dw_pcie_dbi_ro_wr_en(pci); 957 958 writel(PCIE_CAP_SLOT_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP); 959 960 val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); 961 val &= ~PCI_EXP_LNKCAP_ASPMS; 962 writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); 963 964 writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset + 965 PCI_EXP_DEVCTL2); 966 967 dw_pcie_dbi_ro_wr_dis(pci); 968 969 return 0; 970 } 971 972 static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie) 973 { 974 struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; 975 struct dw_pcie *pci = pcie->pci; 976 struct device *dev = pci->dev; 977 int ret; 978 979 res->rst = devm_reset_control_array_get_exclusive(dev); 980 if (IS_ERR(res->rst)) 981 return PTR_ERR(res->rst); 982 983 res->supplies[0].supply = "vdda"; 984 res->supplies[1].supply = "vddpe-3v3"; 985 ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies), 986 res->supplies); 987 if (ret) 988 return ret; 989 990 res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); 991 if (res->num_clks < 0) { 992 dev_err(dev, "Failed to get clocks\n"); 993 return res->num_clks; 994 } 995 996 return 0; 997 } 998 999 static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) 1000 { 1001 struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; 1002 struct dw_pcie *pci = pcie->pci; 1003 struct device *dev = pci->dev; 1004 u32 val; 1005 int ret; 1006 1007 ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies); 1008 if (ret < 0) { 1009 dev_err(dev, "cannot enable regulators\n"); 1010 return ret; 1011 } 1012 1013 ret = clk_bulk_prepare_enable(res->num_clks, res->clks); 1014 if (ret < 0) 1015 goto err_disable_regulators; 1016 1017 ret = reset_control_assert(res->rst); 1018 if (ret) { 1019 dev_err(dev, "reset assert failed (%d)\n", ret); 1020 goto err_disable_clocks; 1021 } 1022 1023 usleep_range(1000, 1500); 1024 1025 ret = reset_control_deassert(res->rst); 1026 if (ret) { 1027 dev_err(dev, "reset deassert failed (%d)\n", ret); 1028 goto err_disable_clocks; 1029 } 1030 1031 /* Wait for reset to complete, required on SM8450 */ 1032 usleep_range(1000, 1500); 1033 1034 /* configure PCIe to RC mode */ 1035 writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE); 1036 1037 /* Force PHY out of lowest power state */ 1038 val = readl(pcie->parf + PARF_PHY_CTRL); 1039 val &= ~PHY_TEST_PWR_DOWN; 1040 writel(val, pcie->parf + PARF_PHY_CTRL); 1041 1042 qcom_pcie_configure_dbi_atu_base(pcie); 1043 1044 /* MAC PHY_POWERDOWN MUX DISABLE */ 1045 val = readl(pcie->parf + PARF_SYS_CTRL); 1046 val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN; 1047 writel(val, pcie->parf + PARF_SYS_CTRL); 1048 1049 val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); 1050 val |= BYPASS; 1051 writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); 1052 1053 /* Enable L1 and L1SS */ 1054 val = readl(pcie->parf + PARF_PM_CTRL); 1055 val &= ~REQ_NOT_ENTR_L1; 1056 writel(val, pcie->parf + PARF_PM_CTRL); 1057 1058 pci->l1ss_support = true; 1059 1060 val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); 1061 val |= EN; 1062 writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); 1063 1064 return 0; 1065 err_disable_clocks: 1066 clk_bulk_disable_unprepare(res->num_clks, res->clks); 1067 err_disable_regulators: 1068 regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); 1069 1070 return ret; 1071 } 1072 1073 static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie) 1074 { 1075 const struct qcom_pcie_cfg *pcie_cfg = pcie->cfg; 1076 1077 if (pcie_cfg->override_no_snoop) 1078 writel(WR_NO_SNOOP_OVERRIDE_EN | RD_NO_SNOOP_OVERRIDE_EN, 1079 pcie->parf + PARF_NO_SNOOP_OVERRIDE); 1080 1081 qcom_pcie_set_slot_nccs(pcie->pci); 1082 1083 return 0; 1084 } 1085 1086 static int qcom_pcie_enable_aspm(struct pci_dev *pdev, void *userdata) 1087 { 1088 /* 1089 * Downstream devices need to be in D0 state before enabling PCI PM 1090 * substates. 1091 */ 1092 pci_set_power_state_locked(pdev, PCI_D0); 1093 pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL); 1094 1095 return 0; 1096 } 1097 1098 static void qcom_pcie_host_post_init_2_7_0(struct qcom_pcie *pcie) 1099 { 1100 struct dw_pcie_rp *pp = &pcie->pci->pp; 1101 1102 pci_walk_bus(pp->bridge->bus, qcom_pcie_enable_aspm, NULL); 1103 } 1104 1105 static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie) 1106 { 1107 struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; 1108 u32 val; 1109 1110 /* Force PHY to lowest power state */ 1111 val = readl(pcie->parf + PARF_PHY_CTRL); 1112 val |= PHY_TEST_PWR_DOWN; 1113 writel(val, pcie->parf + PARF_PHY_CTRL); 1114 1115 clk_bulk_disable_unprepare(res->num_clks, res->clks); 1116 1117 regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); 1118 } 1119 1120 static int qcom_pcie_config_sid_1_9_0(struct qcom_pcie *pcie) 1121 { 1122 /* iommu map structure */ 1123 struct { 1124 u32 bdf; 1125 u32 phandle; 1126 u32 smmu_sid; 1127 u32 smmu_sid_len; 1128 } *map; 1129 void __iomem *bdf_to_sid_base = pcie->parf + PARF_BDF_TO_SID_TABLE_N; 1130 struct device *dev = pcie->pci->dev; 1131 u8 qcom_pcie_crc8_table[CRC8_TABLE_SIZE]; 1132 int i, nr_map, size = 0; 1133 u32 smmu_sid_base; 1134 u32 val; 1135 1136 of_get_property(dev->of_node, "iommu-map", &size); 1137 if (!size) 1138 return 0; 1139 1140 /* Enable BDF to SID translation by disabling bypass mode (default) */ 1141 val = readl(pcie->parf + PARF_BDF_TO_SID_CFG); 1142 val &= ~BDF_TO_SID_BYPASS; 1143 writel(val, pcie->parf + PARF_BDF_TO_SID_CFG); 1144 1145 map = kzalloc(size, GFP_KERNEL); 1146 if (!map) 1147 return -ENOMEM; 1148 1149 of_property_read_u32_array(dev->of_node, "iommu-map", (u32 *)map, 1150 size / sizeof(u32)); 1151 1152 nr_map = size / (sizeof(*map)); 1153 1154 crc8_populate_msb(qcom_pcie_crc8_table, QCOM_PCIE_CRC8_POLYNOMIAL); 1155 1156 /* Registers need to be zero out first */ 1157 memset_io(bdf_to_sid_base, 0, CRC8_TABLE_SIZE * sizeof(u32)); 1158 1159 /* Extract the SMMU SID base from the first entry of iommu-map */ 1160 smmu_sid_base = map[0].smmu_sid; 1161 1162 /* Look for an available entry to hold the mapping */ 1163 for (i = 0; i < nr_map; i++) { 1164 __be16 bdf_be = cpu_to_be16(map[i].bdf); 1165 u32 val; 1166 u8 hash; 1167 1168 hash = crc8(qcom_pcie_crc8_table, (u8 *)&bdf_be, sizeof(bdf_be), 0); 1169 1170 val = readl(bdf_to_sid_base + hash * sizeof(u32)); 1171 1172 /* If the register is already populated, look for next available entry */ 1173 while (val) { 1174 u8 current_hash = hash++; 1175 u8 next_mask = 0xff; 1176 1177 /* If NEXT field is NULL then update it with next hash */ 1178 if (!(val & next_mask)) { 1179 val |= (u32)hash; 1180 writel(val, bdf_to_sid_base + current_hash * sizeof(u32)); 1181 } 1182 1183 val = readl(bdf_to_sid_base + hash * sizeof(u32)); 1184 } 1185 1186 /* BDF [31:16] | SID [15:8] | NEXT [7:0] */ 1187 val = map[i].bdf << 16 | (map[i].smmu_sid - smmu_sid_base) << 8 | 0; 1188 writel(val, bdf_to_sid_base + hash * sizeof(u32)); 1189 } 1190 1191 kfree(map); 1192 1193 return 0; 1194 } 1195 1196 static int qcom_pcie_get_resources_2_9_0(struct qcom_pcie *pcie) 1197 { 1198 struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0; 1199 struct dw_pcie *pci = pcie->pci; 1200 struct device *dev = pci->dev; 1201 1202 res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); 1203 if (res->num_clks < 0) { 1204 dev_err(dev, "Failed to get clocks\n"); 1205 return res->num_clks; 1206 } 1207 1208 res->rst = devm_reset_control_array_get_exclusive(dev); 1209 if (IS_ERR(res->rst)) 1210 return PTR_ERR(res->rst); 1211 1212 return 0; 1213 } 1214 1215 static void qcom_pcie_deinit_2_9_0(struct qcom_pcie *pcie) 1216 { 1217 struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0; 1218 u32 val; 1219 1220 /* Force PHY to lowest power state */ 1221 val = readl(pcie->parf + PARF_PHY_CTRL); 1222 val |= PHY_TEST_PWR_DOWN; 1223 writel(val, pcie->parf + PARF_PHY_CTRL); 1224 1225 clk_bulk_disable_unprepare(res->num_clks, res->clks); 1226 } 1227 1228 static int qcom_pcie_init_2_9_0(struct qcom_pcie *pcie) 1229 { 1230 struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0; 1231 struct device *dev = pcie->pci->dev; 1232 int ret; 1233 1234 ret = reset_control_assert(res->rst); 1235 if (ret) { 1236 dev_err(dev, "reset assert failed (%d)\n", ret); 1237 return ret; 1238 } 1239 1240 /* 1241 * Delay periods before and after reset deassert are working values 1242 * from downstream Codeaurora kernel 1243 */ 1244 usleep_range(2000, 2500); 1245 1246 ret = reset_control_deassert(res->rst); 1247 if (ret) { 1248 dev_err(dev, "reset deassert failed (%d)\n", ret); 1249 return ret; 1250 } 1251 1252 usleep_range(2000, 2500); 1253 1254 return clk_bulk_prepare_enable(res->num_clks, res->clks); 1255 } 1256 1257 static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie) 1258 { 1259 struct dw_pcie *pci = pcie->pci; 1260 u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 1261 u32 val; 1262 int i; 1263 1264 /* Force PHY out of lowest power state */ 1265 val = readl(pcie->parf + PARF_PHY_CTRL); 1266 val &= ~PHY_TEST_PWR_DOWN; 1267 writel(val, pcie->parf + PARF_PHY_CTRL); 1268 1269 qcom_pcie_configure_dbi_atu_base(pcie); 1270 1271 writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE); 1272 writel(BYPASS | MSTR_AXI_CLK_EN | AHB_CLK_EN, 1273 pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); 1274 writel(GEN3_RELATED_OFF_RXEQ_RGRDLESS_RXTS | 1275 GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL, 1276 pci->dbi_base + GEN3_RELATED_OFF); 1277 1278 writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS | 1279 SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS | 1280 AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS, 1281 pcie->parf + PARF_SYS_CTRL); 1282 1283 writel(0, pcie->parf + PARF_Q2A_FLUSH); 1284 1285 dw_pcie_dbi_ro_wr_en(pci); 1286 1287 writel(PCIE_CAP_SLOT_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP); 1288 1289 val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); 1290 val &= ~PCI_EXP_LNKCAP_ASPMS; 1291 writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); 1292 1293 writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset + 1294 PCI_EXP_DEVCTL2); 1295 1296 dw_pcie_dbi_ro_wr_dis(pci); 1297 1298 for (i = 0; i < 256; i++) 1299 writel(0, pcie->parf + PARF_BDF_TO_SID_TABLE_N + (4 * i)); 1300 1301 return 0; 1302 } 1303 1304 static bool qcom_pcie_link_up(struct dw_pcie *pci) 1305 { 1306 u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 1307 u16 val = readw(pci->dbi_base + offset + PCI_EXP_LNKSTA); 1308 1309 return val & PCI_EXP_LNKSTA_DLLLA; 1310 } 1311 1312 static enum dw_pcie_ltssm qcom_pcie_get_ltssm(struct dw_pcie *pci) 1313 { 1314 struct qcom_pcie *pcie = to_qcom_pcie(pci); 1315 u32 val; 1316 1317 if (pcie->cfg->ops->get_ltssm) 1318 return pcie->cfg->ops->get_ltssm(pcie); 1319 1320 val = readl(pcie->parf + PARF_LTSSM); 1321 1322 return (enum dw_pcie_ltssm)FIELD_GET(PARF_LTSSM_STATE_MASK, val); 1323 } 1324 1325 static void qcom_pcie_phy_power_off(struct qcom_pcie *pcie) 1326 { 1327 struct qcom_pcie_port *port; 1328 1329 list_for_each_entry(port, &pcie->ports, list) 1330 phy_power_off(port->phy); 1331 } 1332 1333 static int qcom_pcie_phy_power_on(struct qcom_pcie *pcie) 1334 { 1335 struct qcom_pcie_port *port; 1336 int ret; 1337 1338 list_for_each_entry(port, &pcie->ports, list) { 1339 ret = phy_set_mode_ext(port->phy, PHY_MODE_PCIE, PHY_MODE_PCIE_RC); 1340 if (ret) 1341 return ret; 1342 1343 ret = phy_power_on(port->phy); 1344 if (ret) { 1345 qcom_pcie_phy_power_off(pcie); 1346 return ret; 1347 } 1348 } 1349 1350 return 0; 1351 } 1352 1353 static void qcom_pcie_configure_ports(struct qcom_pcie *pcie) 1354 { 1355 struct qcom_pcie_port *port; 1356 1357 list_for_each_entry(port, &pcie->ports, list) 1358 dw_pcie_program_t_power_on(pcie->pci, port->l1ss_t_power_on); 1359 } 1360 1361 static int qcom_pcie_host_init(struct dw_pcie_rp *pp) 1362 { 1363 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 1364 struct qcom_pcie *pcie = to_qcom_pcie(pci); 1365 int ret; 1366 1367 qcom_pcie_perst_assert(pcie); 1368 1369 ret = pcie->cfg->ops->init(pcie); 1370 if (ret) 1371 return ret; 1372 1373 ret = qcom_pcie_phy_power_on(pcie); 1374 if (ret) 1375 goto err_deinit; 1376 1377 if (!pci->suspended) { 1378 ret = pci_pwrctrl_create_devices(pci->dev); 1379 if (ret) 1380 goto err_disable_phy; 1381 } 1382 1383 if (!pp->skip_pwrctrl_off) { 1384 ret = pci_pwrctrl_power_on_devices(pci->dev); 1385 if (ret) 1386 goto err_pwrctrl_destroy; 1387 } 1388 1389 if (pcie->cfg->ops->post_init) { 1390 ret = pcie->cfg->ops->post_init(pcie); 1391 if (ret) 1392 goto err_pwrctrl_power_off; 1393 } 1394 1395 qcom_pcie_clear_aspm_l0s(pcie->pci); 1396 dw_pcie_remove_capability(pcie->pci, PCI_CAP_ID_MSIX); 1397 dw_pcie_remove_ext_capability(pcie->pci, PCI_EXT_CAP_ID_DPC); 1398 1399 qcom_pcie_configure_ports(pcie); 1400 1401 qcom_pcie_perst_deassert(pcie); 1402 1403 if (pcie->cfg->ops->config_sid) { 1404 ret = pcie->cfg->ops->config_sid(pcie); 1405 if (ret) 1406 goto err_assert_reset; 1407 } 1408 1409 return 0; 1410 1411 err_assert_reset: 1412 qcom_pcie_perst_assert(pcie); 1413 err_pwrctrl_power_off: 1414 if (!pp->skip_pwrctrl_off) 1415 pci_pwrctrl_power_off_devices(pci->dev); 1416 err_pwrctrl_destroy: 1417 if (ret != -EPROBE_DEFER && !pci->suspended) 1418 pci_pwrctrl_destroy_devices(pci->dev); 1419 err_disable_phy: 1420 qcom_pcie_phy_power_off(pcie); 1421 err_deinit: 1422 pcie->cfg->ops->deinit(pcie); 1423 1424 return ret; 1425 } 1426 1427 static void qcom_pcie_host_deinit(struct dw_pcie_rp *pp) 1428 { 1429 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 1430 struct qcom_pcie *pcie = to_qcom_pcie(pci); 1431 1432 qcom_pcie_perst_assert(pcie); 1433 1434 if (!pci->pp.skip_pwrctrl_off) { 1435 /* 1436 * No need to destroy pwrctrl devices as this function only 1437 * gets called during system suspend as of now. 1438 */ 1439 pci_pwrctrl_power_off_devices(pci->dev); 1440 } 1441 1442 qcom_pcie_phy_power_off(pcie); 1443 pcie->cfg->ops->deinit(pcie); 1444 } 1445 1446 static void qcom_pcie_host_post_init(struct dw_pcie_rp *pp) 1447 { 1448 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 1449 struct qcom_pcie *pcie = to_qcom_pcie(pci); 1450 1451 /* 1452 * During system suspend, the Qcom RC driver may turn off the 1453 * analog circuitry of PHY and remove controller votes to save 1454 * power. If the link is in L1SS and the endpoint asserts CLKREQ# 1455 * to exit L1SS, the time required to wake the system and restore 1456 * the PHY/REFCLK may exceed the L1SS exit timing (L10_REFCLK_ON + 1457 * T_COMMONMODE), resulting in Link Down (LDn) and a reset of the 1458 * endpoint. Set this flag to indicate this limitation to client 1459 * drivers so that they can avoid relying on device state being 1460 * preserved during system suspend. 1461 */ 1462 pp->bridge->broken_l1ss_resume = true; 1463 1464 if (pcie->cfg->ops->host_post_init) 1465 pcie->cfg->ops->host_post_init(pcie); 1466 } 1467 1468 static void qcom_pcie_host_pme_turn_off(struct dw_pcie_rp *pp) 1469 { 1470 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 1471 1472 writel(ELBI_SYS_CTRL_PME_TURNOFF_MSG, pci->elbi_base + ELBI_SYS_CTRL); 1473 } 1474 1475 static const struct dw_pcie_host_ops qcom_pcie_dw_ops = { 1476 .init = qcom_pcie_host_init, 1477 .deinit = qcom_pcie_host_deinit, 1478 .post_init = qcom_pcie_host_post_init, 1479 .pme_turn_off = qcom_pcie_host_pme_turn_off, 1480 }; 1481 1482 /* Qcom IP rev.: 2.1.0 Synopsys IP rev.: 4.01a */ 1483 static const struct qcom_pcie_ops ops_2_1_0 = { 1484 .get_resources = qcom_pcie_get_resources_2_1_0, 1485 .init = qcom_pcie_init_2_1_0, 1486 .post_init = qcom_pcie_post_init_2_1_0, 1487 .deinit = qcom_pcie_deinit_2_1_0, 1488 .ltssm_enable = qcom_pcie_2_1_0_ltssm_enable, 1489 .get_ltssm = qcom_pcie_2_1_0_get_ltssm, 1490 }; 1491 1492 /* Qcom IP rev.: 1.0.0 Synopsys IP rev.: 4.11a */ 1493 static const struct qcom_pcie_ops ops_1_0_0 = { 1494 .get_resources = qcom_pcie_get_resources_1_0_0, 1495 .init = qcom_pcie_init_1_0_0, 1496 .post_init = qcom_pcie_post_init_1_0_0, 1497 .deinit = qcom_pcie_deinit_1_0_0, 1498 .ltssm_enable = qcom_pcie_2_1_0_ltssm_enable, 1499 .get_ltssm = qcom_pcie_2_1_0_get_ltssm, 1500 }; 1501 1502 /* Qcom IP rev.: 2.3.2 Synopsys IP rev.: 4.21a */ 1503 static const struct qcom_pcie_ops ops_2_3_2 = { 1504 .get_resources = qcom_pcie_get_resources_2_3_2, 1505 .init = qcom_pcie_init_2_3_2, 1506 .post_init = qcom_pcie_post_init_2_3_2, 1507 .deinit = qcom_pcie_deinit_2_3_2, 1508 .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, 1509 }; 1510 1511 /* Qcom IP rev.: 2.4.0 Synopsys IP rev.: 4.20a */ 1512 static const struct qcom_pcie_ops ops_2_4_0 = { 1513 .get_resources = qcom_pcie_get_resources_2_4_0, 1514 .init = qcom_pcie_init_2_4_0, 1515 .post_init = qcom_pcie_post_init_2_3_2, 1516 .deinit = qcom_pcie_deinit_2_4_0, 1517 .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, 1518 }; 1519 1520 /* Qcom IP rev.: 2.3.3 Synopsys IP rev.: 4.30a */ 1521 static const struct qcom_pcie_ops ops_2_3_3 = { 1522 .get_resources = qcom_pcie_get_resources_2_3_3, 1523 .init = qcom_pcie_init_2_3_3, 1524 .post_init = qcom_pcie_post_init_2_3_3, 1525 .deinit = qcom_pcie_deinit_2_3_3, 1526 .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, 1527 }; 1528 1529 /* Qcom IP rev.: 2.7.0 Synopsys IP rev.: 4.30a */ 1530 static const struct qcom_pcie_ops ops_2_7_0 = { 1531 .get_resources = qcom_pcie_get_resources_2_7_0, 1532 .init = qcom_pcie_init_2_7_0, 1533 .post_init = qcom_pcie_post_init_2_7_0, 1534 .deinit = qcom_pcie_deinit_2_7_0, 1535 .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, 1536 }; 1537 1538 /* Qcom IP rev.: 1.9.0 */ 1539 static const struct qcom_pcie_ops ops_1_9_0 = { 1540 .get_resources = qcom_pcie_get_resources_2_7_0, 1541 .init = qcom_pcie_init_2_7_0, 1542 .post_init = qcom_pcie_post_init_2_7_0, 1543 .host_post_init = qcom_pcie_host_post_init_2_7_0, 1544 .deinit = qcom_pcie_deinit_2_7_0, 1545 .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, 1546 .config_sid = qcom_pcie_config_sid_1_9_0, 1547 }; 1548 1549 /* Qcom IP rev.: 1.21.0 Synopsys IP rev.: 5.60a */ 1550 static const struct qcom_pcie_ops ops_1_21_0 = { 1551 .get_resources = qcom_pcie_get_resources_2_7_0, 1552 .init = qcom_pcie_init_2_7_0, 1553 .post_init = qcom_pcie_post_init_2_7_0, 1554 .host_post_init = qcom_pcie_host_post_init_2_7_0, 1555 .deinit = qcom_pcie_deinit_2_7_0, 1556 .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, 1557 }; 1558 1559 /* Qcom IP rev.: 2.9.0 Synopsys IP rev.: 5.00a */ 1560 static const struct qcom_pcie_ops ops_2_9_0 = { 1561 .get_resources = qcom_pcie_get_resources_2_9_0, 1562 .init = qcom_pcie_init_2_9_0, 1563 .post_init = qcom_pcie_post_init_2_9_0, 1564 .deinit = qcom_pcie_deinit_2_9_0, 1565 .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, 1566 }; 1567 1568 static const struct qcom_pcie_cfg cfg_1_0_0 = { 1569 .ops = &ops_1_0_0, 1570 }; 1571 1572 static const struct qcom_pcie_cfg cfg_1_9_0 = { 1573 .ops = &ops_1_9_0, 1574 }; 1575 1576 static const struct qcom_pcie_cfg cfg_1_34_0 = { 1577 .ops = &ops_1_9_0, 1578 .override_no_snoop = true, 1579 .no_l0s = true, 1580 }; 1581 1582 static const struct qcom_pcie_cfg cfg_2_1_0 = { 1583 .ops = &ops_2_1_0, 1584 }; 1585 1586 static const struct qcom_pcie_cfg cfg_2_3_2 = { 1587 .ops = &ops_2_3_2, 1588 .no_l0s = true, 1589 }; 1590 1591 static const struct qcom_pcie_cfg cfg_2_3_3 = { 1592 .ops = &ops_2_3_3, 1593 }; 1594 1595 static const struct qcom_pcie_cfg cfg_2_4_0 = { 1596 .ops = &ops_2_4_0, 1597 }; 1598 1599 static const struct qcom_pcie_cfg cfg_2_7_0 = { 1600 .ops = &ops_2_7_0, 1601 }; 1602 1603 static const struct qcom_pcie_cfg cfg_2_9_0 = { 1604 .ops = &ops_2_9_0, 1605 }; 1606 1607 static const struct qcom_pcie_cfg cfg_sc8280xp = { 1608 .ops = &ops_1_21_0, 1609 .no_l0s = true, 1610 }; 1611 1612 static const struct qcom_pcie_cfg cfg_fw_managed = { 1613 .firmware_managed = true, 1614 }; 1615 1616 static const struct dw_pcie_ops dw_pcie_ops = { 1617 .link_up = qcom_pcie_link_up, 1618 .start_link = qcom_pcie_start_link, 1619 .get_ltssm = qcom_pcie_get_ltssm, 1620 }; 1621 1622 static int qcom_pcie_icc_init(struct qcom_pcie *pcie) 1623 { 1624 struct dw_pcie *pci = pcie->pci; 1625 int ret; 1626 1627 pcie->icc_mem = devm_of_icc_get(pci->dev, "pcie-mem"); 1628 if (IS_ERR(pcie->icc_mem)) 1629 return PTR_ERR(pcie->icc_mem); 1630 1631 pcie->icc_cpu = devm_of_icc_get(pci->dev, "cpu-pcie"); 1632 if (IS_ERR(pcie->icc_cpu)) 1633 return PTR_ERR(pcie->icc_cpu); 1634 /* 1635 * Some Qualcomm platforms require interconnect bandwidth constraints 1636 * to be set before enabling interconnect clocks. 1637 * 1638 * Set an initial peak bandwidth corresponding to single-lane Gen 1 1639 * for the pcie-mem path. 1640 */ 1641 ret = icc_set_bw(pcie->icc_mem, 0, QCOM_PCIE_LINK_SPEED_TO_BW(1)); 1642 if (ret) { 1643 dev_err(pci->dev, "Failed to set bandwidth for PCIe-MEM interconnect path: %d\n", 1644 ret); 1645 return ret; 1646 } 1647 1648 /* 1649 * Since the CPU-PCIe path is only used for activities like register 1650 * access of the host controller and endpoint Config/BAR space access, 1651 * HW team has recommended to use a minimal bandwidth of 1KBps just to 1652 * keep the path active. 1653 */ 1654 ret = icc_set_bw(pcie->icc_cpu, 0, kBps_to_icc(1)); 1655 if (ret) { 1656 dev_err(pci->dev, "Failed to set bandwidth for CPU-PCIe interconnect path: %d\n", 1657 ret); 1658 icc_set_bw(pcie->icc_mem, 0, 0); 1659 return ret; 1660 } 1661 1662 return 0; 1663 } 1664 1665 static void qcom_pcie_icc_opp_update(struct qcom_pcie *pcie) 1666 { 1667 u32 offset, status, width, speed; 1668 struct dw_pcie *pci = pcie->pci; 1669 struct dev_pm_opp_key key = {}; 1670 unsigned long freq_kbps; 1671 struct dev_pm_opp *opp; 1672 int ret, freq_mbps; 1673 1674 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 1675 status = readw(pci->dbi_base + offset + PCI_EXP_LNKSTA); 1676 1677 /* Only update constraints if link is up. */ 1678 if (!(status & PCI_EXP_LNKSTA_DLLLA)) 1679 return; 1680 1681 speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, status); 1682 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, status); 1683 1684 if (pcie->icc_mem) { 1685 ret = icc_set_bw(pcie->icc_mem, 0, 1686 width * QCOM_PCIE_LINK_SPEED_TO_BW(speed)); 1687 if (ret) { 1688 dev_err(pci->dev, "Failed to set bandwidth for PCIe-MEM interconnect path: %d\n", 1689 ret); 1690 } 1691 } else if (pcie->use_pm_opp) { 1692 freq_mbps = pcie_dev_speed_mbps(pcie_get_link_speed(speed)); 1693 if (freq_mbps < 0) 1694 return; 1695 1696 freq_kbps = freq_mbps * KILO; 1697 opp = dev_pm_opp_find_level_exact(pci->dev, speed); 1698 if (IS_ERR(opp)) { 1699 /* opp-level is not defined use only frequency */ 1700 opp = dev_pm_opp_find_freq_exact(pci->dev, freq_kbps * width, 1701 true); 1702 } else { 1703 /* put opp-level OPP */ 1704 dev_pm_opp_put(opp); 1705 1706 key.freq = freq_kbps * width; 1707 key.level = speed; 1708 key.bw = 0; 1709 opp = dev_pm_opp_find_key_exact(pci->dev, &key, true); 1710 } 1711 if (!IS_ERR(opp)) { 1712 ret = dev_pm_opp_set_opp(pci->dev, opp); 1713 if (ret) 1714 dev_err(pci->dev, "Failed to set OPP for freq (%lu): %d\n", 1715 freq_kbps * width, ret); 1716 dev_pm_opp_put(opp); 1717 } 1718 } 1719 } 1720 1721 static int qcom_pcie_set_max_opp(struct device *dev) 1722 { 1723 unsigned long max_freq = ULONG_MAX; 1724 struct dev_pm_opp *opp; 1725 int ret; 1726 1727 opp = dev_pm_opp_find_freq_floor(dev, &max_freq); 1728 if (IS_ERR(opp)) 1729 return PTR_ERR(opp); 1730 1731 ret = dev_pm_opp_set_opp(dev, opp); 1732 dev_pm_opp_put(opp); 1733 1734 return ret; 1735 } 1736 1737 static int qcom_pcie_link_transition_count(struct seq_file *s, void *data) 1738 { 1739 struct qcom_pcie *pcie = (struct qcom_pcie *)dev_get_drvdata(s->private); 1740 1741 seq_printf(s, "L0s transition count: %u\n", 1742 readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L0S)); 1743 1744 seq_printf(s, "L1 transition count: %u\n", 1745 readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L1)); 1746 1747 seq_printf(s, "L1.1 transition count: %u\n", 1748 readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1)); 1749 1750 seq_printf(s, "L1.2 transition count: %u\n", 1751 readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2)); 1752 1753 seq_printf(s, "L2 transition count: %u\n", 1754 readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L2)); 1755 1756 return 0; 1757 } 1758 1759 static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie) 1760 { 1761 struct dw_pcie *pci = pcie->pci; 1762 struct device *dev = pci->dev; 1763 char *name; 1764 1765 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); 1766 if (!name) 1767 return; 1768 1769 pcie->debugfs = debugfs_create_dir(name, NULL); 1770 debugfs_create_devm_seqfile(dev, "link_transition_count", pcie->debugfs, 1771 qcom_pcie_link_transition_count); 1772 } 1773 1774 static void qcom_pci_free_msi(void *ptr) 1775 { 1776 struct dw_pcie_rp *pp = (struct dw_pcie_rp *)ptr; 1777 1778 if (pp && pp->use_imsi_rx) 1779 dw_pcie_free_msi(pp); 1780 } 1781 1782 static int qcom_pcie_ecam_host_init(struct pci_config_window *cfg) 1783 { 1784 struct device *dev = cfg->parent; 1785 struct dw_pcie_rp *pp; 1786 struct dw_pcie *pci; 1787 int ret; 1788 1789 pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); 1790 if (!pci) 1791 return -ENOMEM; 1792 1793 pci->dev = dev; 1794 pp = &pci->pp; 1795 pci->dbi_base = cfg->win; 1796 pp->num_vectors = MSI_DEF_NUM_VECTORS; 1797 1798 /* 1799 * dw_pcie_msi_host_init() is called directly here, bypassing 1800 * dw_pcie_host_init() where pp->lock is normally initialized. 1801 */ 1802 raw_spin_lock_init(&pp->lock); 1803 1804 ret = dw_pcie_msi_host_init(pp); 1805 if (ret) 1806 return ret; 1807 1808 pp->use_imsi_rx = true; 1809 dw_pcie_msi_init(pp); 1810 1811 return devm_add_action_or_reset(dev, qcom_pci_free_msi, pp); 1812 } 1813 1814 static const struct pci_ecam_ops pci_qcom_ecam_ops = { 1815 .init = qcom_pcie_ecam_host_init, 1816 .pci_ops = { 1817 .map_bus = pci_ecam_map_bus, 1818 .read = pci_generic_config_read, 1819 .write = pci_generic_config_write, 1820 } 1821 }; 1822 1823 /* Parse PERST# from all nodes in depth first manner starting from @np */ 1824 static int qcom_pcie_parse_perst(struct qcom_pcie *pcie, 1825 struct qcom_pcie_port *port, 1826 struct device_node *np) 1827 { 1828 struct device *dev = pcie->pci->dev; 1829 struct qcom_pcie_perst *perst; 1830 struct gpio_desc *reset; 1831 int ret; 1832 1833 if (pcie->reset) { 1834 dev_warn_once(dev, 1835 "Reusing PERST# from Root Complex node. DT needs to be fixed!\n"); 1836 reset = pcie->reset; 1837 goto skip_perst_parsing; 1838 } 1839 1840 if (!of_find_property(np, "reset-gpios", NULL)) 1841 goto parse_child_node; 1842 1843 reset = devm_fwnode_gpiod_get(dev, of_fwnode_handle(np), "reset", 1844 GPIOD_OUT_HIGH, "PERST#"); 1845 if (IS_ERR(reset)) { 1846 /* 1847 * FIXME: GPIOLIB currently supports exclusive GPIO access only. 1848 * Non exclusive access is broken. But shared PERST# requires 1849 * non-exclusive access. So once GPIOLIB properly supports it, 1850 * implement it here. 1851 */ 1852 if (PTR_ERR(reset) == -EBUSY) 1853 dev_err(dev, "Shared PERST# is not supported\n"); 1854 1855 return PTR_ERR(reset); 1856 } 1857 1858 skip_perst_parsing: 1859 perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL); 1860 if (!perst) 1861 return -ENOMEM; 1862 1863 INIT_LIST_HEAD(&perst->list); 1864 perst->desc = reset; 1865 list_add_tail(&perst->list, &port->perst); 1866 1867 parse_child_node: 1868 for_each_available_child_of_node_scoped(np, child) { 1869 ret = qcom_pcie_parse_perst(pcie, port, child); 1870 if (ret) 1871 return ret; 1872 } 1873 1874 return 0; 1875 } 1876 1877 static int qcom_pcie_parse_port(struct qcom_pcie *pcie, struct device_node *node) 1878 { 1879 struct device *dev = pcie->pci->dev; 1880 struct qcom_pcie_port *port; 1881 struct phy *phy; 1882 int ret; 1883 1884 phy = devm_of_phy_get(dev, node, NULL); 1885 if (IS_ERR(phy)) 1886 return PTR_ERR(phy); 1887 1888 port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); 1889 if (!port) 1890 return -ENOMEM; 1891 1892 ret = phy_init(phy); 1893 if (ret) 1894 return ret; 1895 1896 INIT_LIST_HEAD(&port->perst); 1897 1898 ret = qcom_pcie_parse_perst(pcie, port, node); 1899 if (ret) 1900 return ret; 1901 1902 /* TODO: Move to DWC core after multi Root Port support is added */ 1903 of_property_read_u32(node, "t-power-on-us", &port->l1ss_t_power_on); 1904 1905 port->phy = phy; 1906 INIT_LIST_HEAD(&port->list); 1907 list_add_tail(&port->list, &pcie->ports); 1908 1909 return 0; 1910 } 1911 1912 static int qcom_pcie_parse_ports(struct qcom_pcie *pcie) 1913 { 1914 struct qcom_pcie_perst *perst, *tmp_perst; 1915 struct qcom_pcie_port *port, *tmp_port; 1916 struct device *dev = pcie->pci->dev; 1917 int ret = -ENODEV; 1918 1919 if (of_find_property(dev->of_node, "perst-gpios", NULL)) { 1920 pcie->reset = devm_gpiod_get_optional(dev, "perst", 1921 GPIOD_OUT_HIGH); 1922 if (IS_ERR(pcie->reset)) 1923 return PTR_ERR(pcie->reset); 1924 } 1925 1926 for_each_available_child_of_node_scoped(dev->of_node, of_port) { 1927 if (!of_node_is_type(of_port, "pci")) 1928 continue; 1929 ret = qcom_pcie_parse_port(pcie, of_port); 1930 if (ret) 1931 goto err_port_del; 1932 } 1933 1934 return ret; 1935 1936 err_port_del: 1937 list_for_each_entry_safe(port, tmp_port, &pcie->ports, list) { 1938 list_for_each_entry_safe(perst, tmp_perst, &port->perst, list) 1939 list_del(&perst->list); 1940 phy_exit(port->phy); 1941 list_del(&port->list); 1942 } 1943 1944 return ret; 1945 } 1946 1947 static int qcom_pcie_parse_legacy_binding(struct qcom_pcie *pcie) 1948 { 1949 struct device *dev = pcie->pci->dev; 1950 struct qcom_pcie_perst *perst; 1951 struct qcom_pcie_port *port; 1952 struct phy *phy; 1953 int ret; 1954 1955 phy = devm_phy_optional_get(dev, "pciephy"); 1956 if (IS_ERR(phy)) 1957 return PTR_ERR(phy); 1958 1959 ret = phy_init(phy); 1960 if (ret) 1961 return ret; 1962 1963 port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); 1964 if (!port) 1965 return -ENOMEM; 1966 1967 perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL); 1968 if (!perst) 1969 return -ENOMEM; 1970 1971 port->phy = phy; 1972 INIT_LIST_HEAD(&port->list); 1973 list_add_tail(&port->list, &pcie->ports); 1974 1975 perst->desc = pcie->reset; 1976 INIT_LIST_HEAD(&port->perst); 1977 INIT_LIST_HEAD(&perst->list); 1978 list_add_tail(&perst->list, &port->perst); 1979 1980 return 0; 1981 } 1982 1983 static int qcom_pcie_probe(struct platform_device *pdev) 1984 { 1985 struct qcom_pcie_perst *perst, *tmp_perst; 1986 struct qcom_pcie_port *port, *tmp_port; 1987 const struct qcom_pcie_cfg *pcie_cfg; 1988 struct device *dev = &pdev->dev; 1989 struct qcom_pcie *pcie; 1990 struct dw_pcie_rp *pp; 1991 struct resource *res; 1992 struct dw_pcie *pci; 1993 int ret; 1994 1995 pcie_cfg = of_device_get_match_data(dev); 1996 if (!pcie_cfg) { 1997 dev_err(dev, "No platform data\n"); 1998 return -ENODATA; 1999 } 2000 2001 if (!pcie_cfg->firmware_managed && !pcie_cfg->ops) { 2002 dev_err(dev, "No platform ops\n"); 2003 return -ENODATA; 2004 } 2005 2006 pm_runtime_enable(dev); 2007 ret = pm_runtime_get_sync(dev); 2008 if (ret < 0) 2009 goto err_pm_runtime_put; 2010 2011 if (pcie_cfg->firmware_managed) { 2012 struct pci_host_bridge *bridge; 2013 struct pci_config_window *cfg; 2014 2015 bridge = devm_pci_alloc_host_bridge(dev, 0); 2016 if (!bridge) { 2017 ret = -ENOMEM; 2018 goto err_pm_runtime_put; 2019 } 2020 2021 /* Parse and map our ECAM configuration space area */ 2022 cfg = pci_host_common_ecam_create(dev, bridge, 2023 &pci_qcom_ecam_ops); 2024 if (IS_ERR(cfg)) { 2025 ret = PTR_ERR(cfg); 2026 goto err_pm_runtime_put; 2027 } 2028 2029 bridge->sysdata = cfg; 2030 bridge->ops = (struct pci_ops *)&pci_qcom_ecam_ops.pci_ops; 2031 bridge->msi_domain = true; 2032 2033 ret = pci_host_probe(bridge); 2034 if (ret) 2035 goto err_pm_runtime_put; 2036 2037 return 0; 2038 } 2039 2040 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); 2041 if (!pcie) { 2042 ret = -ENOMEM; 2043 goto err_pm_runtime_put; 2044 } 2045 2046 pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); 2047 if (!pci) { 2048 ret = -ENOMEM; 2049 goto err_pm_runtime_put; 2050 } 2051 2052 INIT_LIST_HEAD(&pcie->ports); 2053 2054 pci->dev = dev; 2055 pci->ops = &dw_pcie_ops; 2056 pp = &pci->pp; 2057 2058 pcie->pci = pci; 2059 2060 pcie->cfg = pcie_cfg; 2061 2062 pcie->parf = devm_platform_ioremap_resource_byname(pdev, "parf"); 2063 if (IS_ERR(pcie->parf)) { 2064 ret = PTR_ERR(pcie->parf); 2065 goto err_pm_runtime_put; 2066 } 2067 2068 /* MHI region is optional */ 2069 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mhi"); 2070 if (res) { 2071 pcie->mhi = devm_ioremap_resource(dev, res); 2072 if (IS_ERR(pcie->mhi)) { 2073 ret = PTR_ERR(pcie->mhi); 2074 goto err_pm_runtime_put; 2075 } 2076 } 2077 2078 /* OPP table is optional */ 2079 ret = devm_pm_opp_of_add_table(dev); 2080 if (ret && ret != -ENODEV) { 2081 dev_err_probe(dev, ret, "Failed to add OPP table\n"); 2082 goto err_pm_runtime_put; 2083 } 2084 2085 /* 2086 * Before the PCIe link is initialized, vote for highest OPP in the OPP 2087 * table, so that we are voting for maximum voltage corner for the 2088 * link to come up in maximum supported speed. At the end of the 2089 * probe(), OPP will be updated using qcom_pcie_icc_opp_update(). 2090 */ 2091 if (!ret) { 2092 ret = qcom_pcie_set_max_opp(dev); 2093 if (ret) { 2094 dev_err_probe(dev, ret, "Failed to set max OPP\n"); 2095 goto err_pm_runtime_put; 2096 } 2097 2098 pcie->use_pm_opp = true; 2099 } else { 2100 /* Skip ICC init if OPP is supported as it is handled by OPP */ 2101 ret = qcom_pcie_icc_init(pcie); 2102 if (ret) 2103 goto err_pm_runtime_put; 2104 } 2105 2106 ret = pcie->cfg->ops->get_resources(pcie); 2107 if (ret) 2108 goto err_pm_runtime_put; 2109 2110 pp->ops = &qcom_pcie_dw_ops; 2111 2112 ret = qcom_pcie_parse_ports(pcie); 2113 if (ret) { 2114 if (ret != -ENODEV) { 2115 dev_err_probe(pci->dev, ret, 2116 "Failed to parse Root Port: %d\n", ret); 2117 goto err_pm_runtime_put; 2118 } 2119 2120 /* 2121 * In the case of properties not populated in Root Port node, 2122 * fallback to the legacy method of parsing the Host Bridge 2123 * node. This is to maintain DT backwards compatibility. 2124 */ 2125 ret = qcom_pcie_parse_legacy_binding(pcie); 2126 if (ret) 2127 goto err_pm_runtime_put; 2128 } 2129 2130 platform_set_drvdata(pdev, pcie); 2131 2132 ret = dw_pcie_host_init(pp); 2133 if (ret) { 2134 dev_err_probe(dev, ret, "cannot initialize host\n"); 2135 goto err_phy_exit; 2136 } 2137 2138 qcom_pcie_icc_opp_update(pcie); 2139 2140 if (pcie->mhi) 2141 qcom_pcie_init_debugfs(pcie); 2142 2143 return 0; 2144 2145 err_phy_exit: 2146 list_for_each_entry_safe(port, tmp_port, &pcie->ports, list) { 2147 list_for_each_entry_safe(perst, tmp_perst, &port->perst, list) 2148 list_del(&perst->list); 2149 phy_exit(port->phy); 2150 list_del(&port->list); 2151 } 2152 err_pm_runtime_put: 2153 pm_runtime_put(dev); 2154 pm_runtime_disable(dev); 2155 2156 return ret; 2157 } 2158 2159 static int qcom_pcie_suspend_noirq(struct device *dev) 2160 { 2161 struct qcom_pcie *pcie; 2162 int ret = 0; 2163 2164 pcie = dev_get_drvdata(dev); 2165 if (!pcie) 2166 return 0; 2167 2168 ret = dw_pcie_suspend_noirq(pcie->pci); 2169 if (ret) 2170 return ret; 2171 2172 if (pcie->pci->suspended) { 2173 ret = icc_disable(pcie->icc_mem); 2174 if (ret) 2175 dev_err(dev, "Failed to disable PCIe-MEM interconnect path: %d\n", ret); 2176 2177 ret = icc_disable(pcie->icc_cpu); 2178 if (ret) 2179 dev_err(dev, "Failed to disable CPU-PCIe interconnect path: %d\n", ret); 2180 2181 if (pcie->use_pm_opp) 2182 dev_pm_opp_set_opp(pcie->pci->dev, NULL); 2183 } else { 2184 /* 2185 * Set minimum bandwidth required to keep data path 2186 * functional during suspend. 2187 */ 2188 if (pcie->icc_mem) { 2189 ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1)); 2190 if (ret) { 2191 dev_err(dev, 2192 "Failed to set bandwidth for PCIe-MEM interconnect path: %d\n", 2193 ret); 2194 return ret; 2195 } 2196 } 2197 2198 /* 2199 * Only disable CPU-PCIe interconnect path if the suspend 2200 * is non-S2RAM. On some platforms, DBI access can happen 2201 * very late during S2RAM and a non-active CPU-PCIe 2202 * interconnect path may lead to NoC error. 2203 */ 2204 if (pm_suspend_target_state != PM_SUSPEND_MEM) { 2205 ret = icc_disable(pcie->icc_cpu); 2206 if (ret) 2207 dev_err(dev, "Failed to disable CPU-PCIe interconnect path: %d\n", 2208 ret); 2209 2210 if (pcie->use_pm_opp) 2211 dev_pm_opp_set_opp(pcie->pci->dev, NULL); 2212 } 2213 } 2214 return ret; 2215 } 2216 2217 static int qcom_pcie_resume_noirq(struct device *dev) 2218 { 2219 struct qcom_pcie *pcie; 2220 int ret; 2221 2222 pcie = dev_get_drvdata(dev); 2223 if (!pcie) 2224 return 0; 2225 2226 if (pcie->pci->suspended) { 2227 if (pcie->use_pm_opp) { 2228 ret = qcom_pcie_set_max_opp(dev); 2229 if (ret) { 2230 dev_err(dev, "Failed to set max OPP: %d\n", ret); 2231 return ret; 2232 } 2233 } 2234 2235 ret = icc_enable(pcie->icc_cpu); 2236 if (ret) { 2237 dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n", ret); 2238 return ret; 2239 } 2240 2241 ret = icc_enable(pcie->icc_mem); 2242 if (ret) { 2243 dev_err(dev, "Failed to enable PCIe-MEM interconnect path: %d\n", ret); 2244 goto disable_icc_cpu; 2245 } 2246 2247 /* 2248 * Ignore -ENODEV & -EIO here since it is expected when no 2249 * endpoint is connected to the PCIe link. 2250 */ 2251 ret = dw_pcie_resume_noirq(pcie->pci); 2252 if (ret && ret != -ENODEV && ret != -EIO) 2253 goto disable_icc_mem; 2254 } else { 2255 if (pm_suspend_target_state != PM_SUSPEND_MEM) { 2256 if (pcie->use_pm_opp) { 2257 ret = qcom_pcie_set_max_opp(dev); 2258 if (ret) { 2259 dev_err(dev, "Failed to set max OPP: %d\n", ret); 2260 return ret; 2261 } 2262 } 2263 2264 ret = icc_enable(pcie->icc_cpu); 2265 if (ret) { 2266 dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n", 2267 ret); 2268 return ret; 2269 } 2270 } 2271 } 2272 2273 qcom_pcie_icc_opp_update(pcie); 2274 2275 return 0; 2276 disable_icc_mem: 2277 icc_disable(pcie->icc_mem); 2278 disable_icc_cpu: 2279 icc_disable(pcie->icc_cpu); 2280 2281 return ret; 2282 } 2283 2284 static const struct of_device_id qcom_pcie_match[] = { 2285 { .compatible = "qcom,pcie-apq8064", .data = &cfg_2_1_0 }, 2286 { .compatible = "qcom,pcie-apq8084", .data = &cfg_1_0_0 }, 2287 { .compatible = "qcom,pcie-ipq4019", .data = &cfg_2_4_0 }, 2288 { .compatible = "qcom,pcie-ipq5018", .data = &cfg_2_9_0 }, 2289 { .compatible = "qcom,pcie-ipq6018", .data = &cfg_2_9_0 }, 2290 { .compatible = "qcom,pcie-ipq8064", .data = &cfg_2_1_0 }, 2291 { .compatible = "qcom,pcie-ipq8064-v2", .data = &cfg_2_1_0 }, 2292 { .compatible = "qcom,pcie-ipq8074", .data = &cfg_2_3_3 }, 2293 { .compatible = "qcom,pcie-ipq8074-gen3", .data = &cfg_2_9_0 }, 2294 { .compatible = "qcom,pcie-ipq9574", .data = &cfg_2_9_0 }, 2295 { .compatible = "qcom,pcie-msm8996", .data = &cfg_2_3_2 }, 2296 { .compatible = "qcom,pcie-qcs404", .data = &cfg_2_4_0 }, 2297 { .compatible = "qcom,pcie-sa8255p", .data = &cfg_fw_managed }, 2298 { .compatible = "qcom,pcie-sa8540p", .data = &cfg_sc8280xp }, 2299 { .compatible = "qcom,pcie-sa8775p", .data = &cfg_1_34_0}, 2300 { .compatible = "qcom,pcie-sc7280", .data = &cfg_1_9_0 }, 2301 { .compatible = "qcom,pcie-sc8180x", .data = &cfg_1_9_0 }, 2302 { .compatible = "qcom,pcie-sc8280xp", .data = &cfg_sc8280xp }, 2303 { .compatible = "qcom,pcie-sdm845", .data = &cfg_2_7_0 }, 2304 { .compatible = "qcom,pcie-sdx55", .data = &cfg_1_9_0 }, 2305 { .compatible = "qcom,pcie-sm8150", .data = &cfg_1_9_0 }, 2306 { .compatible = "qcom,pcie-sm8250", .data = &cfg_1_9_0 }, 2307 { .compatible = "qcom,pcie-sm8350", .data = &cfg_1_9_0 }, 2308 { .compatible = "qcom,pcie-sm8450-pcie0", .data = &cfg_1_9_0 }, 2309 { .compatible = "qcom,pcie-sm8450-pcie1", .data = &cfg_1_9_0 }, 2310 { .compatible = "qcom,pcie-sm8550", .data = &cfg_1_9_0 }, 2311 { .compatible = "qcom,pcie-x1e80100", .data = &cfg_sc8280xp }, 2312 { } 2313 }; 2314 2315 static void qcom_fixup_class(struct pci_dev *dev) 2316 { 2317 dev->class = PCI_CLASS_BRIDGE_PCI_NORMAL; 2318 } 2319 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0101, qcom_fixup_class); 2320 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0104, qcom_fixup_class); 2321 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0106, qcom_fixup_class); 2322 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0107, qcom_fixup_class); 2323 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0302, qcom_fixup_class); 2324 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1000, qcom_fixup_class); 2325 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class); 2326 2327 static const struct dev_pm_ops qcom_pcie_pm_ops = { 2328 NOIRQ_SYSTEM_SLEEP_PM_OPS(qcom_pcie_suspend_noirq, qcom_pcie_resume_noirq) 2329 }; 2330 2331 static struct platform_driver qcom_pcie_driver = { 2332 .probe = qcom_pcie_probe, 2333 .driver = { 2334 .name = "qcom-pcie", 2335 .suppress_bind_attrs = true, 2336 .of_match_table = qcom_pcie_match, 2337 .pm = &qcom_pcie_pm_ops, 2338 .probe_type = PROBE_PREFER_ASYNCHRONOUS, 2339 }, 2340 }; 2341 builtin_platform_driver(qcom_pcie_driver); 2342