1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCIe host controller driver for Rockchip SoCs. 4 * 5 * Copyright (C) 2021 Rockchip Electronics Co., Ltd. 6 * http://www.rock-chips.com 7 * 8 * Author: Simon Xue <xxm@rock-chips.com> 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/clk.h> 13 #include <linux/gpio/consumer.h> 14 #include <linux/hw_bitfield.h> 15 #include <linux/irqchip/chained_irq.h> 16 #include <linux/irqdomain.h> 17 #include <linux/mfd/syscon.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_irq.h> 21 #include <linux/phy/phy.h> 22 #include <linux/platform_device.h> 23 #include <linux/regmap.h> 24 #include <linux/reset.h> 25 26 #include "../../pci.h" 27 #include "pcie-designware.h" 28 29 /* 30 * The upper 16 bits of PCIE_CLIENT_CONFIG are a write 31 * mask for the lower 16 bits. 32 */ 33 34 #define to_rockchip_pcie(x) dev_get_drvdata((x)->dev) 35 36 /* General Control Register */ 37 #define PCIE_CLIENT_GENERAL_CON 0x0 38 #define PCIE_CLIENT_MODE_MASK GENMASK(7, 4) 39 #define PCIE_CLIENT_MODE_EP 0x0UL 40 #define PCIE_CLIENT_MODE_RC 0x4UL 41 #define PCIE_CLIENT_SET_MODE(x) FIELD_PREP_WM16(PCIE_CLIENT_MODE_MASK, (x)) 42 #define PCIE_CLIENT_LD_RQ_RST_GRT FIELD_PREP_WM16(BIT(3), 1) 43 #define PCIE_CLIENT_ENABLE_LTSSM FIELD_PREP_WM16(BIT(2), 1) 44 #define PCIE_CLIENT_DISABLE_LTSSM FIELD_PREP_WM16(BIT(2), 0) 45 46 /* Interrupt Status Register Related to Legacy Interrupt */ 47 #define PCIE_CLIENT_INTR_STATUS_LEGACY 0x8 48 49 /* Interrupt Status Register Related to Miscellaneous Operation */ 50 #define PCIE_CLIENT_INTR_STATUS_MISC 0x10 51 #define PCIE_RDLH_LINK_UP_CHGED BIT(1) 52 #define PCIE_LINK_REQ_RST_NOT_INT BIT(2) 53 54 /* Interrupt Mask Register Related to Legacy Interrupt */ 55 #define PCIE_CLIENT_INTR_MASK_LEGACY 0x1c 56 #define PCIE_INTR_MASK GENMASK(7, 0) 57 #define PCIE_INTR_CLAMP(_x) ((BIT((_x)) & PCIE_INTR_MASK)) 58 #define PCIE_INTR_LEGACY_MASK(x) (PCIE_INTR_CLAMP((x)) | \ 59 (PCIE_INTR_CLAMP((x)) << 16)) 60 #define PCIE_INTR_LEGACY_UNMASK(x) (PCIE_INTR_CLAMP((x)) << 16) 61 62 /* Interrupt Mask Register Related to Miscellaneous Operation */ 63 #define PCIE_CLIENT_INTR_MASK_MISC 0x24 64 65 /* Power Management Control Register */ 66 #define PCIE_CLIENT_POWER_CON 0x2c 67 #define PCIE_CLKREQ_READY FIELD_PREP_WM16(BIT(0), 1) 68 #define PCIE_CLKREQ_NOT_READY FIELD_PREP_WM16(BIT(0), 0) 69 #define PCIE_CLKREQ_PULL_DOWN FIELD_PREP_WM16(GENMASK(13, 12), 1) 70 71 /* Hot Reset Control Register */ 72 #define PCIE_CLIENT_HOT_RESET_CTRL 0x180 73 #define PCIE_LTSSM_APP_DLY2_EN BIT(1) 74 #define PCIE_LTSSM_APP_DLY2_DONE BIT(3) 75 #define PCIE_LTSSM_ENABLE_ENHANCE BIT(4) 76 77 /* LTSSM Status Register */ 78 #define PCIE_CLIENT_LTSSM_STATUS 0x300 79 #define PCIE_LINKUP 0x3 80 #define PCIE_LINKUP_MASK GENMASK(17, 16) 81 #define PCIE_LTSSM_STATUS_MASK GENMASK(5, 0) 82 83 struct rockchip_pcie { 84 struct dw_pcie pci; 85 void __iomem *apb_base; 86 struct phy *phy; 87 struct clk_bulk_data *clks; 88 unsigned int clk_cnt; 89 struct reset_control *rst; 90 struct gpio_desc *rst_gpio; 91 struct irq_domain *irq_domain; 92 const struct rockchip_pcie_of_data *data; 93 bool supports_clkreq; 94 }; 95 96 struct rockchip_pcie_of_data { 97 enum dw_pcie_device_mode mode; 98 const struct pci_epc_features *epc_features; 99 }; 100 101 static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, u32 reg) 102 { 103 return readl_relaxed(rockchip->apb_base + reg); 104 } 105 106 static void rockchip_pcie_writel_apb(struct rockchip_pcie *rockchip, u32 val, 107 u32 reg) 108 { 109 writel_relaxed(val, rockchip->apb_base + reg); 110 } 111 112 static void rockchip_pcie_intx_handler(struct irq_desc *desc) 113 { 114 struct irq_chip *chip = irq_desc_get_chip(desc); 115 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 116 unsigned long reg, hwirq; 117 118 chained_irq_enter(chip, desc); 119 120 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_LEGACY); 121 122 for_each_set_bit(hwirq, ®, 4) 123 generic_handle_domain_irq(rockchip->irq_domain, hwirq); 124 125 chained_irq_exit(chip, desc); 126 } 127 128 static void rockchip_intx_mask(struct irq_data *data) 129 { 130 rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data), 131 PCIE_INTR_LEGACY_MASK(data->hwirq), 132 PCIE_CLIENT_INTR_MASK_LEGACY); 133 }; 134 135 static void rockchip_intx_unmask(struct irq_data *data) 136 { 137 rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data), 138 PCIE_INTR_LEGACY_UNMASK(data->hwirq), 139 PCIE_CLIENT_INTR_MASK_LEGACY); 140 }; 141 142 static struct irq_chip rockchip_intx_irq_chip = { 143 .name = "INTx", 144 .irq_mask = rockchip_intx_mask, 145 .irq_unmask = rockchip_intx_unmask, 146 .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, 147 }; 148 149 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 150 irq_hw_number_t hwirq) 151 { 152 irq_set_chip_and_handler(irq, &rockchip_intx_irq_chip, handle_level_irq); 153 irq_set_chip_data(irq, domain->host_data); 154 155 return 0; 156 } 157 158 static const struct irq_domain_ops intx_domain_ops = { 159 .map = rockchip_pcie_intx_map, 160 }; 161 162 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 163 { 164 struct device *dev = rockchip->pci.dev; 165 struct device_node *intc; 166 167 intc = of_get_child_by_name(dev->of_node, "legacy-interrupt-controller"); 168 if (!intc) { 169 dev_err(dev, "missing child interrupt-controller node\n"); 170 return -EINVAL; 171 } 172 173 rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX, 174 &intx_domain_ops, rockchip); 175 of_node_put(intc); 176 if (!rockchip->irq_domain) { 177 dev_err(dev, "failed to get a INTx IRQ domain\n"); 178 return -EINVAL; 179 } 180 181 return 0; 182 } 183 184 static u32 rockchip_pcie_get_ltssm(struct rockchip_pcie *rockchip) 185 { 186 return rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS); 187 } 188 189 static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip) 190 { 191 rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM, 192 PCIE_CLIENT_GENERAL_CON); 193 } 194 195 static void rockchip_pcie_disable_ltssm(struct rockchip_pcie *rockchip) 196 { 197 rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_DISABLE_LTSSM, 198 PCIE_CLIENT_GENERAL_CON); 199 } 200 201 static bool rockchip_pcie_link_up(struct dw_pcie *pci) 202 { 203 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); 204 u32 val = rockchip_pcie_get_ltssm(rockchip); 205 206 return FIELD_GET(PCIE_LINKUP_MASK, val) == PCIE_LINKUP; 207 } 208 209 /* 210 * See e.g. section '11.6.6.4 L1 Substate' in the RK3588 TRM V1.0 for the steps 211 * needed to support L1 substates. Currently, just enable L1 substates for RC 212 * mode if CLKREQ# is properly connected and supports-clkreq is present in DT. 213 * For EP mode, there are more things should be done to actually save power in 214 * L1 substates, so disable L1 substates until there is proper support. 215 */ 216 static void rockchip_pcie_configure_l1ss(struct dw_pcie *pci) 217 { 218 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); 219 220 /* Enable L1 substates if CLKREQ# is properly connected */ 221 if (rockchip->supports_clkreq) { 222 rockchip_pcie_writel_apb(rockchip, PCIE_CLKREQ_READY, 223 PCIE_CLIENT_POWER_CON); 224 pci->l1ss_support = true; 225 return; 226 } 227 228 /* 229 * Otherwise, assert CLKREQ# unconditionally. Since 230 * pci->l1ss_support is not set, the DWC core will prevent L1 231 * Substates support from being advertised. 232 */ 233 rockchip_pcie_writel_apb(rockchip, 234 PCIE_CLKREQ_PULL_DOWN | PCIE_CLKREQ_NOT_READY, 235 PCIE_CLIENT_POWER_CON); 236 } 237 238 static void rockchip_pcie_enable_l0s(struct dw_pcie *pci) 239 { 240 u32 cap, lnkcap; 241 242 /* Enable L0S capability for all SoCs */ 243 cap = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 244 if (cap) { 245 lnkcap = dw_pcie_readl_dbi(pci, cap + PCI_EXP_LNKCAP); 246 lnkcap |= PCI_EXP_LNKCAP_ASPM_L0S; 247 dw_pcie_dbi_ro_wr_en(pci); 248 dw_pcie_writel_dbi(pci, cap + PCI_EXP_LNKCAP, lnkcap); 249 dw_pcie_dbi_ro_wr_dis(pci); 250 } 251 } 252 253 static int rockchip_pcie_start_link(struct dw_pcie *pci) 254 { 255 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); 256 257 /* Reset device */ 258 gpiod_set_value_cansleep(rockchip->rst_gpio, 0); 259 260 rockchip_pcie_enable_ltssm(rockchip); 261 262 /* 263 * PCIe requires the refclk to be stable for 100µs prior to releasing 264 * PERST. See table 2-4 in section 2.6.2 AC Specifications of the PCI 265 * Express Card Electromechanical Specification, 1.1. However, we don't 266 * know if the refclk is coming from RC's PHY or external OSC. If it's 267 * from RC, so enabling LTSSM is the just right place to release #PERST. 268 * We need more extra time as before, rather than setting just 269 * 100us as we don't know how long should the device need to reset. 270 */ 271 msleep(PCIE_T_PVPERL_MS); 272 gpiod_set_value_cansleep(rockchip->rst_gpio, 1); 273 274 return 0; 275 } 276 277 static void rockchip_pcie_stop_link(struct dw_pcie *pci) 278 { 279 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); 280 281 rockchip_pcie_disable_ltssm(rockchip); 282 } 283 284 static int rockchip_pcie_host_init(struct dw_pcie_rp *pp) 285 { 286 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 287 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); 288 struct device *dev = rockchip->pci.dev; 289 int irq, ret; 290 291 irq = of_irq_get_byname(dev->of_node, "legacy"); 292 if (irq < 0) 293 return irq; 294 295 ret = rockchip_pcie_init_irq_domain(rockchip); 296 if (ret < 0) 297 dev_err(dev, "failed to init irq domain\n"); 298 299 irq_set_chained_handler_and_data(irq, rockchip_pcie_intx_handler, 300 rockchip); 301 302 rockchip_pcie_configure_l1ss(pci); 303 rockchip_pcie_enable_l0s(pci); 304 305 return 0; 306 } 307 308 static const struct dw_pcie_host_ops rockchip_pcie_host_ops = { 309 .init = rockchip_pcie_host_init, 310 }; 311 312 /* 313 * ATS does not work on RK3588 when running in EP mode. 314 * 315 * After the host has enabled ATS on the EP side, it will send an IOTLB 316 * invalidation request to the EP side. However, the RK3588 will never send 317 * a completion back and eventually the host will print an IOTLB_INV_TIMEOUT 318 * error, and the EP will not be operational. If we hide the ATS capability, 319 * things work as expected. 320 */ 321 static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep) 322 { 323 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 324 struct device *dev = pci->dev; 325 326 /* Only hide the ATS capability for RK3588 running in EP mode. */ 327 if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep")) 328 return; 329 330 if (dw_pcie_ep_hide_ext_capability(pci, PCI_EXT_CAP_ID_SECPCI, 331 PCI_EXT_CAP_ID_ATS)) 332 dev_err(dev, "failed to hide ATS capability\n"); 333 } 334 335 static void rockchip_pcie_ep_init(struct dw_pcie_ep *ep) 336 { 337 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 338 enum pci_barno bar; 339 340 rockchip_pcie_enable_l0s(pci); 341 rockchip_pcie_ep_hide_broken_ats_cap_rk3588(ep); 342 343 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) 344 dw_pcie_ep_reset_bar(pci, bar); 345 }; 346 347 static int rockchip_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, 348 unsigned int type, u16 interrupt_num) 349 { 350 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 351 352 switch (type) { 353 case PCI_IRQ_INTX: 354 return dw_pcie_ep_raise_intx_irq(ep, func_no); 355 case PCI_IRQ_MSI: 356 return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); 357 case PCI_IRQ_MSIX: 358 return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); 359 default: 360 dev_err(pci->dev, "UNKNOWN IRQ type\n"); 361 } 362 363 return 0; 364 } 365 366 static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = { 367 .linkup_notifier = true, 368 .msi_capable = true, 369 .msix_capable = true, 370 .align = SZ_64K, 371 .bar[BAR_0] = { .type = BAR_RESIZABLE, }, 372 .bar[BAR_1] = { .type = BAR_RESIZABLE, }, 373 .bar[BAR_2] = { .type = BAR_RESIZABLE, }, 374 .bar[BAR_3] = { .type = BAR_RESIZABLE, }, 375 .bar[BAR_4] = { .type = BAR_RESIZABLE, }, 376 .bar[BAR_5] = { .type = BAR_RESIZABLE, }, 377 }; 378 379 /* 380 * BAR4 on rk3588 exposes the ATU Port Logic Structure to the host regardless of 381 * iATU settings for BAR4. This means that BAR4 cannot be used by an EPF driver, 382 * so mark it as RESERVED. (rockchip_pcie_ep_init() will disable all BARs by 383 * default.) If the host could write to BAR4, the iATU settings (for all other 384 * BARs) would be overwritten, resulting in (all other BARs) no longer working. 385 */ 386 static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = { 387 .linkup_notifier = true, 388 .msi_capable = true, 389 .msix_capable = true, 390 .align = SZ_64K, 391 .bar[BAR_0] = { .type = BAR_RESIZABLE, }, 392 .bar[BAR_1] = { .type = BAR_RESIZABLE, }, 393 .bar[BAR_2] = { .type = BAR_RESIZABLE, }, 394 .bar[BAR_3] = { .type = BAR_RESIZABLE, }, 395 .bar[BAR_4] = { .type = BAR_RESERVED, }, 396 .bar[BAR_5] = { .type = BAR_RESIZABLE, }, 397 }; 398 399 static const struct pci_epc_features * 400 rockchip_pcie_get_features(struct dw_pcie_ep *ep) 401 { 402 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 403 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); 404 405 return rockchip->data->epc_features; 406 } 407 408 static const struct dw_pcie_ep_ops rockchip_pcie_ep_ops = { 409 .init = rockchip_pcie_ep_init, 410 .raise_irq = rockchip_pcie_raise_irq, 411 .get_features = rockchip_pcie_get_features, 412 }; 413 414 static int rockchip_pcie_clk_init(struct rockchip_pcie *rockchip) 415 { 416 struct device *dev = rockchip->pci.dev; 417 int ret; 418 419 ret = devm_clk_bulk_get_all(dev, &rockchip->clks); 420 if (ret < 0) 421 return dev_err_probe(dev, ret, "failed to get clocks\n"); 422 423 rockchip->clk_cnt = ret; 424 425 ret = clk_bulk_prepare_enable(rockchip->clk_cnt, rockchip->clks); 426 if (ret) 427 return dev_err_probe(dev, ret, "failed to enable clocks\n"); 428 429 return 0; 430 } 431 432 static int rockchip_pcie_resource_get(struct platform_device *pdev, 433 struct rockchip_pcie *rockchip) 434 { 435 rockchip->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb"); 436 if (IS_ERR(rockchip->apb_base)) 437 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->apb_base), 438 "failed to map apb registers\n"); 439 440 rockchip->rst_gpio = devm_gpiod_get_optional(&pdev->dev, "reset", 441 GPIOD_OUT_LOW); 442 if (IS_ERR(rockchip->rst_gpio)) 443 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst_gpio), 444 "failed to get reset gpio\n"); 445 446 rockchip->rst = devm_reset_control_array_get_exclusive(&pdev->dev); 447 if (IS_ERR(rockchip->rst)) 448 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst), 449 "failed to get reset lines\n"); 450 451 rockchip->supports_clkreq = of_property_read_bool(pdev->dev.of_node, 452 "supports-clkreq"); 453 454 return 0; 455 } 456 457 static int rockchip_pcie_phy_init(struct rockchip_pcie *rockchip) 458 { 459 struct device *dev = rockchip->pci.dev; 460 int ret; 461 462 rockchip->phy = devm_phy_get(dev, "pcie-phy"); 463 if (IS_ERR(rockchip->phy)) 464 return dev_err_probe(dev, PTR_ERR(rockchip->phy), 465 "missing PHY\n"); 466 467 ret = phy_init(rockchip->phy); 468 if (ret < 0) 469 return ret; 470 471 ret = phy_power_on(rockchip->phy); 472 if (ret) 473 phy_exit(rockchip->phy); 474 475 return ret; 476 } 477 478 static void rockchip_pcie_phy_deinit(struct rockchip_pcie *rockchip) 479 { 480 phy_power_off(rockchip->phy); 481 phy_exit(rockchip->phy); 482 } 483 484 static const struct dw_pcie_ops dw_pcie_ops = { 485 .link_up = rockchip_pcie_link_up, 486 .start_link = rockchip_pcie_start_link, 487 .stop_link = rockchip_pcie_stop_link, 488 }; 489 490 static irqreturn_t rockchip_pcie_rc_sys_irq_thread(int irq, void *arg) 491 { 492 struct rockchip_pcie *rockchip = arg; 493 struct dw_pcie *pci = &rockchip->pci; 494 struct dw_pcie_rp *pp = &pci->pp; 495 struct device *dev = pci->dev; 496 u32 reg; 497 498 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC); 499 rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); 500 501 dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); 502 dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip)); 503 504 if (reg & PCIE_RDLH_LINK_UP_CHGED) { 505 if (rockchip_pcie_link_up(pci)) { 506 msleep(PCIE_RESET_CONFIG_WAIT_MS); 507 dev_dbg(dev, "Received Link up event. Starting enumeration!\n"); 508 /* Rescan the bus to enumerate endpoint devices */ 509 pci_lock_rescan_remove(); 510 pci_rescan_bus(pp->bridge->bus); 511 pci_unlock_rescan_remove(); 512 } 513 } 514 515 return IRQ_HANDLED; 516 } 517 518 static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) 519 { 520 struct rockchip_pcie *rockchip = arg; 521 struct dw_pcie *pci = &rockchip->pci; 522 struct device *dev = pci->dev; 523 u32 reg, val; 524 525 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC); 526 rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); 527 528 dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); 529 dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip)); 530 531 if (reg & PCIE_LINK_REQ_RST_NOT_INT) { 532 dev_dbg(dev, "hot reset or link-down reset\n"); 533 dw_pcie_ep_linkdown(&pci->ep); 534 /* Stop delaying link training. */ 535 val = FIELD_PREP_WM16(PCIE_LTSSM_APP_DLY2_DONE, 1); 536 rockchip_pcie_writel_apb(rockchip, val, 537 PCIE_CLIENT_HOT_RESET_CTRL); 538 } 539 540 if (reg & PCIE_RDLH_LINK_UP_CHGED) { 541 if (rockchip_pcie_link_up(pci)) { 542 dev_dbg(dev, "link up\n"); 543 dw_pcie_ep_linkup(&pci->ep); 544 } 545 } 546 547 return IRQ_HANDLED; 548 } 549 550 static int rockchip_pcie_configure_rc(struct platform_device *pdev, 551 struct rockchip_pcie *rockchip) 552 { 553 struct device *dev = &pdev->dev; 554 struct dw_pcie_rp *pp; 555 int irq, ret; 556 u32 val; 557 558 if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_HOST)) 559 return -ENODEV; 560 561 irq = platform_get_irq_byname(pdev, "sys"); 562 if (irq < 0) 563 return irq; 564 565 ret = devm_request_threaded_irq(dev, irq, NULL, 566 rockchip_pcie_rc_sys_irq_thread, 567 IRQF_ONESHOT, "pcie-sys-rc", rockchip); 568 if (ret) { 569 dev_err(dev, "failed to request PCIe sys IRQ\n"); 570 return ret; 571 } 572 573 /* LTSSM enable control mode */ 574 val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1); 575 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); 576 577 rockchip_pcie_writel_apb(rockchip, 578 PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_RC), 579 PCIE_CLIENT_GENERAL_CON); 580 581 pp = &rockchip->pci.pp; 582 pp->ops = &rockchip_pcie_host_ops; 583 pp->use_linkup_irq = true; 584 585 ret = dw_pcie_host_init(pp); 586 if (ret) { 587 dev_err(dev, "failed to initialize host\n"); 588 return ret; 589 } 590 591 /* unmask DLL up/down indicator */ 592 val = FIELD_PREP_WM16(PCIE_RDLH_LINK_UP_CHGED, 0); 593 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC); 594 595 return ret; 596 } 597 598 static int rockchip_pcie_configure_ep(struct platform_device *pdev, 599 struct rockchip_pcie *rockchip) 600 { 601 struct device *dev = &pdev->dev; 602 int irq, ret; 603 u32 val; 604 605 if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_EP)) 606 return -ENODEV; 607 608 irq = platform_get_irq_byname(pdev, "sys"); 609 if (irq < 0) 610 return irq; 611 612 ret = devm_request_threaded_irq(dev, irq, NULL, 613 rockchip_pcie_ep_sys_irq_thread, 614 IRQF_ONESHOT, "pcie-sys-ep", rockchip); 615 if (ret) { 616 dev_err(dev, "failed to request PCIe sys IRQ\n"); 617 return ret; 618 } 619 620 /* 621 * LTSSM enable control mode, and automatically delay link training on 622 * hot reset/link-down reset. 623 */ 624 val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1) | 625 FIELD_PREP_WM16(PCIE_LTSSM_APP_DLY2_EN, 1); 626 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); 627 628 rockchip_pcie_writel_apb(rockchip, 629 PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_EP), 630 PCIE_CLIENT_GENERAL_CON); 631 632 rockchip->pci.ep.ops = &rockchip_pcie_ep_ops; 633 rockchip->pci.ep.page_size = SZ_64K; 634 635 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 636 637 ret = dw_pcie_ep_init(&rockchip->pci.ep); 638 if (ret) { 639 dev_err(dev, "failed to initialize endpoint\n"); 640 return ret; 641 } 642 643 ret = dw_pcie_ep_init_registers(&rockchip->pci.ep); 644 if (ret) { 645 dev_err(dev, "failed to initialize DWC endpoint registers\n"); 646 dw_pcie_ep_deinit(&rockchip->pci.ep); 647 return ret; 648 } 649 650 pci_epc_init_notify(rockchip->pci.ep.epc); 651 652 /* unmask DLL up/down indicator and hot reset/link-down reset */ 653 val = FIELD_PREP_WM16(PCIE_RDLH_LINK_UP_CHGED, 0) | 654 FIELD_PREP_WM16(PCIE_LINK_REQ_RST_NOT_INT, 0); 655 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC); 656 657 return ret; 658 } 659 660 static int rockchip_pcie_probe(struct platform_device *pdev) 661 { 662 struct device *dev = &pdev->dev; 663 struct rockchip_pcie *rockchip; 664 const struct rockchip_pcie_of_data *data; 665 int ret; 666 667 data = of_device_get_match_data(dev); 668 if (!data) 669 return -EINVAL; 670 671 rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL); 672 if (!rockchip) 673 return -ENOMEM; 674 675 platform_set_drvdata(pdev, rockchip); 676 677 rockchip->pci.dev = dev; 678 rockchip->pci.ops = &dw_pcie_ops; 679 rockchip->data = data; 680 681 /* Default N_FTS value (210) is broken, override it to 255 */ 682 rockchip->pci.n_fts[0] = 255; /* Gen1 */ 683 rockchip->pci.n_fts[1] = 255; /* Gen2+ */ 684 685 ret = rockchip_pcie_resource_get(pdev, rockchip); 686 if (ret) 687 return ret; 688 689 ret = reset_control_assert(rockchip->rst); 690 if (ret) 691 return ret; 692 693 /* DON'T MOVE ME: must be enable before PHY init */ 694 ret = devm_regulator_get_enable_optional(dev, "vpcie3v3"); 695 if (ret < 0 && ret != -ENODEV) 696 return dev_err_probe(dev, ret, 697 "failed to enable vpcie3v3 regulator\n"); 698 699 ret = rockchip_pcie_phy_init(rockchip); 700 if (ret) 701 return dev_err_probe(dev, ret, 702 "failed to initialize the phy\n"); 703 704 ret = reset_control_deassert(rockchip->rst); 705 if (ret) 706 goto deinit_phy; 707 708 ret = rockchip_pcie_clk_init(rockchip); 709 if (ret) 710 goto deinit_phy; 711 712 switch (data->mode) { 713 case DW_PCIE_RC_TYPE: 714 ret = rockchip_pcie_configure_rc(pdev, rockchip); 715 if (ret) 716 goto deinit_clk; 717 break; 718 case DW_PCIE_EP_TYPE: 719 ret = rockchip_pcie_configure_ep(pdev, rockchip); 720 if (ret) 721 goto deinit_clk; 722 break; 723 default: 724 dev_err(dev, "INVALID device type %d\n", data->mode); 725 ret = -EINVAL; 726 goto deinit_clk; 727 } 728 729 return 0; 730 731 deinit_clk: 732 clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks); 733 deinit_phy: 734 rockchip_pcie_phy_deinit(rockchip); 735 736 return ret; 737 } 738 739 static const struct rockchip_pcie_of_data rockchip_pcie_rc_of_data_rk3568 = { 740 .mode = DW_PCIE_RC_TYPE, 741 }; 742 743 static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3568 = { 744 .mode = DW_PCIE_EP_TYPE, 745 .epc_features = &rockchip_pcie_epc_features_rk3568, 746 }; 747 748 static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3588 = { 749 .mode = DW_PCIE_EP_TYPE, 750 .epc_features = &rockchip_pcie_epc_features_rk3588, 751 }; 752 753 static const struct of_device_id rockchip_pcie_of_match[] = { 754 { 755 .compatible = "rockchip,rk3568-pcie", 756 .data = &rockchip_pcie_rc_of_data_rk3568, 757 }, 758 { 759 .compatible = "rockchip,rk3568-pcie-ep", 760 .data = &rockchip_pcie_ep_of_data_rk3568, 761 }, 762 { 763 .compatible = "rockchip,rk3588-pcie-ep", 764 .data = &rockchip_pcie_ep_of_data_rk3588, 765 }, 766 {}, 767 }; 768 769 static struct platform_driver rockchip_pcie_driver = { 770 .driver = { 771 .name = "rockchip-dw-pcie", 772 .of_match_table = rockchip_pcie_of_match, 773 .suppress_bind_attrs = true, 774 }, 775 .probe = rockchip_pcie_probe, 776 }; 777 builtin_platform_driver(rockchip_pcie_driver); 778