1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe host controller driver 4 * 5 * Copyright (c) 2016 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Wenrui Li <wenrui.li@rock-chips.com> 9 * 10 * Bits taken from Synopsys DesignWare Host controller driver and 11 * ARM PCI Host generic driver. 12 */ 13 14 #include <linux/bitrev.h> 15 #include <linux/clk.h> 16 #include <linux/delay.h> 17 #include <linux/gpio/consumer.h> 18 #include <linux/init.h> 19 #include <linux/interrupt.h> 20 #include <linux/iopoll.h> 21 #include <linux/irq.h> 22 #include <linux/irqchip/chained_irq.h> 23 #include <linux/irqdomain.h> 24 #include <linux/kernel.h> 25 #include <linux/mfd/syscon.h> 26 #include <linux/module.h> 27 #include <linux/of.h> 28 #include <linux/of_pci.h> 29 #include <linux/pci.h> 30 #include <linux/pci_ids.h> 31 #include <linux/phy/phy.h> 32 #include <linux/platform_device.h> 33 #include <linux/reset.h> 34 #include <linux/regmap.h> 35 36 #include "../pci.h" 37 #include "pcie-rockchip.h" 38 39 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) 40 { 41 u32 status; 42 43 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 44 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 45 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 46 } 47 48 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) 49 { 50 u32 status; 51 52 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 53 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; 54 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 55 } 56 57 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) 58 { 59 u32 val; 60 61 /* Update Tx credit maximum update interval */ 62 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); 63 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; 64 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ 65 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); 66 } 67 68 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, 69 struct pci_bus *bus, int dev) 70 { 71 /* 72 * Access only one slot on each root port. 73 * Do not read more than one device on the bus directly attached 74 * to RC's downstream side. 75 */ 76 if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) 77 return dev == 0; 78 79 return 1; 80 } 81 82 static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) 83 { 84 u32 val; 85 u8 map; 86 87 if (rockchip->legacy_phy) 88 return GENMASK(MAX_LANE_NUM - 1, 0); 89 90 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); 91 map = val & PCIE_CORE_LANE_MAP_MASK; 92 93 /* The link may be using a reverse-indexed mapping. */ 94 if (val & PCIE_CORE_LANE_MAP_REVERSE) 95 map = bitrev8(map) >> 4; 96 97 return map; 98 } 99 100 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, 101 int where, int size, u32 *val) 102 { 103 void __iomem *addr; 104 105 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; 106 107 if (!IS_ALIGNED((uintptr_t)addr, size)) { 108 *val = 0; 109 return PCIBIOS_BAD_REGISTER_NUMBER; 110 } 111 112 if (size == 4) { 113 *val = readl(addr); 114 } else if (size == 2) { 115 *val = readw(addr); 116 } else if (size == 1) { 117 *val = readb(addr); 118 } else { 119 *val = 0; 120 return PCIBIOS_BAD_REGISTER_NUMBER; 121 } 122 return PCIBIOS_SUCCESSFUL; 123 } 124 125 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, 126 int where, int size, u32 val) 127 { 128 u32 mask, tmp, offset; 129 void __iomem *addr; 130 131 offset = where & ~0x3; 132 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; 133 134 if (size == 4) { 135 writel(val, addr); 136 return PCIBIOS_SUCCESSFUL; 137 } 138 139 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 140 141 /* 142 * N.B. This read/modify/write isn't safe in general because it can 143 * corrupt RW1C bits in adjacent registers. But the hardware 144 * doesn't support smaller writes. 145 */ 146 tmp = readl(addr) & mask; 147 tmp |= val << ((where & 0x3) * 8); 148 writel(tmp, addr); 149 150 return PCIBIOS_SUCCESSFUL; 151 } 152 153 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, 154 struct pci_bus *bus, u32 devfn, 155 int where, int size, u32 *val) 156 { 157 void __iomem *addr; 158 159 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 160 161 if (!IS_ALIGNED((uintptr_t)addr, size)) { 162 *val = 0; 163 return PCIBIOS_BAD_REGISTER_NUMBER; 164 } 165 166 if (pci_is_root_bus(bus->parent)) 167 rockchip_pcie_cfg_configuration_accesses(rockchip, 168 AXI_WRAPPER_TYPE0_CFG); 169 else 170 rockchip_pcie_cfg_configuration_accesses(rockchip, 171 AXI_WRAPPER_TYPE1_CFG); 172 173 if (size == 4) { 174 *val = readl(addr); 175 } else if (size == 2) { 176 *val = readw(addr); 177 } else if (size == 1) { 178 *val = readb(addr); 179 } else { 180 *val = 0; 181 return PCIBIOS_BAD_REGISTER_NUMBER; 182 } 183 return PCIBIOS_SUCCESSFUL; 184 } 185 186 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, 187 struct pci_bus *bus, u32 devfn, 188 int where, int size, u32 val) 189 { 190 void __iomem *addr; 191 192 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 193 194 if (!IS_ALIGNED((uintptr_t)addr, size)) 195 return PCIBIOS_BAD_REGISTER_NUMBER; 196 197 if (pci_is_root_bus(bus->parent)) 198 rockchip_pcie_cfg_configuration_accesses(rockchip, 199 AXI_WRAPPER_TYPE0_CFG); 200 else 201 rockchip_pcie_cfg_configuration_accesses(rockchip, 202 AXI_WRAPPER_TYPE1_CFG); 203 204 if (size == 4) 205 writel(val, addr); 206 else if (size == 2) 207 writew(val, addr); 208 else if (size == 1) 209 writeb(val, addr); 210 else 211 return PCIBIOS_BAD_REGISTER_NUMBER; 212 213 return PCIBIOS_SUCCESSFUL; 214 } 215 216 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, 217 int size, u32 *val) 218 { 219 struct rockchip_pcie *rockchip = bus->sysdata; 220 221 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 222 return PCIBIOS_DEVICE_NOT_FOUND; 223 224 if (pci_is_root_bus(bus)) 225 return rockchip_pcie_rd_own_conf(rockchip, where, size, val); 226 227 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, 228 val); 229 } 230 231 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, 232 int where, int size, u32 val) 233 { 234 struct rockchip_pcie *rockchip = bus->sysdata; 235 236 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 237 return PCIBIOS_DEVICE_NOT_FOUND; 238 239 if (pci_is_root_bus(bus)) 240 return rockchip_pcie_wr_own_conf(rockchip, where, size, val); 241 242 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, 243 val); 244 } 245 246 static struct pci_ops rockchip_pcie_ops = { 247 .read = rockchip_pcie_rd_conf, 248 .write = rockchip_pcie_wr_conf, 249 }; 250 251 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) 252 { 253 int curr; 254 u32 status, scale, power; 255 256 if (IS_ERR(rockchip->vpcie3v3)) 257 return; 258 259 /* 260 * Set RC's captured slot power limit and scale if 261 * vpcie3v3 available. The default values are both zero 262 * which means the software should set these two according 263 * to the actual power supply. 264 */ 265 curr = regulator_get_current_limit(rockchip->vpcie3v3); 266 if (curr <= 0) 267 return; 268 269 scale = 3; /* 0.001x */ 270 curr = curr / 1000; /* convert to mA */ 271 power = (curr * 3300) / 1000; /* milliwatt */ 272 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { 273 if (!scale) { 274 dev_warn(rockchip->dev, "invalid power supply\n"); 275 return; 276 } 277 scale--; 278 power = power / 10; 279 } 280 281 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); 282 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | 283 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); 284 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR); 285 } 286 287 /** 288 * rockchip_pcie_host_init_port - Initialize hardware 289 * @rockchip: PCIe port information 290 */ 291 static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) 292 { 293 struct device *dev = rockchip->dev; 294 int err, i = MAX_LANE_NUM; 295 u32 status; 296 297 gpiod_set_value_cansleep(rockchip->ep_gpio, 0); 298 299 err = rockchip_pcie_init_port(rockchip); 300 if (err) 301 return err; 302 303 /* Fix the transmitted FTS count desired to exit from L0s. */ 304 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); 305 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | 306 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); 307 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); 308 309 rockchip_pcie_set_power_limit(rockchip); 310 311 /* Set RC's clock architecture as common clock */ 312 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 313 status |= PCI_EXP_LNKSTA_SLC << 16; 314 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 315 316 /* Set RC's RCB to 128 */ 317 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 318 status |= PCI_EXP_LNKCTL_RCB; 319 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 320 321 /* Enable Gen1 training */ 322 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 323 PCIE_CLIENT_CONFIG); 324 325 msleep(PCIE_T_PVPERL_MS); 326 gpiod_set_value_cansleep(rockchip->ep_gpio, 1); 327 328 /* 500ms timeout value should be enough for Gen1/2 training */ 329 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, 330 status, PCIE_LINK_UP(status), 20, 331 500 * USEC_PER_MSEC); 332 if (err) { 333 dev_err(dev, "PCIe link training gen1 timeout!\n"); 334 goto err_power_off_phy; 335 } 336 337 if (rockchip->link_gen == 2) { 338 /* 339 * Enable retrain for gen2. This should be configured only after 340 * gen1 finished. 341 */ 342 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 343 status |= PCI_EXP_LNKCTL_RL; 344 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 345 346 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, 347 status, PCIE_LINK_IS_GEN2(status), 20, 348 500 * USEC_PER_MSEC); 349 if (err) 350 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n"); 351 } 352 353 /* Check the final link width from negotiated lane counter from MGMT */ 354 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); 355 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> 356 PCIE_CORE_PL_CONF_LANE_SHIFT); 357 dev_dbg(dev, "current link width is x%d\n", status); 358 359 /* Power off unused lane(s) */ 360 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); 361 for (i = 0; i < MAX_LANE_NUM; i++) { 362 if (!(rockchip->lanes_map & BIT(i))) { 363 dev_dbg(dev, "idling lane %d\n", i); 364 phy_power_off(rockchip->phys[i]); 365 } 366 } 367 368 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, 369 PCIE_CORE_CONFIG_VENDOR); 370 rockchip_pcie_write(rockchip, 371 PCI_CLASS_BRIDGE_PCI_NORMAL << 8, 372 PCIE_RC_CONFIG_RID_CCR); 373 374 /* Clear THP cap's next cap pointer to remove L1 substate cap */ 375 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); 376 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; 377 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); 378 379 /* Clear L0s from RC's link cap */ 380 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { 381 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); 382 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; 383 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP); 384 } 385 386 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); 387 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; 388 status |= PCIE_RC_CONFIG_DCSR_MPS_256; 389 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR); 390 391 return 0; 392 err_power_off_phy: 393 while (i--) 394 phy_power_off(rockchip->phys[i]); 395 i = MAX_LANE_NUM; 396 while (i--) 397 phy_exit(rockchip->phys[i]); 398 return err; 399 } 400 401 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) 402 { 403 struct rockchip_pcie *rockchip = arg; 404 struct device *dev = rockchip->dev; 405 u32 reg; 406 u32 sub_reg; 407 408 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 409 if (reg & PCIE_CLIENT_INT_LOCAL) { 410 dev_dbg(dev, "local interrupt received\n"); 411 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); 412 if (sub_reg & PCIE_CORE_INT_PRFPE) 413 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); 414 415 if (sub_reg & PCIE_CORE_INT_CRFPE) 416 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); 417 418 if (sub_reg & PCIE_CORE_INT_RRPE) 419 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); 420 421 if (sub_reg & PCIE_CORE_INT_PRFO) 422 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); 423 424 if (sub_reg & PCIE_CORE_INT_CRFO) 425 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); 426 427 if (sub_reg & PCIE_CORE_INT_RT) 428 dev_dbg(dev, "replay timer timed out\n"); 429 430 if (sub_reg & PCIE_CORE_INT_RTR) 431 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); 432 433 if (sub_reg & PCIE_CORE_INT_PE) 434 dev_dbg(dev, "phy error detected on receive side\n"); 435 436 if (sub_reg & PCIE_CORE_INT_MTR) 437 dev_dbg(dev, "malformed TLP received from the link\n"); 438 439 if (sub_reg & PCIE_CORE_INT_UCR) 440 dev_dbg(dev, "malformed TLP received from the link\n"); 441 442 if (sub_reg & PCIE_CORE_INT_FCE) 443 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); 444 445 if (sub_reg & PCIE_CORE_INT_CT) 446 dev_dbg(dev, "a request timed out waiting for completion\n"); 447 448 if (sub_reg & PCIE_CORE_INT_UTC) 449 dev_dbg(dev, "unmapped TC error\n"); 450 451 if (sub_reg & PCIE_CORE_INT_MMVC) 452 dev_dbg(dev, "MSI mask register changes\n"); 453 454 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); 455 } else if (reg & PCIE_CLIENT_INT_PHY) { 456 dev_dbg(dev, "phy link changes\n"); 457 rockchip_pcie_update_txcredit_mui(rockchip); 458 rockchip_pcie_clr_bw_int(rockchip); 459 } 460 461 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, 462 PCIE_CLIENT_INT_STATUS); 463 464 return IRQ_HANDLED; 465 } 466 467 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) 468 { 469 struct rockchip_pcie *rockchip = arg; 470 struct device *dev = rockchip->dev; 471 u32 reg; 472 473 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 474 if (reg & PCIE_CLIENT_INT_LEGACY_DONE) 475 dev_dbg(dev, "legacy done interrupt received\n"); 476 477 if (reg & PCIE_CLIENT_INT_MSG) 478 dev_dbg(dev, "message done interrupt received\n"); 479 480 if (reg & PCIE_CLIENT_INT_HOT_RST) 481 dev_dbg(dev, "hot reset interrupt received\n"); 482 483 if (reg & PCIE_CLIENT_INT_DPA) 484 dev_dbg(dev, "dpa interrupt received\n"); 485 486 if (reg & PCIE_CLIENT_INT_FATAL_ERR) 487 dev_dbg(dev, "fatal error interrupt received\n"); 488 489 if (reg & PCIE_CLIENT_INT_NFATAL_ERR) 490 dev_dbg(dev, "no fatal error interrupt received\n"); 491 492 if (reg & PCIE_CLIENT_INT_CORR_ERR) 493 dev_dbg(dev, "correctable error interrupt received\n"); 494 495 if (reg & PCIE_CLIENT_INT_PHY) 496 dev_dbg(dev, "phy interrupt received\n"); 497 498 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | 499 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | 500 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | 501 PCIE_CLIENT_INT_NFATAL_ERR | 502 PCIE_CLIENT_INT_CORR_ERR | 503 PCIE_CLIENT_INT_PHY), 504 PCIE_CLIENT_INT_STATUS); 505 506 return IRQ_HANDLED; 507 } 508 509 static void rockchip_pcie_intx_handler(struct irq_desc *desc) 510 { 511 struct irq_chip *chip = irq_desc_get_chip(desc); 512 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 513 struct device *dev = rockchip->dev; 514 u32 reg; 515 u32 hwirq; 516 int ret; 517 518 chained_irq_enter(chip, desc); 519 520 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 521 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; 522 523 while (reg) { 524 hwirq = ffs(reg) - 1; 525 reg &= ~BIT(hwirq); 526 527 ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq); 528 if (ret) 529 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); 530 } 531 532 chained_irq_exit(chip, desc); 533 } 534 535 static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) 536 { 537 int irq, err; 538 struct device *dev = rockchip->dev; 539 struct platform_device *pdev = to_platform_device(dev); 540 541 irq = platform_get_irq_byname(pdev, "sys"); 542 if (irq < 0) 543 return irq; 544 545 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, 546 IRQF_SHARED, "pcie-sys", rockchip); 547 if (err) { 548 dev_err(dev, "failed to request PCIe subsystem IRQ\n"); 549 return err; 550 } 551 552 irq = platform_get_irq_byname(pdev, "legacy"); 553 if (irq < 0) 554 return irq; 555 556 irq_set_chained_handler_and_data(irq, 557 rockchip_pcie_intx_handler, 558 rockchip); 559 560 irq = platform_get_irq_byname(pdev, "client"); 561 if (irq < 0) 562 return irq; 563 564 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, 565 IRQF_SHARED, "pcie-client", rockchip); 566 if (err) { 567 dev_err(dev, "failed to request PCIe client IRQ\n"); 568 return err; 569 } 570 571 return 0; 572 } 573 574 /** 575 * rockchip_pcie_parse_host_dt - Parse Device Tree 576 * @rockchip: PCIe port information 577 * 578 * Return: '0' on success and error value on failure 579 */ 580 static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) 581 { 582 struct device *dev = rockchip->dev; 583 int err; 584 585 err = rockchip_pcie_parse_dt(rockchip); 586 if (err) 587 return err; 588 589 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); 590 if (IS_ERR(rockchip->vpcie12v)) { 591 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) 592 return PTR_ERR(rockchip->vpcie12v); 593 dev_info(dev, "no vpcie12v regulator found\n"); 594 } 595 596 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); 597 if (IS_ERR(rockchip->vpcie3v3)) { 598 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) 599 return PTR_ERR(rockchip->vpcie3v3); 600 dev_info(dev, "no vpcie3v3 regulator found\n"); 601 } 602 603 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); 604 if (IS_ERR(rockchip->vpcie1v8)) 605 return PTR_ERR(rockchip->vpcie1v8); 606 607 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); 608 if (IS_ERR(rockchip->vpcie0v9)) 609 return PTR_ERR(rockchip->vpcie0v9); 610 611 return 0; 612 } 613 614 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) 615 { 616 struct device *dev = rockchip->dev; 617 int err; 618 619 if (!IS_ERR(rockchip->vpcie12v)) { 620 err = regulator_enable(rockchip->vpcie12v); 621 if (err) { 622 dev_err(dev, "fail to enable vpcie12v regulator\n"); 623 goto err_out; 624 } 625 } 626 627 if (!IS_ERR(rockchip->vpcie3v3)) { 628 err = regulator_enable(rockchip->vpcie3v3); 629 if (err) { 630 dev_err(dev, "fail to enable vpcie3v3 regulator\n"); 631 goto err_disable_12v; 632 } 633 } 634 635 err = regulator_enable(rockchip->vpcie1v8); 636 if (err) { 637 dev_err(dev, "fail to enable vpcie1v8 regulator\n"); 638 goto err_disable_3v3; 639 } 640 641 err = regulator_enable(rockchip->vpcie0v9); 642 if (err) { 643 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 644 goto err_disable_1v8; 645 } 646 647 return 0; 648 649 err_disable_1v8: 650 regulator_disable(rockchip->vpcie1v8); 651 err_disable_3v3: 652 if (!IS_ERR(rockchip->vpcie3v3)) 653 regulator_disable(rockchip->vpcie3v3); 654 err_disable_12v: 655 if (!IS_ERR(rockchip->vpcie12v)) 656 regulator_disable(rockchip->vpcie12v); 657 err_out: 658 return err; 659 } 660 661 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) 662 { 663 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & 664 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); 665 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), 666 PCIE_CORE_INT_MASK); 667 668 rockchip_pcie_enable_bw_int(rockchip); 669 } 670 671 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 672 irq_hw_number_t hwirq) 673 { 674 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); 675 irq_set_chip_data(irq, domain->host_data); 676 677 return 0; 678 } 679 680 static const struct irq_domain_ops intx_domain_ops = { 681 .map = rockchip_pcie_intx_map, 682 }; 683 684 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 685 { 686 struct device *dev = rockchip->dev; 687 struct device_node *intc = of_get_next_child(dev->of_node, NULL); 688 689 if (!intc) { 690 dev_err(dev, "missing child interrupt-controller node\n"); 691 return -EINVAL; 692 } 693 694 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX, 695 &intx_domain_ops, rockchip); 696 of_node_put(intc); 697 if (!rockchip->irq_domain) { 698 dev_err(dev, "failed to get a INTx IRQ domain\n"); 699 return -EINVAL; 700 } 701 702 return 0; 703 } 704 705 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, 706 int region_no, int type, u8 num_pass_bits, 707 u32 lower_addr, u32 upper_addr) 708 { 709 u32 ob_addr_0; 710 u32 ob_addr_1; 711 u32 ob_desc_0; 712 u32 aw_offset; 713 714 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) 715 return -EINVAL; 716 if (num_pass_bits + 1 < 8) 717 return -EINVAL; 718 if (num_pass_bits > 63) 719 return -EINVAL; 720 if (region_no == 0) { 721 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) 722 return -EINVAL; 723 } 724 if (region_no != 0) { 725 if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) 726 return -EINVAL; 727 } 728 729 aw_offset = (region_no << OB_REG_SIZE_SHIFT); 730 731 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; 732 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; 733 ob_addr_1 = upper_addr; 734 ob_desc_0 = (1 << 23 | type); 735 736 rockchip_pcie_write(rockchip, ob_addr_0, 737 PCIE_CORE_OB_REGION_ADDR0 + aw_offset); 738 rockchip_pcie_write(rockchip, ob_addr_1, 739 PCIE_CORE_OB_REGION_ADDR1 + aw_offset); 740 rockchip_pcie_write(rockchip, ob_desc_0, 741 PCIE_CORE_OB_REGION_DESC0 + aw_offset); 742 rockchip_pcie_write(rockchip, 0, 743 PCIE_CORE_OB_REGION_DESC1 + aw_offset); 744 745 return 0; 746 } 747 748 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, 749 int region_no, u8 num_pass_bits, 750 u32 lower_addr, u32 upper_addr) 751 { 752 u32 ib_addr_0; 753 u32 ib_addr_1; 754 u32 aw_offset; 755 756 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) 757 return -EINVAL; 758 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) 759 return -EINVAL; 760 if (num_pass_bits > 63) 761 return -EINVAL; 762 763 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); 764 765 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; 766 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; 767 ib_addr_1 = upper_addr; 768 769 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); 770 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); 771 772 return 0; 773 } 774 775 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) 776 { 777 struct device *dev = rockchip->dev; 778 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 779 struct resource_entry *entry; 780 u64 pci_addr, size; 781 int offset; 782 int err; 783 int reg_no; 784 785 rockchip_pcie_cfg_configuration_accesses(rockchip, 786 AXI_WRAPPER_TYPE0_CFG); 787 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); 788 if (!entry) 789 return -ENODEV; 790 791 size = resource_size(entry->res); 792 pci_addr = entry->res->start - entry->offset; 793 rockchip->msg_bus_addr = pci_addr; 794 795 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 796 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, 797 AXI_WRAPPER_MEM_WRITE, 798 20 - 1, 799 pci_addr + (reg_no << 20), 800 0); 801 if (err) { 802 dev_err(dev, "program RC mem outbound ATU failed\n"); 803 return err; 804 } 805 } 806 807 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); 808 if (err) { 809 dev_err(dev, "program RC mem inbound ATU failed\n"); 810 return err; 811 } 812 813 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); 814 if (!entry) 815 return -ENODEV; 816 817 /* store the register number offset to program RC io outbound ATU */ 818 offset = size >> 20; 819 820 size = resource_size(entry->res); 821 pci_addr = entry->res->start - entry->offset; 822 823 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 824 err = rockchip_pcie_prog_ob_atu(rockchip, 825 reg_no + 1 + offset, 826 AXI_WRAPPER_IO_WRITE, 827 20 - 1, 828 pci_addr + (reg_no << 20), 829 0); 830 if (err) { 831 dev_err(dev, "program RC io outbound ATU failed\n"); 832 return err; 833 } 834 } 835 836 /* assign message regions */ 837 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, 838 AXI_WRAPPER_NOR_MSG, 839 20 - 1, 0, 0); 840 841 rockchip->msg_bus_addr += ((reg_no + offset) << 20); 842 return err; 843 } 844 845 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) 846 { 847 u32 value; 848 int err; 849 850 /* send PME_TURN_OFF message */ 851 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); 852 853 /* read LTSSM and wait for falling into L2 link state */ 854 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, 855 value, PCIE_LINK_IS_L2(value), 20, 856 jiffies_to_usecs(5 * HZ)); 857 if (err) { 858 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); 859 return err; 860 } 861 862 return 0; 863 } 864 865 static int rockchip_pcie_suspend_noirq(struct device *dev) 866 { 867 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 868 int ret; 869 870 /* disable core and cli int since we don't need to ack PME_ACK */ 871 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | 872 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); 873 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); 874 875 ret = rockchip_pcie_wait_l2(rockchip); 876 if (ret) { 877 rockchip_pcie_enable_interrupts(rockchip); 878 return ret; 879 } 880 881 rockchip_pcie_deinit_phys(rockchip); 882 883 rockchip_pcie_disable_clocks(rockchip); 884 885 regulator_disable(rockchip->vpcie0v9); 886 887 return ret; 888 } 889 890 static int rockchip_pcie_resume_noirq(struct device *dev) 891 { 892 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 893 int err; 894 895 err = regulator_enable(rockchip->vpcie0v9); 896 if (err) { 897 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 898 return err; 899 } 900 901 err = rockchip_pcie_enable_clocks(rockchip); 902 if (err) 903 goto err_disable_0v9; 904 905 err = rockchip_pcie_host_init_port(rockchip); 906 if (err) 907 goto err_pcie_resume; 908 909 err = rockchip_pcie_cfg_atu(rockchip); 910 if (err) 911 goto err_err_deinit_port; 912 913 /* Need this to enter L1 again */ 914 rockchip_pcie_update_txcredit_mui(rockchip); 915 rockchip_pcie_enable_interrupts(rockchip); 916 917 return 0; 918 919 err_err_deinit_port: 920 rockchip_pcie_deinit_phys(rockchip); 921 err_pcie_resume: 922 rockchip_pcie_disable_clocks(rockchip); 923 err_disable_0v9: 924 regulator_disable(rockchip->vpcie0v9); 925 return err; 926 } 927 928 static int rockchip_pcie_probe(struct platform_device *pdev) 929 { 930 struct rockchip_pcie *rockchip; 931 struct device *dev = &pdev->dev; 932 struct pci_host_bridge *bridge; 933 int err; 934 935 if (!dev->of_node) 936 return -ENODEV; 937 938 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); 939 if (!bridge) 940 return -ENOMEM; 941 942 rockchip = pci_host_bridge_priv(bridge); 943 944 platform_set_drvdata(pdev, rockchip); 945 rockchip->dev = dev; 946 rockchip->is_rc = true; 947 948 err = rockchip_pcie_parse_host_dt(rockchip); 949 if (err) 950 return err; 951 952 err = rockchip_pcie_enable_clocks(rockchip); 953 if (err) 954 return err; 955 956 err = rockchip_pcie_set_vpcie(rockchip); 957 if (err) { 958 dev_err(dev, "failed to set vpcie regulator\n"); 959 goto err_set_vpcie; 960 } 961 962 err = rockchip_pcie_host_init_port(rockchip); 963 if (err) 964 goto err_vpcie; 965 966 err = rockchip_pcie_init_irq_domain(rockchip); 967 if (err < 0) 968 goto err_deinit_port; 969 970 err = rockchip_pcie_cfg_atu(rockchip); 971 if (err) 972 goto err_remove_irq_domain; 973 974 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); 975 if (!rockchip->msg_region) { 976 err = -ENOMEM; 977 goto err_remove_irq_domain; 978 } 979 980 bridge->sysdata = rockchip; 981 bridge->ops = &rockchip_pcie_ops; 982 983 err = rockchip_pcie_setup_irq(rockchip); 984 if (err) 985 goto err_remove_irq_domain; 986 987 rockchip_pcie_enable_interrupts(rockchip); 988 989 err = pci_host_probe(bridge); 990 if (err < 0) 991 goto err_remove_irq_domain; 992 993 return 0; 994 995 err_remove_irq_domain: 996 irq_domain_remove(rockchip->irq_domain); 997 err_deinit_port: 998 rockchip_pcie_deinit_phys(rockchip); 999 err_vpcie: 1000 if (!IS_ERR(rockchip->vpcie12v)) 1001 regulator_disable(rockchip->vpcie12v); 1002 if (!IS_ERR(rockchip->vpcie3v3)) 1003 regulator_disable(rockchip->vpcie3v3); 1004 regulator_disable(rockchip->vpcie1v8); 1005 regulator_disable(rockchip->vpcie0v9); 1006 err_set_vpcie: 1007 rockchip_pcie_disable_clocks(rockchip); 1008 return err; 1009 } 1010 1011 static void rockchip_pcie_remove(struct platform_device *pdev) 1012 { 1013 struct device *dev = &pdev->dev; 1014 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 1015 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 1016 1017 pci_stop_root_bus(bridge->bus); 1018 pci_remove_root_bus(bridge->bus); 1019 irq_domain_remove(rockchip->irq_domain); 1020 1021 rockchip_pcie_deinit_phys(rockchip); 1022 1023 rockchip_pcie_disable_clocks(rockchip); 1024 1025 if (!IS_ERR(rockchip->vpcie12v)) 1026 regulator_disable(rockchip->vpcie12v); 1027 if (!IS_ERR(rockchip->vpcie3v3)) 1028 regulator_disable(rockchip->vpcie3v3); 1029 regulator_disable(rockchip->vpcie1v8); 1030 regulator_disable(rockchip->vpcie0v9); 1031 } 1032 1033 static const struct dev_pm_ops rockchip_pcie_pm_ops = { 1034 NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, 1035 rockchip_pcie_resume_noirq) 1036 }; 1037 1038 static const struct of_device_id rockchip_pcie_of_match[] = { 1039 { .compatible = "rockchip,rk3399-pcie", }, 1040 {} 1041 }; 1042 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); 1043 1044 static struct platform_driver rockchip_pcie_driver = { 1045 .driver = { 1046 .name = "rockchip-pcie", 1047 .of_match_table = rockchip_pcie_of_match, 1048 .pm = &rockchip_pcie_pm_ops, 1049 }, 1050 .probe = rockchip_pcie_probe, 1051 .remove_new = rockchip_pcie_remove, 1052 }; 1053 module_platform_driver(rockchip_pcie_driver); 1054 1055 MODULE_AUTHOR("Rockchip Inc"); 1056 MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); 1057 MODULE_LICENSE("GPL v2"); 1058