1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe host controller driver 4 * 5 * Copyright (c) 2016 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Wenrui Li <wenrui.li@rock-chips.com> 9 * 10 * Bits taken from Synopsys DesignWare Host controller driver and 11 * ARM PCI Host generic driver. 12 */ 13 14 #include <linux/bitfield.h> 15 #include <linux/bitrev.h> 16 #include <linux/gpio/consumer.h> 17 #include <linux/interrupt.h> 18 #include <linux/iopoll.h> 19 #include <linux/irq.h> 20 #include <linux/irqchip/chained_irq.h> 21 #include <linux/irqdomain.h> 22 #include <linux/module.h> 23 #include <linux/of.h> 24 #include <linux/of_pci.h> 25 #include <linux/phy/phy.h> 26 #include <linux/platform_device.h> 27 #include <linux/regulator/consumer.h> 28 29 #include "../pci.h" 30 #include "pcie-rockchip.h" 31 32 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) 33 { 34 u32 status; 35 36 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); 37 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 38 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); 39 } 40 41 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) 42 { 43 u32 status; 44 45 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); 46 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; 47 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); 48 } 49 50 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) 51 { 52 u32 val; 53 54 /* Update Tx credit maximum update interval */ 55 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); 56 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; 57 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ 58 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); 59 } 60 61 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, 62 struct pci_bus *bus, int dev) 63 { 64 /* 65 * Access only one slot on each root port. 66 * Do not read more than one device on the bus directly attached 67 * to RC's downstream side. 68 */ 69 if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) 70 return dev == 0; 71 72 return 1; 73 } 74 75 static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) 76 { 77 u32 val; 78 u8 map; 79 80 if (rockchip->legacy_phy) 81 return GENMASK(MAX_LANE_NUM - 1, 0); 82 83 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); 84 map = val & PCIE_CORE_LANE_MAP_MASK; 85 86 /* The link may be using a reverse-indexed mapping. */ 87 if (val & PCIE_CORE_LANE_MAP_REVERSE) 88 map = bitrev8(map) >> 4; 89 90 return map; 91 } 92 93 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, 94 int where, int size, u32 *val) 95 { 96 void __iomem *addr; 97 98 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; 99 100 if (!IS_ALIGNED((uintptr_t)addr, size)) { 101 *val = 0; 102 return PCIBIOS_BAD_REGISTER_NUMBER; 103 } 104 105 if (size == 4) { 106 *val = readl(addr); 107 } else if (size == 2) { 108 *val = readw(addr); 109 } else if (size == 1) { 110 *val = readb(addr); 111 } else { 112 *val = 0; 113 return PCIBIOS_BAD_REGISTER_NUMBER; 114 } 115 return PCIBIOS_SUCCESSFUL; 116 } 117 118 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, 119 int where, int size, u32 val) 120 { 121 u32 mask, tmp, offset; 122 void __iomem *addr; 123 124 offset = where & ~0x3; 125 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; 126 127 if (size == 4) { 128 writel(val, addr); 129 return PCIBIOS_SUCCESSFUL; 130 } 131 132 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 133 134 /* 135 * N.B. This read/modify/write isn't safe in general because it can 136 * corrupt RW1C bits in adjacent registers. But the hardware 137 * doesn't support smaller writes. 138 */ 139 tmp = readl(addr) & mask; 140 tmp |= val << ((where & 0x3) * 8); 141 writel(tmp, addr); 142 143 return PCIBIOS_SUCCESSFUL; 144 } 145 146 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, 147 struct pci_bus *bus, u32 devfn, 148 int where, int size, u32 *val) 149 { 150 void __iomem *addr; 151 152 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 153 154 if (!IS_ALIGNED((uintptr_t)addr, size)) { 155 *val = 0; 156 return PCIBIOS_BAD_REGISTER_NUMBER; 157 } 158 159 if (pci_is_root_bus(bus->parent)) 160 rockchip_pcie_cfg_configuration_accesses(rockchip, 161 AXI_WRAPPER_TYPE0_CFG); 162 else 163 rockchip_pcie_cfg_configuration_accesses(rockchip, 164 AXI_WRAPPER_TYPE1_CFG); 165 166 if (size == 4) { 167 *val = readl(addr); 168 } else if (size == 2) { 169 *val = readw(addr); 170 } else if (size == 1) { 171 *val = readb(addr); 172 } else { 173 *val = 0; 174 return PCIBIOS_BAD_REGISTER_NUMBER; 175 } 176 return PCIBIOS_SUCCESSFUL; 177 } 178 179 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, 180 struct pci_bus *bus, u32 devfn, 181 int where, int size, u32 val) 182 { 183 void __iomem *addr; 184 185 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 186 187 if (!IS_ALIGNED((uintptr_t)addr, size)) 188 return PCIBIOS_BAD_REGISTER_NUMBER; 189 190 if (pci_is_root_bus(bus->parent)) 191 rockchip_pcie_cfg_configuration_accesses(rockchip, 192 AXI_WRAPPER_TYPE0_CFG); 193 else 194 rockchip_pcie_cfg_configuration_accesses(rockchip, 195 AXI_WRAPPER_TYPE1_CFG); 196 197 if (size == 4) 198 writel(val, addr); 199 else if (size == 2) 200 writew(val, addr); 201 else if (size == 1) 202 writeb(val, addr); 203 else 204 return PCIBIOS_BAD_REGISTER_NUMBER; 205 206 return PCIBIOS_SUCCESSFUL; 207 } 208 209 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, 210 int size, u32 *val) 211 { 212 struct rockchip_pcie *rockchip = bus->sysdata; 213 214 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 215 return PCIBIOS_DEVICE_NOT_FOUND; 216 217 if (pci_is_root_bus(bus)) 218 return rockchip_pcie_rd_own_conf(rockchip, where, size, val); 219 220 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, 221 val); 222 } 223 224 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, 225 int where, int size, u32 val) 226 { 227 struct rockchip_pcie *rockchip = bus->sysdata; 228 229 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 230 return PCIBIOS_DEVICE_NOT_FOUND; 231 232 if (pci_is_root_bus(bus)) 233 return rockchip_pcie_wr_own_conf(rockchip, where, size, val); 234 235 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, 236 val); 237 } 238 239 static struct pci_ops rockchip_pcie_ops = { 240 .read = rockchip_pcie_rd_conf, 241 .write = rockchip_pcie_wr_conf, 242 }; 243 244 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) 245 { 246 int curr; 247 u32 status, scale, power; 248 249 if (IS_ERR(rockchip->vpcie3v3)) 250 return; 251 252 /* 253 * Set RC's captured slot power limit and scale if 254 * vpcie3v3 available. The default values are both zero 255 * which means the software should set these two according 256 * to the actual power supply. 257 */ 258 curr = regulator_get_current_limit(rockchip->vpcie3v3); 259 if (curr <= 0) 260 return; 261 262 scale = 3; /* 0.001x */ 263 curr = curr / 1000; /* convert to mA */ 264 power = (curr * 3300) / 1000; /* milliwatt */ 265 while (power > FIELD_MAX(PCI_EXP_DEVCAP_PWR_VAL)) { 266 if (!scale) { 267 dev_warn(rockchip->dev, "invalid power supply\n"); 268 return; 269 } 270 scale--; 271 power = power / 10; 272 } 273 274 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP); 275 status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_VAL, power); 276 status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_SCL, scale); 277 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP); 278 } 279 280 /** 281 * rockchip_pcie_host_init_port - Initialize hardware 282 * @rockchip: PCIe port information 283 */ 284 static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) 285 { 286 struct device *dev = rockchip->dev; 287 int err, i = MAX_LANE_NUM; 288 u32 status; 289 290 gpiod_set_value_cansleep(rockchip->perst_gpio, 0); 291 292 err = rockchip_pcie_init_port(rockchip); 293 if (err) 294 return err; 295 296 /* Fix the transmitted FTS count desired to exit from L0s. */ 297 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); 298 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | 299 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); 300 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); 301 302 rockchip_pcie_set_power_limit(rockchip); 303 304 /* Set RC's clock architecture as common clock */ 305 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); 306 status |= PCI_EXP_LNKSTA_SLC << 16; 307 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); 308 309 /* Set RC's RCB to 128 */ 310 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); 311 status |= PCI_EXP_LNKCTL_RCB; 312 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); 313 314 /* Enable Gen1 training */ 315 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 316 PCIE_CLIENT_CONFIG); 317 318 msleep(PCIE_T_PVPERL_MS); 319 gpiod_set_value_cansleep(rockchip->perst_gpio, 1); 320 321 msleep(PCIE_RESET_CONFIG_WAIT_MS); 322 323 /* 500ms timeout value should be enough for Gen1/2 training */ 324 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, 325 status, PCIE_LINK_UP(status), 20, 326 500 * USEC_PER_MSEC); 327 if (err) { 328 dev_err(dev, "PCIe link training gen1 timeout!\n"); 329 goto err_power_off_phy; 330 } 331 332 /* Check the final link width from negotiated lane counter from MGMT */ 333 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); 334 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> 335 PCIE_CORE_PL_CONF_LANE_SHIFT); 336 dev_dbg(dev, "current link width is x%d\n", status); 337 338 /* Power off unused lane(s) */ 339 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); 340 for (i = 0; i < MAX_LANE_NUM; i++) { 341 if (!(rockchip->lanes_map & BIT(i))) { 342 dev_dbg(dev, "idling lane %d\n", i); 343 phy_power_off(rockchip->phys[i]); 344 } 345 } 346 347 rockchip_pcie_write(rockchip, PCI_VENDOR_ID_ROCKCHIP, 348 PCIE_CORE_CONFIG_VENDOR); 349 rockchip_pcie_write(rockchip, 350 PCI_CLASS_BRIDGE_PCI_NORMAL << 8, 351 PCIE_RC_CONFIG_RID_CCR); 352 353 /* Clear THP cap's next cap pointer to remove L1 substate cap */ 354 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); 355 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; 356 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); 357 358 /* Clear L0s from RC's link cap */ 359 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { 360 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP); 361 status &= ~PCI_EXP_LNKCAP_ASPM_L0S; 362 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP); 363 } 364 365 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL); 366 status &= ~PCI_EXP_DEVCTL_PAYLOAD; 367 status |= PCI_EXP_DEVCTL_PAYLOAD_256B; 368 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL); 369 370 return 0; 371 err_power_off_phy: 372 while (i--) 373 phy_power_off(rockchip->phys[i]); 374 i = MAX_LANE_NUM; 375 while (i--) 376 phy_exit(rockchip->phys[i]); 377 return err; 378 } 379 380 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) 381 { 382 struct rockchip_pcie *rockchip = arg; 383 struct device *dev = rockchip->dev; 384 u32 reg; 385 u32 sub_reg; 386 387 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 388 if (reg & PCIE_CLIENT_INT_LOCAL) { 389 dev_dbg(dev, "local interrupt received\n"); 390 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); 391 if (sub_reg & PCIE_CORE_INT_PRFPE) 392 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); 393 394 if (sub_reg & PCIE_CORE_INT_CRFPE) 395 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); 396 397 if (sub_reg & PCIE_CORE_INT_RRPE) 398 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); 399 400 if (sub_reg & PCIE_CORE_INT_PRFO) 401 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); 402 403 if (sub_reg & PCIE_CORE_INT_CRFO) 404 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); 405 406 if (sub_reg & PCIE_CORE_INT_RT) 407 dev_dbg(dev, "replay timer timed out\n"); 408 409 if (sub_reg & PCIE_CORE_INT_RTR) 410 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); 411 412 if (sub_reg & PCIE_CORE_INT_PE) 413 dev_dbg(dev, "phy error detected on receive side\n"); 414 415 if (sub_reg & PCIE_CORE_INT_MTR) 416 dev_dbg(dev, "malformed TLP received from the link\n"); 417 418 if (sub_reg & PCIE_CORE_INT_UCR) 419 dev_dbg(dev, "Unexpected Completion received from the link\n"); 420 421 if (sub_reg & PCIE_CORE_INT_FCE) 422 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); 423 424 if (sub_reg & PCIE_CORE_INT_CT) 425 dev_dbg(dev, "a request timed out waiting for completion\n"); 426 427 if (sub_reg & PCIE_CORE_INT_UTC) 428 dev_dbg(dev, "unmapped TC error\n"); 429 430 if (sub_reg & PCIE_CORE_INT_MMVC) 431 dev_dbg(dev, "MSI mask register changes\n"); 432 433 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); 434 } else if (reg & PCIE_CLIENT_INT_PHY) { 435 dev_dbg(dev, "phy link changes\n"); 436 rockchip_pcie_update_txcredit_mui(rockchip); 437 rockchip_pcie_clr_bw_int(rockchip); 438 } 439 440 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, 441 PCIE_CLIENT_INT_STATUS); 442 443 return IRQ_HANDLED; 444 } 445 446 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) 447 { 448 struct rockchip_pcie *rockchip = arg; 449 struct device *dev = rockchip->dev; 450 u32 reg; 451 452 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 453 if (reg & PCIE_CLIENT_INT_LEGACY_DONE) 454 dev_dbg(dev, "legacy done interrupt received\n"); 455 456 if (reg & PCIE_CLIENT_INT_MSG) 457 dev_dbg(dev, "message done interrupt received\n"); 458 459 if (reg & PCIE_CLIENT_INT_HOT_RST) 460 dev_dbg(dev, "hot reset interrupt received\n"); 461 462 if (reg & PCIE_CLIENT_INT_DPA) 463 dev_dbg(dev, "dpa interrupt received\n"); 464 465 if (reg & PCIE_CLIENT_INT_FATAL_ERR) 466 dev_dbg(dev, "fatal error interrupt received\n"); 467 468 if (reg & PCIE_CLIENT_INT_NFATAL_ERR) 469 dev_dbg(dev, "non fatal error interrupt received\n"); 470 471 if (reg & PCIE_CLIENT_INT_CORR_ERR) 472 dev_dbg(dev, "correctable error interrupt received\n"); 473 474 if (reg & PCIE_CLIENT_INT_PHY) 475 dev_dbg(dev, "phy interrupt received\n"); 476 477 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | 478 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | 479 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | 480 PCIE_CLIENT_INT_NFATAL_ERR | 481 PCIE_CLIENT_INT_CORR_ERR | 482 PCIE_CLIENT_INT_PHY), 483 PCIE_CLIENT_INT_STATUS); 484 485 return IRQ_HANDLED; 486 } 487 488 static void rockchip_pcie_intx_handler(struct irq_desc *desc) 489 { 490 struct irq_chip *chip = irq_desc_get_chip(desc); 491 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 492 struct device *dev = rockchip->dev; 493 u32 reg; 494 u32 hwirq; 495 int ret; 496 497 chained_irq_enter(chip, desc); 498 499 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 500 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; 501 502 while (reg) { 503 hwirq = ffs(reg) - 1; 504 reg &= ~BIT(hwirq); 505 506 ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq); 507 if (ret) 508 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); 509 } 510 511 chained_irq_exit(chip, desc); 512 } 513 514 static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) 515 { 516 int irq, err; 517 struct device *dev = rockchip->dev; 518 struct platform_device *pdev = to_platform_device(dev); 519 520 irq = platform_get_irq_byname(pdev, "sys"); 521 if (irq < 0) 522 return irq; 523 524 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, 525 IRQF_SHARED, "pcie-sys", rockchip); 526 if (err) { 527 dev_err(dev, "failed to request PCIe subsystem IRQ\n"); 528 return err; 529 } 530 531 irq = platform_get_irq_byname(pdev, "legacy"); 532 if (irq < 0) 533 return irq; 534 535 irq_set_chained_handler_and_data(irq, 536 rockchip_pcie_intx_handler, 537 rockchip); 538 539 irq = platform_get_irq_byname(pdev, "client"); 540 if (irq < 0) 541 return irq; 542 543 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, 544 IRQF_SHARED, "pcie-client", rockchip); 545 if (err) { 546 dev_err(dev, "failed to request PCIe client IRQ\n"); 547 return err; 548 } 549 550 return 0; 551 } 552 553 /** 554 * rockchip_pcie_parse_host_dt - Parse Device Tree 555 * @rockchip: PCIe port information 556 * 557 * Return: '0' on success and error value on failure 558 */ 559 static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) 560 { 561 struct device *dev = rockchip->dev; 562 int err; 563 564 err = rockchip_pcie_parse_dt(rockchip); 565 if (err) 566 return err; 567 568 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); 569 if (IS_ERR(rockchip->vpcie12v)) { 570 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) 571 return PTR_ERR(rockchip->vpcie12v); 572 dev_info(dev, "no vpcie12v regulator found\n"); 573 } 574 575 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); 576 if (IS_ERR(rockchip->vpcie3v3)) { 577 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) 578 return PTR_ERR(rockchip->vpcie3v3); 579 dev_info(dev, "no vpcie3v3 regulator found\n"); 580 } 581 582 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); 583 if (IS_ERR(rockchip->vpcie1v8)) 584 return PTR_ERR(rockchip->vpcie1v8); 585 586 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); 587 if (IS_ERR(rockchip->vpcie0v9)) 588 return PTR_ERR(rockchip->vpcie0v9); 589 590 return 0; 591 } 592 593 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) 594 { 595 struct device *dev = rockchip->dev; 596 int err; 597 598 if (!IS_ERR(rockchip->vpcie12v)) { 599 err = regulator_enable(rockchip->vpcie12v); 600 if (err) { 601 dev_err(dev, "fail to enable vpcie12v regulator\n"); 602 goto err_out; 603 } 604 } 605 606 if (!IS_ERR(rockchip->vpcie3v3)) { 607 err = regulator_enable(rockchip->vpcie3v3); 608 if (err) { 609 dev_err(dev, "fail to enable vpcie3v3 regulator\n"); 610 goto err_disable_12v; 611 } 612 } 613 614 err = regulator_enable(rockchip->vpcie1v8); 615 if (err) { 616 dev_err(dev, "fail to enable vpcie1v8 regulator\n"); 617 goto err_disable_3v3; 618 } 619 620 err = regulator_enable(rockchip->vpcie0v9); 621 if (err) { 622 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 623 goto err_disable_1v8; 624 } 625 626 return 0; 627 628 err_disable_1v8: 629 regulator_disable(rockchip->vpcie1v8); 630 err_disable_3v3: 631 if (!IS_ERR(rockchip->vpcie3v3)) 632 regulator_disable(rockchip->vpcie3v3); 633 err_disable_12v: 634 if (!IS_ERR(rockchip->vpcie12v)) 635 regulator_disable(rockchip->vpcie12v); 636 err_out: 637 return err; 638 } 639 640 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) 641 { 642 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & 643 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); 644 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), 645 PCIE_CORE_INT_MASK); 646 647 rockchip_pcie_enable_bw_int(rockchip); 648 } 649 650 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 651 irq_hw_number_t hwirq) 652 { 653 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); 654 irq_set_chip_data(irq, domain->host_data); 655 656 return 0; 657 } 658 659 static const struct irq_domain_ops intx_domain_ops = { 660 .map = rockchip_pcie_intx_map, 661 }; 662 663 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 664 { 665 struct device *dev = rockchip->dev; 666 struct device_node *intc = of_get_next_child(dev->of_node, NULL); 667 668 if (!intc) { 669 dev_err(dev, "missing child interrupt-controller node\n"); 670 return -EINVAL; 671 } 672 673 rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX, 674 &intx_domain_ops, rockchip); 675 of_node_put(intc); 676 if (!rockchip->irq_domain) { 677 dev_err(dev, "failed to get a INTx IRQ domain\n"); 678 return -EINVAL; 679 } 680 681 return 0; 682 } 683 684 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, 685 int region_no, int type, u8 num_pass_bits, 686 u32 lower_addr, u32 upper_addr) 687 { 688 u32 ob_addr_0; 689 u32 ob_addr_1; 690 u32 ob_desc_0; 691 u32 aw_offset; 692 693 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) 694 return -EINVAL; 695 if (num_pass_bits + 1 < 8) 696 return -EINVAL; 697 if (num_pass_bits > 63) 698 return -EINVAL; 699 if (region_no == 0) { 700 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) 701 return -EINVAL; 702 } 703 if (region_no != 0) { 704 if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) 705 return -EINVAL; 706 } 707 708 aw_offset = (region_no << OB_REG_SIZE_SHIFT); 709 710 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; 711 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; 712 ob_addr_1 = upper_addr; 713 ob_desc_0 = (1 << 23 | type); 714 715 rockchip_pcie_write(rockchip, ob_addr_0, 716 PCIE_CORE_OB_REGION_ADDR0 + aw_offset); 717 rockchip_pcie_write(rockchip, ob_addr_1, 718 PCIE_CORE_OB_REGION_ADDR1 + aw_offset); 719 rockchip_pcie_write(rockchip, ob_desc_0, 720 PCIE_CORE_OB_REGION_DESC0 + aw_offset); 721 rockchip_pcie_write(rockchip, 0, 722 PCIE_CORE_OB_REGION_DESC1 + aw_offset); 723 724 return 0; 725 } 726 727 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, 728 int region_no, u8 num_pass_bits, 729 u32 lower_addr, u32 upper_addr) 730 { 731 u32 ib_addr_0; 732 u32 ib_addr_1; 733 u32 aw_offset; 734 735 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) 736 return -EINVAL; 737 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) 738 return -EINVAL; 739 if (num_pass_bits > 63) 740 return -EINVAL; 741 742 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); 743 744 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; 745 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; 746 ib_addr_1 = upper_addr; 747 748 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); 749 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); 750 751 return 0; 752 } 753 754 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) 755 { 756 struct device *dev = rockchip->dev; 757 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 758 struct resource_entry *entry; 759 u64 pci_addr, size; 760 int offset; 761 int err; 762 int reg_no; 763 764 rockchip_pcie_cfg_configuration_accesses(rockchip, 765 AXI_WRAPPER_TYPE0_CFG); 766 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); 767 if (!entry) 768 return -ENODEV; 769 770 size = resource_size(entry->res); 771 pci_addr = entry->res->start - entry->offset; 772 rockchip->msg_bus_addr = pci_addr; 773 774 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 775 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, 776 AXI_WRAPPER_MEM_WRITE, 777 20 - 1, 778 pci_addr + (reg_no << 20), 779 0); 780 if (err) { 781 dev_err(dev, "program RC mem outbound ATU failed\n"); 782 return err; 783 } 784 } 785 786 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); 787 if (err) { 788 dev_err(dev, "program RC mem inbound ATU failed\n"); 789 return err; 790 } 791 792 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); 793 if (!entry) 794 return -ENODEV; 795 796 /* store the register number offset to program RC io outbound ATU */ 797 offset = size >> 20; 798 799 size = resource_size(entry->res); 800 pci_addr = entry->res->start - entry->offset; 801 802 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 803 err = rockchip_pcie_prog_ob_atu(rockchip, 804 reg_no + 1 + offset, 805 AXI_WRAPPER_IO_WRITE, 806 20 - 1, 807 pci_addr + (reg_no << 20), 808 0); 809 if (err) { 810 dev_err(dev, "program RC io outbound ATU failed\n"); 811 return err; 812 } 813 } 814 815 /* assign message regions */ 816 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, 817 AXI_WRAPPER_NOR_MSG, 818 20 - 1, 0, 0); 819 820 rockchip->msg_bus_addr += ((reg_no + offset) << 20); 821 return err; 822 } 823 824 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) 825 { 826 u32 value; 827 int err; 828 829 /* send PME_TURN_OFF message */ 830 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); 831 832 /* read LTSSM and wait for falling into L2 link state */ 833 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, 834 value, PCIE_LINK_IS_L2(value), 20, 835 jiffies_to_usecs(5 * HZ)); 836 if (err) { 837 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); 838 return err; 839 } 840 841 return 0; 842 } 843 844 static int rockchip_pcie_suspend_noirq(struct device *dev) 845 { 846 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 847 int ret; 848 849 /* disable core and cli int since we don't need to ack PME_ACK */ 850 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | 851 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); 852 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); 853 854 ret = rockchip_pcie_wait_l2(rockchip); 855 if (ret) { 856 rockchip_pcie_enable_interrupts(rockchip); 857 return ret; 858 } 859 860 rockchip_pcie_deinit_phys(rockchip); 861 862 rockchip_pcie_disable_clocks(rockchip); 863 864 regulator_disable(rockchip->vpcie0v9); 865 866 return ret; 867 } 868 869 static int rockchip_pcie_resume_noirq(struct device *dev) 870 { 871 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 872 int err; 873 874 err = regulator_enable(rockchip->vpcie0v9); 875 if (err) { 876 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 877 return err; 878 } 879 880 err = rockchip_pcie_enable_clocks(rockchip); 881 if (err) 882 goto err_disable_0v9; 883 884 err = rockchip_pcie_host_init_port(rockchip); 885 if (err) 886 goto err_pcie_resume; 887 888 err = rockchip_pcie_cfg_atu(rockchip); 889 if (err) 890 goto err_err_deinit_port; 891 892 /* Need this to enter L1 again */ 893 rockchip_pcie_update_txcredit_mui(rockchip); 894 rockchip_pcie_enable_interrupts(rockchip); 895 896 return 0; 897 898 err_err_deinit_port: 899 rockchip_pcie_deinit_phys(rockchip); 900 err_pcie_resume: 901 rockchip_pcie_disable_clocks(rockchip); 902 err_disable_0v9: 903 regulator_disable(rockchip->vpcie0v9); 904 return err; 905 } 906 907 static int rockchip_pcie_probe(struct platform_device *pdev) 908 { 909 struct rockchip_pcie *rockchip; 910 struct device *dev = &pdev->dev; 911 struct pci_host_bridge *bridge; 912 int err; 913 914 if (!dev->of_node) 915 return -ENODEV; 916 917 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); 918 if (!bridge) 919 return -ENOMEM; 920 921 rockchip = pci_host_bridge_priv(bridge); 922 923 platform_set_drvdata(pdev, rockchip); 924 rockchip->dev = dev; 925 rockchip->is_rc = true; 926 927 err = rockchip_pcie_parse_host_dt(rockchip); 928 if (err) 929 return err; 930 931 err = rockchip_pcie_enable_clocks(rockchip); 932 if (err) 933 return err; 934 935 err = rockchip_pcie_set_vpcie(rockchip); 936 if (err) { 937 dev_err(dev, "failed to set vpcie regulator\n"); 938 goto err_set_vpcie; 939 } 940 941 err = rockchip_pcie_host_init_port(rockchip); 942 if (err) 943 goto err_vpcie; 944 945 err = rockchip_pcie_init_irq_domain(rockchip); 946 if (err < 0) 947 goto err_deinit_port; 948 949 err = rockchip_pcie_cfg_atu(rockchip); 950 if (err) 951 goto err_remove_irq_domain; 952 953 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); 954 if (!rockchip->msg_region) { 955 err = -ENOMEM; 956 goto err_remove_irq_domain; 957 } 958 959 bridge->sysdata = rockchip; 960 bridge->ops = &rockchip_pcie_ops; 961 962 err = rockchip_pcie_setup_irq(rockchip); 963 if (err) 964 goto err_remove_irq_domain; 965 966 rockchip_pcie_enable_interrupts(rockchip); 967 968 err = pci_host_probe(bridge); 969 if (err < 0) 970 goto err_remove_irq_domain; 971 972 return 0; 973 974 err_remove_irq_domain: 975 irq_domain_remove(rockchip->irq_domain); 976 err_deinit_port: 977 rockchip_pcie_deinit_phys(rockchip); 978 err_vpcie: 979 if (!IS_ERR(rockchip->vpcie12v)) 980 regulator_disable(rockchip->vpcie12v); 981 if (!IS_ERR(rockchip->vpcie3v3)) 982 regulator_disable(rockchip->vpcie3v3); 983 regulator_disable(rockchip->vpcie1v8); 984 regulator_disable(rockchip->vpcie0v9); 985 err_set_vpcie: 986 rockchip_pcie_disable_clocks(rockchip); 987 return err; 988 } 989 990 static void rockchip_pcie_remove(struct platform_device *pdev) 991 { 992 struct device *dev = &pdev->dev; 993 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 994 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 995 996 pci_lock_rescan_remove(); 997 pci_stop_root_bus(bridge->bus); 998 pci_remove_root_bus(bridge->bus); 999 pci_unlock_rescan_remove(); 1000 irq_domain_remove(rockchip->irq_domain); 1001 1002 rockchip_pcie_deinit_phys(rockchip); 1003 1004 rockchip_pcie_disable_clocks(rockchip); 1005 1006 if (!IS_ERR(rockchip->vpcie12v)) 1007 regulator_disable(rockchip->vpcie12v); 1008 if (!IS_ERR(rockchip->vpcie3v3)) 1009 regulator_disable(rockchip->vpcie3v3); 1010 regulator_disable(rockchip->vpcie1v8); 1011 regulator_disable(rockchip->vpcie0v9); 1012 } 1013 1014 static const struct dev_pm_ops rockchip_pcie_pm_ops = { 1015 NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, 1016 rockchip_pcie_resume_noirq) 1017 }; 1018 1019 static const struct of_device_id rockchip_pcie_of_match[] = { 1020 { .compatible = "rockchip,rk3399-pcie", }, 1021 {} 1022 }; 1023 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); 1024 1025 static struct platform_driver rockchip_pcie_driver = { 1026 .driver = { 1027 .name = "rockchip-pcie", 1028 .of_match_table = rockchip_pcie_of_match, 1029 .pm = &rockchip_pcie_pm_ops, 1030 }, 1031 .probe = rockchip_pcie_probe, 1032 .remove = rockchip_pcie_remove, 1033 }; 1034 module_platform_driver(rockchip_pcie_driver); 1035 1036 MODULE_AUTHOR("Rockchip Inc"); 1037 MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); 1038 MODULE_LICENSE("GPL v2"); 1039