1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * MediaTek PCIe host controller driver. 4 * 5 * Copyright (c) 2020 MediaTek Inc. 6 * Author: Jianjun Wang <jianjun.wang@mediatek.com> 7 */ 8 9 #include <linux/clk.h> 10 #include <linux/delay.h> 11 #include <linux/iopoll.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip/chained_irq.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/msi.h> 18 #include <linux/pci.h> 19 #include <linux/phy/phy.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm_domain.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/reset.h> 24 25 #include "../pci.h" 26 27 #define PCIE_SETTING_REG 0x80 28 #define PCIE_PCI_IDS_1 0x9c 29 #define PCI_CLASS(class) (class << 8) 30 #define PCIE_RC_MODE BIT(0) 31 32 #define PCIE_CFGNUM_REG 0x140 33 #define PCIE_CFG_DEVFN(devfn) ((devfn) & GENMASK(7, 0)) 34 #define PCIE_CFG_BUS(bus) (((bus) << 8) & GENMASK(15, 8)) 35 #define PCIE_CFG_BYTE_EN(bytes) (((bytes) << 16) & GENMASK(19, 16)) 36 #define PCIE_CFG_FORCE_BYTE_EN BIT(20) 37 #define PCIE_CFG_OFFSET_ADDR 0x1000 38 #define PCIE_CFG_HEADER(bus, devfn) \ 39 (PCIE_CFG_BUS(bus) | PCIE_CFG_DEVFN(devfn)) 40 41 #define PCIE_RST_CTRL_REG 0x148 42 #define PCIE_MAC_RSTB BIT(0) 43 #define PCIE_PHY_RSTB BIT(1) 44 #define PCIE_BRG_RSTB BIT(2) 45 #define PCIE_PE_RSTB BIT(3) 46 47 #define PCIE_LTSSM_STATUS_REG 0x150 48 #define PCIE_LTSSM_STATE_MASK GENMASK(28, 24) 49 #define PCIE_LTSSM_STATE(val) ((val & PCIE_LTSSM_STATE_MASK) >> 24) 50 #define PCIE_LTSSM_STATE_L2_IDLE 0x14 51 52 #define PCIE_LINK_STATUS_REG 0x154 53 #define PCIE_PORT_LINKUP BIT(8) 54 55 #define PCIE_MSI_SET_NUM 8 56 #define PCIE_MSI_IRQS_PER_SET 32 57 #define PCIE_MSI_IRQS_NUM \ 58 (PCIE_MSI_IRQS_PER_SET * PCIE_MSI_SET_NUM) 59 60 #define PCIE_INT_ENABLE_REG 0x180 61 #define PCIE_MSI_ENABLE GENMASK(PCIE_MSI_SET_NUM + 8 - 1, 8) 62 #define PCIE_MSI_SHIFT 8 63 #define PCIE_INTX_SHIFT 24 64 #define PCIE_INTX_ENABLE \ 65 GENMASK(PCIE_INTX_SHIFT + PCI_NUM_INTX - 1, PCIE_INTX_SHIFT) 66 67 #define PCIE_INT_STATUS_REG 0x184 68 #define PCIE_MSI_SET_ENABLE_REG 0x190 69 #define PCIE_MSI_SET_ENABLE GENMASK(PCIE_MSI_SET_NUM - 1, 0) 70 71 #define PCIE_MSI_SET_BASE_REG 0xc00 72 #define PCIE_MSI_SET_OFFSET 0x10 73 #define PCIE_MSI_SET_STATUS_OFFSET 0x04 74 #define PCIE_MSI_SET_ENABLE_OFFSET 0x08 75 76 #define PCIE_MSI_SET_ADDR_HI_BASE 0xc80 77 #define PCIE_MSI_SET_ADDR_HI_OFFSET 0x04 78 79 #define PCIE_ICMD_PM_REG 0x198 80 #define PCIE_TURN_OFF_LINK BIT(4) 81 82 #define PCIE_MISC_CTRL_REG 0x348 83 #define PCIE_DISABLE_DVFSRC_VLT_REQ BIT(1) 84 85 #define PCIE_TRANS_TABLE_BASE_REG 0x800 86 #define PCIE_ATR_SRC_ADDR_MSB_OFFSET 0x4 87 #define PCIE_ATR_TRSL_ADDR_LSB_OFFSET 0x8 88 #define PCIE_ATR_TRSL_ADDR_MSB_OFFSET 0xc 89 #define PCIE_ATR_TRSL_PARAM_OFFSET 0x10 90 #define PCIE_ATR_TLB_SET_OFFSET 0x20 91 92 #define PCIE_MAX_TRANS_TABLES 8 93 #define PCIE_ATR_EN BIT(0) 94 #define PCIE_ATR_SIZE(size) \ 95 (((((size) - 1) << 1) & GENMASK(6, 1)) | PCIE_ATR_EN) 96 #define PCIE_ATR_ID(id) ((id) & GENMASK(3, 0)) 97 #define PCIE_ATR_TYPE_MEM PCIE_ATR_ID(0) 98 #define PCIE_ATR_TYPE_IO PCIE_ATR_ID(1) 99 #define PCIE_ATR_TLP_TYPE(type) (((type) << 16) & GENMASK(18, 16)) 100 #define PCIE_ATR_TLP_TYPE_MEM PCIE_ATR_TLP_TYPE(0) 101 #define PCIE_ATR_TLP_TYPE_IO PCIE_ATR_TLP_TYPE(2) 102 103 /** 104 * struct mtk_msi_set - MSI information for each set 105 * @base: IO mapped register base 106 * @msg_addr: MSI message address 107 * @saved_irq_state: IRQ enable state saved at suspend time 108 */ 109 struct mtk_msi_set { 110 void __iomem *base; 111 phys_addr_t msg_addr; 112 u32 saved_irq_state; 113 }; 114 115 /** 116 * struct mtk_gen3_pcie - PCIe port information 117 * @dev: pointer to PCIe device 118 * @base: IO mapped register base 119 * @reg_base: physical register base 120 * @mac_reset: MAC reset control 121 * @phy_reset: PHY reset control 122 * @phy: PHY controller block 123 * @clks: PCIe clocks 124 * @num_clks: PCIe clocks count for this port 125 * @irq: PCIe controller interrupt number 126 * @saved_irq_state: IRQ enable state saved at suspend time 127 * @irq_lock: lock protecting IRQ register access 128 * @intx_domain: legacy INTx IRQ domain 129 * @msi_domain: MSI IRQ domain 130 * @msi_bottom_domain: MSI IRQ bottom domain 131 * @msi_sets: MSI sets information 132 * @lock: lock protecting IRQ bit map 133 * @msi_irq_in_use: bit map for assigned MSI IRQ 134 */ 135 struct mtk_gen3_pcie { 136 struct device *dev; 137 void __iomem *base; 138 phys_addr_t reg_base; 139 struct reset_control *mac_reset; 140 struct reset_control *phy_reset; 141 struct phy *phy; 142 struct clk_bulk_data *clks; 143 int num_clks; 144 145 int irq; 146 u32 saved_irq_state; 147 raw_spinlock_t irq_lock; 148 struct irq_domain *intx_domain; 149 struct irq_domain *msi_domain; 150 struct irq_domain *msi_bottom_domain; 151 struct mtk_msi_set msi_sets[PCIE_MSI_SET_NUM]; 152 struct mutex lock; 153 DECLARE_BITMAP(msi_irq_in_use, PCIE_MSI_IRQS_NUM); 154 }; 155 156 /* LTSSM state in PCIE_LTSSM_STATUS_REG bit[28:24] */ 157 static const char *const ltssm_str[] = { 158 "detect.quiet", /* 0x00 */ 159 "detect.active", /* 0x01 */ 160 "polling.active", /* 0x02 */ 161 "polling.compliance", /* 0x03 */ 162 "polling.configuration", /* 0x04 */ 163 "config.linkwidthstart", /* 0x05 */ 164 "config.linkwidthaccept", /* 0x06 */ 165 "config.lanenumwait", /* 0x07 */ 166 "config.lanenumaccept", /* 0x08 */ 167 "config.complete", /* 0x09 */ 168 "config.idle", /* 0x0A */ 169 "recovery.receiverlock", /* 0x0B */ 170 "recovery.equalization", /* 0x0C */ 171 "recovery.speed", /* 0x0D */ 172 "recovery.receiverconfig", /* 0x0E */ 173 "recovery.idle", /* 0x0F */ 174 "L0", /* 0x10 */ 175 "L0s", /* 0x11 */ 176 "L1.entry", /* 0x12 */ 177 "L1.idle", /* 0x13 */ 178 "L2.idle", /* 0x14 */ 179 "L2.transmitwake", /* 0x15 */ 180 "disable", /* 0x16 */ 181 "loopback.entry", /* 0x17 */ 182 "loopback.active", /* 0x18 */ 183 "loopback.exit", /* 0x19 */ 184 "hotreset", /* 0x1A */ 185 }; 186 187 /** 188 * mtk_pcie_config_tlp_header() - Configure a configuration TLP header 189 * @bus: PCI bus to query 190 * @devfn: device/function number 191 * @where: offset in config space 192 * @size: data size in TLP header 193 * 194 * Set byte enable field and device information in configuration TLP header. 195 */ 196 static void mtk_pcie_config_tlp_header(struct pci_bus *bus, unsigned int devfn, 197 int where, int size) 198 { 199 struct mtk_gen3_pcie *pcie = bus->sysdata; 200 int bytes; 201 u32 val; 202 203 bytes = (GENMASK(size - 1, 0) & 0xf) << (where & 0x3); 204 205 val = PCIE_CFG_FORCE_BYTE_EN | PCIE_CFG_BYTE_EN(bytes) | 206 PCIE_CFG_HEADER(bus->number, devfn); 207 208 writel_relaxed(val, pcie->base + PCIE_CFGNUM_REG); 209 } 210 211 static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, 212 int where) 213 { 214 struct mtk_gen3_pcie *pcie = bus->sysdata; 215 216 return pcie->base + PCIE_CFG_OFFSET_ADDR + where; 217 } 218 219 static int mtk_pcie_config_read(struct pci_bus *bus, unsigned int devfn, 220 int where, int size, u32 *val) 221 { 222 mtk_pcie_config_tlp_header(bus, devfn, where, size); 223 224 return pci_generic_config_read32(bus, devfn, where, size, val); 225 } 226 227 static int mtk_pcie_config_write(struct pci_bus *bus, unsigned int devfn, 228 int where, int size, u32 val) 229 { 230 mtk_pcie_config_tlp_header(bus, devfn, where, size); 231 232 if (size <= 2) 233 val <<= (where & 0x3) * 8; 234 235 return pci_generic_config_write32(bus, devfn, where, 4, val); 236 } 237 238 static struct pci_ops mtk_pcie_ops = { 239 .map_bus = mtk_pcie_map_bus, 240 .read = mtk_pcie_config_read, 241 .write = mtk_pcie_config_write, 242 }; 243 244 static int mtk_pcie_set_trans_table(struct mtk_gen3_pcie *pcie, 245 resource_size_t cpu_addr, 246 resource_size_t pci_addr, 247 resource_size_t size, 248 unsigned long type, int num) 249 { 250 void __iomem *table; 251 u32 val; 252 253 if (num >= PCIE_MAX_TRANS_TABLES) { 254 dev_err(pcie->dev, "not enough translate table for addr: %#llx, limited to [%d]\n", 255 (unsigned long long)cpu_addr, PCIE_MAX_TRANS_TABLES); 256 return -ENODEV; 257 } 258 259 table = pcie->base + PCIE_TRANS_TABLE_BASE_REG + 260 num * PCIE_ATR_TLB_SET_OFFSET; 261 262 writel_relaxed(lower_32_bits(cpu_addr) | PCIE_ATR_SIZE(fls(size) - 1), 263 table); 264 writel_relaxed(upper_32_bits(cpu_addr), 265 table + PCIE_ATR_SRC_ADDR_MSB_OFFSET); 266 writel_relaxed(lower_32_bits(pci_addr), 267 table + PCIE_ATR_TRSL_ADDR_LSB_OFFSET); 268 writel_relaxed(upper_32_bits(pci_addr), 269 table + PCIE_ATR_TRSL_ADDR_MSB_OFFSET); 270 271 if (type == IORESOURCE_IO) 272 val = PCIE_ATR_TYPE_IO | PCIE_ATR_TLP_TYPE_IO; 273 else 274 val = PCIE_ATR_TYPE_MEM | PCIE_ATR_TLP_TYPE_MEM; 275 276 writel_relaxed(val, table + PCIE_ATR_TRSL_PARAM_OFFSET); 277 278 return 0; 279 } 280 281 static void mtk_pcie_enable_msi(struct mtk_gen3_pcie *pcie) 282 { 283 int i; 284 u32 val; 285 286 for (i = 0; i < PCIE_MSI_SET_NUM; i++) { 287 struct mtk_msi_set *msi_set = &pcie->msi_sets[i]; 288 289 msi_set->base = pcie->base + PCIE_MSI_SET_BASE_REG + 290 i * PCIE_MSI_SET_OFFSET; 291 msi_set->msg_addr = pcie->reg_base + PCIE_MSI_SET_BASE_REG + 292 i * PCIE_MSI_SET_OFFSET; 293 294 /* Configure the MSI capture address */ 295 writel_relaxed(lower_32_bits(msi_set->msg_addr), msi_set->base); 296 writel_relaxed(upper_32_bits(msi_set->msg_addr), 297 pcie->base + PCIE_MSI_SET_ADDR_HI_BASE + 298 i * PCIE_MSI_SET_ADDR_HI_OFFSET); 299 } 300 301 val = readl_relaxed(pcie->base + PCIE_MSI_SET_ENABLE_REG); 302 val |= PCIE_MSI_SET_ENABLE; 303 writel_relaxed(val, pcie->base + PCIE_MSI_SET_ENABLE_REG); 304 305 val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); 306 val |= PCIE_MSI_ENABLE; 307 writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG); 308 } 309 310 static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie) 311 { 312 struct resource_entry *entry; 313 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 314 unsigned int table_index = 0; 315 int err; 316 u32 val; 317 318 /* Set as RC mode */ 319 val = readl_relaxed(pcie->base + PCIE_SETTING_REG); 320 val |= PCIE_RC_MODE; 321 writel_relaxed(val, pcie->base + PCIE_SETTING_REG); 322 323 /* Set class code */ 324 val = readl_relaxed(pcie->base + PCIE_PCI_IDS_1); 325 val &= ~GENMASK(31, 8); 326 val |= PCI_CLASS(PCI_CLASS_BRIDGE_PCI_NORMAL); 327 writel_relaxed(val, pcie->base + PCIE_PCI_IDS_1); 328 329 /* Mask all INTx interrupts */ 330 val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); 331 val &= ~PCIE_INTX_ENABLE; 332 writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG); 333 334 /* Disable DVFSRC voltage request */ 335 val = readl_relaxed(pcie->base + PCIE_MISC_CTRL_REG); 336 val |= PCIE_DISABLE_DVFSRC_VLT_REQ; 337 writel_relaxed(val, pcie->base + PCIE_MISC_CTRL_REG); 338 339 /* Assert all reset signals */ 340 val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG); 341 val |= PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB | PCIE_PE_RSTB; 342 writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG); 343 344 /* 345 * Described in PCIe CEM specification sections 2.2 (PERST# Signal) 346 * and 2.2.1 (Initial Power-Up (G3 to S0)). 347 * The deassertion of PERST# should be delayed 100ms (TPVPERL) 348 * for the power and clock to become stable. 349 */ 350 msleep(100); 351 352 /* De-assert reset signals */ 353 val &= ~(PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB | PCIE_PE_RSTB); 354 writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG); 355 356 /* Check if the link is up or not */ 357 err = readl_poll_timeout(pcie->base + PCIE_LINK_STATUS_REG, val, 358 !!(val & PCIE_PORT_LINKUP), 20, 359 PCI_PM_D3COLD_WAIT * USEC_PER_MSEC); 360 if (err) { 361 const char *ltssm_state; 362 int ltssm_index; 363 364 val = readl_relaxed(pcie->base + PCIE_LTSSM_STATUS_REG); 365 ltssm_index = PCIE_LTSSM_STATE(val); 366 ltssm_state = ltssm_index >= ARRAY_SIZE(ltssm_str) ? 367 "Unknown state" : ltssm_str[ltssm_index]; 368 dev_err(pcie->dev, 369 "PCIe link down, current LTSSM state: %s (%#x)\n", 370 ltssm_state, val); 371 return err; 372 } 373 374 mtk_pcie_enable_msi(pcie); 375 376 /* Set PCIe translation windows */ 377 resource_list_for_each_entry(entry, &host->windows) { 378 struct resource *res = entry->res; 379 unsigned long type = resource_type(res); 380 resource_size_t cpu_addr; 381 resource_size_t pci_addr; 382 resource_size_t size; 383 const char *range_type; 384 385 if (type == IORESOURCE_IO) { 386 cpu_addr = pci_pio_to_address(res->start); 387 range_type = "IO"; 388 } else if (type == IORESOURCE_MEM) { 389 cpu_addr = res->start; 390 range_type = "MEM"; 391 } else { 392 continue; 393 } 394 395 pci_addr = res->start - entry->offset; 396 size = resource_size(res); 397 err = mtk_pcie_set_trans_table(pcie, cpu_addr, pci_addr, size, 398 type, table_index); 399 if (err) 400 return err; 401 402 dev_dbg(pcie->dev, "set %s trans window[%d]: cpu_addr = %#llx, pci_addr = %#llx, size = %#llx\n", 403 range_type, table_index, (unsigned long long)cpu_addr, 404 (unsigned long long)pci_addr, (unsigned long long)size); 405 406 table_index++; 407 } 408 409 return 0; 410 } 411 412 static int mtk_pcie_set_affinity(struct irq_data *data, 413 const struct cpumask *mask, bool force) 414 { 415 return -EINVAL; 416 } 417 418 static void mtk_pcie_msi_irq_mask(struct irq_data *data) 419 { 420 pci_msi_mask_irq(data); 421 irq_chip_mask_parent(data); 422 } 423 424 static void mtk_pcie_msi_irq_unmask(struct irq_data *data) 425 { 426 pci_msi_unmask_irq(data); 427 irq_chip_unmask_parent(data); 428 } 429 430 static struct irq_chip mtk_msi_irq_chip = { 431 .irq_ack = irq_chip_ack_parent, 432 .irq_mask = mtk_pcie_msi_irq_mask, 433 .irq_unmask = mtk_pcie_msi_irq_unmask, 434 .name = "MSI", 435 }; 436 437 static struct msi_domain_info mtk_msi_domain_info = { 438 .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | 439 MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI), 440 .chip = &mtk_msi_irq_chip, 441 }; 442 443 static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) 444 { 445 struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data); 446 struct mtk_gen3_pcie *pcie = data->domain->host_data; 447 unsigned long hwirq; 448 449 hwirq = data->hwirq % PCIE_MSI_IRQS_PER_SET; 450 451 msg->address_hi = upper_32_bits(msi_set->msg_addr); 452 msg->address_lo = lower_32_bits(msi_set->msg_addr); 453 msg->data = hwirq; 454 dev_dbg(pcie->dev, "msi#%#lx address_hi %#x address_lo %#x data %d\n", 455 hwirq, msg->address_hi, msg->address_lo, msg->data); 456 } 457 458 static void mtk_msi_bottom_irq_ack(struct irq_data *data) 459 { 460 struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data); 461 unsigned long hwirq; 462 463 hwirq = data->hwirq % PCIE_MSI_IRQS_PER_SET; 464 465 writel_relaxed(BIT(hwirq), msi_set->base + PCIE_MSI_SET_STATUS_OFFSET); 466 } 467 468 static void mtk_msi_bottom_irq_mask(struct irq_data *data) 469 { 470 struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data); 471 struct mtk_gen3_pcie *pcie = data->domain->host_data; 472 unsigned long hwirq, flags; 473 u32 val; 474 475 hwirq = data->hwirq % PCIE_MSI_IRQS_PER_SET; 476 477 raw_spin_lock_irqsave(&pcie->irq_lock, flags); 478 val = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); 479 val &= ~BIT(hwirq); 480 writel_relaxed(val, msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); 481 raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); 482 } 483 484 static void mtk_msi_bottom_irq_unmask(struct irq_data *data) 485 { 486 struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data); 487 struct mtk_gen3_pcie *pcie = data->domain->host_data; 488 unsigned long hwirq, flags; 489 u32 val; 490 491 hwirq = data->hwirq % PCIE_MSI_IRQS_PER_SET; 492 493 raw_spin_lock_irqsave(&pcie->irq_lock, flags); 494 val = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); 495 val |= BIT(hwirq); 496 writel_relaxed(val, msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); 497 raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); 498 } 499 500 static struct irq_chip mtk_msi_bottom_irq_chip = { 501 .irq_ack = mtk_msi_bottom_irq_ack, 502 .irq_mask = mtk_msi_bottom_irq_mask, 503 .irq_unmask = mtk_msi_bottom_irq_unmask, 504 .irq_compose_msi_msg = mtk_compose_msi_msg, 505 .irq_set_affinity = mtk_pcie_set_affinity, 506 .name = "MSI", 507 }; 508 509 static int mtk_msi_bottom_domain_alloc(struct irq_domain *domain, 510 unsigned int virq, unsigned int nr_irqs, 511 void *arg) 512 { 513 struct mtk_gen3_pcie *pcie = domain->host_data; 514 struct mtk_msi_set *msi_set; 515 int i, hwirq, set_idx; 516 517 mutex_lock(&pcie->lock); 518 519 hwirq = bitmap_find_free_region(pcie->msi_irq_in_use, PCIE_MSI_IRQS_NUM, 520 order_base_2(nr_irqs)); 521 522 mutex_unlock(&pcie->lock); 523 524 if (hwirq < 0) 525 return -ENOSPC; 526 527 set_idx = hwirq / PCIE_MSI_IRQS_PER_SET; 528 msi_set = &pcie->msi_sets[set_idx]; 529 530 for (i = 0; i < nr_irqs; i++) 531 irq_domain_set_info(domain, virq + i, hwirq + i, 532 &mtk_msi_bottom_irq_chip, msi_set, 533 handle_edge_irq, NULL, NULL); 534 535 return 0; 536 } 537 538 static void mtk_msi_bottom_domain_free(struct irq_domain *domain, 539 unsigned int virq, unsigned int nr_irqs) 540 { 541 struct mtk_gen3_pcie *pcie = domain->host_data; 542 struct irq_data *data = irq_domain_get_irq_data(domain, virq); 543 544 mutex_lock(&pcie->lock); 545 546 bitmap_release_region(pcie->msi_irq_in_use, data->hwirq, 547 order_base_2(nr_irqs)); 548 549 mutex_unlock(&pcie->lock); 550 551 irq_domain_free_irqs_common(domain, virq, nr_irqs); 552 } 553 554 static const struct irq_domain_ops mtk_msi_bottom_domain_ops = { 555 .alloc = mtk_msi_bottom_domain_alloc, 556 .free = mtk_msi_bottom_domain_free, 557 }; 558 559 static void mtk_intx_mask(struct irq_data *data) 560 { 561 struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data); 562 unsigned long flags; 563 u32 val; 564 565 raw_spin_lock_irqsave(&pcie->irq_lock, flags); 566 val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); 567 val &= ~BIT(data->hwirq + PCIE_INTX_SHIFT); 568 writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG); 569 raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); 570 } 571 572 static void mtk_intx_unmask(struct irq_data *data) 573 { 574 struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data); 575 unsigned long flags; 576 u32 val; 577 578 raw_spin_lock_irqsave(&pcie->irq_lock, flags); 579 val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); 580 val |= BIT(data->hwirq + PCIE_INTX_SHIFT); 581 writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG); 582 raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); 583 } 584 585 /** 586 * mtk_intx_eoi() - Clear INTx IRQ status at the end of interrupt 587 * @data: pointer to chip specific data 588 * 589 * As an emulated level IRQ, its interrupt status will remain 590 * until the corresponding de-assert message is received; hence that 591 * the status can only be cleared when the interrupt has been serviced. 592 */ 593 static void mtk_intx_eoi(struct irq_data *data) 594 { 595 struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data); 596 unsigned long hwirq; 597 598 hwirq = data->hwirq + PCIE_INTX_SHIFT; 599 writel_relaxed(BIT(hwirq), pcie->base + PCIE_INT_STATUS_REG); 600 } 601 602 static struct irq_chip mtk_intx_irq_chip = { 603 .irq_mask = mtk_intx_mask, 604 .irq_unmask = mtk_intx_unmask, 605 .irq_eoi = mtk_intx_eoi, 606 .irq_set_affinity = mtk_pcie_set_affinity, 607 .name = "INTx", 608 }; 609 610 static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 611 irq_hw_number_t hwirq) 612 { 613 irq_set_chip_data(irq, domain->host_data); 614 irq_set_chip_and_handler_name(irq, &mtk_intx_irq_chip, 615 handle_fasteoi_irq, "INTx"); 616 return 0; 617 } 618 619 static const struct irq_domain_ops intx_domain_ops = { 620 .map = mtk_pcie_intx_map, 621 }; 622 623 static int mtk_pcie_init_irq_domains(struct mtk_gen3_pcie *pcie) 624 { 625 struct device *dev = pcie->dev; 626 struct device_node *intc_node, *node = dev->of_node; 627 int ret; 628 629 raw_spin_lock_init(&pcie->irq_lock); 630 631 /* Setup INTx */ 632 intc_node = of_get_child_by_name(node, "interrupt-controller"); 633 if (!intc_node) { 634 dev_err(dev, "missing interrupt-controller node\n"); 635 return -ENODEV; 636 } 637 638 pcie->intx_domain = irq_domain_add_linear(intc_node, PCI_NUM_INTX, 639 &intx_domain_ops, pcie); 640 if (!pcie->intx_domain) { 641 dev_err(dev, "failed to create INTx IRQ domain\n"); 642 ret = -ENODEV; 643 goto out_put_node; 644 } 645 646 /* Setup MSI */ 647 mutex_init(&pcie->lock); 648 649 pcie->msi_bottom_domain = irq_domain_add_linear(node, PCIE_MSI_IRQS_NUM, 650 &mtk_msi_bottom_domain_ops, pcie); 651 if (!pcie->msi_bottom_domain) { 652 dev_err(dev, "failed to create MSI bottom domain\n"); 653 ret = -ENODEV; 654 goto err_msi_bottom_domain; 655 } 656 657 pcie->msi_domain = pci_msi_create_irq_domain(dev->fwnode, 658 &mtk_msi_domain_info, 659 pcie->msi_bottom_domain); 660 if (!pcie->msi_domain) { 661 dev_err(dev, "failed to create MSI domain\n"); 662 ret = -ENODEV; 663 goto err_msi_domain; 664 } 665 666 of_node_put(intc_node); 667 return 0; 668 669 err_msi_domain: 670 irq_domain_remove(pcie->msi_bottom_domain); 671 err_msi_bottom_domain: 672 irq_domain_remove(pcie->intx_domain); 673 out_put_node: 674 of_node_put(intc_node); 675 return ret; 676 } 677 678 static void mtk_pcie_irq_teardown(struct mtk_gen3_pcie *pcie) 679 { 680 irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); 681 682 if (pcie->intx_domain) 683 irq_domain_remove(pcie->intx_domain); 684 685 if (pcie->msi_domain) 686 irq_domain_remove(pcie->msi_domain); 687 688 if (pcie->msi_bottom_domain) 689 irq_domain_remove(pcie->msi_bottom_domain); 690 691 irq_dispose_mapping(pcie->irq); 692 } 693 694 static void mtk_pcie_msi_handler(struct mtk_gen3_pcie *pcie, int set_idx) 695 { 696 struct mtk_msi_set *msi_set = &pcie->msi_sets[set_idx]; 697 unsigned long msi_enable, msi_status; 698 irq_hw_number_t bit, hwirq; 699 700 msi_enable = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); 701 702 do { 703 msi_status = readl_relaxed(msi_set->base + 704 PCIE_MSI_SET_STATUS_OFFSET); 705 msi_status &= msi_enable; 706 if (!msi_status) 707 break; 708 709 for_each_set_bit(bit, &msi_status, PCIE_MSI_IRQS_PER_SET) { 710 hwirq = bit + set_idx * PCIE_MSI_IRQS_PER_SET; 711 generic_handle_domain_irq(pcie->msi_bottom_domain, hwirq); 712 } 713 } while (true); 714 } 715 716 static void mtk_pcie_irq_handler(struct irq_desc *desc) 717 { 718 struct mtk_gen3_pcie *pcie = irq_desc_get_handler_data(desc); 719 struct irq_chip *irqchip = irq_desc_get_chip(desc); 720 unsigned long status; 721 irq_hw_number_t irq_bit = PCIE_INTX_SHIFT; 722 723 chained_irq_enter(irqchip, desc); 724 725 status = readl_relaxed(pcie->base + PCIE_INT_STATUS_REG); 726 for_each_set_bit_from(irq_bit, &status, PCI_NUM_INTX + 727 PCIE_INTX_SHIFT) 728 generic_handle_domain_irq(pcie->intx_domain, 729 irq_bit - PCIE_INTX_SHIFT); 730 731 irq_bit = PCIE_MSI_SHIFT; 732 for_each_set_bit_from(irq_bit, &status, PCIE_MSI_SET_NUM + 733 PCIE_MSI_SHIFT) { 734 mtk_pcie_msi_handler(pcie, irq_bit - PCIE_MSI_SHIFT); 735 736 writel_relaxed(BIT(irq_bit), pcie->base + PCIE_INT_STATUS_REG); 737 } 738 739 chained_irq_exit(irqchip, desc); 740 } 741 742 static int mtk_pcie_setup_irq(struct mtk_gen3_pcie *pcie) 743 { 744 struct device *dev = pcie->dev; 745 struct platform_device *pdev = to_platform_device(dev); 746 int err; 747 748 err = mtk_pcie_init_irq_domains(pcie); 749 if (err) 750 return err; 751 752 pcie->irq = platform_get_irq(pdev, 0); 753 if (pcie->irq < 0) 754 return pcie->irq; 755 756 irq_set_chained_handler_and_data(pcie->irq, mtk_pcie_irq_handler, pcie); 757 758 return 0; 759 } 760 761 static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie) 762 { 763 struct device *dev = pcie->dev; 764 struct platform_device *pdev = to_platform_device(dev); 765 struct resource *regs; 766 int ret; 767 768 regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-mac"); 769 if (!regs) 770 return -EINVAL; 771 pcie->base = devm_ioremap_resource(dev, regs); 772 if (IS_ERR(pcie->base)) { 773 dev_err(dev, "failed to map register base\n"); 774 return PTR_ERR(pcie->base); 775 } 776 777 pcie->reg_base = regs->start; 778 779 pcie->phy_reset = devm_reset_control_get_optional_exclusive(dev, "phy"); 780 if (IS_ERR(pcie->phy_reset)) { 781 ret = PTR_ERR(pcie->phy_reset); 782 if (ret != -EPROBE_DEFER) 783 dev_err(dev, "failed to get PHY reset\n"); 784 785 return ret; 786 } 787 788 pcie->mac_reset = devm_reset_control_get_optional_exclusive(dev, "mac"); 789 if (IS_ERR(pcie->mac_reset)) { 790 ret = PTR_ERR(pcie->mac_reset); 791 if (ret != -EPROBE_DEFER) 792 dev_err(dev, "failed to get MAC reset\n"); 793 794 return ret; 795 } 796 797 pcie->phy = devm_phy_optional_get(dev, "pcie-phy"); 798 if (IS_ERR(pcie->phy)) { 799 ret = PTR_ERR(pcie->phy); 800 if (ret != -EPROBE_DEFER) 801 dev_err(dev, "failed to get PHY\n"); 802 803 return ret; 804 } 805 806 pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks); 807 if (pcie->num_clks < 0) { 808 dev_err(dev, "failed to get clocks\n"); 809 return pcie->num_clks; 810 } 811 812 return 0; 813 } 814 815 static int mtk_pcie_power_up(struct mtk_gen3_pcie *pcie) 816 { 817 struct device *dev = pcie->dev; 818 int err; 819 820 /* PHY power on and enable pipe clock */ 821 reset_control_deassert(pcie->phy_reset); 822 823 err = phy_init(pcie->phy); 824 if (err) { 825 dev_err(dev, "failed to initialize PHY\n"); 826 goto err_phy_init; 827 } 828 829 err = phy_power_on(pcie->phy); 830 if (err) { 831 dev_err(dev, "failed to power on PHY\n"); 832 goto err_phy_on; 833 } 834 835 /* MAC power on and enable transaction layer clocks */ 836 reset_control_deassert(pcie->mac_reset); 837 838 pm_runtime_enable(dev); 839 pm_runtime_get_sync(dev); 840 841 err = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks); 842 if (err) { 843 dev_err(dev, "failed to enable clocks\n"); 844 goto err_clk_init; 845 } 846 847 return 0; 848 849 err_clk_init: 850 pm_runtime_put_sync(dev); 851 pm_runtime_disable(dev); 852 reset_control_assert(pcie->mac_reset); 853 phy_power_off(pcie->phy); 854 err_phy_on: 855 phy_exit(pcie->phy); 856 err_phy_init: 857 reset_control_assert(pcie->phy_reset); 858 859 return err; 860 } 861 862 static void mtk_pcie_power_down(struct mtk_gen3_pcie *pcie) 863 { 864 clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks); 865 866 pm_runtime_put_sync(pcie->dev); 867 pm_runtime_disable(pcie->dev); 868 reset_control_assert(pcie->mac_reset); 869 870 phy_power_off(pcie->phy); 871 phy_exit(pcie->phy); 872 reset_control_assert(pcie->phy_reset); 873 } 874 875 static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie) 876 { 877 int err; 878 879 err = mtk_pcie_parse_port(pcie); 880 if (err) 881 return err; 882 883 /* 884 * The controller may have been left out of reset by the bootloader 885 * so make sure that we get a clean start by asserting resets here. 886 */ 887 reset_control_assert(pcie->phy_reset); 888 reset_control_assert(pcie->mac_reset); 889 usleep_range(10, 20); 890 891 /* Don't touch the hardware registers before power up */ 892 err = mtk_pcie_power_up(pcie); 893 if (err) 894 return err; 895 896 /* Try link up */ 897 err = mtk_pcie_startup_port(pcie); 898 if (err) 899 goto err_setup; 900 901 err = mtk_pcie_setup_irq(pcie); 902 if (err) 903 goto err_setup; 904 905 return 0; 906 907 err_setup: 908 mtk_pcie_power_down(pcie); 909 910 return err; 911 } 912 913 static int mtk_pcie_probe(struct platform_device *pdev) 914 { 915 struct device *dev = &pdev->dev; 916 struct mtk_gen3_pcie *pcie; 917 struct pci_host_bridge *host; 918 int err; 919 920 host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); 921 if (!host) 922 return -ENOMEM; 923 924 pcie = pci_host_bridge_priv(host); 925 926 pcie->dev = dev; 927 platform_set_drvdata(pdev, pcie); 928 929 err = mtk_pcie_setup(pcie); 930 if (err) 931 return err; 932 933 host->ops = &mtk_pcie_ops; 934 host->sysdata = pcie; 935 936 err = pci_host_probe(host); 937 if (err) { 938 mtk_pcie_irq_teardown(pcie); 939 mtk_pcie_power_down(pcie); 940 return err; 941 } 942 943 return 0; 944 } 945 946 static void mtk_pcie_remove(struct platform_device *pdev) 947 { 948 struct mtk_gen3_pcie *pcie = platform_get_drvdata(pdev); 949 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 950 951 pci_lock_rescan_remove(); 952 pci_stop_root_bus(host->bus); 953 pci_remove_root_bus(host->bus); 954 pci_unlock_rescan_remove(); 955 956 mtk_pcie_irq_teardown(pcie); 957 mtk_pcie_power_down(pcie); 958 } 959 960 static void mtk_pcie_irq_save(struct mtk_gen3_pcie *pcie) 961 { 962 int i; 963 964 raw_spin_lock(&pcie->irq_lock); 965 966 pcie->saved_irq_state = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); 967 968 for (i = 0; i < PCIE_MSI_SET_NUM; i++) { 969 struct mtk_msi_set *msi_set = &pcie->msi_sets[i]; 970 971 msi_set->saved_irq_state = readl_relaxed(msi_set->base + 972 PCIE_MSI_SET_ENABLE_OFFSET); 973 } 974 975 raw_spin_unlock(&pcie->irq_lock); 976 } 977 978 static void mtk_pcie_irq_restore(struct mtk_gen3_pcie *pcie) 979 { 980 int i; 981 982 raw_spin_lock(&pcie->irq_lock); 983 984 writel_relaxed(pcie->saved_irq_state, pcie->base + PCIE_INT_ENABLE_REG); 985 986 for (i = 0; i < PCIE_MSI_SET_NUM; i++) { 987 struct mtk_msi_set *msi_set = &pcie->msi_sets[i]; 988 989 writel_relaxed(msi_set->saved_irq_state, 990 msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); 991 } 992 993 raw_spin_unlock(&pcie->irq_lock); 994 } 995 996 static int mtk_pcie_turn_off_link(struct mtk_gen3_pcie *pcie) 997 { 998 u32 val; 999 1000 val = readl_relaxed(pcie->base + PCIE_ICMD_PM_REG); 1001 val |= PCIE_TURN_OFF_LINK; 1002 writel_relaxed(val, pcie->base + PCIE_ICMD_PM_REG); 1003 1004 /* Check the link is L2 */ 1005 return readl_poll_timeout(pcie->base + PCIE_LTSSM_STATUS_REG, val, 1006 (PCIE_LTSSM_STATE(val) == 1007 PCIE_LTSSM_STATE_L2_IDLE), 20, 1008 50 * USEC_PER_MSEC); 1009 } 1010 1011 static int mtk_pcie_suspend_noirq(struct device *dev) 1012 { 1013 struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev); 1014 int err; 1015 u32 val; 1016 1017 /* Trigger link to L2 state */ 1018 err = mtk_pcie_turn_off_link(pcie); 1019 if (err) { 1020 dev_err(pcie->dev, "cannot enter L2 state\n"); 1021 return err; 1022 } 1023 1024 /* Pull down the PERST# pin */ 1025 val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG); 1026 val |= PCIE_PE_RSTB; 1027 writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG); 1028 1029 dev_dbg(pcie->dev, "entered L2 states successfully"); 1030 1031 mtk_pcie_irq_save(pcie); 1032 mtk_pcie_power_down(pcie); 1033 1034 return 0; 1035 } 1036 1037 static int mtk_pcie_resume_noirq(struct device *dev) 1038 { 1039 struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev); 1040 int err; 1041 1042 err = mtk_pcie_power_up(pcie); 1043 if (err) 1044 return err; 1045 1046 err = mtk_pcie_startup_port(pcie); 1047 if (err) { 1048 mtk_pcie_power_down(pcie); 1049 return err; 1050 } 1051 1052 mtk_pcie_irq_restore(pcie); 1053 1054 return 0; 1055 } 1056 1057 static const struct dev_pm_ops mtk_pcie_pm_ops = { 1058 NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq, 1059 mtk_pcie_resume_noirq) 1060 }; 1061 1062 static const struct of_device_id mtk_pcie_of_match[] = { 1063 { .compatible = "mediatek,mt8192-pcie" }, 1064 {}, 1065 }; 1066 MODULE_DEVICE_TABLE(of, mtk_pcie_of_match); 1067 1068 static struct platform_driver mtk_pcie_driver = { 1069 .probe = mtk_pcie_probe, 1070 .remove_new = mtk_pcie_remove, 1071 .driver = { 1072 .name = "mtk-pcie-gen3", 1073 .of_match_table = mtk_pcie_of_match, 1074 .pm = &mtk_pcie_pm_ops, 1075 }, 1076 }; 1077 1078 module_platform_driver(mtk_pcie_driver); 1079 MODULE_LICENSE("GPL v2"); 1080