1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCIe host bridge driver for Apple system-on-chips. 4 * 5 * The HW is ECAM compliant, so once the controller is initialized, 6 * the driver mostly deals MSI mapping and handling of per-port 7 * interrupts (INTx, management and error signals). 8 * 9 * Initialization requires enabling power and clocks, along with a 10 * number of register pokes. 11 * 12 * Copyright (C) 2021 Alyssa Rosenzweig <alyssa@rosenzweig.io> 13 * Copyright (C) 2021 Google LLC 14 * Copyright (C) 2021 Corellium LLC 15 * Copyright (C) 2021 Mark Kettenis <kettenis@openbsd.org> 16 * 17 * Author: Alyssa Rosenzweig <alyssa@rosenzweig.io> 18 * Author: Marc Zyngier <maz@kernel.org> 19 */ 20 21 #include <linux/bitfield.h> 22 #include <linux/gpio/consumer.h> 23 #include <linux/kernel.h> 24 #include <linux/iopoll.h> 25 #include <linux/irqchip/chained_irq.h> 26 #include <linux/irqchip/irq-msi-lib.h> 27 #include <linux/irqdomain.h> 28 #include <linux/list.h> 29 #include <linux/module.h> 30 #include <linux/msi.h> 31 #include <linux/of_irq.h> 32 #include <linux/pci-ecam.h> 33 34 #include "pci-host-common.h" 35 36 /* T8103 (original M1) and related SoCs */ 37 #define CORE_RC_PHYIF_CTL 0x00024 38 #define CORE_RC_PHYIF_CTL_RUN BIT(0) 39 #define CORE_RC_PHYIF_STAT 0x00028 40 #define CORE_RC_PHYIF_STAT_REFCLK BIT(4) 41 #define CORE_RC_CTL 0x00050 42 #define CORE_RC_CTL_RUN BIT(0) 43 #define CORE_RC_STAT 0x00058 44 #define CORE_RC_STAT_READY BIT(0) 45 #define CORE_FABRIC_STAT 0x04000 46 #define CORE_FABRIC_STAT_MASK 0x001F001F 47 48 #define CORE_PHY_DEFAULT_BASE(port) (0x84000 + 0x4000 * (port)) 49 50 #define PHY_LANE_CFG 0x00000 51 #define PHY_LANE_CFG_REFCLK0REQ BIT(0) 52 #define PHY_LANE_CFG_REFCLK1REQ BIT(1) 53 #define PHY_LANE_CFG_REFCLK0ACK BIT(2) 54 #define PHY_LANE_CFG_REFCLK1ACK BIT(3) 55 #define PHY_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) 56 #define PHY_LANE_CFG_REFCLKCGEN (BIT(30) | BIT(31)) 57 #define PHY_LANE_CTL 0x00004 58 #define PHY_LANE_CTL_CFGACC BIT(15) 59 60 #define PORT_LTSSMCTL 0x00080 61 #define PORT_LTSSMCTL_START BIT(0) 62 #define PORT_INTSTAT 0x00100 63 #define PORT_INT_TUNNEL_ERR 31 64 #define PORT_INT_CPL_TIMEOUT 23 65 #define PORT_INT_RID2SID_MAPERR 22 66 #define PORT_INT_CPL_ABORT 21 67 #define PORT_INT_MSI_BAD_DATA 19 68 #define PORT_INT_MSI_ERR 18 69 #define PORT_INT_REQADDR_GT32 17 70 #define PORT_INT_AF_TIMEOUT 15 71 #define PORT_INT_LINK_DOWN 14 72 #define PORT_INT_LINK_UP 12 73 #define PORT_INT_LINK_BWMGMT 11 74 #define PORT_INT_AER_MASK (15 << 4) 75 #define PORT_INT_PORT_ERR 4 76 #define PORT_INT_INTx(i) i 77 #define PORT_INT_INTx_MASK 15 78 #define PORT_INTMSK 0x00104 79 #define PORT_INTMSKSET 0x00108 80 #define PORT_INTMSKCLR 0x0010c 81 #define PORT_MSICFG 0x00124 82 #define PORT_MSICFG_EN BIT(0) 83 #define PORT_MSICFG_L2MSINUM_SHIFT 4 84 #define PORT_MSIBASE 0x00128 85 #define PORT_MSIBASE_1_SHIFT 16 86 #define PORT_MSIADDR 0x00168 87 #define PORT_LINKSTS 0x00208 88 #define PORT_LINKSTS_UP BIT(0) 89 #define PORT_LINKSTS_BUSY BIT(2) 90 #define PORT_LINKCMDSTS 0x00210 91 #define PORT_OUTS_NPREQS 0x00284 92 #define PORT_OUTS_NPREQS_REQ BIT(24) 93 #define PORT_OUTS_NPREQS_CPL BIT(16) 94 #define PORT_RXWR_FIFO 0x00288 95 #define PORT_RXWR_FIFO_HDR GENMASK(15, 10) 96 #define PORT_RXWR_FIFO_DATA GENMASK(9, 0) 97 #define PORT_RXRD_FIFO 0x0028C 98 #define PORT_RXRD_FIFO_REQ GENMASK(6, 0) 99 #define PORT_OUTS_CPLS 0x00290 100 #define PORT_OUTS_CPLS_SHRD GENMASK(14, 8) 101 #define PORT_OUTS_CPLS_WAIT GENMASK(6, 0) 102 #define PORT_APPCLK 0x00800 103 #define PORT_APPCLK_EN BIT(0) 104 #define PORT_APPCLK_CGDIS BIT(8) 105 #define PORT_STATUS 0x00804 106 #define PORT_STATUS_READY BIT(0) 107 #define PORT_REFCLK 0x00810 108 #define PORT_REFCLK_EN BIT(0) 109 #define PORT_REFCLK_CGDIS BIT(8) 110 #define PORT_PERST 0x00814 111 #define PORT_PERST_OFF BIT(0) 112 #define PORT_RID2SID 0x00828 113 #define PORT_RID2SID_VALID BIT(31) 114 #define PORT_RID2SID_SID_SHIFT 16 115 #define PORT_RID2SID_BUS_SHIFT 8 116 #define PORT_RID2SID_DEV_SHIFT 3 117 #define PORT_RID2SID_FUNC_SHIFT 0 118 #define PORT_OUTS_PREQS_HDR 0x00980 119 #define PORT_OUTS_PREQS_HDR_MASK GENMASK(9, 0) 120 #define PORT_OUTS_PREQS_DATA 0x00984 121 #define PORT_OUTS_PREQS_DATA_MASK GENMASK(15, 0) 122 #define PORT_TUNCTRL 0x00988 123 #define PORT_TUNCTRL_PERST_ON BIT(0) 124 #define PORT_TUNCTRL_PERST_ACK_REQ BIT(1) 125 #define PORT_TUNSTAT 0x0098c 126 #define PORT_TUNSTAT_PERST_ON BIT(0) 127 #define PORT_TUNSTAT_PERST_ACK_PEND BIT(1) 128 #define PORT_PREFMEM_ENABLE 0x00994 129 130 /* T602x (M2-pro and co) */ 131 #define PORT_T602X_MSIADDR 0x016c 132 #define PORT_T602X_MSIADDR_HI 0x0170 133 #define PORT_T602X_PERST 0x082c 134 #define PORT_T602X_RID2SID 0x3000 135 #define PORT_T602X_MSIMAP 0x3800 136 137 #define PORT_MSIMAP_ENABLE BIT(31) 138 #define PORT_MSIMAP_TARGET GENMASK(7, 0) 139 140 /* 141 * The doorbell address is set to 0xfffff000, which by convention 142 * matches what MacOS does, and it is possible to use any other 143 * address (in the bottom 4GB, as the base register is only 32bit). 144 * However, it has to be excluded from the IOVA range, and the DART 145 * driver has to know about it. 146 */ 147 #define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 148 149 struct hw_info { 150 u32 phy_lane_ctl; 151 u32 port_msiaddr; 152 u32 port_msiaddr_hi; 153 u32 port_refclk; 154 u32 port_perst; 155 u32 port_rid2sid; 156 u32 port_msimap; 157 u32 max_rid2sid; 158 }; 159 160 static const struct hw_info t8103_hw = { 161 .phy_lane_ctl = PHY_LANE_CTL, 162 .port_msiaddr = PORT_MSIADDR, 163 .port_msiaddr_hi = 0, 164 .port_refclk = PORT_REFCLK, 165 .port_perst = PORT_PERST, 166 .port_rid2sid = PORT_RID2SID, 167 .port_msimap = 0, 168 .max_rid2sid = 64, 169 }; 170 171 static const struct hw_info t602x_hw = { 172 .phy_lane_ctl = 0, 173 .port_msiaddr = PORT_T602X_MSIADDR, 174 .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, 175 .port_refclk = 0, 176 .port_perst = PORT_T602X_PERST, 177 .port_rid2sid = PORT_T602X_RID2SID, 178 .port_msimap = PORT_T602X_MSIMAP, 179 /* 16 on t602x, guess for autodetect on future HW */ 180 .max_rid2sid = 512, 181 }; 182 183 struct apple_pcie { 184 struct mutex lock; 185 struct device *dev; 186 void __iomem *base; 187 const struct hw_info *hw; 188 unsigned long *bitmap; 189 struct list_head ports; 190 struct list_head entry; 191 struct completion event; 192 struct irq_fwspec fwspec; 193 u32 nvecs; 194 }; 195 196 struct apple_pcie_port { 197 raw_spinlock_t lock; 198 struct apple_pcie *pcie; 199 struct device_node *np; 200 void __iomem *base; 201 void __iomem *phy; 202 struct irq_domain *domain; 203 struct list_head entry; 204 unsigned long *sid_map; 205 int sid_map_sz; 206 int idx; 207 }; 208 209 static LIST_HEAD(pcie_list); 210 static DEFINE_MUTEX(pcie_list_lock); 211 212 static void rmw_set(u32 set, void __iomem *addr) 213 { 214 writel_relaxed(readl_relaxed(addr) | set, addr); 215 } 216 217 static void rmw_clear(u32 clr, void __iomem *addr) 218 { 219 writel_relaxed(readl_relaxed(addr) & ~clr, addr); 220 } 221 222 static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) 223 { 224 msg->address_hi = upper_32_bits(DOORBELL_ADDR); 225 msg->address_lo = lower_32_bits(DOORBELL_ADDR); 226 msg->data = data->hwirq; 227 } 228 229 static struct irq_chip apple_msi_bottom_chip = { 230 .name = "MSI", 231 .irq_mask = irq_chip_mask_parent, 232 .irq_unmask = irq_chip_unmask_parent, 233 .irq_eoi = irq_chip_eoi_parent, 234 .irq_set_affinity = irq_chip_set_affinity_parent, 235 .irq_set_type = irq_chip_set_type_parent, 236 .irq_compose_msi_msg = apple_msi_compose_msg, 237 }; 238 239 static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, 240 unsigned int nr_irqs, void *args) 241 { 242 struct apple_pcie *pcie = domain->host_data; 243 struct irq_fwspec fwspec = pcie->fwspec; 244 unsigned int i; 245 int ret, hwirq; 246 247 mutex_lock(&pcie->lock); 248 249 hwirq = bitmap_find_free_region(pcie->bitmap, pcie->nvecs, 250 order_base_2(nr_irqs)); 251 252 mutex_unlock(&pcie->lock); 253 254 if (hwirq < 0) 255 return -ENOSPC; 256 257 fwspec.param[fwspec.param_count - 2] += hwirq; 258 259 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); 260 if (ret) 261 return ret; 262 263 for (i = 0; i < nr_irqs; i++) { 264 irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, 265 &apple_msi_bottom_chip, pcie); 266 } 267 268 return 0; 269 } 270 271 static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq, 272 unsigned int nr_irqs) 273 { 274 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 275 struct apple_pcie *pcie = domain->host_data; 276 277 mutex_lock(&pcie->lock); 278 279 bitmap_release_region(pcie->bitmap, d->hwirq, order_base_2(nr_irqs)); 280 281 mutex_unlock(&pcie->lock); 282 } 283 284 static const struct irq_domain_ops apple_msi_domain_ops = { 285 .alloc = apple_msi_domain_alloc, 286 .free = apple_msi_domain_free, 287 }; 288 289 static void apple_port_irq_mask(struct irq_data *data) 290 { 291 struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); 292 293 guard(raw_spinlock_irqsave)(&port->lock); 294 rmw_set(BIT(data->hwirq), port->base + PORT_INTMSK); 295 } 296 297 static void apple_port_irq_unmask(struct irq_data *data) 298 { 299 struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); 300 301 guard(raw_spinlock_irqsave)(&port->lock); 302 rmw_clear(BIT(data->hwirq), port->base + PORT_INTMSK); 303 } 304 305 static bool hwirq_is_intx(unsigned int hwirq) 306 { 307 return BIT(hwirq) & PORT_INT_INTx_MASK; 308 } 309 310 static void apple_port_irq_ack(struct irq_data *data) 311 { 312 struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); 313 314 if (!hwirq_is_intx(data->hwirq)) 315 writel_relaxed(BIT(data->hwirq), port->base + PORT_INTSTAT); 316 } 317 318 static int apple_port_irq_set_type(struct irq_data *data, unsigned int type) 319 { 320 /* 321 * It doesn't seem that there is any way to configure the 322 * trigger, so assume INTx have to be level (as per the spec), 323 * and the rest is edge (which looks likely). 324 */ 325 if (hwirq_is_intx(data->hwirq) ^ !!(type & IRQ_TYPE_LEVEL_MASK)) 326 return -EINVAL; 327 328 irqd_set_trigger_type(data, type); 329 return 0; 330 } 331 332 static struct irq_chip apple_port_irqchip = { 333 .name = "PCIe", 334 .irq_ack = apple_port_irq_ack, 335 .irq_mask = apple_port_irq_mask, 336 .irq_unmask = apple_port_irq_unmask, 337 .irq_set_type = apple_port_irq_set_type, 338 }; 339 340 static int apple_port_irq_domain_alloc(struct irq_domain *domain, 341 unsigned int virq, unsigned int nr_irqs, 342 void *args) 343 { 344 struct apple_pcie_port *port = domain->host_data; 345 struct irq_fwspec *fwspec = args; 346 int i; 347 348 for (i = 0; i < nr_irqs; i++) { 349 irq_flow_handler_t flow = handle_edge_irq; 350 unsigned int type = IRQ_TYPE_EDGE_RISING; 351 352 if (hwirq_is_intx(fwspec->param[0] + i)) { 353 flow = handle_level_irq; 354 type = IRQ_TYPE_LEVEL_HIGH; 355 } 356 357 irq_domain_set_info(domain, virq + i, fwspec->param[0] + i, 358 &apple_port_irqchip, port, flow, 359 NULL, NULL); 360 361 irq_set_irq_type(virq + i, type); 362 } 363 364 return 0; 365 } 366 367 static void apple_port_irq_domain_free(struct irq_domain *domain, 368 unsigned int virq, unsigned int nr_irqs) 369 { 370 int i; 371 372 for (i = 0; i < nr_irqs; i++) { 373 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 374 375 irq_set_handler(virq + i, NULL); 376 irq_domain_reset_irq_data(d); 377 } 378 } 379 380 static const struct irq_domain_ops apple_port_irq_domain_ops = { 381 .translate = irq_domain_translate_onecell, 382 .alloc = apple_port_irq_domain_alloc, 383 .free = apple_port_irq_domain_free, 384 }; 385 386 static void apple_port_irq_handler(struct irq_desc *desc) 387 { 388 struct apple_pcie_port *port = irq_desc_get_handler_data(desc); 389 struct irq_chip *chip = irq_desc_get_chip(desc); 390 unsigned long stat; 391 int i; 392 393 chained_irq_enter(chip, desc); 394 395 stat = readl_relaxed(port->base + PORT_INTSTAT); 396 397 for_each_set_bit(i, &stat, 32) 398 generic_handle_domain_irq(port->domain, i); 399 400 chained_irq_exit(chip, desc); 401 } 402 403 static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) 404 { 405 struct fwnode_handle *fwnode = &port->np->fwnode; 406 struct apple_pcie *pcie = port->pcie; 407 unsigned int irq; 408 u32 val = 0; 409 410 /* FIXME: consider moving each interrupt under each port */ 411 irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), 412 port->idx); 413 if (!irq) 414 return -ENXIO; 415 416 port->domain = irq_domain_create_linear(fwnode, 32, 417 &apple_port_irq_domain_ops, 418 port); 419 if (!port->domain) 420 return -ENOMEM; 421 422 /* Disable all interrupts */ 423 writel_relaxed(~0, port->base + PORT_INTMSK); 424 writel_relaxed(~0, port->base + PORT_INTSTAT); 425 writel_relaxed(~0, port->base + PORT_LINKCMDSTS); 426 427 irq_set_chained_handler_and_data(irq, apple_port_irq_handler, port); 428 429 /* Configure MSI base address */ 430 BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR)); 431 writel_relaxed(lower_32_bits(DOORBELL_ADDR), 432 port->base + pcie->hw->port_msiaddr); 433 if (pcie->hw->port_msiaddr_hi) 434 writel_relaxed(0, port->base + pcie->hw->port_msiaddr_hi); 435 436 /* Enable MSIs, shared between all ports */ 437 if (pcie->hw->port_msimap) { 438 for (int i = 0; i < pcie->nvecs; i++) 439 writel_relaxed(FIELD_PREP(PORT_MSIMAP_TARGET, i) | 440 PORT_MSIMAP_ENABLE, 441 port->base + pcie->hw->port_msimap + 4 * i); 442 } else { 443 writel_relaxed(0, port->base + PORT_MSIBASE); 444 val = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; 445 } 446 447 writel_relaxed(val | PORT_MSICFG_EN, port->base + PORT_MSICFG); 448 return 0; 449 } 450 451 static irqreturn_t apple_pcie_port_irq(int irq, void *data) 452 { 453 struct apple_pcie_port *port = data; 454 unsigned int hwirq = irq_domain_get_irq_data(port->domain, irq)->hwirq; 455 456 switch (hwirq) { 457 case PORT_INT_LINK_UP: 458 dev_info_ratelimited(port->pcie->dev, "Link up on %pOF\n", 459 port->np); 460 complete_all(&port->pcie->event); 461 break; 462 case PORT_INT_LINK_DOWN: 463 dev_info_ratelimited(port->pcie->dev, "Link down on %pOF\n", 464 port->np); 465 break; 466 default: 467 return IRQ_NONE; 468 } 469 470 return IRQ_HANDLED; 471 } 472 473 static int apple_pcie_port_register_irqs(struct apple_pcie_port *port) 474 { 475 static struct { 476 unsigned int hwirq; 477 const char *name; 478 } port_irqs[] = { 479 { PORT_INT_LINK_UP, "Link up", }, 480 { PORT_INT_LINK_DOWN, "Link down", }, 481 }; 482 int i; 483 484 for (i = 0; i < ARRAY_SIZE(port_irqs); i++) { 485 struct irq_fwspec fwspec = { 486 .fwnode = &port->np->fwnode, 487 .param_count = 1, 488 .param = { 489 [0] = port_irqs[i].hwirq, 490 }, 491 }; 492 unsigned int irq; 493 int ret; 494 495 irq = irq_domain_alloc_irqs(port->domain, 1, NUMA_NO_NODE, 496 &fwspec); 497 if (WARN_ON(!irq)) 498 continue; 499 500 ret = request_irq(irq, apple_pcie_port_irq, 0, 501 port_irqs[i].name, port); 502 WARN_ON(ret); 503 } 504 505 return 0; 506 } 507 508 static int apple_pcie_setup_refclk(struct apple_pcie *pcie, 509 struct apple_pcie_port *port) 510 { 511 u32 stat; 512 int res; 513 514 if (pcie->hw->phy_lane_ctl) 515 rmw_set(PHY_LANE_CTL_CFGACC, port->phy + pcie->hw->phy_lane_ctl); 516 517 rmw_set(PHY_LANE_CFG_REFCLK0REQ, port->phy + PHY_LANE_CFG); 518 519 res = readl_relaxed_poll_timeout(port->phy + PHY_LANE_CFG, 520 stat, stat & PHY_LANE_CFG_REFCLK0ACK, 521 100, 50000); 522 if (res < 0) 523 return res; 524 525 rmw_set(PHY_LANE_CFG_REFCLK1REQ, port->phy + PHY_LANE_CFG); 526 res = readl_relaxed_poll_timeout(port->phy + PHY_LANE_CFG, 527 stat, stat & PHY_LANE_CFG_REFCLK1ACK, 528 100, 50000); 529 530 if (res < 0) 531 return res; 532 533 if (pcie->hw->phy_lane_ctl) 534 rmw_clear(PHY_LANE_CTL_CFGACC, port->phy + pcie->hw->phy_lane_ctl); 535 536 rmw_set(PHY_LANE_CFG_REFCLKEN, port->phy + PHY_LANE_CFG); 537 538 if (pcie->hw->port_refclk) 539 rmw_set(PORT_REFCLK_EN, port->base + pcie->hw->port_refclk); 540 541 return 0; 542 } 543 544 static void __iomem *port_rid2sid_addr(struct apple_pcie_port *port, int idx) 545 { 546 return port->base + port->pcie->hw->port_rid2sid + 4 * idx; 547 } 548 549 static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, 550 int idx, u32 val) 551 { 552 writel_relaxed(val, port_rid2sid_addr(port, idx)); 553 /* Read back to ensure completion of the write */ 554 return readl_relaxed(port_rid2sid_addr(port, idx)); 555 } 556 557 static int apple_pcie_setup_port(struct apple_pcie *pcie, 558 struct device_node *np) 559 { 560 struct platform_device *platform = to_platform_device(pcie->dev); 561 struct apple_pcie_port *port; 562 struct gpio_desc *reset; 563 struct resource *res; 564 char name[16]; 565 u32 stat, idx; 566 int ret, i; 567 568 reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", 569 GPIOD_OUT_LOW, "PERST#"); 570 if (IS_ERR(reset)) 571 return PTR_ERR(reset); 572 573 port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); 574 if (!port) 575 return -ENOMEM; 576 577 port->sid_map = devm_bitmap_zalloc(pcie->dev, pcie->hw->max_rid2sid, GFP_KERNEL); 578 if (!port->sid_map) 579 return -ENOMEM; 580 581 ret = of_property_read_u32_index(np, "reg", 0, &idx); 582 if (ret) 583 return ret; 584 585 /* Use the first reg entry to work out the port index */ 586 port->idx = idx >> 11; 587 port->pcie = pcie; 588 port->np = np; 589 590 raw_spin_lock_init(&port->lock); 591 592 snprintf(name, sizeof(name), "port%d", port->idx); 593 res = platform_get_resource_byname(platform, IORESOURCE_MEM, name); 594 if (!res) 595 res = platform_get_resource(platform, IORESOURCE_MEM, port->idx + 2); 596 597 port->base = devm_ioremap_resource(&platform->dev, res); 598 if (IS_ERR(port->base)) 599 return PTR_ERR(port->base); 600 601 snprintf(name, sizeof(name), "phy%d", port->idx); 602 res = platform_get_resource_byname(platform, IORESOURCE_MEM, name); 603 if (res) 604 port->phy = devm_ioremap_resource(&platform->dev, res); 605 else 606 port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); 607 608 rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); 609 610 /* Assert PERST# before setting up the clock */ 611 gpiod_set_value_cansleep(reset, 1); 612 613 ret = apple_pcie_setup_refclk(pcie, port); 614 if (ret < 0) 615 return ret; 616 617 /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */ 618 usleep_range(100, 200); 619 620 /* Deassert PERST# */ 621 rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); 622 gpiod_set_value_cansleep(reset, 0); 623 624 /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ 625 msleep(100); 626 627 ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, 628 stat & PORT_STATUS_READY, 100, 250000); 629 if (ret < 0) { 630 dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); 631 return ret; 632 } 633 634 if (pcie->hw->port_refclk) 635 rmw_clear(PORT_REFCLK_CGDIS, port->base + pcie->hw->port_refclk); 636 else 637 rmw_set(PHY_LANE_CFG_REFCLKCGEN, port->phy + PHY_LANE_CFG); 638 639 rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); 640 641 ret = apple_pcie_port_setup_irq(port); 642 if (ret) 643 return ret; 644 645 /* Reset all RID/SID mappings, and check for RAZ/WI registers */ 646 for (i = 0; i < pcie->hw->max_rid2sid; i++) { 647 if (apple_pcie_rid2sid_write(port, i, 0xbad1d) != 0xbad1d) 648 break; 649 apple_pcie_rid2sid_write(port, i, 0); 650 } 651 652 dev_dbg(pcie->dev, "%pOF: %d RID/SID mapping entries\n", np, i); 653 654 port->sid_map_sz = i; 655 656 list_add_tail(&port->entry, &pcie->ports); 657 init_completion(&pcie->event); 658 659 /* In the success path, we keep a reference to np around */ 660 of_node_get(np); 661 662 ret = apple_pcie_port_register_irqs(port); 663 WARN_ON(ret); 664 665 writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); 666 667 if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) 668 dev_warn(pcie->dev, "%pOF link didn't come up\n", np); 669 670 return 0; 671 } 672 673 static const struct msi_parent_ops apple_msi_parent_ops = { 674 .supported_flags = (MSI_GENERIC_FLAGS_MASK | 675 MSI_FLAG_PCI_MSIX | 676 MSI_FLAG_MULTI_PCI_MSI), 677 .required_flags = (MSI_FLAG_USE_DEF_DOM_OPS | 678 MSI_FLAG_USE_DEF_CHIP_OPS | 679 MSI_FLAG_PCI_MSI_MASK_PARENT), 680 .chip_flags = MSI_CHIP_FLAG_SET_EOI, 681 .bus_select_token = DOMAIN_BUS_PCI_MSI, 682 .init_dev_msi_info = msi_lib_init_dev_msi_info, 683 }; 684 685 static int apple_msi_init(struct apple_pcie *pcie) 686 { 687 struct fwnode_handle *fwnode = dev_fwnode(pcie->dev); 688 struct irq_domain_info info = { 689 .fwnode = fwnode, 690 .ops = &apple_msi_domain_ops, 691 .size = pcie->nvecs, 692 .host_data = pcie, 693 }; 694 struct of_phandle_args args = {}; 695 int ret; 696 697 ret = of_parse_phandle_with_args(to_of_node(fwnode), "msi-ranges", 698 "#interrupt-cells", 0, &args); 699 if (ret) 700 return ret; 701 702 ret = of_property_read_u32_index(to_of_node(fwnode), "msi-ranges", 703 args.args_count + 1, &pcie->nvecs); 704 if (ret) 705 return ret; 706 707 of_phandle_args_to_fwspec(args.np, args.args, args.args_count, 708 &pcie->fwspec); 709 710 pcie->bitmap = devm_bitmap_zalloc(pcie->dev, pcie->nvecs, GFP_KERNEL); 711 if (!pcie->bitmap) 712 return -ENOMEM; 713 714 info.parent = irq_find_matching_fwspec(&pcie->fwspec, DOMAIN_BUS_WIRED); 715 if (!info.parent) { 716 dev_err(pcie->dev, "failed to find parent domain\n"); 717 return -ENXIO; 718 } 719 720 if (!msi_create_parent_irq_domain(&info, &apple_msi_parent_ops)) { 721 dev_err(pcie->dev, "failed to create IRQ domain\n"); 722 return -ENOMEM; 723 } 724 return 0; 725 } 726 727 static void apple_pcie_register(struct apple_pcie *pcie) 728 { 729 guard(mutex)(&pcie_list_lock); 730 731 list_add_tail(&pcie->entry, &pcie_list); 732 } 733 734 static void apple_pcie_unregister(struct apple_pcie *pcie) 735 { 736 guard(mutex)(&pcie_list_lock); 737 738 list_del(&pcie->entry); 739 } 740 741 static struct apple_pcie *apple_pcie_lookup(struct device *dev) 742 { 743 struct apple_pcie *pcie; 744 745 guard(mutex)(&pcie_list_lock); 746 747 list_for_each_entry(pcie, &pcie_list, entry) { 748 if (pcie->dev == dev) 749 return pcie; 750 } 751 752 return NULL; 753 } 754 755 static struct apple_pcie_port *apple_pcie_get_port(struct pci_dev *pdev) 756 { 757 struct pci_config_window *cfg = pdev->sysdata; 758 struct apple_pcie *pcie; 759 struct pci_dev *port_pdev; 760 struct apple_pcie_port *port; 761 762 pcie = apple_pcie_lookup(cfg->parent); 763 if (WARN_ON(!pcie)) 764 return NULL; 765 766 /* Find the root port this device is on */ 767 port_pdev = pcie_find_root_port(pdev); 768 769 /* If finding the port itself, nothing to do */ 770 if (WARN_ON(!port_pdev) || pdev == port_pdev) 771 return NULL; 772 773 list_for_each_entry(port, &pcie->ports, entry) { 774 if (port->idx == PCI_SLOT(port_pdev->devfn)) 775 return port; 776 } 777 778 return NULL; 779 } 780 781 static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev) 782 { 783 u32 sid, rid = pci_dev_id(pdev); 784 struct apple_pcie_port *port; 785 int idx, err; 786 787 port = apple_pcie_get_port(pdev); 788 if (!port) 789 return 0; 790 791 dev_dbg(&pdev->dev, "added to bus %s, index %d\n", 792 pci_name(pdev->bus->self), port->idx); 793 794 err = of_map_id(port->pcie->dev->of_node, rid, "iommu-map", 795 "iommu-map-mask", NULL, &sid); 796 if (err) 797 return err; 798 799 mutex_lock(&port->pcie->lock); 800 801 idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0); 802 if (idx >= 0) { 803 apple_pcie_rid2sid_write(port, idx, 804 PORT_RID2SID_VALID | 805 (sid << PORT_RID2SID_SID_SHIFT) | rid); 806 807 dev_dbg(&pdev->dev, "mapping RID%x to SID%x (index %d)\n", 808 rid, sid, idx); 809 } 810 811 mutex_unlock(&port->pcie->lock); 812 813 return idx >= 0 ? 0 : -ENOSPC; 814 } 815 816 static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev) 817 { 818 struct apple_pcie_port *port; 819 u32 rid = pci_dev_id(pdev); 820 int idx; 821 822 port = apple_pcie_get_port(pdev); 823 if (!port) 824 return; 825 826 mutex_lock(&port->pcie->lock); 827 828 for_each_set_bit(idx, port->sid_map, port->sid_map_sz) { 829 u32 val; 830 831 val = readl_relaxed(port_rid2sid_addr(port, idx)); 832 if ((val & 0xffff) == rid) { 833 apple_pcie_rid2sid_write(port, idx, 0); 834 bitmap_release_region(port->sid_map, idx, 0); 835 dev_dbg(&pdev->dev, "Released %x (%d)\n", val, idx); 836 break; 837 } 838 } 839 840 mutex_unlock(&port->pcie->lock); 841 } 842 843 static int apple_pcie_init(struct pci_config_window *cfg) 844 { 845 struct device *dev = cfg->parent; 846 struct apple_pcie *pcie; 847 int ret; 848 849 pcie = apple_pcie_lookup(dev); 850 if (WARN_ON(!pcie)) 851 return -ENOENT; 852 853 for_each_available_child_of_node_scoped(dev->of_node, of_port) { 854 ret = apple_pcie_setup_port(pcie, of_port); 855 if (ret) { 856 dev_err(dev, "Port %pOF setup fail: %d\n", of_port, ret); 857 return ret; 858 } 859 } 860 861 return 0; 862 } 863 864 static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { 865 .init = apple_pcie_init, 866 .enable_device = apple_pcie_enable_device, 867 .disable_device = apple_pcie_disable_device, 868 .pci_ops = { 869 .map_bus = pci_ecam_map_bus, 870 .read = pci_generic_config_read, 871 .write = pci_generic_config_write, 872 } 873 }; 874 875 static int apple_pcie_probe(struct platform_device *pdev) 876 { 877 struct device *dev = &pdev->dev; 878 struct apple_pcie *pcie; 879 int ret; 880 881 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); 882 if (!pcie) 883 return -ENOMEM; 884 885 pcie->dev = dev; 886 pcie->hw = of_device_get_match_data(dev); 887 if (!pcie->hw) 888 return -ENODEV; 889 pcie->base = devm_platform_ioremap_resource(pdev, 1); 890 if (IS_ERR(pcie->base)) 891 return PTR_ERR(pcie->base); 892 893 mutex_init(&pcie->lock); 894 INIT_LIST_HEAD(&pcie->ports); 895 896 ret = apple_msi_init(pcie); 897 if (ret) 898 return ret; 899 900 apple_pcie_register(pcie); 901 902 ret = pci_host_common_init(pdev, &apple_pcie_cfg_ecam_ops); 903 if (ret) 904 apple_pcie_unregister(pcie); 905 906 return ret; 907 } 908 909 static const struct of_device_id apple_pcie_of_match[] = { 910 { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, 911 { .compatible = "apple,pcie", .data = &t8103_hw }, 912 { } 913 }; 914 MODULE_DEVICE_TABLE(of, apple_pcie_of_match); 915 916 static struct platform_driver apple_pcie_driver = { 917 .probe = apple_pcie_probe, 918 .driver = { 919 .name = "pcie-apple", 920 .of_match_table = apple_pcie_of_match, 921 .suppress_bind_attrs = true, 922 }, 923 }; 924 module_platform_driver(apple_pcie_driver); 925 926 MODULE_DESCRIPTION("Apple PCIe host bridge driver"); 927 MODULE_LICENSE("GPL v2"); 928