1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe host controller driver 4 * 5 * Copyright (C) 2013 Samsung Electronics Co., Ltd. 6 * https://www.samsung.com 7 * 8 * Author: Jingoo Han <jg1.han@samsung.com> 9 */ 10 11 #include <linux/align.h> 12 #include <linux/iopoll.h> 13 #include <linux/irqchip/chained_irq.h> 14 #include <linux/irqchip/irq-msi-lib.h> 15 #include <linux/irqdomain.h> 16 #include <linux/msi.h> 17 #include <linux/of_address.h> 18 #include <linux/of_pci.h> 19 #include <linux/pci_regs.h> 20 #include <linux/platform_device.h> 21 22 #include "../../pci.h" 23 #include "pcie-designware.h" 24 25 static struct pci_ops dw_pcie_ops; 26 static struct pci_ops dw_pcie_ecam_ops; 27 static struct pci_ops dw_child_pcie_ops; 28 29 #define DW_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ 30 MSI_FLAG_USE_DEF_CHIP_OPS | \ 31 MSI_FLAG_NO_AFFINITY | \ 32 MSI_FLAG_PCI_MSI_MASK_PARENT) 33 #define DW_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \ 34 MSI_FLAG_PCI_MSIX | \ 35 MSI_GENERIC_FLAGS_MASK) 36 37 #define IS_256MB_ALIGNED(x) IS_ALIGNED(x, SZ_256M) 38 39 static const struct msi_parent_ops dw_pcie_msi_parent_ops = { 40 .required_flags = DW_PCIE_MSI_FLAGS_REQUIRED, 41 .supported_flags = DW_PCIE_MSI_FLAGS_SUPPORTED, 42 .bus_select_token = DOMAIN_BUS_PCI_MSI, 43 .chip_flags = MSI_CHIP_FLAG_SET_ACK, 44 .prefix = "DW-", 45 .init_dev_msi_info = msi_lib_init_dev_msi_info, 46 }; 47 48 /* MSI int handler */ 49 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp) 50 { 51 int i, pos; 52 unsigned long val; 53 u32 status, num_ctrls; 54 irqreturn_t ret = IRQ_NONE; 55 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 56 57 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; 58 59 for (i = 0; i < num_ctrls; i++) { 60 status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS + 61 (i * MSI_REG_CTRL_BLOCK_SIZE)); 62 if (!status) 63 continue; 64 65 ret = IRQ_HANDLED; 66 val = status; 67 pos = 0; 68 while ((pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL, 69 pos)) != MAX_MSI_IRQS_PER_CTRL) { 70 generic_handle_domain_irq(pp->irq_domain, 71 (i * MAX_MSI_IRQS_PER_CTRL) + 72 pos); 73 pos++; 74 } 75 } 76 77 return ret; 78 } 79 80 /* Chained MSI interrupt service routine */ 81 static void dw_chained_msi_isr(struct irq_desc *desc) 82 { 83 struct irq_chip *chip = irq_desc_get_chip(desc); 84 struct dw_pcie_rp *pp; 85 86 chained_irq_enter(chip, desc); 87 88 pp = irq_desc_get_handler_data(desc); 89 dw_handle_msi_irq(pp); 90 91 chained_irq_exit(chip, desc); 92 } 93 94 static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg) 95 { 96 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); 97 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 98 u64 msi_target; 99 100 msi_target = (u64)pp->msi_data; 101 102 msg->address_lo = lower_32_bits(msi_target); 103 msg->address_hi = upper_32_bits(msi_target); 104 105 msg->data = d->hwirq; 106 107 dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n", 108 (int)d->hwirq, msg->address_hi, msg->address_lo); 109 } 110 111 static void dw_pci_bottom_mask(struct irq_data *d) 112 { 113 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); 114 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 115 unsigned int res, bit, ctrl; 116 unsigned long flags; 117 118 raw_spin_lock_irqsave(&pp->lock, flags); 119 120 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; 121 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; 122 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; 123 124 pp->irq_mask[ctrl] |= BIT(bit); 125 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); 126 127 raw_spin_unlock_irqrestore(&pp->lock, flags); 128 } 129 130 static void dw_pci_bottom_unmask(struct irq_data *d) 131 { 132 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); 133 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 134 unsigned int res, bit, ctrl; 135 unsigned long flags; 136 137 raw_spin_lock_irqsave(&pp->lock, flags); 138 139 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; 140 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; 141 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; 142 143 pp->irq_mask[ctrl] &= ~BIT(bit); 144 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); 145 146 raw_spin_unlock_irqrestore(&pp->lock, flags); 147 } 148 149 static void dw_pci_bottom_ack(struct irq_data *d) 150 { 151 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); 152 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 153 unsigned int res, bit, ctrl; 154 155 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; 156 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; 157 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; 158 159 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit)); 160 } 161 162 static struct irq_chip dw_pci_msi_bottom_irq_chip = { 163 .name = "DWPCI-MSI", 164 .irq_ack = dw_pci_bottom_ack, 165 .irq_compose_msi_msg = dw_pci_setup_msi_msg, 166 .irq_mask = dw_pci_bottom_mask, 167 .irq_unmask = dw_pci_bottom_unmask, 168 }; 169 170 static int dw_pcie_irq_domain_alloc(struct irq_domain *domain, 171 unsigned int virq, unsigned int nr_irqs, 172 void *args) 173 { 174 struct dw_pcie_rp *pp = domain->host_data; 175 unsigned long flags; 176 u32 i; 177 int bit; 178 179 raw_spin_lock_irqsave(&pp->lock, flags); 180 181 bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors, 182 order_base_2(nr_irqs)); 183 184 raw_spin_unlock_irqrestore(&pp->lock, flags); 185 186 if (bit < 0) 187 return -ENOSPC; 188 189 for (i = 0; i < nr_irqs; i++) 190 irq_domain_set_info(domain, virq + i, bit + i, 191 pp->msi_irq_chip, 192 pp, handle_edge_irq, 193 NULL, NULL); 194 195 return 0; 196 } 197 198 static void dw_pcie_irq_domain_free(struct irq_domain *domain, 199 unsigned int virq, unsigned int nr_irqs) 200 { 201 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 202 struct dw_pcie_rp *pp = domain->host_data; 203 unsigned long flags; 204 205 raw_spin_lock_irqsave(&pp->lock, flags); 206 207 bitmap_release_region(pp->msi_irq_in_use, d->hwirq, 208 order_base_2(nr_irqs)); 209 210 raw_spin_unlock_irqrestore(&pp->lock, flags); 211 } 212 213 static const struct irq_domain_ops dw_pcie_msi_domain_ops = { 214 .alloc = dw_pcie_irq_domain_alloc, 215 .free = dw_pcie_irq_domain_free, 216 }; 217 218 int dw_pcie_allocate_domains(struct dw_pcie_rp *pp) 219 { 220 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 221 struct irq_domain_info info = { 222 .fwnode = dev_fwnode(pci->dev), 223 .ops = &dw_pcie_msi_domain_ops, 224 .size = pp->num_vectors, 225 .host_data = pp, 226 }; 227 228 pp->irq_domain = msi_create_parent_irq_domain(&info, &dw_pcie_msi_parent_ops); 229 if (!pp->irq_domain) { 230 dev_err(pci->dev, "Failed to create IRQ domain\n"); 231 return -ENOMEM; 232 } 233 234 return 0; 235 } 236 237 void dw_pcie_free_msi(struct dw_pcie_rp *pp) 238 { 239 u32 ctrl; 240 241 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) { 242 if (pp->msi_irq[ctrl] > 0) 243 irq_set_chained_handler_and_data(pp->msi_irq[ctrl], 244 NULL, NULL); 245 } 246 247 irq_domain_remove(pp->irq_domain); 248 } 249 EXPORT_SYMBOL_GPL(dw_pcie_free_msi); 250 251 void dw_pcie_msi_init(struct dw_pcie_rp *pp) 252 { 253 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 254 u64 msi_target = (u64)pp->msi_data; 255 u32 ctrl, num_ctrls; 256 257 if (!pci_msi_enabled() || !pp->has_msi_ctrl) 258 return; 259 260 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; 261 262 /* Initialize IRQ Status array */ 263 for (ctrl = 0; ctrl < num_ctrls; ctrl++) { 264 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + 265 (ctrl * MSI_REG_CTRL_BLOCK_SIZE), 266 pp->irq_mask[ctrl]); 267 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE + 268 (ctrl * MSI_REG_CTRL_BLOCK_SIZE), 269 ~0); 270 } 271 272 /* Program the msi_data */ 273 dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target)); 274 dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target)); 275 } 276 EXPORT_SYMBOL_GPL(dw_pcie_msi_init); 277 278 static int dw_pcie_parse_split_msi_irq(struct dw_pcie_rp *pp) 279 { 280 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 281 struct device *dev = pci->dev; 282 struct platform_device *pdev = to_platform_device(dev); 283 u32 ctrl, max_vectors; 284 int irq; 285 286 /* Parse any "msiX" IRQs described in the devicetree */ 287 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) { 288 char msi_name[] = "msiX"; 289 290 msi_name[3] = '0' + ctrl; 291 irq = platform_get_irq_byname_optional(pdev, msi_name); 292 if (irq == -ENXIO) 293 break; 294 if (irq < 0) 295 return dev_err_probe(dev, irq, 296 "Failed to parse MSI IRQ '%s'\n", 297 msi_name); 298 299 pp->msi_irq[ctrl] = irq; 300 } 301 302 /* If no "msiX" IRQs, caller should fallback to "msi" IRQ */ 303 if (ctrl == 0) 304 return -ENXIO; 305 306 max_vectors = ctrl * MAX_MSI_IRQS_PER_CTRL; 307 if (pp->num_vectors > max_vectors) { 308 dev_warn(dev, "Exceeding number of MSI vectors, limiting to %u\n", 309 max_vectors); 310 pp->num_vectors = max_vectors; 311 } 312 if (!pp->num_vectors) 313 pp->num_vectors = max_vectors; 314 315 return 0; 316 } 317 318 int dw_pcie_msi_host_init(struct dw_pcie_rp *pp) 319 { 320 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 321 struct device *dev = pci->dev; 322 struct platform_device *pdev = to_platform_device(dev); 323 u64 *msi_vaddr = NULL; 324 int ret; 325 u32 ctrl, num_ctrls; 326 327 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) 328 pp->irq_mask[ctrl] = ~0; 329 330 if (!pp->msi_irq[0]) { 331 ret = dw_pcie_parse_split_msi_irq(pp); 332 if (ret < 0 && ret != -ENXIO) 333 return ret; 334 } 335 336 if (!pp->num_vectors) 337 pp->num_vectors = MSI_DEF_NUM_VECTORS; 338 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; 339 340 if (!pp->msi_irq[0]) { 341 pp->msi_irq[0] = platform_get_irq_byname_optional(pdev, "msi"); 342 if (pp->msi_irq[0] < 0) { 343 pp->msi_irq[0] = platform_get_irq(pdev, 0); 344 if (pp->msi_irq[0] < 0) 345 return pp->msi_irq[0]; 346 } 347 } 348 349 dev_dbg(dev, "Using %d MSI vectors\n", pp->num_vectors); 350 351 pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip; 352 353 ret = dw_pcie_allocate_domains(pp); 354 if (ret) 355 return ret; 356 357 for (ctrl = 0; ctrl < num_ctrls; ctrl++) { 358 if (pp->msi_irq[ctrl] > 0) 359 irq_set_chained_handler_and_data(pp->msi_irq[ctrl], 360 dw_chained_msi_isr, pp); 361 } 362 363 /* 364 * Even though the iMSI-RX Module supports 64-bit addresses some 365 * peripheral PCIe devices may lack 64-bit message support. In 366 * order not to miss MSI TLPs from those devices the MSI target 367 * address has to be within the lowest 4GB. 368 * 369 * Note until there is a better alternative found the reservation is 370 * done by allocating from the artificially limited DMA-coherent 371 * memory. 372 */ 373 ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); 374 if (!ret) 375 msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, 376 GFP_KERNEL); 377 378 if (!msi_vaddr) { 379 dev_warn(dev, "Failed to allocate 32-bit MSI address\n"); 380 dma_set_coherent_mask(dev, DMA_BIT_MASK(64)); 381 msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, 382 GFP_KERNEL); 383 if (!msi_vaddr) { 384 dev_err(dev, "Failed to allocate MSI address\n"); 385 dw_pcie_free_msi(pp); 386 return -ENOMEM; 387 } 388 } 389 390 return 0; 391 } 392 EXPORT_SYMBOL_GPL(dw_pcie_msi_host_init); 393 394 static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp) 395 { 396 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 397 struct resource_entry *win; 398 struct resource *res; 399 400 win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM); 401 if (win) { 402 res = devm_kzalloc(pci->dev, sizeof(*res), GFP_KERNEL); 403 if (!res) 404 return; 405 406 /* 407 * Allocate MSG TLP region of size 'region_align' at the end of 408 * the host bridge window. 409 */ 410 res->start = win->res->end - pci->region_align + 1; 411 res->end = win->res->end; 412 res->name = "msg"; 413 res->flags = win->res->flags | IORESOURCE_BUSY; 414 415 if (!devm_request_resource(pci->dev, win->res, res)) 416 pp->msg_res = res; 417 } 418 } 419 420 static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp) 421 { 422 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 423 struct dw_pcie_ob_atu_cfg atu = {0}; 424 resource_size_t bus_range_max; 425 struct resource_entry *bus; 426 int ret; 427 428 bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); 429 430 /* 431 * Root bus under the host bridge doesn't require any iATU configuration 432 * as DBI region will be used to access root bus config space. 433 * Immediate bus under Root Bus, needs type 0 iATU configuration and 434 * remaining buses need type 1 iATU configuration. 435 */ 436 atu.index = 0; 437 atu.type = PCIE_ATU_TYPE_CFG0; 438 atu.parent_bus_addr = pp->cfg0_base + SZ_1M; 439 /* 1MiB is to cover 1 (bus) * 32 (devices) * 8 (functions) */ 440 atu.size = SZ_1M; 441 atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE; 442 ret = dw_pcie_prog_outbound_atu(pci, &atu); 443 if (ret) 444 return ret; 445 446 bus_range_max = resource_size(bus->res); 447 448 if (bus_range_max < 2) 449 return 0; 450 451 /* Configure remaining buses in type 1 iATU configuration */ 452 atu.index = 1; 453 atu.type = PCIE_ATU_TYPE_CFG1; 454 atu.parent_bus_addr = pp->cfg0_base + SZ_2M; 455 atu.size = (SZ_1M * bus_range_max) - SZ_2M; 456 atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE; 457 458 return dw_pcie_prog_outbound_atu(pci, &atu); 459 } 460 461 static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res) 462 { 463 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 464 struct device *dev = pci->dev; 465 struct resource_entry *bus; 466 467 bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); 468 if (!bus) 469 return -ENODEV; 470 471 pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops); 472 if (IS_ERR(pp->cfg)) 473 return PTR_ERR(pp->cfg); 474 475 return 0; 476 } 477 478 static bool dw_pcie_ecam_enabled(struct dw_pcie_rp *pp, struct resource *config_res) 479 { 480 struct resource *bus_range; 481 u64 nr_buses; 482 483 /* Vendor glue drivers may implement their own ECAM mechanism */ 484 if (pp->native_ecam) 485 return false; 486 487 /* 488 * PCIe spec r6.0, sec 7.2.2 mandates the base address used for ECAM to 489 * be aligned on a 2^(n+20) byte boundary, where n is the number of bits 490 * used for representing 'bus' in BDF. Since the DWC cores always use 8 491 * bits for representing 'bus', the base address has to be aligned to 492 * 2^28 byte boundary, which is 256 MiB. 493 */ 494 if (!IS_256MB_ALIGNED(config_res->start)) 495 return false; 496 497 bus_range = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS)->res; 498 if (!bus_range) 499 return false; 500 501 nr_buses = resource_size(config_res) >> PCIE_ECAM_BUS_SHIFT; 502 503 return nr_buses >= resource_size(bus_range); 504 } 505 506 static int dw_pcie_host_get_resources(struct dw_pcie_rp *pp) 507 { 508 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 509 struct device *dev = pci->dev; 510 struct platform_device *pdev = to_platform_device(dev); 511 struct resource_entry *win; 512 struct resource *res; 513 int ret; 514 515 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); 516 if (!res) { 517 dev_err(dev, "Missing \"config\" reg space\n"); 518 return -ENODEV; 519 } 520 521 pp->cfg0_size = resource_size(res); 522 pp->cfg0_base = res->start; 523 524 pp->ecam_enabled = dw_pcie_ecam_enabled(pp, res); 525 if (pp->ecam_enabled) { 526 ret = dw_pcie_create_ecam_window(pp, res); 527 if (ret) 528 return ret; 529 530 pp->bridge->ops = &dw_pcie_ecam_ops; 531 pp->bridge->sysdata = pp->cfg; 532 pp->cfg->priv = pp; 533 } else { 534 pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res); 535 if (IS_ERR(pp->va_cfg0_base)) 536 return PTR_ERR(pp->va_cfg0_base); 537 538 /* Set default bus ops */ 539 pp->bridge->ops = &dw_pcie_ops; 540 pp->bridge->child_ops = &dw_child_pcie_ops; 541 pp->bridge->sysdata = pp; 542 } 543 544 ret = dw_pcie_get_resources(pci); 545 if (ret) { 546 if (pp->cfg) 547 pci_ecam_free(pp->cfg); 548 return ret; 549 } 550 551 /* Get the I/O range from DT */ 552 win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_IO); 553 if (win) { 554 pp->io_size = resource_size(win->res); 555 pp->io_bus_addr = win->res->start - win->offset; 556 pp->io_base = pci_pio_to_address(win->res->start); 557 } 558 559 /* 560 * visconti_pcie_cpu_addr_fixup() uses pp->io_base, so we have to 561 * call dw_pcie_parent_bus_offset() after setting pp->io_base. 562 */ 563 pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "config", 564 pp->cfg0_base); 565 return 0; 566 } 567 568 int dw_pcie_host_init(struct dw_pcie_rp *pp) 569 { 570 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 571 struct device *dev = pci->dev; 572 struct device_node *np = dev->of_node; 573 struct pci_host_bridge *bridge; 574 int ret; 575 576 raw_spin_lock_init(&pp->lock); 577 578 bridge = devm_pci_alloc_host_bridge(dev, 0); 579 if (!bridge) 580 return -ENOMEM; 581 582 pp->bridge = bridge; 583 584 ret = dw_pcie_host_get_resources(pp); 585 if (ret) 586 return ret; 587 588 if (pp->ops->init) { 589 ret = pp->ops->init(pp); 590 if (ret) 591 goto err_free_ecam; 592 } 593 594 if (pci_msi_enabled()) { 595 pp->has_msi_ctrl = !(pp->ops->msi_init || 596 of_property_present(np, "msi-parent") || 597 of_property_present(np, "msi-map")); 598 599 /* 600 * For the has_msi_ctrl case the default assignment is handled 601 * in the dw_pcie_msi_host_init(). 602 */ 603 if (!pp->has_msi_ctrl && !pp->num_vectors) { 604 pp->num_vectors = MSI_DEF_NUM_VECTORS; 605 } else if (pp->num_vectors > MAX_MSI_IRQS) { 606 dev_err(dev, "Invalid number of vectors\n"); 607 ret = -EINVAL; 608 goto err_deinit_host; 609 } 610 611 if (pp->ops->msi_init) { 612 ret = pp->ops->msi_init(pp); 613 if (ret < 0) 614 goto err_deinit_host; 615 } else if (pp->has_msi_ctrl) { 616 ret = dw_pcie_msi_host_init(pp); 617 if (ret < 0) 618 goto err_deinit_host; 619 } 620 } 621 622 dw_pcie_version_detect(pci); 623 624 dw_pcie_iatu_detect(pci); 625 626 if (pci->num_lanes < 1) 627 pci->num_lanes = dw_pcie_link_get_max_link_width(pci); 628 629 ret = of_pci_get_equalization_presets(dev, &pp->presets, pci->num_lanes); 630 if (ret) 631 goto err_free_msi; 632 633 if (pp->ecam_enabled) { 634 ret = dw_pcie_config_ecam_iatu(pp); 635 if (ret) { 636 dev_err(dev, "Failed to configure iATU in ECAM mode\n"); 637 goto err_free_msi; 638 } 639 } 640 641 /* 642 * Allocate the resource for MSG TLP before programming the iATU 643 * outbound window in dw_pcie_setup_rc(). Since the allocation depends 644 * on the value of 'region_align', this has to be done after 645 * dw_pcie_iatu_detect(). 646 * 647 * Glue drivers need to set 'use_atu_msg' before dw_pcie_host_init() to 648 * make use of the generic MSG TLP implementation. 649 */ 650 if (pp->use_atu_msg) 651 dw_pcie_host_request_msg_tlp_res(pp); 652 653 ret = dw_pcie_edma_detect(pci); 654 if (ret) 655 goto err_free_msi; 656 657 ret = dw_pcie_setup_rc(pp); 658 if (ret) 659 goto err_remove_edma; 660 661 if (!dw_pcie_link_up(pci)) { 662 ret = dw_pcie_start_link(pci); 663 if (ret) 664 goto err_remove_edma; 665 } 666 667 /* 668 * Note: Skip the link up delay only when a Link Up IRQ is present. 669 * If there is no Link Up IRQ, we should not bypass the delay 670 * because that would require users to manually rescan for devices. 671 */ 672 if (!pp->use_linkup_irq) 673 /* Ignore errors, the link may come up later */ 674 dw_pcie_wait_for_link(pci); 675 676 ret = pci_host_probe(bridge); 677 if (ret) 678 goto err_stop_link; 679 680 if (pp->ops->post_init) 681 pp->ops->post_init(pp); 682 683 dwc_pcie_debugfs_init(pci, DW_PCIE_RC_TYPE); 684 685 return 0; 686 687 err_stop_link: 688 dw_pcie_stop_link(pci); 689 690 err_remove_edma: 691 dw_pcie_edma_remove(pci); 692 693 err_free_msi: 694 if (pp->has_msi_ctrl) 695 dw_pcie_free_msi(pp); 696 697 err_deinit_host: 698 if (pp->ops->deinit) 699 pp->ops->deinit(pp); 700 701 err_free_ecam: 702 if (pp->cfg) 703 pci_ecam_free(pp->cfg); 704 705 return ret; 706 } 707 EXPORT_SYMBOL_GPL(dw_pcie_host_init); 708 709 void dw_pcie_host_deinit(struct dw_pcie_rp *pp) 710 { 711 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 712 713 dwc_pcie_debugfs_deinit(pci); 714 715 pci_stop_root_bus(pp->bridge->bus); 716 pci_remove_root_bus(pp->bridge->bus); 717 718 dw_pcie_stop_link(pci); 719 720 dw_pcie_edma_remove(pci); 721 722 if (pp->has_msi_ctrl) 723 dw_pcie_free_msi(pp); 724 725 if (pp->ops->deinit) 726 pp->ops->deinit(pp); 727 728 if (pp->cfg) 729 pci_ecam_free(pp->cfg); 730 } 731 EXPORT_SYMBOL_GPL(dw_pcie_host_deinit); 732 733 static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus, 734 unsigned int devfn, int where) 735 { 736 struct dw_pcie_rp *pp = bus->sysdata; 737 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 738 struct dw_pcie_ob_atu_cfg atu = { 0 }; 739 int type, ret; 740 u32 busdev; 741 742 /* 743 * Checking whether the link is up here is a last line of defense 744 * against platforms that forward errors on the system bus as 745 * SError upon PCI configuration transactions issued when the link 746 * is down. This check is racy by definition and does not stop 747 * the system from triggering an SError if the link goes down 748 * after this check is performed. 749 */ 750 if (!dw_pcie_link_up(pci)) 751 return NULL; 752 753 busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) | 754 PCIE_ATU_FUNC(PCI_FUNC(devfn)); 755 756 if (pci_is_root_bus(bus->parent)) 757 type = PCIE_ATU_TYPE_CFG0; 758 else 759 type = PCIE_ATU_TYPE_CFG1; 760 761 atu.type = type; 762 atu.parent_bus_addr = pp->cfg0_base - pci->parent_bus_offset; 763 atu.pci_addr = busdev; 764 atu.size = pp->cfg0_size; 765 766 ret = dw_pcie_prog_outbound_atu(pci, &atu); 767 if (ret) 768 return NULL; 769 770 return pp->va_cfg0_base + where; 771 } 772 773 static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn, 774 int where, int size, u32 *val) 775 { 776 struct dw_pcie_rp *pp = bus->sysdata; 777 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 778 struct dw_pcie_ob_atu_cfg atu = { 0 }; 779 int ret; 780 781 ret = pci_generic_config_read(bus, devfn, where, size, val); 782 if (ret != PCIBIOS_SUCCESSFUL) 783 return ret; 784 785 if (pp->cfg0_io_shared) { 786 atu.type = PCIE_ATU_TYPE_IO; 787 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; 788 atu.pci_addr = pp->io_bus_addr; 789 atu.size = pp->io_size; 790 791 ret = dw_pcie_prog_outbound_atu(pci, &atu); 792 if (ret) 793 return PCIBIOS_SET_FAILED; 794 } 795 796 return PCIBIOS_SUCCESSFUL; 797 } 798 799 static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn, 800 int where, int size, u32 val) 801 { 802 struct dw_pcie_rp *pp = bus->sysdata; 803 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 804 struct dw_pcie_ob_atu_cfg atu = { 0 }; 805 int ret; 806 807 ret = pci_generic_config_write(bus, devfn, where, size, val); 808 if (ret != PCIBIOS_SUCCESSFUL) 809 return ret; 810 811 if (pp->cfg0_io_shared) { 812 atu.type = PCIE_ATU_TYPE_IO; 813 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; 814 atu.pci_addr = pp->io_bus_addr; 815 atu.size = pp->io_size; 816 817 ret = dw_pcie_prog_outbound_atu(pci, &atu); 818 if (ret) 819 return PCIBIOS_SET_FAILED; 820 } 821 822 return PCIBIOS_SUCCESSFUL; 823 } 824 825 static struct pci_ops dw_child_pcie_ops = { 826 .map_bus = dw_pcie_other_conf_map_bus, 827 .read = dw_pcie_rd_other_conf, 828 .write = dw_pcie_wr_other_conf, 829 }; 830 831 void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) 832 { 833 struct dw_pcie_rp *pp = bus->sysdata; 834 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 835 836 if (PCI_SLOT(devfn) > 0) 837 return NULL; 838 839 return pci->dbi_base + where; 840 } 841 EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus); 842 843 static void __iomem *dw_pcie_ecam_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) 844 { 845 struct pci_config_window *cfg = bus->sysdata; 846 struct dw_pcie_rp *pp = cfg->priv; 847 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 848 unsigned int busn = bus->number; 849 850 if (busn > 0) 851 return pci_ecam_map_bus(bus, devfn, where); 852 853 if (PCI_SLOT(devfn) > 0) 854 return NULL; 855 856 return pci->dbi_base + where; 857 } 858 859 static struct pci_ops dw_pcie_ops = { 860 .map_bus = dw_pcie_own_conf_map_bus, 861 .read = pci_generic_config_read, 862 .write = pci_generic_config_write, 863 }; 864 865 static struct pci_ops dw_pcie_ecam_ops = { 866 .map_bus = dw_pcie_ecam_conf_map_bus, 867 .read = pci_generic_config_read, 868 .write = pci_generic_config_write, 869 }; 870 871 static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp) 872 { 873 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 874 struct dw_pcie_ob_atu_cfg atu = { 0 }; 875 struct resource_entry *entry; 876 int i, ret; 877 878 /* Note the very first outbound ATU is used for CFG IOs */ 879 if (!pci->num_ob_windows) { 880 dev_err(pci->dev, "No outbound iATU found\n"); 881 return -EINVAL; 882 } 883 884 /* 885 * Ensure all out/inbound windows are disabled before proceeding with 886 * the MEM/IO (dma-)ranges setups. 887 */ 888 for (i = 0; i < pci->num_ob_windows; i++) 889 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, i); 890 891 for (i = 0; i < pci->num_ib_windows; i++) 892 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, i); 893 894 i = 0; 895 resource_list_for_each_entry(entry, &pp->bridge->windows) { 896 if (resource_type(entry->res) != IORESOURCE_MEM) 897 continue; 898 899 if (pci->num_ob_windows <= ++i) 900 break; 901 902 atu.index = i; 903 atu.type = PCIE_ATU_TYPE_MEM; 904 atu.parent_bus_addr = entry->res->start - pci->parent_bus_offset; 905 atu.pci_addr = entry->res->start - entry->offset; 906 907 /* Adjust iATU size if MSG TLP region was allocated before */ 908 if (pp->msg_res && pp->msg_res->parent == entry->res) 909 atu.size = resource_size(entry->res) - 910 resource_size(pp->msg_res); 911 else 912 atu.size = resource_size(entry->res); 913 914 ret = dw_pcie_prog_outbound_atu(pci, &atu); 915 if (ret) { 916 dev_err(pci->dev, "Failed to set MEM range %pr\n", 917 entry->res); 918 return ret; 919 } 920 } 921 922 if (pp->io_size) { 923 if (pci->num_ob_windows > ++i) { 924 atu.index = i; 925 atu.type = PCIE_ATU_TYPE_IO; 926 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; 927 atu.pci_addr = pp->io_bus_addr; 928 atu.size = pp->io_size; 929 930 ret = dw_pcie_prog_outbound_atu(pci, &atu); 931 if (ret) { 932 dev_err(pci->dev, "Failed to set IO range %pr\n", 933 entry->res); 934 return ret; 935 } 936 } else { 937 pp->cfg0_io_shared = true; 938 } 939 } 940 941 if (pci->num_ob_windows <= i) 942 dev_warn(pci->dev, "Ranges exceed outbound iATU size (%d)\n", 943 pci->num_ob_windows); 944 945 pp->msg_atu_index = i; 946 947 i = 0; 948 resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) { 949 if (resource_type(entry->res) != IORESOURCE_MEM) 950 continue; 951 952 if (pci->num_ib_windows <= i) 953 break; 954 955 ret = dw_pcie_prog_inbound_atu(pci, i++, PCIE_ATU_TYPE_MEM, 956 entry->res->start, 957 entry->res->start - entry->offset, 958 resource_size(entry->res)); 959 if (ret) { 960 dev_err(pci->dev, "Failed to set DMA range %pr\n", 961 entry->res); 962 return ret; 963 } 964 } 965 966 if (pci->num_ib_windows <= i) 967 dev_warn(pci->dev, "Dma-ranges exceed inbound iATU size (%u)\n", 968 pci->num_ib_windows); 969 970 return 0; 971 } 972 973 static void dw_pcie_program_presets(struct dw_pcie_rp *pp, enum pci_bus_speed speed) 974 { 975 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 976 u8 lane_eq_offset, lane_reg_size, cap_id; 977 u8 *presets; 978 u32 cap; 979 int i; 980 981 if (speed == PCIE_SPEED_8_0GT) { 982 presets = (u8 *)pp->presets.eq_presets_8gts; 983 lane_eq_offset = PCI_SECPCI_LE_CTRL; 984 cap_id = PCI_EXT_CAP_ID_SECPCI; 985 /* For data rate of 8 GT/S each lane equalization control is 16bits wide*/ 986 lane_reg_size = 0x2; 987 } else if (speed == PCIE_SPEED_16_0GT) { 988 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_16GTS - 1]; 989 lane_eq_offset = PCI_PL_16GT_LE_CTRL; 990 cap_id = PCI_EXT_CAP_ID_PL_16GT; 991 lane_reg_size = 0x1; 992 } else if (speed == PCIE_SPEED_32_0GT) { 993 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_32GTS - 1]; 994 lane_eq_offset = PCI_PL_32GT_LE_CTRL; 995 cap_id = PCI_EXT_CAP_ID_PL_32GT; 996 lane_reg_size = 0x1; 997 } else if (speed == PCIE_SPEED_64_0GT) { 998 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_64GTS - 1]; 999 lane_eq_offset = PCI_PL_64GT_LE_CTRL; 1000 cap_id = PCI_EXT_CAP_ID_PL_64GT; 1001 lane_reg_size = 0x1; 1002 } else { 1003 return; 1004 } 1005 1006 if (presets[0] == PCI_EQ_RESV) 1007 return; 1008 1009 cap = dw_pcie_find_ext_capability(pci, cap_id); 1010 if (!cap) 1011 return; 1012 1013 /* 1014 * Write preset values to the registers byte-by-byte for the given 1015 * number of lanes and register size. 1016 */ 1017 for (i = 0; i < pci->num_lanes * lane_reg_size; i++) 1018 dw_pcie_writeb_dbi(pci, cap + lane_eq_offset + i, presets[i]); 1019 } 1020 1021 static void dw_pcie_config_presets(struct dw_pcie_rp *pp) 1022 { 1023 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 1024 enum pci_bus_speed speed = pcie_link_speed[pci->max_link_speed]; 1025 1026 /* 1027 * Lane equalization settings need to be applied for all data rates the 1028 * controller supports and for all supported lanes. 1029 */ 1030 1031 if (speed >= PCIE_SPEED_8_0GT) 1032 dw_pcie_program_presets(pp, PCIE_SPEED_8_0GT); 1033 1034 if (speed >= PCIE_SPEED_16_0GT) 1035 dw_pcie_program_presets(pp, PCIE_SPEED_16_0GT); 1036 1037 if (speed >= PCIE_SPEED_32_0GT) 1038 dw_pcie_program_presets(pp, PCIE_SPEED_32_0GT); 1039 1040 if (speed >= PCIE_SPEED_64_0GT) 1041 dw_pcie_program_presets(pp, PCIE_SPEED_64_0GT); 1042 } 1043 1044 int dw_pcie_setup_rc(struct dw_pcie_rp *pp) 1045 { 1046 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 1047 u32 val; 1048 int ret; 1049 1050 /* 1051 * Enable DBI read-only registers for writing/updating configuration. 1052 * Write permission gets disabled towards the end of this function. 1053 */ 1054 dw_pcie_dbi_ro_wr_en(pci); 1055 1056 dw_pcie_setup(pci); 1057 1058 dw_pcie_msi_init(pp); 1059 1060 /* Setup RC BARs */ 1061 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004); 1062 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000); 1063 1064 /* Setup interrupt pins */ 1065 val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE); 1066 val &= 0xffff00ff; 1067 val |= 0x00000100; 1068 dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val); 1069 1070 /* Setup bus numbers */ 1071 val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS); 1072 val &= 0xff000000; 1073 val |= 0x00ff0100; 1074 dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val); 1075 1076 /* Setup command register */ 1077 val = dw_pcie_readl_dbi(pci, PCI_COMMAND); 1078 val &= 0xffff0000; 1079 val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY | 1080 PCI_COMMAND_MASTER | PCI_COMMAND_SERR; 1081 dw_pcie_writel_dbi(pci, PCI_COMMAND, val); 1082 1083 dw_pcie_config_presets(pp); 1084 /* 1085 * If the platform provides its own child bus config accesses, it means 1086 * the platform uses its own address translation component rather than 1087 * ATU, so we should not program the ATU here. 1088 */ 1089 if (pp->bridge->child_ops == &dw_child_pcie_ops) { 1090 ret = dw_pcie_iatu_setup(pp); 1091 if (ret) 1092 return ret; 1093 } 1094 1095 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); 1096 1097 /* Program correct class for RC */ 1098 dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); 1099 1100 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); 1101 val |= PORT_LOGIC_SPEED_CHANGE; 1102 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); 1103 1104 dw_pcie_dbi_ro_wr_dis(pci); 1105 1106 return 0; 1107 } 1108 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc); 1109 1110 static int dw_pcie_pme_turn_off(struct dw_pcie *pci) 1111 { 1112 struct dw_pcie_ob_atu_cfg atu = { 0 }; 1113 void __iomem *mem; 1114 int ret; 1115 1116 if (pci->num_ob_windows <= pci->pp.msg_atu_index) 1117 return -ENOSPC; 1118 1119 if (!pci->pp.msg_res) 1120 return -ENOSPC; 1121 1122 atu.code = PCIE_MSG_CODE_PME_TURN_OFF; 1123 atu.routing = PCIE_MSG_TYPE_R_BC; 1124 atu.type = PCIE_ATU_TYPE_MSG; 1125 atu.size = resource_size(pci->pp.msg_res); 1126 atu.index = pci->pp.msg_atu_index; 1127 1128 atu.parent_bus_addr = pci->pp.msg_res->start - pci->parent_bus_offset; 1129 1130 ret = dw_pcie_prog_outbound_atu(pci, &atu); 1131 if (ret) 1132 return ret; 1133 1134 mem = ioremap(pci->pp.msg_res->start, pci->region_align); 1135 if (!mem) 1136 return -ENOMEM; 1137 1138 /* A dummy write is converted to a Msg TLP */ 1139 writel(0, mem); 1140 1141 iounmap(mem); 1142 1143 return 0; 1144 } 1145 1146 int dw_pcie_suspend_noirq(struct dw_pcie *pci) 1147 { 1148 u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 1149 u32 val; 1150 int ret; 1151 1152 /* 1153 * If L1SS is supported, then do not put the link into L2 as some 1154 * devices such as NVMe expect low resume latency. 1155 */ 1156 if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1) 1157 return 0; 1158 1159 if (pci->pp.ops->pme_turn_off) { 1160 pci->pp.ops->pme_turn_off(&pci->pp); 1161 } else { 1162 ret = dw_pcie_pme_turn_off(pci); 1163 if (ret) 1164 return ret; 1165 } 1166 1167 ret = read_poll_timeout(dw_pcie_get_ltssm, val, 1168 val == DW_PCIE_LTSSM_L2_IDLE || 1169 val <= DW_PCIE_LTSSM_DETECT_WAIT, 1170 PCIE_PME_TO_L2_TIMEOUT_US/10, 1171 PCIE_PME_TO_L2_TIMEOUT_US, false, pci); 1172 if (ret) { 1173 /* Only log message when LTSSM isn't in DETECT or POLL */ 1174 dev_err(pci->dev, "Timeout waiting for L2 entry! LTSSM: 0x%x\n", val); 1175 return ret; 1176 } 1177 1178 /* 1179 * Per PCIe r6.0, sec 5.3.3.2.1, software should wait at least 1180 * 100ns after L2/L3 Ready before turning off refclock and 1181 * main power. This is harmless when no endpoint is connected. 1182 */ 1183 udelay(1); 1184 1185 dw_pcie_stop_link(pci); 1186 if (pci->pp.ops->deinit) 1187 pci->pp.ops->deinit(&pci->pp); 1188 1189 pci->suspended = true; 1190 1191 return ret; 1192 } 1193 EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq); 1194 1195 int dw_pcie_resume_noirq(struct dw_pcie *pci) 1196 { 1197 int ret; 1198 1199 if (!pci->suspended) 1200 return 0; 1201 1202 pci->suspended = false; 1203 1204 if (pci->pp.ops->init) { 1205 ret = pci->pp.ops->init(&pci->pp); 1206 if (ret) { 1207 dev_err(pci->dev, "Host init failed: %d\n", ret); 1208 return ret; 1209 } 1210 } 1211 1212 dw_pcie_setup_rc(&pci->pp); 1213 1214 ret = dw_pcie_start_link(pci); 1215 if (ret) 1216 return ret; 1217 1218 ret = dw_pcie_wait_for_link(pci); 1219 if (ret) 1220 return ret; 1221 1222 return ret; 1223 } 1224 EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq); 1225