1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe host controller driver 4 * 5 * Copyright (C) 2013 Samsung Electronics Co., Ltd. 6 * https://www.samsung.com 7 * 8 * Author: Jingoo Han <jg1.han@samsung.com> 9 */ 10 11 #include <linux/align.h> 12 #include <linux/iopoll.h> 13 #include <linux/irqchip/chained_irq.h> 14 #include <linux/irqchip/irq-msi-lib.h> 15 #include <linux/irqdomain.h> 16 #include <linux/msi.h> 17 #include <linux/of_address.h> 18 #include <linux/of_pci.h> 19 #include <linux/pci_regs.h> 20 #include <linux/platform_device.h> 21 22 #include "../../pci.h" 23 #include "pcie-designware.h" 24 25 static struct pci_ops dw_pcie_ops; 26 static struct pci_ops dw_pcie_ecam_ops; 27 static struct pci_ops dw_child_pcie_ops; 28 29 #define DW_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ 30 MSI_FLAG_USE_DEF_CHIP_OPS | \ 31 MSI_FLAG_NO_AFFINITY | \ 32 MSI_FLAG_PCI_MSI_MASK_PARENT) 33 #define DW_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \ 34 MSI_FLAG_PCI_MSIX | \ 35 MSI_GENERIC_FLAGS_MASK) 36 37 #define IS_256MB_ALIGNED(x) IS_ALIGNED(x, SZ_256M) 38 39 static const struct msi_parent_ops dw_pcie_msi_parent_ops = { 40 .required_flags = DW_PCIE_MSI_FLAGS_REQUIRED, 41 .supported_flags = DW_PCIE_MSI_FLAGS_SUPPORTED, 42 .bus_select_token = DOMAIN_BUS_PCI_MSI, 43 .chip_flags = MSI_CHIP_FLAG_SET_ACK, 44 .prefix = "DW-", 45 .init_dev_msi_info = msi_lib_init_dev_msi_info, 46 }; 47 48 /* MSI int handler */ 49 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp) 50 { 51 int i, pos; 52 unsigned long val; 53 u32 status, num_ctrls; 54 irqreturn_t ret = IRQ_NONE; 55 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 56 57 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; 58 59 for (i = 0; i < num_ctrls; i++) { 60 status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS + 61 (i * MSI_REG_CTRL_BLOCK_SIZE)); 62 if (!status) 63 continue; 64 65 ret = IRQ_HANDLED; 66 val = status; 67 pos = 0; 68 while ((pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL, 69 pos)) != MAX_MSI_IRQS_PER_CTRL) { 70 generic_handle_domain_irq(pp->irq_domain, 71 (i * MAX_MSI_IRQS_PER_CTRL) + 72 pos); 73 pos++; 74 } 75 } 76 77 return ret; 78 } 79 80 /* Chained MSI interrupt service routine */ 81 static void dw_chained_msi_isr(struct irq_desc *desc) 82 { 83 struct irq_chip *chip = irq_desc_get_chip(desc); 84 struct dw_pcie_rp *pp; 85 86 chained_irq_enter(chip, desc); 87 88 pp = irq_desc_get_handler_data(desc); 89 dw_handle_msi_irq(pp); 90 91 chained_irq_exit(chip, desc); 92 } 93 94 static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg) 95 { 96 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); 97 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 98 u64 msi_target; 99 100 msi_target = (u64)pp->msi_data; 101 102 msg->address_lo = lower_32_bits(msi_target); 103 msg->address_hi = upper_32_bits(msi_target); 104 105 msg->data = d->hwirq; 106 107 dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n", 108 (int)d->hwirq, msg->address_hi, msg->address_lo); 109 } 110 111 static void dw_pci_bottom_mask(struct irq_data *d) 112 { 113 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); 114 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 115 unsigned int res, bit, ctrl; 116 unsigned long flags; 117 118 raw_spin_lock_irqsave(&pp->lock, flags); 119 120 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; 121 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; 122 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; 123 124 pp->irq_mask[ctrl] |= BIT(bit); 125 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); 126 127 raw_spin_unlock_irqrestore(&pp->lock, flags); 128 } 129 130 static void dw_pci_bottom_unmask(struct irq_data *d) 131 { 132 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); 133 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 134 unsigned int res, bit, ctrl; 135 unsigned long flags; 136 137 raw_spin_lock_irqsave(&pp->lock, flags); 138 139 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; 140 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; 141 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; 142 143 pp->irq_mask[ctrl] &= ~BIT(bit); 144 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); 145 146 raw_spin_unlock_irqrestore(&pp->lock, flags); 147 } 148 149 static void dw_pci_bottom_ack(struct irq_data *d) 150 { 151 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); 152 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 153 unsigned int res, bit, ctrl; 154 155 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; 156 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; 157 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; 158 159 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit)); 160 } 161 162 static struct irq_chip dw_pci_msi_bottom_irq_chip = { 163 .name = "DWPCI-MSI", 164 .irq_ack = dw_pci_bottom_ack, 165 .irq_compose_msi_msg = dw_pci_setup_msi_msg, 166 .irq_mask = dw_pci_bottom_mask, 167 .irq_unmask = dw_pci_bottom_unmask, 168 }; 169 170 static int dw_pcie_irq_domain_alloc(struct irq_domain *domain, 171 unsigned int virq, unsigned int nr_irqs, 172 void *args) 173 { 174 struct dw_pcie_rp *pp = domain->host_data; 175 unsigned long flags; 176 u32 i; 177 int bit; 178 179 raw_spin_lock_irqsave(&pp->lock, flags); 180 181 bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors, 182 order_base_2(nr_irqs)); 183 184 raw_spin_unlock_irqrestore(&pp->lock, flags); 185 186 if (bit < 0) 187 return -ENOSPC; 188 189 for (i = 0; i < nr_irqs; i++) 190 irq_domain_set_info(domain, virq + i, bit + i, 191 pp->msi_irq_chip, 192 pp, handle_edge_irq, 193 NULL, NULL); 194 195 return 0; 196 } 197 198 static void dw_pcie_irq_domain_free(struct irq_domain *domain, 199 unsigned int virq, unsigned int nr_irqs) 200 { 201 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 202 struct dw_pcie_rp *pp = domain->host_data; 203 unsigned long flags; 204 205 raw_spin_lock_irqsave(&pp->lock, flags); 206 207 bitmap_release_region(pp->msi_irq_in_use, d->hwirq, 208 order_base_2(nr_irqs)); 209 210 raw_spin_unlock_irqrestore(&pp->lock, flags); 211 } 212 213 static const struct irq_domain_ops dw_pcie_msi_domain_ops = { 214 .alloc = dw_pcie_irq_domain_alloc, 215 .free = dw_pcie_irq_domain_free, 216 }; 217 218 int dw_pcie_allocate_domains(struct dw_pcie_rp *pp) 219 { 220 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 221 struct irq_domain_info info = { 222 .fwnode = dev_fwnode(pci->dev), 223 .ops = &dw_pcie_msi_domain_ops, 224 .size = pp->num_vectors, 225 .host_data = pp, 226 }; 227 228 pp->irq_domain = msi_create_parent_irq_domain(&info, &dw_pcie_msi_parent_ops); 229 if (!pp->irq_domain) { 230 dev_err(pci->dev, "Failed to create IRQ domain\n"); 231 return -ENOMEM; 232 } 233 234 return 0; 235 } 236 EXPORT_SYMBOL_GPL(dw_pcie_allocate_domains); 237 238 void dw_pcie_free_msi(struct dw_pcie_rp *pp) 239 { 240 u32 ctrl; 241 242 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) { 243 if (pp->msi_irq[ctrl] > 0) 244 irq_set_chained_handler_and_data(pp->msi_irq[ctrl], 245 NULL, NULL); 246 } 247 248 irq_domain_remove(pp->irq_domain); 249 } 250 EXPORT_SYMBOL_GPL(dw_pcie_free_msi); 251 252 void dw_pcie_msi_init(struct dw_pcie_rp *pp) 253 { 254 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 255 u64 msi_target = (u64)pp->msi_data; 256 u32 ctrl, num_ctrls; 257 258 if (!pci_msi_enabled() || !pp->has_msi_ctrl) 259 return; 260 261 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; 262 263 /* Initialize IRQ Status array */ 264 for (ctrl = 0; ctrl < num_ctrls; ctrl++) { 265 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + 266 (ctrl * MSI_REG_CTRL_BLOCK_SIZE), 267 pp->irq_mask[ctrl]); 268 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE + 269 (ctrl * MSI_REG_CTRL_BLOCK_SIZE), 270 ~0); 271 } 272 273 /* Program the msi_data */ 274 dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target)); 275 dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target)); 276 } 277 EXPORT_SYMBOL_GPL(dw_pcie_msi_init); 278 279 static int dw_pcie_parse_split_msi_irq(struct dw_pcie_rp *pp) 280 { 281 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 282 struct device *dev = pci->dev; 283 struct platform_device *pdev = to_platform_device(dev); 284 u32 ctrl, max_vectors; 285 int irq; 286 287 /* Parse any "msiX" IRQs described in the devicetree */ 288 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) { 289 char msi_name[] = "msiX"; 290 291 msi_name[3] = '0' + ctrl; 292 irq = platform_get_irq_byname_optional(pdev, msi_name); 293 if (irq == -ENXIO) 294 break; 295 if (irq < 0) 296 return dev_err_probe(dev, irq, 297 "Failed to parse MSI IRQ '%s'\n", 298 msi_name); 299 300 pp->msi_irq[ctrl] = irq; 301 } 302 303 /* If no "msiX" IRQs, caller should fallback to "msi" IRQ */ 304 if (ctrl == 0) 305 return -ENXIO; 306 307 max_vectors = ctrl * MAX_MSI_IRQS_PER_CTRL; 308 if (pp->num_vectors > max_vectors) { 309 dev_warn(dev, "Exceeding number of MSI vectors, limiting to %u\n", 310 max_vectors); 311 pp->num_vectors = max_vectors; 312 } 313 if (!pp->num_vectors) 314 pp->num_vectors = max_vectors; 315 316 return 0; 317 } 318 319 int dw_pcie_msi_host_init(struct dw_pcie_rp *pp) 320 { 321 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 322 struct device *dev = pci->dev; 323 struct platform_device *pdev = to_platform_device(dev); 324 u64 *msi_vaddr = NULL; 325 int ret; 326 u32 ctrl, num_ctrls; 327 328 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) 329 pp->irq_mask[ctrl] = ~0; 330 331 if (!pp->msi_irq[0]) { 332 ret = dw_pcie_parse_split_msi_irq(pp); 333 if (ret < 0 && ret != -ENXIO) 334 return ret; 335 } 336 337 if (!pp->num_vectors) 338 pp->num_vectors = MSI_DEF_NUM_VECTORS; 339 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; 340 341 if (!pp->msi_irq[0]) { 342 pp->msi_irq[0] = platform_get_irq_byname_optional(pdev, "msi"); 343 if (pp->msi_irq[0] < 0) { 344 pp->msi_irq[0] = platform_get_irq(pdev, 0); 345 if (pp->msi_irq[0] < 0) 346 return pp->msi_irq[0]; 347 } 348 } 349 350 dev_dbg(dev, "Using %d MSI vectors\n", pp->num_vectors); 351 352 pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip; 353 354 ret = dw_pcie_allocate_domains(pp); 355 if (ret) 356 return ret; 357 358 for (ctrl = 0; ctrl < num_ctrls; ctrl++) { 359 if (pp->msi_irq[ctrl] > 0) 360 irq_set_chained_handler_and_data(pp->msi_irq[ctrl], 361 dw_chained_msi_isr, pp); 362 } 363 364 /* 365 * Even though the iMSI-RX Module supports 64-bit addresses some 366 * peripheral PCIe devices may lack 64-bit message support. In 367 * order not to miss MSI TLPs from those devices the MSI target 368 * address has to be within the lowest 4GB. 369 * 370 * Note until there is a better alternative found the reservation is 371 * done by allocating from the artificially limited DMA-coherent 372 * memory. 373 */ 374 ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); 375 if (!ret) 376 msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, 377 GFP_KERNEL); 378 379 if (!msi_vaddr) { 380 dev_warn(dev, "Failed to allocate 32-bit MSI address\n"); 381 dma_set_coherent_mask(dev, DMA_BIT_MASK(64)); 382 msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, 383 GFP_KERNEL); 384 if (!msi_vaddr) { 385 dev_err(dev, "Failed to allocate MSI address\n"); 386 dw_pcie_free_msi(pp); 387 return -ENOMEM; 388 } 389 } 390 391 return 0; 392 } 393 EXPORT_SYMBOL_GPL(dw_pcie_msi_host_init); 394 395 static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp) 396 { 397 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 398 struct resource_entry *win; 399 struct resource *res; 400 401 win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM); 402 if (win) { 403 res = devm_kzalloc(pci->dev, sizeof(*res), GFP_KERNEL); 404 if (!res) 405 return; 406 407 /* 408 * Allocate MSG TLP region of size 'region_align' at the end of 409 * the host bridge window. 410 */ 411 res->start = win->res->end - pci->region_align + 1; 412 res->end = win->res->end; 413 res->name = "msg"; 414 res->flags = win->res->flags | IORESOURCE_BUSY; 415 416 if (!devm_request_resource(pci->dev, win->res, res)) 417 pp->msg_res = res; 418 } 419 } 420 421 static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp) 422 { 423 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 424 struct dw_pcie_ob_atu_cfg atu = {0}; 425 resource_size_t bus_range_max; 426 struct resource_entry *bus; 427 int ret; 428 429 bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); 430 431 /* 432 * Root bus under the host bridge doesn't require any iATU configuration 433 * as DBI region will be used to access root bus config space. 434 * Immediate bus under Root Bus, needs type 0 iATU configuration and 435 * remaining buses need type 1 iATU configuration. 436 */ 437 atu.index = 0; 438 atu.type = PCIE_ATU_TYPE_CFG0; 439 atu.parent_bus_addr = pp->cfg0_base + SZ_1M; 440 /* 1MiB is to cover 1 (bus) * 32 (devices) * 8 (functions) */ 441 atu.size = SZ_1M; 442 atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE; 443 ret = dw_pcie_prog_outbound_atu(pci, &atu); 444 if (ret) 445 return ret; 446 447 bus_range_max = resource_size(bus->res); 448 449 if (bus_range_max < 2) 450 return 0; 451 452 /* Configure remaining buses in type 1 iATU configuration */ 453 atu.index = 1; 454 atu.type = PCIE_ATU_TYPE_CFG1; 455 atu.parent_bus_addr = pp->cfg0_base + SZ_2M; 456 atu.size = (SZ_1M * bus_range_max) - SZ_2M; 457 atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE; 458 459 return dw_pcie_prog_outbound_atu(pci, &atu); 460 } 461 462 static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res) 463 { 464 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 465 struct device *dev = pci->dev; 466 struct resource_entry *bus; 467 468 bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); 469 if (!bus) 470 return -ENODEV; 471 472 pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops); 473 if (IS_ERR(pp->cfg)) 474 return PTR_ERR(pp->cfg); 475 476 return 0; 477 } 478 479 static bool dw_pcie_ecam_enabled(struct dw_pcie_rp *pp, struct resource *config_res) 480 { 481 struct resource *bus_range; 482 u64 nr_buses; 483 484 /* Vendor glue drivers may implement their own ECAM mechanism */ 485 if (pp->native_ecam) 486 return false; 487 488 /* 489 * PCIe spec r6.0, sec 7.2.2 mandates the base address used for ECAM to 490 * be aligned on a 2^(n+20) byte boundary, where n is the number of bits 491 * used for representing 'bus' in BDF. Since the DWC cores always use 8 492 * bits for representing 'bus', the base address has to be aligned to 493 * 2^28 byte boundary, which is 256 MiB. 494 */ 495 if (!IS_256MB_ALIGNED(config_res->start)) 496 return false; 497 498 bus_range = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS)->res; 499 if (!bus_range) 500 return false; 501 502 nr_buses = resource_size(config_res) >> PCIE_ECAM_BUS_SHIFT; 503 504 return nr_buses >= resource_size(bus_range); 505 } 506 507 static int dw_pcie_host_get_resources(struct dw_pcie_rp *pp) 508 { 509 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 510 struct device *dev = pci->dev; 511 struct platform_device *pdev = to_platform_device(dev); 512 struct resource_entry *win; 513 struct resource *res; 514 int ret; 515 516 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); 517 if (!res) { 518 dev_err(dev, "Missing \"config\" reg space\n"); 519 return -ENODEV; 520 } 521 522 pp->cfg0_size = resource_size(res); 523 pp->cfg0_base = res->start; 524 525 pp->ecam_enabled = dw_pcie_ecam_enabled(pp, res); 526 if (pp->ecam_enabled) { 527 ret = dw_pcie_create_ecam_window(pp, res); 528 if (ret) 529 return ret; 530 531 pp->bridge->ops = &dw_pcie_ecam_ops; 532 pp->bridge->sysdata = pp->cfg; 533 pp->cfg->priv = pp; 534 } else { 535 pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res); 536 if (IS_ERR(pp->va_cfg0_base)) 537 return PTR_ERR(pp->va_cfg0_base); 538 539 /* Set default bus ops */ 540 pp->bridge->ops = &dw_pcie_ops; 541 pp->bridge->child_ops = &dw_child_pcie_ops; 542 pp->bridge->sysdata = pp; 543 } 544 545 ret = dw_pcie_get_resources(pci); 546 if (ret) { 547 if (pp->cfg) 548 pci_ecam_free(pp->cfg); 549 return ret; 550 } 551 552 /* Get the I/O range from DT */ 553 win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_IO); 554 if (win) { 555 pp->io_size = resource_size(win->res); 556 pp->io_bus_addr = win->res->start - win->offset; 557 pp->io_base = pci_pio_to_address(win->res->start); 558 } 559 560 /* 561 * visconti_pcie_cpu_addr_fixup() uses pp->io_base, so we have to 562 * call dw_pcie_parent_bus_offset() after setting pp->io_base. 563 */ 564 pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "config", 565 pp->cfg0_base); 566 return 0; 567 } 568 569 int dw_pcie_host_init(struct dw_pcie_rp *pp) 570 { 571 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 572 struct device *dev = pci->dev; 573 struct device_node *np = dev->of_node; 574 struct pci_host_bridge *bridge; 575 int ret; 576 577 raw_spin_lock_init(&pp->lock); 578 579 bridge = devm_pci_alloc_host_bridge(dev, 0); 580 if (!bridge) 581 return -ENOMEM; 582 583 pp->bridge = bridge; 584 585 ret = dw_pcie_host_get_resources(pp); 586 if (ret) 587 return ret; 588 589 if (pp->ops->init) { 590 ret = pp->ops->init(pp); 591 if (ret) 592 goto err_free_ecam; 593 } 594 595 if (pci_msi_enabled()) { 596 pp->has_msi_ctrl = !(pp->ops->msi_init || 597 of_property_present(np, "msi-parent") || 598 of_property_present(np, "msi-map")); 599 600 /* 601 * For the has_msi_ctrl case the default assignment is handled 602 * in the dw_pcie_msi_host_init(). 603 */ 604 if (!pp->has_msi_ctrl && !pp->num_vectors) { 605 pp->num_vectors = MSI_DEF_NUM_VECTORS; 606 } else if (pp->num_vectors > MAX_MSI_IRQS) { 607 dev_err(dev, "Invalid number of vectors\n"); 608 ret = -EINVAL; 609 goto err_deinit_host; 610 } 611 612 if (pp->ops->msi_init) { 613 ret = pp->ops->msi_init(pp); 614 if (ret < 0) 615 goto err_deinit_host; 616 } else if (pp->has_msi_ctrl) { 617 ret = dw_pcie_msi_host_init(pp); 618 if (ret < 0) 619 goto err_deinit_host; 620 } 621 } 622 623 dw_pcie_version_detect(pci); 624 625 dw_pcie_iatu_detect(pci); 626 627 if (pci->num_lanes < 1) 628 pci->num_lanes = dw_pcie_link_get_max_link_width(pci); 629 630 ret = of_pci_get_equalization_presets(dev, &pp->presets, pci->num_lanes); 631 if (ret) 632 goto err_free_msi; 633 634 if (pp->ecam_enabled) { 635 ret = dw_pcie_config_ecam_iatu(pp); 636 if (ret) { 637 dev_err(dev, "Failed to configure iATU in ECAM mode\n"); 638 goto err_free_msi; 639 } 640 } 641 642 /* 643 * Allocate the resource for MSG TLP before programming the iATU 644 * outbound window in dw_pcie_setup_rc(). Since the allocation depends 645 * on the value of 'region_align', this has to be done after 646 * dw_pcie_iatu_detect(). 647 * 648 * Glue drivers need to set 'use_atu_msg' before dw_pcie_host_init() to 649 * make use of the generic MSG TLP implementation. 650 */ 651 if (pp->use_atu_msg) 652 dw_pcie_host_request_msg_tlp_res(pp); 653 654 ret = dw_pcie_edma_detect(pci); 655 if (ret) 656 goto err_free_msi; 657 658 ret = dw_pcie_setup_rc(pp); 659 if (ret) 660 goto err_remove_edma; 661 662 if (!dw_pcie_link_up(pci)) { 663 ret = dw_pcie_start_link(pci); 664 if (ret) 665 goto err_remove_edma; 666 } 667 668 /* 669 * Note: Skip the link up delay only when a Link Up IRQ is present. 670 * If there is no Link Up IRQ, we should not bypass the delay 671 * because that would require users to manually rescan for devices. 672 */ 673 if (!pp->use_linkup_irq) 674 /* Ignore errors, the link may come up later */ 675 dw_pcie_wait_for_link(pci); 676 677 ret = pci_host_probe(bridge); 678 if (ret) 679 goto err_stop_link; 680 681 if (pp->ops->post_init) 682 pp->ops->post_init(pp); 683 684 dwc_pcie_debugfs_init(pci, DW_PCIE_RC_TYPE); 685 686 return 0; 687 688 err_stop_link: 689 dw_pcie_stop_link(pci); 690 691 err_remove_edma: 692 dw_pcie_edma_remove(pci); 693 694 err_free_msi: 695 if (pp->has_msi_ctrl) 696 dw_pcie_free_msi(pp); 697 698 err_deinit_host: 699 if (pp->ops->deinit) 700 pp->ops->deinit(pp); 701 702 err_free_ecam: 703 if (pp->cfg) 704 pci_ecam_free(pp->cfg); 705 706 return ret; 707 } 708 EXPORT_SYMBOL_GPL(dw_pcie_host_init); 709 710 void dw_pcie_host_deinit(struct dw_pcie_rp *pp) 711 { 712 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 713 714 dwc_pcie_debugfs_deinit(pci); 715 716 pci_stop_root_bus(pp->bridge->bus); 717 pci_remove_root_bus(pp->bridge->bus); 718 719 dw_pcie_stop_link(pci); 720 721 dw_pcie_edma_remove(pci); 722 723 if (pp->has_msi_ctrl) 724 dw_pcie_free_msi(pp); 725 726 if (pp->ops->deinit) 727 pp->ops->deinit(pp); 728 729 if (pp->cfg) 730 pci_ecam_free(pp->cfg); 731 } 732 EXPORT_SYMBOL_GPL(dw_pcie_host_deinit); 733 734 static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus, 735 unsigned int devfn, int where) 736 { 737 struct dw_pcie_rp *pp = bus->sysdata; 738 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 739 struct dw_pcie_ob_atu_cfg atu = { 0 }; 740 int type, ret; 741 u32 busdev; 742 743 /* 744 * Checking whether the link is up here is a last line of defense 745 * against platforms that forward errors on the system bus as 746 * SError upon PCI configuration transactions issued when the link 747 * is down. This check is racy by definition and does not stop 748 * the system from triggering an SError if the link goes down 749 * after this check is performed. 750 */ 751 if (!dw_pcie_link_up(pci)) 752 return NULL; 753 754 busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) | 755 PCIE_ATU_FUNC(PCI_FUNC(devfn)); 756 757 if (pci_is_root_bus(bus->parent)) 758 type = PCIE_ATU_TYPE_CFG0; 759 else 760 type = PCIE_ATU_TYPE_CFG1; 761 762 atu.type = type; 763 atu.parent_bus_addr = pp->cfg0_base - pci->parent_bus_offset; 764 atu.pci_addr = busdev; 765 atu.size = pp->cfg0_size; 766 767 ret = dw_pcie_prog_outbound_atu(pci, &atu); 768 if (ret) 769 return NULL; 770 771 return pp->va_cfg0_base + where; 772 } 773 774 static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn, 775 int where, int size, u32 *val) 776 { 777 struct dw_pcie_rp *pp = bus->sysdata; 778 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 779 struct dw_pcie_ob_atu_cfg atu = { 0 }; 780 int ret; 781 782 ret = pci_generic_config_read(bus, devfn, where, size, val); 783 if (ret != PCIBIOS_SUCCESSFUL) 784 return ret; 785 786 if (pp->cfg0_io_shared) { 787 atu.type = PCIE_ATU_TYPE_IO; 788 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; 789 atu.pci_addr = pp->io_bus_addr; 790 atu.size = pp->io_size; 791 792 ret = dw_pcie_prog_outbound_atu(pci, &atu); 793 if (ret) 794 return PCIBIOS_SET_FAILED; 795 } 796 797 return PCIBIOS_SUCCESSFUL; 798 } 799 800 static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn, 801 int where, int size, u32 val) 802 { 803 struct dw_pcie_rp *pp = bus->sysdata; 804 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 805 struct dw_pcie_ob_atu_cfg atu = { 0 }; 806 int ret; 807 808 ret = pci_generic_config_write(bus, devfn, where, size, val); 809 if (ret != PCIBIOS_SUCCESSFUL) 810 return ret; 811 812 if (pp->cfg0_io_shared) { 813 atu.type = PCIE_ATU_TYPE_IO; 814 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; 815 atu.pci_addr = pp->io_bus_addr; 816 atu.size = pp->io_size; 817 818 ret = dw_pcie_prog_outbound_atu(pci, &atu); 819 if (ret) 820 return PCIBIOS_SET_FAILED; 821 } 822 823 return PCIBIOS_SUCCESSFUL; 824 } 825 826 static struct pci_ops dw_child_pcie_ops = { 827 .map_bus = dw_pcie_other_conf_map_bus, 828 .read = dw_pcie_rd_other_conf, 829 .write = dw_pcie_wr_other_conf, 830 }; 831 832 void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) 833 { 834 struct dw_pcie_rp *pp = bus->sysdata; 835 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 836 837 if (PCI_SLOT(devfn) > 0) 838 return NULL; 839 840 return pci->dbi_base + where; 841 } 842 EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus); 843 844 static void __iomem *dw_pcie_ecam_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) 845 { 846 struct pci_config_window *cfg = bus->sysdata; 847 struct dw_pcie_rp *pp = cfg->priv; 848 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 849 unsigned int busn = bus->number; 850 851 if (busn > 0) 852 return pci_ecam_map_bus(bus, devfn, where); 853 854 if (PCI_SLOT(devfn) > 0) 855 return NULL; 856 857 return pci->dbi_base + where; 858 } 859 860 static int dw_pcie_op_assert_perst(struct pci_bus *bus, bool assert) 861 { 862 struct dw_pcie_rp *pp = bus->sysdata; 863 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 864 865 return dw_pcie_assert_perst(pci, assert); 866 } 867 868 static struct pci_ops dw_pcie_ops = { 869 .map_bus = dw_pcie_own_conf_map_bus, 870 .read = pci_generic_config_read, 871 .write = pci_generic_config_write, 872 .assert_perst = dw_pcie_op_assert_perst, 873 }; 874 875 static struct pci_ops dw_pcie_ecam_ops = { 876 .map_bus = dw_pcie_ecam_conf_map_bus, 877 .read = pci_generic_config_read, 878 .write = pci_generic_config_write, 879 }; 880 881 static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp) 882 { 883 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 884 struct dw_pcie_ob_atu_cfg atu = { 0 }; 885 struct resource_entry *entry; 886 int i, ret; 887 888 /* Note the very first outbound ATU is used for CFG IOs */ 889 if (!pci->num_ob_windows) { 890 dev_err(pci->dev, "No outbound iATU found\n"); 891 return -EINVAL; 892 } 893 894 /* 895 * Ensure all out/inbound windows are disabled before proceeding with 896 * the MEM/IO (dma-)ranges setups. 897 */ 898 for (i = 0; i < pci->num_ob_windows; i++) 899 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, i); 900 901 for (i = 0; i < pci->num_ib_windows; i++) 902 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, i); 903 904 i = 0; 905 resource_list_for_each_entry(entry, &pp->bridge->windows) { 906 if (resource_type(entry->res) != IORESOURCE_MEM) 907 continue; 908 909 if (pci->num_ob_windows <= ++i) 910 break; 911 912 atu.index = i; 913 atu.type = PCIE_ATU_TYPE_MEM; 914 atu.parent_bus_addr = entry->res->start - pci->parent_bus_offset; 915 atu.pci_addr = entry->res->start - entry->offset; 916 917 /* Adjust iATU size if MSG TLP region was allocated before */ 918 if (pp->msg_res && pp->msg_res->parent == entry->res) 919 atu.size = resource_size(entry->res) - 920 resource_size(pp->msg_res); 921 else 922 atu.size = resource_size(entry->res); 923 924 ret = dw_pcie_prog_outbound_atu(pci, &atu); 925 if (ret) { 926 dev_err(pci->dev, "Failed to set MEM range %pr\n", 927 entry->res); 928 return ret; 929 } 930 } 931 932 if (pp->io_size) { 933 if (pci->num_ob_windows > ++i) { 934 atu.index = i; 935 atu.type = PCIE_ATU_TYPE_IO; 936 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; 937 atu.pci_addr = pp->io_bus_addr; 938 atu.size = pp->io_size; 939 940 ret = dw_pcie_prog_outbound_atu(pci, &atu); 941 if (ret) { 942 dev_err(pci->dev, "Failed to set IO range %pr\n", 943 entry->res); 944 return ret; 945 } 946 } else { 947 pp->cfg0_io_shared = true; 948 } 949 } 950 951 if (pci->num_ob_windows <= i) 952 dev_warn(pci->dev, "Ranges exceed outbound iATU size (%d)\n", 953 pci->num_ob_windows); 954 955 pp->msg_atu_index = i; 956 957 i = 0; 958 resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) { 959 if (resource_type(entry->res) != IORESOURCE_MEM) 960 continue; 961 962 if (pci->num_ib_windows <= i) 963 break; 964 965 ret = dw_pcie_prog_inbound_atu(pci, i++, PCIE_ATU_TYPE_MEM, 966 entry->res->start, 967 entry->res->start - entry->offset, 968 resource_size(entry->res)); 969 if (ret) { 970 dev_err(pci->dev, "Failed to set DMA range %pr\n", 971 entry->res); 972 return ret; 973 } 974 } 975 976 if (pci->num_ib_windows <= i) 977 dev_warn(pci->dev, "Dma-ranges exceed inbound iATU size (%u)\n", 978 pci->num_ib_windows); 979 980 return 0; 981 } 982 983 static void dw_pcie_program_presets(struct dw_pcie_rp *pp, enum pci_bus_speed speed) 984 { 985 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 986 u8 lane_eq_offset, lane_reg_size, cap_id; 987 u8 *presets; 988 u32 cap; 989 int i; 990 991 if (speed == PCIE_SPEED_8_0GT) { 992 presets = (u8 *)pp->presets.eq_presets_8gts; 993 lane_eq_offset = PCI_SECPCI_LE_CTRL; 994 cap_id = PCI_EXT_CAP_ID_SECPCI; 995 /* For data rate of 8 GT/S each lane equalization control is 16bits wide*/ 996 lane_reg_size = 0x2; 997 } else if (speed == PCIE_SPEED_16_0GT) { 998 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_16GTS - 1]; 999 lane_eq_offset = PCI_PL_16GT_LE_CTRL; 1000 cap_id = PCI_EXT_CAP_ID_PL_16GT; 1001 lane_reg_size = 0x1; 1002 } else if (speed == PCIE_SPEED_32_0GT) { 1003 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_32GTS - 1]; 1004 lane_eq_offset = PCI_PL_32GT_LE_CTRL; 1005 cap_id = PCI_EXT_CAP_ID_PL_32GT; 1006 lane_reg_size = 0x1; 1007 } else if (speed == PCIE_SPEED_64_0GT) { 1008 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_64GTS - 1]; 1009 lane_eq_offset = PCI_PL_64GT_LE_CTRL; 1010 cap_id = PCI_EXT_CAP_ID_PL_64GT; 1011 lane_reg_size = 0x1; 1012 } else { 1013 return; 1014 } 1015 1016 if (presets[0] == PCI_EQ_RESV) 1017 return; 1018 1019 cap = dw_pcie_find_ext_capability(pci, cap_id); 1020 if (!cap) 1021 return; 1022 1023 /* 1024 * Write preset values to the registers byte-by-byte for the given 1025 * number of lanes and register size. 1026 */ 1027 for (i = 0; i < pci->num_lanes * lane_reg_size; i++) 1028 dw_pcie_writeb_dbi(pci, cap + lane_eq_offset + i, presets[i]); 1029 } 1030 1031 static void dw_pcie_config_presets(struct dw_pcie_rp *pp) 1032 { 1033 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 1034 enum pci_bus_speed speed = pcie_link_speed[pci->max_link_speed]; 1035 1036 /* 1037 * Lane equalization settings need to be applied for all data rates the 1038 * controller supports and for all supported lanes. 1039 */ 1040 1041 if (speed >= PCIE_SPEED_8_0GT) 1042 dw_pcie_program_presets(pp, PCIE_SPEED_8_0GT); 1043 1044 if (speed >= PCIE_SPEED_16_0GT) 1045 dw_pcie_program_presets(pp, PCIE_SPEED_16_0GT); 1046 1047 if (speed >= PCIE_SPEED_32_0GT) 1048 dw_pcie_program_presets(pp, PCIE_SPEED_32_0GT); 1049 1050 if (speed >= PCIE_SPEED_64_0GT) 1051 dw_pcie_program_presets(pp, PCIE_SPEED_64_0GT); 1052 } 1053 1054 int dw_pcie_setup_rc(struct dw_pcie_rp *pp) 1055 { 1056 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 1057 u32 val; 1058 int ret; 1059 1060 /* 1061 * Enable DBI read-only registers for writing/updating configuration. 1062 * Write permission gets disabled towards the end of this function. 1063 */ 1064 dw_pcie_dbi_ro_wr_en(pci); 1065 1066 dw_pcie_setup(pci); 1067 1068 dw_pcie_msi_init(pp); 1069 1070 /* Setup RC BARs */ 1071 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004); 1072 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000); 1073 1074 /* Setup interrupt pins */ 1075 val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE); 1076 val &= 0xffff00ff; 1077 val |= 0x00000100; 1078 dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val); 1079 1080 /* Setup bus numbers */ 1081 val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS); 1082 val &= 0xff000000; 1083 val |= 0x00ff0100; 1084 dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val); 1085 1086 /* Setup command register */ 1087 val = dw_pcie_readl_dbi(pci, PCI_COMMAND); 1088 val &= 0xffff0000; 1089 val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY | 1090 PCI_COMMAND_MASTER | PCI_COMMAND_SERR; 1091 dw_pcie_writel_dbi(pci, PCI_COMMAND, val); 1092 1093 dw_pcie_hide_unsupported_l1ss(pci); 1094 1095 dw_pcie_config_presets(pp); 1096 /* 1097 * If the platform provides its own child bus config accesses, it means 1098 * the platform uses its own address translation component rather than 1099 * ATU, so we should not program the ATU here. 1100 */ 1101 if (pp->bridge->child_ops == &dw_child_pcie_ops) { 1102 ret = dw_pcie_iatu_setup(pp); 1103 if (ret) 1104 return ret; 1105 } 1106 1107 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); 1108 1109 /* Program correct class for RC */ 1110 dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); 1111 1112 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); 1113 val |= PORT_LOGIC_SPEED_CHANGE; 1114 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); 1115 1116 dw_pcie_dbi_ro_wr_dis(pci); 1117 1118 return 0; 1119 } 1120 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc); 1121 1122 static int dw_pcie_pme_turn_off(struct dw_pcie *pci) 1123 { 1124 struct dw_pcie_ob_atu_cfg atu = { 0 }; 1125 void __iomem *mem; 1126 int ret; 1127 1128 if (pci->num_ob_windows <= pci->pp.msg_atu_index) 1129 return -ENOSPC; 1130 1131 if (!pci->pp.msg_res) 1132 return -ENOSPC; 1133 1134 atu.code = PCIE_MSG_CODE_PME_TURN_OFF; 1135 atu.routing = PCIE_MSG_TYPE_R_BC; 1136 atu.type = PCIE_ATU_TYPE_MSG; 1137 atu.size = resource_size(pci->pp.msg_res); 1138 atu.index = pci->pp.msg_atu_index; 1139 1140 atu.parent_bus_addr = pci->pp.msg_res->start - pci->parent_bus_offset; 1141 1142 ret = dw_pcie_prog_outbound_atu(pci, &atu); 1143 if (ret) 1144 return ret; 1145 1146 mem = ioremap(pci->pp.msg_res->start, pci->region_align); 1147 if (!mem) 1148 return -ENOMEM; 1149 1150 /* A dummy write is converted to a Msg TLP */ 1151 writel(0, mem); 1152 1153 iounmap(mem); 1154 1155 return 0; 1156 } 1157 1158 int dw_pcie_suspend_noirq(struct dw_pcie *pci) 1159 { 1160 u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 1161 u32 val; 1162 int ret; 1163 1164 /* 1165 * If L1SS is supported, then do not put the link into L2 as some 1166 * devices such as NVMe expect low resume latency. 1167 */ 1168 if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1) 1169 return 0; 1170 1171 if (pci->pp.ops->pme_turn_off) { 1172 pci->pp.ops->pme_turn_off(&pci->pp); 1173 } else { 1174 ret = dw_pcie_pme_turn_off(pci); 1175 if (ret) 1176 return ret; 1177 } 1178 1179 ret = read_poll_timeout(dw_pcie_get_ltssm, val, 1180 val == DW_PCIE_LTSSM_L2_IDLE || 1181 val <= DW_PCIE_LTSSM_DETECT_WAIT, 1182 PCIE_PME_TO_L2_TIMEOUT_US/10, 1183 PCIE_PME_TO_L2_TIMEOUT_US, false, pci); 1184 if (ret) { 1185 /* Only log message when LTSSM isn't in DETECT or POLL */ 1186 dev_err(pci->dev, "Timeout waiting for L2 entry! LTSSM: 0x%x\n", val); 1187 return ret; 1188 } 1189 1190 /* 1191 * Per PCIe r6.0, sec 5.3.3.2.1, software should wait at least 1192 * 100ns after L2/L3 Ready before turning off refclock and 1193 * main power. This is harmless when no endpoint is connected. 1194 */ 1195 udelay(1); 1196 1197 dw_pcie_stop_link(pci); 1198 if (pci->pp.ops->deinit) 1199 pci->pp.ops->deinit(&pci->pp); 1200 1201 pci->suspended = true; 1202 1203 return ret; 1204 } 1205 EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq); 1206 1207 int dw_pcie_resume_noirq(struct dw_pcie *pci) 1208 { 1209 int ret; 1210 1211 if (!pci->suspended) 1212 return 0; 1213 1214 pci->suspended = false; 1215 1216 if (pci->pp.ops->init) { 1217 ret = pci->pp.ops->init(&pci->pp); 1218 if (ret) { 1219 dev_err(pci->dev, "Host init failed: %d\n", ret); 1220 return ret; 1221 } 1222 } 1223 1224 dw_pcie_setup_rc(&pci->pp); 1225 1226 ret = dw_pcie_start_link(pci); 1227 if (ret) 1228 return ret; 1229 1230 ret = dw_pcie_wait_for_link(pci); 1231 if (ret) 1232 return ret; 1233 1234 return ret; 1235 } 1236 EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq); 1237