1 // SPDX-License-Identifier: GPL-2.0
3 // Cadence PCIe host controller driver.
4 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
14 #include "pcie-cadence.h"
25 	[RP_BAR0] = 0x1F,
26 	[RP_BAR1] = 0xF,
34 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pci_map_bus()  local
35 	unsigned int busn = bus->number;  in cdns_pci_map_bus()
40 		 * Only the root port (devfn == 0) is connected to this bus.  in cdns_pci_map_bus()
47 		return pcie->reg_base + (where & 0xfff);  in cdns_pci_map_bus()
50 	if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))  in cdns_pci_map_bus()
52 	/* Clear AXI link-down status */  in cdns_pci_map_bus()
53 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);  in cdns_pci_map_bus()
55 	/* Update Output registers for AXI region 0. */  in cdns_pci_map_bus()
59 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);  in cdns_pci_map_bus()
61 	/* Configuration Type 0 or Type 1 access. */  in cdns_pci_map_bus()
63 		CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);  in cdns_pci_map_bus()
68 	if (busn == bridge->busnr + 1)  in cdns_pci_map_bus()
72 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);  in cdns_pci_map_bus()
74 	return rc->cfg_base + (where & 0xfff);  in cdns_pci_map_bus()
84 static int cdns_pcie_host_training_complete(struct cdns_pcie *pcie)  in cdns_pcie_host_training_complete()  argument
93 		lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);  in cdns_pcie_host_training_complete()
96 		usleep_range(0, 1000);  in cdns_pcie_host_training_complete()
100 		return 0;  in cdns_pcie_host_training_complete()
102 	return -ETIMEDOUT;  in cdns_pcie_host_training_complete()
105 static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)  in cdns_pcie_host_wait_for_link()  argument
107 	struct device *dev = pcie->dev;  in cdns_pcie_host_wait_for_link()
111 	for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {  in cdns_pcie_host_wait_for_link()
112 		if (cdns_pcie_link_up(pcie)) {  in cdns_pcie_host_wait_for_link()
114 			return 0;  in cdns_pcie_host_wait_for_link()
119 	return -ETIMEDOUT;  in cdns_pcie_host_wait_for_link()
122 static int cdns_pcie_retrain(struct cdns_pcie *pcie)  in cdns_pcie_retrain()  argument
126 	int ret = 0;  in cdns_pcie_retrain()
130 	 * but the PCIe root port support is > 2.5 GB/s.  in cdns_pcie_retrain()
133 	lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off +  in cdns_pcie_retrain()
138 	lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);  in cdns_pcie_retrain()
140 		lnk_ctl = cdns_pcie_rp_readw(pcie,  in cdns_pcie_retrain()
143 		cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL,  in cdns_pcie_retrain()
146 		ret = cdns_pcie_host_training_complete(pcie);  in cdns_pcie_retrain()
150 		ret = cdns_pcie_host_wait_for_link(pcie);  in cdns_pcie_retrain()
155 static void cdns_pcie_host_disable_ptm_response(struct cdns_pcie *pcie)  in cdns_pcie_host_disable_ptm_response()  argument
159 	val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL);  in cdns_pcie_host_disable_ptm_response()
160 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val & ~CDNS_PCIE_LM_TPM_CTRL_PTMRSEN);  in cdns_pcie_host_disable_ptm_response()
163 static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie)  in cdns_pcie_host_enable_ptm_response()  argument
167 	val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL);  in cdns_pcie_host_enable_ptm_response()
168 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN);  in cdns_pcie_host_enable_ptm_response()
173 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_start_link()  local
176 	ret = cdns_pcie_host_wait_for_link(pcie);  in cdns_pcie_host_start_link()
182 	if (!ret && rc->quirk_retrain_flag)  in cdns_pcie_host_start_link()
183 		ret = cdns_pcie_retrain(pcie);  in cdns_pcie_host_start_link()
190 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_deinit_root_port()  local
193 	cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, 0xffff);  in cdns_pcie_host_deinit_root_port()
194 	cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0xff);  in cdns_pcie_host_deinit_root_port()
195 	cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0xff);  in cdns_pcie_host_deinit_root_port()
196 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, 0xffffffff);  in cdns_pcie_host_deinit_root_port()
197 	cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, 0xffff);  in cdns_pcie_host_deinit_root_port()
205 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);  in cdns_pcie_host_deinit_root_port()
210 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_init_root_port()  local
216 	 * - disable both BAR0 and BAR1.  in cdns_pcie_host_init_root_port()
217 	 * - enable Prefetchable Memory Base and Limit registers in type 1  in cdns_pcie_host_init_root_port()
219 	 * - enable IO Base and Limit registers in type 1 config  in cdns_pcie_host_init_root_port()
229 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);  in cdns_pcie_host_init_root_port()
232 	if (rc->vendor_id != 0xffff) {  in cdns_pcie_host_init_root_port()
233 		id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |  in cdns_pcie_host_init_root_port()
234 			CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);  in cdns_pcie_host_init_root_port()
235 		cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);  in cdns_pcie_host_init_root_port()
238 	if (rc->device_id != 0xffff)  in cdns_pcie_host_init_root_port()
239 		cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);  in cdns_pcie_host_init_root_port()
241 	cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);  in cdns_pcie_host_init_root_port()
242 	cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);  in cdns_pcie_host_init_root_port()
243 	cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);  in cdns_pcie_host_init_root_port()
245 	return 0;  in cdns_pcie_host_init_root_port()
253 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_bar_ib_config()  local
256 	if (!rc->avail_ib_bar[bar])  in cdns_pcie_host_bar_ib_config()
257 		return -EBUSY;  in cdns_pcie_host_bar_ib_config()
259 	rc->avail_ib_bar[bar] = false;  in cdns_pcie_host_bar_ib_config()
265 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);  in cdns_pcie_host_bar_ib_config()
266 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);  in cdns_pcie_host_bar_ib_config()
269 		return 0;  in cdns_pcie_host_bar_ib_config()
271 	value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);  in cdns_pcie_host_bar_ib_config()
288 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);  in cdns_pcie_host_bar_ib_config()
290 	return 0;  in cdns_pcie_host_bar_ib_config()
300 		if (!rc->avail_ib_bar[bar])  in cdns_pcie_host_find_min_bar()
324 		if (!rc->avail_ib_bar[bar])  in cdns_pcie_host_find_max_bar()
345 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_bar_config()  local
346 	struct device *dev = pcie->dev;  in cdns_pcie_host_bar_config()
351 	cpu_addr = entry->res->start;  in cdns_pcie_host_bar_config()
352 	pci_addr = entry->res->start - entry->offset;  in cdns_pcie_host_bar_config()
353 	flags = entry->res->flags;  in cdns_pcie_host_bar_config()
354 	size = resource_size(entry->res);  in cdns_pcie_host_bar_config()
356 	if (entry->offset) {  in cdns_pcie_host_bar_config()
359 		return -EINVAL;  in cdns_pcie_host_bar_config()
362 	while (size > 0) {  in cdns_pcie_host_bar_config()
395 			return -EINVAL;  in cdns_pcie_host_bar_config()
406 		size -= winsize;  in cdns_pcie_host_bar_config()
410 	return 0;  in cdns_pcie_host_bar_config()
421         return resource_size(entry2->res) - resource_size(entry1->res);  in cdns_pcie_host_dma_ranges_cmp()
426 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_unmap_dma_ranges()  local
432 		if (rc->avail_ib_bar[bar])  in cdns_pcie_host_unmap_dma_ranges()
435 		cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), 0);  in cdns_pcie_host_unmap_dma_ranges()
436 		cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), 0);  in cdns_pcie_host_unmap_dma_ranges()
446 		cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);  in cdns_pcie_host_unmap_dma_ranges()
452 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_map_dma_ranges()  local
453 	struct device *dev = pcie->dev;  in cdns_pcie_host_map_dma_ranges()
454 	struct device_node *np = dev->of_node;  in cdns_pcie_host_map_dma_ranges()
462 		return -ENOMEM;  in cdns_pcie_host_map_dma_ranges()
464 	if (list_empty(&bridge->dma_ranges)) {  in cdns_pcie_host_map_dma_ranges()
465 		of_property_read_u32(np, "cdns,no-bar-match-nbits",  in cdns_pcie_host_map_dma_ranges()
467 		err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,  in cdns_pcie_host_map_dma_ranges()
468 						   (u64)1 << no_bar_nbits, 0);  in cdns_pcie_host_map_dma_ranges()
474 	list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);  in cdns_pcie_host_map_dma_ranges()
476 	resource_list_for_each_entry(entry, &bridge->dma_ranges) {  in cdns_pcie_host_map_dma_ranges()
479 			dev_err(dev, "Fail to configure IB using dma-ranges\n");  in cdns_pcie_host_map_dma_ranges()
484 	return 0;  in cdns_pcie_host_map_dma_ranges()
489 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_deinit_address_translation()  local
497 	 * Reset outbound region 0 which was reserved for configuration space  in cdns_pcie_host_deinit_address_translation()
500 	cdns_pcie_reset_outbound_region(pcie, 0);  in cdns_pcie_host_deinit_address_translation()
504 	resource_list_for_each_entry(entry, &bridge->windows) {  in cdns_pcie_host_deinit_address_translation()
505 		cdns_pcie_reset_outbound_region(pcie, r);  in cdns_pcie_host_deinit_address_translation()
512 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_init_address_translation()  local
514 	struct resource *cfg_res = rc->cfg_res;  in cdns_pcie_host_init_address_translation()
516 	u64 cpu_addr = cfg_res->start;  in cdns_pcie_host_init_address_translation()
518 	int r, busnr = 0;  in cdns_pcie_host_init_address_translation()
520 	entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);  in cdns_pcie_host_init_address_translation()
522 		busnr = entry->res->start;  in cdns_pcie_host_init_address_translation()
525 	 * Reserve region 0 for PCI configure space accesses:  in cdns_pcie_host_init_address_translation()
529 	addr1 = 0; /* Should be programmed to zero. */  in cdns_pcie_host_init_address_translation()
531 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);  in cdns_pcie_host_init_address_translation()
532 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);  in cdns_pcie_host_init_address_translation()
534 	if (pcie->ops && pcie->ops->cpu_addr_fixup)  in cdns_pcie_host_init_address_translation()
535 		cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);  in cdns_pcie_host_init_address_translation()
540 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);  in cdns_pcie_host_init_address_translation()
541 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);  in cdns_pcie_host_init_address_translation()
544 	resource_list_for_each_entry(entry, &bridge->windows) {  in cdns_pcie_host_init_address_translation()
545 		struct resource *res = entry->res;  in cdns_pcie_host_init_address_translation()
546 		u64 pci_addr = res->start - entry->offset;  in cdns_pcie_host_init_address_translation()
549 			cdns_pcie_set_outbound_region(pcie, busnr, 0, r,  in cdns_pcie_host_init_address_translation()
551 						      pci_pio_to_address(res->start),  in cdns_pcie_host_init_address_translation()
555 			cdns_pcie_set_outbound_region(pcie, busnr, 0, r,  in cdns_pcie_host_init_address_translation()
557 						      res->start,  in cdns_pcie_host_init_address_translation()
587 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_link_disable()  local
589 	cdns_pcie_stop_link(pcie);  in cdns_pcie_host_link_disable()
590 	cdns_pcie_host_disable_ptm_response(pcie);  in cdns_pcie_host_link_disable()
595 	struct cdns_pcie *pcie = &rc->pcie;  in cdns_pcie_host_link_setup()  local
596 	struct device *dev = rc->pcie.dev;  in cdns_pcie_host_link_setup()
599 	if (rc->quirk_detect_quiet_flag)  in cdns_pcie_host_link_setup()
600 		cdns_pcie_detect_quiet_min_delay_set(&rc->pcie);  in cdns_pcie_host_link_setup()
602 	cdns_pcie_host_enable_ptm_response(pcie);  in cdns_pcie_host_link_setup()
604 	ret = cdns_pcie_start_link(pcie);  in cdns_pcie_host_link_setup()
612 		dev_dbg(dev, "PCIe link never came up\n");  in cdns_pcie_host_link_setup()
614 	return 0;  in cdns_pcie_host_link_setup()
623 	pci_stop_root_bus(bridge->bus);  in cdns_pcie_host_disable()
624 	pci_remove_root_bus(bridge->bus);  in cdns_pcie_host_disable()
633 	struct device *dev = rc->pcie.dev;  in cdns_pcie_host_setup()
635 	struct device_node *np = dev->of_node;  in cdns_pcie_host_setup()
638 	struct cdns_pcie *pcie;  in cdns_pcie_host_setup()  local
644 		return -ENOMEM;  in cdns_pcie_host_setup()
646 	pcie = &rc->pcie;  in cdns_pcie_host_setup()
647 	pcie->is_rc = true;  in cdns_pcie_host_setup()
649 	rc->vendor_id = 0xffff;  in cdns_pcie_host_setup()
650 	of_property_read_u32(np, "vendor-id", &rc->vendor_id);  in cdns_pcie_host_setup()
652 	rc->device_id = 0xffff;  in cdns_pcie_host_setup()
653 	of_property_read_u32(np, "device-id", &rc->device_id);  in cdns_pcie_host_setup()
655 	pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");  in cdns_pcie_host_setup()
656 	if (IS_ERR(pcie->reg_base)) {  in cdns_pcie_host_setup()
658 		return PTR_ERR(pcie->reg_base);  in cdns_pcie_host_setup()
662 	rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);  in cdns_pcie_host_setup()
663 	if (IS_ERR(rc->cfg_base))  in cdns_pcie_host_setup()
664 		return PTR_ERR(rc->cfg_base);  in cdns_pcie_host_setup()
665 	rc->cfg_res = res;  in cdns_pcie_host_setup()
672 		rc->avail_ib_bar[bar] = true;  in cdns_pcie_host_setup()
678 	if (!bridge->ops)  in cdns_pcie_host_setup()
679 		bridge->ops = &cdns_pcie_host_ops;  in cdns_pcie_host_setup()
686 MODULE_DESCRIPTION("Cadence PCIe host controller driver");
687 MODULE_AUTHOR("Cyrille Pitchen <cyrille.pitchen@free-electrons.com>");