1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe endpoint controller driver 4 * 5 * Copyright (c) 2018 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Simon Xue <xxm@rock-chips.com> 9 */ 10 11 #include <linux/configfs.h> 12 #include <linux/delay.h> 13 #include <linux/kernel.h> 14 #include <linux/of.h> 15 #include <linux/pci-epc.h> 16 #include <linux/platform_device.h> 17 #include <linux/pci-epf.h> 18 #include <linux/sizes.h> 19 20 #include "pcie-rockchip.h" 21 22 /** 23 * struct rockchip_pcie_ep - private data for PCIe endpoint controller driver 24 * @rockchip: Rockchip PCIe controller 25 * @epc: PCI EPC device 26 * @max_regions: maximum number of regions supported by hardware 27 * @ob_region_map: bitmask of mapped outbound regions 28 * @ob_addr: base addresses in the AXI bus where the outbound regions start 29 * @irq_phys_addr: base address on the AXI bus where the MSI/INTX IRQ 30 * dedicated outbound regions is mapped. 31 * @irq_cpu_addr: base address in the CPU space where a write access triggers 32 * the sending of a memory write (MSI) / normal message (INTX 33 * IRQ) TLP through the PCIe bus. 34 * @irq_pci_addr: used to save the current mapping of the MSI/INTX IRQ 35 * dedicated outbound region. 36 * @irq_pci_fn: the latest PCI function that has updated the mapping of 37 * the MSI/INTX IRQ dedicated outbound region. 38 * @irq_pending: bitmask of asserted INTX IRQs. 39 */ 40 struct rockchip_pcie_ep { 41 struct rockchip_pcie rockchip; 42 struct pci_epc *epc; 43 u32 max_regions; 44 unsigned long ob_region_map; 45 phys_addr_t *ob_addr; 46 phys_addr_t irq_phys_addr; 47 void __iomem *irq_cpu_addr; 48 u64 irq_pci_addr; 49 u8 irq_pci_fn; 50 u8 irq_pending; 51 }; 52 53 static void rockchip_pcie_clear_ep_ob_atu(struct rockchip_pcie *rockchip, 54 u32 region) 55 { 56 rockchip_pcie_write(rockchip, 0, 57 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(region)); 58 rockchip_pcie_write(rockchip, 0, 59 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(region)); 60 rockchip_pcie_write(rockchip, 0, 61 ROCKCHIP_PCIE_AT_OB_REGION_DESC0(region)); 62 rockchip_pcie_write(rockchip, 0, 63 ROCKCHIP_PCIE_AT_OB_REGION_DESC1(region)); 64 } 65 66 static void rockchip_pcie_prog_ep_ob_atu(struct rockchip_pcie *rockchip, u8 fn, 67 u32 r, u64 cpu_addr, u64 pci_addr, 68 size_t size) 69 { 70 int num_pass_bits = fls64(size - 1); 71 u32 addr0, addr1, desc0; 72 73 if (num_pass_bits < 8) 74 num_pass_bits = 8; 75 76 addr0 = ((num_pass_bits - 1) & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS) | 77 (lower_32_bits(pci_addr) & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR); 78 addr1 = upper_32_bits(pci_addr); 79 desc0 = ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN(fn) | AXI_WRAPPER_MEM_WRITE; 80 81 /* PCI bus address region */ 82 rockchip_pcie_write(rockchip, addr0, 83 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r)); 84 rockchip_pcie_write(rockchip, addr1, 85 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r)); 86 rockchip_pcie_write(rockchip, desc0, 87 ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r)); 88 rockchip_pcie_write(rockchip, 0, 89 ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r)); 90 } 91 92 static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, 93 struct pci_epf_header *hdr) 94 { 95 u32 reg; 96 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 97 struct rockchip_pcie *rockchip = &ep->rockchip; 98 99 /* All functions share the same vendor ID with function 0 */ 100 if (fn == 0) { 101 rockchip_pcie_write(rockchip, 102 hdr->vendorid | hdr->subsys_vendor_id << 16, 103 PCIE_CORE_CONFIG_VENDOR); 104 } 105 106 reg = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_DID_VID); 107 reg = (reg & 0xFFFF) | (hdr->deviceid << 16); 108 rockchip_pcie_write(rockchip, reg, PCIE_EP_CONFIG_DID_VID); 109 110 rockchip_pcie_write(rockchip, 111 hdr->revid | 112 hdr->progif_code << 8 | 113 hdr->subclass_code << 16 | 114 hdr->baseclass_code << 24, 115 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + PCI_REVISION_ID); 116 rockchip_pcie_write(rockchip, hdr->cache_line_size, 117 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 118 PCI_CACHE_LINE_SIZE); 119 rockchip_pcie_write(rockchip, hdr->subsys_id << 16, 120 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 121 PCI_SUBSYSTEM_VENDOR_ID); 122 rockchip_pcie_write(rockchip, hdr->interrupt_pin << 8, 123 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 124 PCI_INTERRUPT_LINE); 125 126 return 0; 127 } 128 129 static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, 130 struct pci_epf_bar *epf_bar) 131 { 132 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 133 struct rockchip_pcie *rockchip = &ep->rockchip; 134 dma_addr_t bar_phys = epf_bar->phys_addr; 135 enum pci_barno bar = epf_bar->barno; 136 int flags = epf_bar->flags; 137 u32 addr0, addr1, reg, cfg, b, aperture, ctrl; 138 u64 sz; 139 140 /* BAR size is 2^(aperture + 7) */ 141 sz = max_t(size_t, epf_bar->size, MIN_EP_APERTURE); 142 143 /* 144 * roundup_pow_of_two() returns an unsigned long, which is not suited 145 * for 64bit values. 146 */ 147 sz = 1ULL << fls64(sz - 1); 148 aperture = ilog2(sz) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */ 149 150 if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) { 151 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS; 152 } else { 153 bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH); 154 bool is_64bits = !!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64); 155 156 if (is_64bits && (bar & 1)) 157 return -EINVAL; 158 159 if (is_64bits && is_prefetch) 160 ctrl = 161 ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_64BITS; 162 else if (is_prefetch) 163 ctrl = 164 ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_32BITS; 165 else if (is_64bits) 166 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_64BITS; 167 else 168 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_32BITS; 169 } 170 171 if (bar < BAR_4) { 172 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); 173 b = bar; 174 } else { 175 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); 176 b = bar - BAR_4; 177 } 178 179 addr0 = lower_32_bits(bar_phys); 180 addr1 = upper_32_bits(bar_phys); 181 182 cfg = rockchip_pcie_read(rockchip, reg); 183 cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | 184 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); 185 cfg |= (ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) | 186 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl)); 187 188 rockchip_pcie_write(rockchip, cfg, reg); 189 rockchip_pcie_write(rockchip, addr0, 190 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); 191 rockchip_pcie_write(rockchip, addr1, 192 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); 193 194 return 0; 195 } 196 197 static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, 198 struct pci_epf_bar *epf_bar) 199 { 200 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 201 struct rockchip_pcie *rockchip = &ep->rockchip; 202 u32 reg, cfg, b, ctrl; 203 enum pci_barno bar = epf_bar->barno; 204 205 if (bar < BAR_4) { 206 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); 207 b = bar; 208 } else { 209 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); 210 b = bar - BAR_4; 211 } 212 213 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_DISABLED; 214 cfg = rockchip_pcie_read(rockchip, reg); 215 cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | 216 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); 217 cfg |= ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl); 218 219 rockchip_pcie_write(rockchip, cfg, reg); 220 rockchip_pcie_write(rockchip, 0x0, 221 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); 222 rockchip_pcie_write(rockchip, 0x0, 223 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); 224 } 225 226 static inline u32 rockchip_ob_region(phys_addr_t addr) 227 { 228 return (addr >> ilog2(SZ_1M)) & 0x1f; 229 } 230 231 static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, 232 phys_addr_t addr, u64 pci_addr, 233 size_t size) 234 { 235 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 236 struct rockchip_pcie *pcie = &ep->rockchip; 237 u32 r = rockchip_ob_region(addr); 238 239 rockchip_pcie_prog_ep_ob_atu(pcie, fn, r, addr, pci_addr, size); 240 241 set_bit(r, &ep->ob_region_map); 242 ep->ob_addr[r] = addr; 243 244 return 0; 245 } 246 247 static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, 248 phys_addr_t addr) 249 { 250 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 251 struct rockchip_pcie *rockchip = &ep->rockchip; 252 u32 r; 253 254 for (r = 0; r < ep->max_regions; r++) 255 if (ep->ob_addr[r] == addr) 256 break; 257 258 if (r == ep->max_regions) 259 return; 260 261 rockchip_pcie_clear_ep_ob_atu(rockchip, r); 262 263 ep->ob_addr[r] = 0; 264 clear_bit(r, &ep->ob_region_map); 265 } 266 267 static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, 268 u8 multi_msg_cap) 269 { 270 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 271 struct rockchip_pcie *rockchip = &ep->rockchip; 272 u32 flags; 273 274 flags = rockchip_pcie_read(rockchip, 275 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 276 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 277 flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_MASK; 278 flags |= 279 (multi_msg_cap << ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_OFFSET) | 280 (PCI_MSI_FLAGS_64BIT << ROCKCHIP_PCIE_EP_MSI_FLAGS_OFFSET); 281 flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MASK_MSI_CAP; 282 rockchip_pcie_write(rockchip, flags, 283 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 284 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 285 return 0; 286 } 287 288 static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) 289 { 290 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 291 struct rockchip_pcie *rockchip = &ep->rockchip; 292 u32 flags; 293 294 flags = rockchip_pcie_read(rockchip, 295 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 296 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 297 if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) 298 return -EINVAL; 299 300 return ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> 301 ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); 302 } 303 304 static void rockchip_pcie_ep_assert_intx(struct rockchip_pcie_ep *ep, u8 fn, 305 u8 intx, bool do_assert) 306 { 307 struct rockchip_pcie *rockchip = &ep->rockchip; 308 309 intx &= 3; 310 311 if (do_assert) { 312 ep->irq_pending |= BIT(intx); 313 rockchip_pcie_write(rockchip, 314 PCIE_CLIENT_INT_IN_ASSERT | 315 PCIE_CLIENT_INT_PEND_ST_PEND, 316 PCIE_CLIENT_LEGACY_INT_CTRL); 317 } else { 318 ep->irq_pending &= ~BIT(intx); 319 rockchip_pcie_write(rockchip, 320 PCIE_CLIENT_INT_IN_DEASSERT | 321 PCIE_CLIENT_INT_PEND_ST_NORMAL, 322 PCIE_CLIENT_LEGACY_INT_CTRL); 323 } 324 } 325 326 static int rockchip_pcie_ep_send_intx_irq(struct rockchip_pcie_ep *ep, u8 fn, 327 u8 intx) 328 { 329 u16 cmd; 330 331 cmd = rockchip_pcie_read(&ep->rockchip, 332 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 333 ROCKCHIP_PCIE_EP_CMD_STATUS); 334 335 if (cmd & PCI_COMMAND_INTX_DISABLE) 336 return -EINVAL; 337 338 /* 339 * Should add some delay between toggling INTx per TRM vaguely saying 340 * it depends on some cycles of the AHB bus clock to function it. So 341 * add sufficient 1ms here. 342 */ 343 rockchip_pcie_ep_assert_intx(ep, fn, intx, true); 344 mdelay(1); 345 rockchip_pcie_ep_assert_intx(ep, fn, intx, false); 346 return 0; 347 } 348 349 static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn, 350 u8 interrupt_num) 351 { 352 struct rockchip_pcie *rockchip = &ep->rockchip; 353 u32 flags, mme, data, data_mask; 354 u8 msi_count; 355 u64 pci_addr; 356 u32 r; 357 358 /* Check MSI enable bit */ 359 flags = rockchip_pcie_read(&ep->rockchip, 360 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 361 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 362 if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) 363 return -EINVAL; 364 365 /* Get MSI numbers from MME */ 366 mme = ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> 367 ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); 368 msi_count = 1 << mme; 369 if (!interrupt_num || interrupt_num > msi_count) 370 return -EINVAL; 371 372 /* Set MSI private data */ 373 data_mask = msi_count - 1; 374 data = rockchip_pcie_read(rockchip, 375 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 376 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 377 PCI_MSI_DATA_64); 378 data = (data & ~data_mask) | ((interrupt_num - 1) & data_mask); 379 380 /* Get MSI PCI address */ 381 pci_addr = rockchip_pcie_read(rockchip, 382 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 383 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 384 PCI_MSI_ADDRESS_HI); 385 pci_addr <<= 32; 386 pci_addr |= rockchip_pcie_read(rockchip, 387 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 388 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 389 PCI_MSI_ADDRESS_LO); 390 391 /* Set the outbound region if needed. */ 392 if (unlikely(ep->irq_pci_addr != (pci_addr & PCIE_ADDR_MASK) || 393 ep->irq_pci_fn != fn)) { 394 r = rockchip_ob_region(ep->irq_phys_addr); 395 rockchip_pcie_prog_ep_ob_atu(rockchip, fn, r, 396 ep->irq_phys_addr, 397 pci_addr & PCIE_ADDR_MASK, 398 ~PCIE_ADDR_MASK + 1); 399 ep->irq_pci_addr = (pci_addr & PCIE_ADDR_MASK); 400 ep->irq_pci_fn = fn; 401 } 402 403 writew(data, ep->irq_cpu_addr + (pci_addr & ~PCIE_ADDR_MASK)); 404 return 0; 405 } 406 407 static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, 408 unsigned int type, u16 interrupt_num) 409 { 410 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 411 412 switch (type) { 413 case PCI_IRQ_INTX: 414 return rockchip_pcie_ep_send_intx_irq(ep, fn, 0); 415 case PCI_IRQ_MSI: 416 return rockchip_pcie_ep_send_msi_irq(ep, fn, interrupt_num); 417 default: 418 return -EINVAL; 419 } 420 } 421 422 static int rockchip_pcie_ep_start(struct pci_epc *epc) 423 { 424 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 425 struct rockchip_pcie *rockchip = &ep->rockchip; 426 struct pci_epf *epf; 427 u32 cfg; 428 429 cfg = BIT(0); 430 list_for_each_entry(epf, &epc->pci_epf, list) 431 cfg |= BIT(epf->func_no); 432 433 rockchip_pcie_write(rockchip, cfg, PCIE_CORE_PHY_FUNC_CFG); 434 435 return 0; 436 } 437 438 static const struct pci_epc_features rockchip_pcie_epc_features = { 439 .linkup_notifier = false, 440 .msi_capable = true, 441 .msix_capable = false, 442 .align = 256, 443 }; 444 445 static const struct pci_epc_features* 446 rockchip_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 447 { 448 return &rockchip_pcie_epc_features; 449 } 450 451 static const struct pci_epc_ops rockchip_pcie_epc_ops = { 452 .write_header = rockchip_pcie_ep_write_header, 453 .set_bar = rockchip_pcie_ep_set_bar, 454 .clear_bar = rockchip_pcie_ep_clear_bar, 455 .map_addr = rockchip_pcie_ep_map_addr, 456 .unmap_addr = rockchip_pcie_ep_unmap_addr, 457 .set_msi = rockchip_pcie_ep_set_msi, 458 .get_msi = rockchip_pcie_ep_get_msi, 459 .raise_irq = rockchip_pcie_ep_raise_irq, 460 .start = rockchip_pcie_ep_start, 461 .get_features = rockchip_pcie_ep_get_features, 462 }; 463 464 static int rockchip_pcie_parse_ep_dt(struct rockchip_pcie *rockchip, 465 struct rockchip_pcie_ep *ep) 466 { 467 struct device *dev = rockchip->dev; 468 int err; 469 470 err = rockchip_pcie_parse_dt(rockchip); 471 if (err) 472 return err; 473 474 err = rockchip_pcie_get_phys(rockchip); 475 if (err) 476 return err; 477 478 err = of_property_read_u32(dev->of_node, 479 "rockchip,max-outbound-regions", 480 &ep->max_regions); 481 if (err < 0 || ep->max_regions > MAX_REGION_LIMIT) 482 ep->max_regions = MAX_REGION_LIMIT; 483 484 ep->ob_region_map = 0; 485 486 err = of_property_read_u8(dev->of_node, "max-functions", 487 &ep->epc->max_functions); 488 if (err < 0) 489 ep->epc->max_functions = 1; 490 491 return 0; 492 } 493 494 static const struct of_device_id rockchip_pcie_ep_of_match[] = { 495 { .compatible = "rockchip,rk3399-pcie-ep"}, 496 {}, 497 }; 498 499 static int rockchip_pcie_ep_probe(struct platform_device *pdev) 500 { 501 struct device *dev = &pdev->dev; 502 struct rockchip_pcie_ep *ep; 503 struct rockchip_pcie *rockchip; 504 struct pci_epc *epc; 505 size_t max_regions; 506 struct pci_epc_mem_window *windows = NULL; 507 int err, i; 508 u32 cfg_msi, cfg_msix_cp; 509 510 ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); 511 if (!ep) 512 return -ENOMEM; 513 514 rockchip = &ep->rockchip; 515 rockchip->is_rc = false; 516 rockchip->dev = dev; 517 518 epc = devm_pci_epc_create(dev, &rockchip_pcie_epc_ops); 519 if (IS_ERR(epc)) { 520 dev_err(dev, "failed to create epc device\n"); 521 return PTR_ERR(epc); 522 } 523 524 ep->epc = epc; 525 epc_set_drvdata(epc, ep); 526 527 err = rockchip_pcie_parse_ep_dt(rockchip, ep); 528 if (err) 529 return err; 530 531 err = rockchip_pcie_enable_clocks(rockchip); 532 if (err) 533 return err; 534 535 err = rockchip_pcie_init_port(rockchip); 536 if (err) 537 goto err_disable_clocks; 538 539 /* Establish the link automatically */ 540 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 541 PCIE_CLIENT_CONFIG); 542 543 max_regions = ep->max_regions; 544 ep->ob_addr = devm_kcalloc(dev, max_regions, sizeof(*ep->ob_addr), 545 GFP_KERNEL); 546 547 if (!ep->ob_addr) { 548 err = -ENOMEM; 549 goto err_uninit_port; 550 } 551 552 /* Only enable function 0 by default */ 553 rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG); 554 555 windows = devm_kcalloc(dev, ep->max_regions, 556 sizeof(struct pci_epc_mem_window), GFP_KERNEL); 557 if (!windows) { 558 err = -ENOMEM; 559 goto err_uninit_port; 560 } 561 for (i = 0; i < ep->max_regions; i++) { 562 windows[i].phys_base = rockchip->mem_res->start + (SZ_1M * i); 563 windows[i].size = SZ_1M; 564 windows[i].page_size = SZ_1M; 565 } 566 err = pci_epc_multi_mem_init(epc, windows, ep->max_regions); 567 devm_kfree(dev, windows); 568 569 if (err < 0) { 570 dev_err(dev, "failed to initialize the memory space\n"); 571 goto err_uninit_port; 572 } 573 574 ep->irq_cpu_addr = pci_epc_mem_alloc_addr(epc, &ep->irq_phys_addr, 575 SZ_1M); 576 if (!ep->irq_cpu_addr) { 577 dev_err(dev, "failed to reserve memory space for MSI\n"); 578 err = -ENOMEM; 579 goto err_epc_mem_exit; 580 } 581 582 ep->irq_pci_addr = ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR; 583 584 /* 585 * MSI-X is not supported but the controller still advertises the MSI-X 586 * capability by default, which can lead to the Root Complex side 587 * allocating MSI-X vectors which cannot be used. Avoid this by skipping 588 * the MSI-X capability entry in the PCIe capabilities linked-list: get 589 * the next pointer from the MSI-X entry and set that in the MSI 590 * capability entry (which is the previous entry). This way the MSI-X 591 * entry is skipped (left out of the linked-list) and not advertised. 592 */ 593 cfg_msi = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + 594 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 595 596 cfg_msi &= ~ROCKCHIP_PCIE_EP_MSI_CP1_MASK; 597 598 cfg_msix_cp = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + 599 ROCKCHIP_PCIE_EP_MSIX_CAP_REG) & 600 ROCKCHIP_PCIE_EP_MSIX_CAP_CP_MASK; 601 602 cfg_msi |= cfg_msix_cp; 603 604 rockchip_pcie_write(rockchip, cfg_msi, 605 PCIE_EP_CONFIG_BASE + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 606 607 rockchip_pcie_write(rockchip, PCIE_CLIENT_CONF_ENABLE, 608 PCIE_CLIENT_CONFIG); 609 610 pci_epc_init_notify(epc); 611 612 return 0; 613 err_epc_mem_exit: 614 pci_epc_mem_exit(epc); 615 err_uninit_port: 616 rockchip_pcie_deinit_phys(rockchip); 617 err_disable_clocks: 618 rockchip_pcie_disable_clocks(rockchip); 619 return err; 620 } 621 622 static struct platform_driver rockchip_pcie_ep_driver = { 623 .driver = { 624 .name = "rockchip-pcie-ep", 625 .of_match_table = rockchip_pcie_ep_of_match, 626 }, 627 .probe = rockchip_pcie_ep_probe, 628 }; 629 630 builtin_platform_driver(rockchip_pcie_ep_driver); 631