1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * PCIe host controller driver for Rockchip SoCs.
4 *
5 * Copyright (C) 2021 Rockchip Electronics Co., Ltd.
6 * http://www.rock-chips.com
7 *
8 * Author: Simon Xue <xxm@rock-chips.com>
9 */
10
11 #include <linux/bitfield.h>
12 #include <linux/clk.h>
13 #include <linux/gpio/consumer.h>
14 #include <linux/irqchip/chained_irq.h>
15 #include <linux/irqdomain.h>
16 #include <linux/mfd/syscon.h>
17 #include <linux/module.h>
18 #include <linux/of.h>
19 #include <linux/of_irq.h>
20 #include <linux/phy/phy.h>
21 #include <linux/platform_device.h>
22 #include <linux/regmap.h>
23 #include <linux/reset.h>
24
25 #include "../../pci.h"
26 #include "pcie-designware.h"
27
28 /*
29 * The upper 16 bits of PCIE_CLIENT_CONFIG are a write
30 * mask for the lower 16 bits.
31 */
32 #define HIWORD_UPDATE(mask, val) (((mask) << 16) | (val))
33 #define HIWORD_UPDATE_BIT(val) HIWORD_UPDATE(val, val)
34 #define HIWORD_DISABLE_BIT(val) HIWORD_UPDATE(val, ~val)
35
36 #define to_rockchip_pcie(x) dev_get_drvdata((x)->dev)
37
38 /* General Control Register */
39 #define PCIE_CLIENT_GENERAL_CON 0x0
40 #define PCIE_CLIENT_RC_MODE HIWORD_UPDATE_BIT(0x40)
41 #define PCIE_CLIENT_EP_MODE HIWORD_UPDATE(0xf0, 0x0)
42 #define PCIE_CLIENT_ENABLE_LTSSM HIWORD_UPDATE_BIT(0xc)
43 #define PCIE_CLIENT_DISABLE_LTSSM HIWORD_UPDATE(0x0c, 0x8)
44
45 /* Interrupt Status Register Related to Legacy Interrupt */
46 #define PCIE_CLIENT_INTR_STATUS_LEGACY 0x8
47
48 /* Interrupt Status Register Related to Miscellaneous Operation */
49 #define PCIE_CLIENT_INTR_STATUS_MISC 0x10
50 #define PCIE_RDLH_LINK_UP_CHGED BIT(1)
51 #define PCIE_LINK_REQ_RST_NOT_INT BIT(2)
52
53 /* Interrupt Mask Register Related to Legacy Interrupt */
54 #define PCIE_CLIENT_INTR_MASK_LEGACY 0x1c
55
56 /* Interrupt Mask Register Related to Miscellaneous Operation */
57 #define PCIE_CLIENT_INTR_MASK_MISC 0x24
58
59 /* Hot Reset Control Register */
60 #define PCIE_CLIENT_HOT_RESET_CTRL 0x180
61 #define PCIE_LTSSM_APP_DLY2_EN BIT(1)
62 #define PCIE_LTSSM_APP_DLY2_DONE BIT(3)
63 #define PCIE_LTSSM_ENABLE_ENHANCE BIT(4)
64
65 /* LTSSM Status Register */
66 #define PCIE_CLIENT_LTSSM_STATUS 0x300
67 #define PCIE_LINKUP 0x3
68 #define PCIE_LINKUP_MASK GENMASK(17, 16)
69 #define PCIE_LTSSM_STATUS_MASK GENMASK(5, 0)
70
71 struct rockchip_pcie {
72 struct dw_pcie pci;
73 void __iomem *apb_base;
74 struct phy *phy;
75 struct clk_bulk_data *clks;
76 unsigned int clk_cnt;
77 struct reset_control *rst;
78 struct gpio_desc *rst_gpio;
79 struct regulator *vpcie3v3;
80 struct irq_domain *irq_domain;
81 const struct rockchip_pcie_of_data *data;
82 };
83
84 struct rockchip_pcie_of_data {
85 enum dw_pcie_device_mode mode;
86 const struct pci_epc_features *epc_features;
87 };
88
rockchip_pcie_readl_apb(struct rockchip_pcie * rockchip,u32 reg)89 static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, u32 reg)
90 {
91 return readl_relaxed(rockchip->apb_base + reg);
92 }
93
rockchip_pcie_writel_apb(struct rockchip_pcie * rockchip,u32 val,u32 reg)94 static void rockchip_pcie_writel_apb(struct rockchip_pcie *rockchip, u32 val,
95 u32 reg)
96 {
97 writel_relaxed(val, rockchip->apb_base + reg);
98 }
99
rockchip_pcie_intx_handler(struct irq_desc * desc)100 static void rockchip_pcie_intx_handler(struct irq_desc *desc)
101 {
102 struct irq_chip *chip = irq_desc_get_chip(desc);
103 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
104 unsigned long reg, hwirq;
105
106 chained_irq_enter(chip, desc);
107
108 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_LEGACY);
109
110 for_each_set_bit(hwirq, ®, 4)
111 generic_handle_domain_irq(rockchip->irq_domain, hwirq);
112
113 chained_irq_exit(chip, desc);
114 }
115
rockchip_intx_mask(struct irq_data * data)116 static void rockchip_intx_mask(struct irq_data *data)
117 {
118 rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data),
119 HIWORD_UPDATE_BIT(BIT(data->hwirq)),
120 PCIE_CLIENT_INTR_MASK_LEGACY);
121 };
122
rockchip_intx_unmask(struct irq_data * data)123 static void rockchip_intx_unmask(struct irq_data *data)
124 {
125 rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data),
126 HIWORD_DISABLE_BIT(BIT(data->hwirq)),
127 PCIE_CLIENT_INTR_MASK_LEGACY);
128 };
129
130 static struct irq_chip rockchip_intx_irq_chip = {
131 .name = "INTx",
132 .irq_mask = rockchip_intx_mask,
133 .irq_unmask = rockchip_intx_unmask,
134 .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
135 };
136
rockchip_pcie_intx_map(struct irq_domain * domain,unsigned int irq,irq_hw_number_t hwirq)137 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
138 irq_hw_number_t hwirq)
139 {
140 irq_set_chip_and_handler(irq, &rockchip_intx_irq_chip, handle_level_irq);
141 irq_set_chip_data(irq, domain->host_data);
142
143 return 0;
144 }
145
146 static const struct irq_domain_ops intx_domain_ops = {
147 .map = rockchip_pcie_intx_map,
148 };
149
rockchip_pcie_init_irq_domain(struct rockchip_pcie * rockchip)150 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
151 {
152 struct device *dev = rockchip->pci.dev;
153 struct device_node *intc;
154
155 intc = of_get_child_by_name(dev->of_node, "legacy-interrupt-controller");
156 if (!intc) {
157 dev_err(dev, "missing child interrupt-controller node\n");
158 return -EINVAL;
159 }
160
161 rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX,
162 &intx_domain_ops, rockchip);
163 of_node_put(intc);
164 if (!rockchip->irq_domain) {
165 dev_err(dev, "failed to get a INTx IRQ domain\n");
166 return -EINVAL;
167 }
168
169 return 0;
170 }
171
rockchip_pcie_get_ltssm(struct rockchip_pcie * rockchip)172 static u32 rockchip_pcie_get_ltssm(struct rockchip_pcie *rockchip)
173 {
174 return rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS);
175 }
176
rockchip_pcie_enable_ltssm(struct rockchip_pcie * rockchip)177 static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip)
178 {
179 rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM,
180 PCIE_CLIENT_GENERAL_CON);
181 }
182
rockchip_pcie_disable_ltssm(struct rockchip_pcie * rockchip)183 static void rockchip_pcie_disable_ltssm(struct rockchip_pcie *rockchip)
184 {
185 rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_DISABLE_LTSSM,
186 PCIE_CLIENT_GENERAL_CON);
187 }
188
rockchip_pcie_link_up(struct dw_pcie * pci)189 static bool rockchip_pcie_link_up(struct dw_pcie *pci)
190 {
191 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
192 u32 val = rockchip_pcie_get_ltssm(rockchip);
193
194 return FIELD_GET(PCIE_LINKUP_MASK, val) == PCIE_LINKUP;
195 }
196
rockchip_pcie_enable_l0s(struct dw_pcie * pci)197 static void rockchip_pcie_enable_l0s(struct dw_pcie *pci)
198 {
199 u32 cap, lnkcap;
200
201 /* Enable L0S capability for all SoCs */
202 cap = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
203 if (cap) {
204 lnkcap = dw_pcie_readl_dbi(pci, cap + PCI_EXP_LNKCAP);
205 lnkcap |= PCI_EXP_LNKCAP_ASPM_L0S;
206 dw_pcie_dbi_ro_wr_en(pci);
207 dw_pcie_writel_dbi(pci, cap + PCI_EXP_LNKCAP, lnkcap);
208 dw_pcie_dbi_ro_wr_dis(pci);
209 }
210 }
211
rockchip_pcie_start_link(struct dw_pcie * pci)212 static int rockchip_pcie_start_link(struct dw_pcie *pci)
213 {
214 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
215
216 /* Reset device */
217 gpiod_set_value_cansleep(rockchip->rst_gpio, 0);
218
219 rockchip_pcie_enable_ltssm(rockchip);
220
221 /*
222 * PCIe requires the refclk to be stable for 100µs prior to releasing
223 * PERST. See table 2-4 in section 2.6.2 AC Specifications of the PCI
224 * Express Card Electromechanical Specification, 1.1. However, we don't
225 * know if the refclk is coming from RC's PHY or external OSC. If it's
226 * from RC, so enabling LTSSM is the just right place to release #PERST.
227 * We need more extra time as before, rather than setting just
228 * 100us as we don't know how long should the device need to reset.
229 */
230 msleep(PCIE_T_PVPERL_MS);
231 gpiod_set_value_cansleep(rockchip->rst_gpio, 1);
232
233 return 0;
234 }
235
rockchip_pcie_stop_link(struct dw_pcie * pci)236 static void rockchip_pcie_stop_link(struct dw_pcie *pci)
237 {
238 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
239
240 rockchip_pcie_disable_ltssm(rockchip);
241 }
242
rockchip_pcie_host_init(struct dw_pcie_rp * pp)243 static int rockchip_pcie_host_init(struct dw_pcie_rp *pp)
244 {
245 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
246 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
247 struct device *dev = rockchip->pci.dev;
248 int irq, ret;
249
250 irq = of_irq_get_byname(dev->of_node, "legacy");
251 if (irq < 0)
252 return irq;
253
254 ret = rockchip_pcie_init_irq_domain(rockchip);
255 if (ret < 0)
256 dev_err(dev, "failed to init irq domain\n");
257
258 irq_set_chained_handler_and_data(irq, rockchip_pcie_intx_handler,
259 rockchip);
260
261 rockchip_pcie_enable_l0s(pci);
262
263 return 0;
264 }
265
266 static const struct dw_pcie_host_ops rockchip_pcie_host_ops = {
267 .init = rockchip_pcie_host_init,
268 };
269
270 /*
271 * ATS does not work on RK3588 when running in EP mode.
272 *
273 * After the host has enabled ATS on the EP side, it will send an IOTLB
274 * invalidation request to the EP side. However, the RK3588 will never send
275 * a completion back and eventually the host will print an IOTLB_INV_TIMEOUT
276 * error, and the EP will not be operational. If we hide the ATS capability,
277 * things work as expected.
278 */
rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep * ep)279 static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep)
280 {
281 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
282 struct device *dev = pci->dev;
283
284 /* Only hide the ATS capability for RK3588 running in EP mode. */
285 if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep"))
286 return;
287
288 if (dw_pcie_ep_hide_ext_capability(pci, PCI_EXT_CAP_ID_SECPCI,
289 PCI_EXT_CAP_ID_ATS))
290 dev_err(dev, "failed to hide ATS capability\n");
291 }
292
rockchip_pcie_ep_init(struct dw_pcie_ep * ep)293 static void rockchip_pcie_ep_init(struct dw_pcie_ep *ep)
294 {
295 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
296 enum pci_barno bar;
297
298 rockchip_pcie_enable_l0s(pci);
299 rockchip_pcie_ep_hide_broken_ats_cap_rk3588(ep);
300
301 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
302 dw_pcie_ep_reset_bar(pci, bar);
303 };
304
rockchip_pcie_raise_irq(struct dw_pcie_ep * ep,u8 func_no,unsigned int type,u16 interrupt_num)305 static int rockchip_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
306 unsigned int type, u16 interrupt_num)
307 {
308 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
309
310 switch (type) {
311 case PCI_IRQ_INTX:
312 return dw_pcie_ep_raise_intx_irq(ep, func_no);
313 case PCI_IRQ_MSI:
314 return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
315 case PCI_IRQ_MSIX:
316 return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num);
317 default:
318 dev_err(pci->dev, "UNKNOWN IRQ type\n");
319 }
320
321 return 0;
322 }
323
324 static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = {
325 .linkup_notifier = true,
326 .msi_capable = true,
327 .msix_capable = true,
328 .intx_capable = false,
329 .align = SZ_64K,
330 .bar[BAR_0] = { .type = BAR_RESIZABLE, },
331 .bar[BAR_1] = { .type = BAR_RESIZABLE, },
332 .bar[BAR_2] = { .type = BAR_RESIZABLE, },
333 .bar[BAR_3] = { .type = BAR_RESIZABLE, },
334 .bar[BAR_4] = { .type = BAR_RESIZABLE, },
335 .bar[BAR_5] = { .type = BAR_RESIZABLE, },
336 };
337
338 /*
339 * BAR4 on rk3588 exposes the ATU Port Logic Structure to the host regardless of
340 * iATU settings for BAR4. This means that BAR4 cannot be used by an EPF driver,
341 * so mark it as RESERVED. (rockchip_pcie_ep_init() will disable all BARs by
342 * default.) If the host could write to BAR4, the iATU settings (for all other
343 * BARs) would be overwritten, resulting in (all other BARs) no longer working.
344 */
345 static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = {
346 .linkup_notifier = true,
347 .msi_capable = true,
348 .msix_capable = true,
349 .intx_capable = false,
350 .align = SZ_64K,
351 .bar[BAR_0] = { .type = BAR_RESIZABLE, },
352 .bar[BAR_1] = { .type = BAR_RESIZABLE, },
353 .bar[BAR_2] = { .type = BAR_RESIZABLE, },
354 .bar[BAR_3] = { .type = BAR_RESIZABLE, },
355 .bar[BAR_4] = { .type = BAR_RESERVED, },
356 .bar[BAR_5] = { .type = BAR_RESIZABLE, },
357 };
358
359 static const struct pci_epc_features *
rockchip_pcie_get_features(struct dw_pcie_ep * ep)360 rockchip_pcie_get_features(struct dw_pcie_ep *ep)
361 {
362 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
363 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
364
365 return rockchip->data->epc_features;
366 }
367
368 static const struct dw_pcie_ep_ops rockchip_pcie_ep_ops = {
369 .init = rockchip_pcie_ep_init,
370 .raise_irq = rockchip_pcie_raise_irq,
371 .get_features = rockchip_pcie_get_features,
372 };
373
rockchip_pcie_clk_init(struct rockchip_pcie * rockchip)374 static int rockchip_pcie_clk_init(struct rockchip_pcie *rockchip)
375 {
376 struct device *dev = rockchip->pci.dev;
377 int ret;
378
379 ret = devm_clk_bulk_get_all(dev, &rockchip->clks);
380 if (ret < 0)
381 return dev_err_probe(dev, ret, "failed to get clocks\n");
382
383 rockchip->clk_cnt = ret;
384
385 ret = clk_bulk_prepare_enable(rockchip->clk_cnt, rockchip->clks);
386 if (ret)
387 return dev_err_probe(dev, ret, "failed to enable clocks\n");
388
389 return 0;
390 }
391
rockchip_pcie_resource_get(struct platform_device * pdev,struct rockchip_pcie * rockchip)392 static int rockchip_pcie_resource_get(struct platform_device *pdev,
393 struct rockchip_pcie *rockchip)
394 {
395 rockchip->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb");
396 if (IS_ERR(rockchip->apb_base))
397 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->apb_base),
398 "failed to map apb registers\n");
399
400 rockchip->rst_gpio = devm_gpiod_get_optional(&pdev->dev, "reset",
401 GPIOD_OUT_LOW);
402 if (IS_ERR(rockchip->rst_gpio))
403 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst_gpio),
404 "failed to get reset gpio\n");
405
406 rockchip->rst = devm_reset_control_array_get_exclusive(&pdev->dev);
407 if (IS_ERR(rockchip->rst))
408 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst),
409 "failed to get reset lines\n");
410
411 return 0;
412 }
413
rockchip_pcie_phy_init(struct rockchip_pcie * rockchip)414 static int rockchip_pcie_phy_init(struct rockchip_pcie *rockchip)
415 {
416 struct device *dev = rockchip->pci.dev;
417 int ret;
418
419 rockchip->phy = devm_phy_get(dev, "pcie-phy");
420 if (IS_ERR(rockchip->phy))
421 return dev_err_probe(dev, PTR_ERR(rockchip->phy),
422 "missing PHY\n");
423
424 ret = phy_init(rockchip->phy);
425 if (ret < 0)
426 return ret;
427
428 ret = phy_power_on(rockchip->phy);
429 if (ret)
430 phy_exit(rockchip->phy);
431
432 return ret;
433 }
434
rockchip_pcie_phy_deinit(struct rockchip_pcie * rockchip)435 static void rockchip_pcie_phy_deinit(struct rockchip_pcie *rockchip)
436 {
437 phy_power_off(rockchip->phy);
438 phy_exit(rockchip->phy);
439 }
440
441 static const struct dw_pcie_ops dw_pcie_ops = {
442 .link_up = rockchip_pcie_link_up,
443 .start_link = rockchip_pcie_start_link,
444 .stop_link = rockchip_pcie_stop_link,
445 };
446
rockchip_pcie_rc_sys_irq_thread(int irq,void * arg)447 static irqreturn_t rockchip_pcie_rc_sys_irq_thread(int irq, void *arg)
448 {
449 struct rockchip_pcie *rockchip = arg;
450 struct dw_pcie *pci = &rockchip->pci;
451 struct dw_pcie_rp *pp = &pci->pp;
452 struct device *dev = pci->dev;
453 u32 reg;
454
455 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
456 rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
457
458 dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg);
459 dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip));
460
461 if (reg & PCIE_RDLH_LINK_UP_CHGED) {
462 if (rockchip_pcie_link_up(pci)) {
463 msleep(PCIE_RESET_CONFIG_WAIT_MS);
464 dev_dbg(dev, "Received Link up event. Starting enumeration!\n");
465 /* Rescan the bus to enumerate endpoint devices */
466 pci_lock_rescan_remove();
467 pci_rescan_bus(pp->bridge->bus);
468 pci_unlock_rescan_remove();
469 }
470 }
471
472 return IRQ_HANDLED;
473 }
474
rockchip_pcie_ep_sys_irq_thread(int irq,void * arg)475 static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg)
476 {
477 struct rockchip_pcie *rockchip = arg;
478 struct dw_pcie *pci = &rockchip->pci;
479 struct device *dev = pci->dev;
480 u32 reg, val;
481
482 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
483 rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
484
485 dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg);
486 dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip));
487
488 if (reg & PCIE_LINK_REQ_RST_NOT_INT) {
489 dev_dbg(dev, "hot reset or link-down reset\n");
490 dw_pcie_ep_linkdown(&pci->ep);
491 /* Stop delaying link training. */
492 val = HIWORD_UPDATE_BIT(PCIE_LTSSM_APP_DLY2_DONE);
493 rockchip_pcie_writel_apb(rockchip, val,
494 PCIE_CLIENT_HOT_RESET_CTRL);
495 }
496
497 if (reg & PCIE_RDLH_LINK_UP_CHGED) {
498 if (rockchip_pcie_link_up(pci)) {
499 dev_dbg(dev, "link up\n");
500 dw_pcie_ep_linkup(&pci->ep);
501 }
502 }
503
504 return IRQ_HANDLED;
505 }
506
rockchip_pcie_configure_rc(struct platform_device * pdev,struct rockchip_pcie * rockchip)507 static int rockchip_pcie_configure_rc(struct platform_device *pdev,
508 struct rockchip_pcie *rockchip)
509 {
510 struct device *dev = &pdev->dev;
511 struct dw_pcie_rp *pp;
512 int irq, ret;
513 u32 val;
514
515 if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_HOST))
516 return -ENODEV;
517
518 irq = platform_get_irq_byname(pdev, "sys");
519 if (irq < 0)
520 return irq;
521
522 ret = devm_request_threaded_irq(dev, irq, NULL,
523 rockchip_pcie_rc_sys_irq_thread,
524 IRQF_ONESHOT, "pcie-sys-rc", rockchip);
525 if (ret) {
526 dev_err(dev, "failed to request PCIe sys IRQ\n");
527 return ret;
528 }
529
530 /* LTSSM enable control mode */
531 val = HIWORD_UPDATE_BIT(PCIE_LTSSM_ENABLE_ENHANCE);
532 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
533
534 rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_RC_MODE,
535 PCIE_CLIENT_GENERAL_CON);
536
537 pp = &rockchip->pci.pp;
538 pp->ops = &rockchip_pcie_host_ops;
539 pp->use_linkup_irq = true;
540
541 ret = dw_pcie_host_init(pp);
542 if (ret) {
543 dev_err(dev, "failed to initialize host\n");
544 return ret;
545 }
546
547 /* unmask DLL up/down indicator */
548 val = HIWORD_UPDATE(PCIE_RDLH_LINK_UP_CHGED, 0);
549 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC);
550
551 return ret;
552 }
553
rockchip_pcie_configure_ep(struct platform_device * pdev,struct rockchip_pcie * rockchip)554 static int rockchip_pcie_configure_ep(struct platform_device *pdev,
555 struct rockchip_pcie *rockchip)
556 {
557 struct device *dev = &pdev->dev;
558 int irq, ret;
559 u32 val;
560
561 if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_EP))
562 return -ENODEV;
563
564 irq = platform_get_irq_byname(pdev, "sys");
565 if (irq < 0)
566 return irq;
567
568 ret = devm_request_threaded_irq(dev, irq, NULL,
569 rockchip_pcie_ep_sys_irq_thread,
570 IRQF_ONESHOT, "pcie-sys-ep", rockchip);
571 if (ret) {
572 dev_err(dev, "failed to request PCIe sys IRQ\n");
573 return ret;
574 }
575
576 /*
577 * LTSSM enable control mode, and automatically delay link training on
578 * hot reset/link-down reset.
579 */
580 val = HIWORD_UPDATE_BIT(PCIE_LTSSM_ENABLE_ENHANCE | PCIE_LTSSM_APP_DLY2_EN);
581 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
582
583 rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_EP_MODE,
584 PCIE_CLIENT_GENERAL_CON);
585
586 rockchip->pci.ep.ops = &rockchip_pcie_ep_ops;
587 rockchip->pci.ep.page_size = SZ_64K;
588
589 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
590
591 ret = dw_pcie_ep_init(&rockchip->pci.ep);
592 if (ret) {
593 dev_err(dev, "failed to initialize endpoint\n");
594 return ret;
595 }
596
597 ret = dw_pcie_ep_init_registers(&rockchip->pci.ep);
598 if (ret) {
599 dev_err(dev, "failed to initialize DWC endpoint registers\n");
600 dw_pcie_ep_deinit(&rockchip->pci.ep);
601 return ret;
602 }
603
604 pci_epc_init_notify(rockchip->pci.ep.epc);
605
606 /* unmask DLL up/down indicator and hot reset/link-down reset */
607 val = HIWORD_UPDATE(PCIE_RDLH_LINK_UP_CHGED | PCIE_LINK_REQ_RST_NOT_INT, 0);
608 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC);
609
610 return ret;
611 }
612
rockchip_pcie_probe(struct platform_device * pdev)613 static int rockchip_pcie_probe(struct platform_device *pdev)
614 {
615 struct device *dev = &pdev->dev;
616 struct rockchip_pcie *rockchip;
617 const struct rockchip_pcie_of_data *data;
618 int ret;
619
620 data = of_device_get_match_data(dev);
621 if (!data)
622 return -EINVAL;
623
624 rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL);
625 if (!rockchip)
626 return -ENOMEM;
627
628 platform_set_drvdata(pdev, rockchip);
629
630 rockchip->pci.dev = dev;
631 rockchip->pci.ops = &dw_pcie_ops;
632 rockchip->data = data;
633
634 /* Default N_FTS value (210) is broken, override it to 255 */
635 rockchip->pci.n_fts[0] = 255; /* Gen1 */
636 rockchip->pci.n_fts[1] = 255; /* Gen2+ */
637
638 ret = rockchip_pcie_resource_get(pdev, rockchip);
639 if (ret)
640 return ret;
641
642 ret = reset_control_assert(rockchip->rst);
643 if (ret)
644 return ret;
645
646 /* DON'T MOVE ME: must be enable before PHY init */
647 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
648 if (IS_ERR(rockchip->vpcie3v3)) {
649 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV)
650 return dev_err_probe(dev, PTR_ERR(rockchip->vpcie3v3),
651 "failed to get vpcie3v3 regulator\n");
652 rockchip->vpcie3v3 = NULL;
653 } else {
654 ret = regulator_enable(rockchip->vpcie3v3);
655 if (ret)
656 return dev_err_probe(dev, ret,
657 "failed to enable vpcie3v3 regulator\n");
658 }
659
660 ret = rockchip_pcie_phy_init(rockchip);
661 if (ret)
662 goto disable_regulator;
663
664 ret = reset_control_deassert(rockchip->rst);
665 if (ret)
666 goto deinit_phy;
667
668 ret = rockchip_pcie_clk_init(rockchip);
669 if (ret)
670 goto deinit_phy;
671
672 switch (data->mode) {
673 case DW_PCIE_RC_TYPE:
674 ret = rockchip_pcie_configure_rc(pdev, rockchip);
675 if (ret)
676 goto deinit_clk;
677 break;
678 case DW_PCIE_EP_TYPE:
679 ret = rockchip_pcie_configure_ep(pdev, rockchip);
680 if (ret)
681 goto deinit_clk;
682 break;
683 default:
684 dev_err(dev, "INVALID device type %d\n", data->mode);
685 ret = -EINVAL;
686 goto deinit_clk;
687 }
688
689 return 0;
690
691 deinit_clk:
692 clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks);
693 deinit_phy:
694 rockchip_pcie_phy_deinit(rockchip);
695 disable_regulator:
696 if (rockchip->vpcie3v3)
697 regulator_disable(rockchip->vpcie3v3);
698
699 return ret;
700 }
701
702 static const struct rockchip_pcie_of_data rockchip_pcie_rc_of_data_rk3568 = {
703 .mode = DW_PCIE_RC_TYPE,
704 };
705
706 static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3568 = {
707 .mode = DW_PCIE_EP_TYPE,
708 .epc_features = &rockchip_pcie_epc_features_rk3568,
709 };
710
711 static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3588 = {
712 .mode = DW_PCIE_EP_TYPE,
713 .epc_features = &rockchip_pcie_epc_features_rk3588,
714 };
715
716 static const struct of_device_id rockchip_pcie_of_match[] = {
717 {
718 .compatible = "rockchip,rk3568-pcie",
719 .data = &rockchip_pcie_rc_of_data_rk3568,
720 },
721 {
722 .compatible = "rockchip,rk3568-pcie-ep",
723 .data = &rockchip_pcie_ep_of_data_rk3568,
724 },
725 {
726 .compatible = "rockchip,rk3588-pcie-ep",
727 .data = &rockchip_pcie_ep_of_data_rk3588,
728 },
729 {},
730 };
731
732 static struct platform_driver rockchip_pcie_driver = {
733 .driver = {
734 .name = "rockchip-dw-pcie",
735 .of_match_table = rockchip_pcie_of_match,
736 .suppress_bind_attrs = true,
737 },
738 .probe = rockchip_pcie_probe,
739 };
740 builtin_platform_driver(rockchip_pcie_driver);
741