1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * PCIe host controller driver for Rockchip SoCs.
4 *
5 * Copyright (C) 2021 Rockchip Electronics Co., Ltd.
6 * http://www.rock-chips.com
7 *
8 * Author: Simon Xue <xxm@rock-chips.com>
9 */
10
11 #include <linux/bitfield.h>
12 #include <linux/clk.h>
13 #include <linux/gpio/consumer.h>
14 #include <linux/hw_bitfield.h>
15 #include <linux/irqchip/chained_irq.h>
16 #include <linux/irqdomain.h>
17 #include <linux/mfd/syscon.h>
18 #include <linux/module.h>
19 #include <linux/of.h>
20 #include <linux/of_irq.h>
21 #include <linux/phy/phy.h>
22 #include <linux/platform_device.h>
23 #include <linux/regmap.h>
24 #include <linux/reset.h>
25 #include <linux/workqueue.h>
26 #include <trace/events/pci_controller.h>
27
28 #include "../../pci.h"
29 #include "../pci-host-common.h"
30 #include "pcie-designware.h"
31
32 /*
33 * The upper 16 bits of PCIE_CLIENT_CONFIG are a write
34 * mask for the lower 16 bits.
35 */
36
37 #define to_rockchip_pcie(x) dev_get_drvdata((x)->dev)
38
39 /* General Control Register */
40 #define PCIE_CLIENT_GENERAL_CON 0x0
41 #define PCIE_CLIENT_MODE_MASK GENMASK(7, 4)
42 #define PCIE_CLIENT_MODE_EP 0x0UL
43 #define PCIE_CLIENT_MODE_RC 0x4UL
44 #define PCIE_CLIENT_SET_MODE(x) FIELD_PREP_WM16(PCIE_CLIENT_MODE_MASK, (x))
45 #define PCIE_CLIENT_LD_RQ_RST_GRT FIELD_PREP_WM16(BIT(3), 1)
46 #define PCIE_CLIENT_ENABLE_LTSSM FIELD_PREP_WM16(BIT(2), 1)
47 #define PCIE_CLIENT_DISABLE_LTSSM FIELD_PREP_WM16(BIT(2), 0)
48
49 /* Interrupt Status Register Related to Legacy Interrupt */
50 #define PCIE_CLIENT_INTR_STATUS_LEGACY 0x8
51
52 /* Interrupt Status Register Related to Miscellaneous Operation */
53 #define PCIE_CLIENT_INTR_STATUS_MISC 0x10
54 #define PCIE_RDLH_LINK_UP_CHGED BIT(1)
55 #define PCIE_LINK_REQ_RST_NOT_INT BIT(2)
56
57 /* Interrupt Mask Register Related to Legacy Interrupt */
58 #define PCIE_CLIENT_INTR_MASK_LEGACY 0x1c
59 #define PCIE_INTR_MASK GENMASK(7, 0)
60 #define PCIE_INTR_CLAMP(_x) ((BIT((_x)) & PCIE_INTR_MASK))
61 #define PCIE_INTR_LEGACY_MASK(x) (PCIE_INTR_CLAMP((x)) | \
62 (PCIE_INTR_CLAMP((x)) << 16))
63 #define PCIE_INTR_LEGACY_UNMASK(x) (PCIE_INTR_CLAMP((x)) << 16)
64
65 /* Interrupt Mask Register Related to Miscellaneous Operation */
66 #define PCIE_CLIENT_INTR_MASK_MISC 0x24
67
68 /* Power Management Control Register */
69 #define PCIE_CLIENT_POWER_CON 0x2c
70 #define PCIE_CLKREQ_READY FIELD_PREP_WM16(BIT(0), 1)
71 #define PCIE_CLKREQ_NOT_READY FIELD_PREP_WM16(BIT(0), 0)
72 #define PCIE_CLKREQ_PULL_DOWN FIELD_PREP_WM16(GENMASK(13, 12), 1)
73
74 /* RASDES TBA information */
75 #define PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN 0x154
76 #define PCIE_CLIENT_CDM_RASDES_TBA_L1_1 BIT(4)
77 #define PCIE_CLIENT_CDM_RASDES_TBA_L1_2 BIT(5)
78
79 /* Debug FIFO information */
80 #define PCIE_CLIENT_DBG_FIFO_MODE_CON 0x310
81 #define PCIE_CLIENT_DBG_EN 0xffff0007
82 #define PCIE_CLIENT_DBG_DIS 0xffff0000
83 #define PCIE_CLIENT_DBG_FIFO_PTN_HIT_D0 0x320
84 #define PCIE_CLIENT_DBG_FIFO_PTN_HIT_D1 0x324
85 #define PCIE_CLIENT_DBG_FIFO_TRN_HIT_D0 0x328
86 #define PCIE_CLIENT_DBG_FIFO_TRN_HIT_D1 0x32c
87 #define PCIE_CLIENT_DBG_TRANSITION_DATA 0xffff0000
88 #define PCIE_CLIENT_DBG_FIFO_STATUS 0x350
89 #define PCIE_DBG_FIFO_RATE_MASK GENMASK(22, 20)
90 #define PCIE_DBG_FIFO_L1SUB_MASK GENMASK(10, 8)
91 #define PCIE_DBG_LTSSM_HISTORY_CNT 64
92
93 /* Hot Reset Control Register */
94 #define PCIE_CLIENT_HOT_RESET_CTRL 0x180
95 #define PCIE_LTSSM_APP_DLY2_EN BIT(1)
96 #define PCIE_LTSSM_APP_DLY2_DONE BIT(3)
97 #define PCIE_LTSSM_ENABLE_ENHANCE BIT(4)
98
99 /* LTSSM Status Register */
100 #define PCIE_CLIENT_LTSSM_STATUS 0x300
101 #define PCIE_LINKUP 0x3
102 #define PCIE_LINKUP_MASK GENMASK(17, 16)
103 #define PCIE_LTSSM_STATUS_MASK GENMASK(5, 0)
104
105 #define PCIE_TYPE0_HDR_DBI2_OFFSET 0x100000
106
107 struct rockchip_pcie {
108 struct dw_pcie pci;
109 void __iomem *apb_base;
110 struct phy *phy;
111 struct clk_bulk_data *clks;
112 unsigned int clk_cnt;
113 struct reset_control *rst;
114 struct gpio_desc *rst_gpio;
115 struct irq_domain *irq_domain;
116 const struct rockchip_pcie_of_data *data;
117 bool supports_clkreq;
118 struct delayed_work trace_work;
119 };
120
121 struct rockchip_pcie_of_data {
122 enum dw_pcie_device_mode mode;
123 const struct pci_epc_features *epc_features;
124 };
125
126 static int rockchip_pcie_rc_reset_root_port(struct pci_host_bridge *bridge,
127 struct pci_dev *pdev);
128
rockchip_pcie_readl_apb(struct rockchip_pcie * rockchip,u32 reg)129 static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, u32 reg)
130 {
131 return readl_relaxed(rockchip->apb_base + reg);
132 }
133
rockchip_pcie_writel_apb(struct rockchip_pcie * rockchip,u32 val,u32 reg)134 static void rockchip_pcie_writel_apb(struct rockchip_pcie *rockchip, u32 val,
135 u32 reg)
136 {
137 writel_relaxed(val, rockchip->apb_base + reg);
138 }
139
rockchip_pcie_intx_handler(struct irq_desc * desc)140 static void rockchip_pcie_intx_handler(struct irq_desc *desc)
141 {
142 struct irq_chip *chip = irq_desc_get_chip(desc);
143 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
144 unsigned long reg, hwirq;
145
146 chained_irq_enter(chip, desc);
147
148 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_LEGACY);
149
150 for_each_set_bit(hwirq, ®, 4)
151 generic_handle_domain_irq(rockchip->irq_domain, hwirq);
152
153 chained_irq_exit(chip, desc);
154 }
155
rockchip_intx_mask(struct irq_data * data)156 static void rockchip_intx_mask(struct irq_data *data)
157 {
158 rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data),
159 PCIE_INTR_LEGACY_MASK(data->hwirq),
160 PCIE_CLIENT_INTR_MASK_LEGACY);
161 };
162
rockchip_intx_unmask(struct irq_data * data)163 static void rockchip_intx_unmask(struct irq_data *data)
164 {
165 rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data),
166 PCIE_INTR_LEGACY_UNMASK(data->hwirq),
167 PCIE_CLIENT_INTR_MASK_LEGACY);
168 };
169
170 static struct irq_chip rockchip_intx_irq_chip = {
171 .name = "INTx",
172 .irq_mask = rockchip_intx_mask,
173 .irq_unmask = rockchip_intx_unmask,
174 .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
175 };
176
rockchip_pcie_intx_map(struct irq_domain * domain,unsigned int irq,irq_hw_number_t hwirq)177 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
178 irq_hw_number_t hwirq)
179 {
180 irq_set_chip_and_handler(irq, &rockchip_intx_irq_chip, handle_level_irq);
181 irq_set_chip_data(irq, domain->host_data);
182
183 return 0;
184 }
185
186 static const struct irq_domain_ops intx_domain_ops = {
187 .map = rockchip_pcie_intx_map,
188 };
189
rockchip_pcie_init_irq_domain(struct rockchip_pcie * rockchip)190 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
191 {
192 struct device *dev = rockchip->pci.dev;
193 struct device_node *intc;
194
195 intc = of_get_child_by_name(dev->of_node, "legacy-interrupt-controller");
196 if (!intc) {
197 dev_err(dev, "missing child interrupt-controller node\n");
198 return -EINVAL;
199 }
200
201 rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX,
202 &intx_domain_ops, rockchip);
203 of_node_put(intc);
204 if (!rockchip->irq_domain) {
205 dev_err(dev, "failed to get a INTx IRQ domain\n");
206 return -EINVAL;
207 }
208
209 return 0;
210 }
211
rockchip_pcie_get_ltssm_reg(struct rockchip_pcie * rockchip)212 static u32 rockchip_pcie_get_ltssm_reg(struct rockchip_pcie *rockchip)
213 {
214 return rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS);
215 }
216
rockchip_pcie_get_ltssm(struct dw_pcie * pci)217 static enum dw_pcie_ltssm rockchip_pcie_get_ltssm(struct dw_pcie *pci)
218 {
219 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
220 u32 val = rockchip_pcie_readl_apb(rockchip,
221 PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN);
222
223 if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_1)
224 return DW_PCIE_LTSSM_L1_1;
225
226 if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_2)
227 return DW_PCIE_LTSSM_L1_2;
228
229 return rockchip_pcie_get_ltssm_reg(rockchip) & PCIE_LTSSM_STATUS_MASK;
230 }
231
232 #ifdef CONFIG_TRACING
rockchip_pcie_ltssm_trace_work(struct work_struct * work)233 static void rockchip_pcie_ltssm_trace_work(struct work_struct *work)
234 {
235 struct rockchip_pcie *rockchip = container_of(work,
236 struct rockchip_pcie,
237 trace_work.work);
238 struct dw_pcie *pci = &rockchip->pci;
239 enum dw_pcie_ltssm state;
240 u32 i, l1ss, prev_val = DW_PCIE_LTSSM_UNKNOWN, rate, val;
241
242 if (!trace_pcie_ltssm_state_transition_enabled())
243 goto skip_trace;
244
245 for (i = 0; i < PCIE_DBG_LTSSM_HISTORY_CNT; i++) {
246 val = rockchip_pcie_readl_apb(rockchip,
247 PCIE_CLIENT_DBG_FIFO_STATUS);
248 rate = FIELD_GET(PCIE_DBG_FIFO_RATE_MASK, val);
249 l1ss = FIELD_GET(PCIE_DBG_FIFO_L1SUB_MASK, val);
250 val = FIELD_GET(PCIE_LTSSM_STATUS_MASK, val);
251
252 /*
253 * Hardware Mechanism: The ring FIFO employs two tracking
254 * counters:
255 * - 'last-read-point': maintains the user's last read position
256 * - 'last-valid-point': tracks the HW's last state update
257 *
258 * Software Handling: When two consecutive LTSSM states are
259 * identical, it indicates invalid subsequent data in the FIFO.
260 * In this case, we skip the remaining entries. The dual counter
261 * design ensures that on the next state transition, reading can
262 * resume from the last user position.
263 */
264 if ((i > 0 && val == prev_val) || val > DW_PCIE_LTSSM_RCVRY_EQ3)
265 break;
266
267 state = prev_val = val;
268 if (val == DW_PCIE_LTSSM_L1_IDLE) {
269 if (l1ss == 2)
270 state = DW_PCIE_LTSSM_L1_2;
271 else if (l1ss == 1)
272 state = DW_PCIE_LTSSM_L1_1;
273 }
274
275 trace_pcie_ltssm_state_transition(dev_name(pci->dev),
276 dw_pcie_ltssm_status_string(state),
277 ((rate + 1) > pci->max_link_speed) ?
278 PCI_SPEED_UNKNOWN : PCIE_SPEED_2_5GT + rate);
279 }
280
281 skip_trace:
282 schedule_delayed_work(&rockchip->trace_work, msecs_to_jiffies(5000));
283 }
284
rockchip_pcie_ltssm_trace(struct rockchip_pcie * rockchip,bool enable)285 static void rockchip_pcie_ltssm_trace(struct rockchip_pcie *rockchip,
286 bool enable)
287 {
288 if (enable) {
289 rockchip_pcie_writel_apb(rockchip,
290 PCIE_CLIENT_DBG_TRANSITION_DATA,
291 PCIE_CLIENT_DBG_FIFO_PTN_HIT_D0);
292 rockchip_pcie_writel_apb(rockchip,
293 PCIE_CLIENT_DBG_TRANSITION_DATA,
294 PCIE_CLIENT_DBG_FIFO_PTN_HIT_D1);
295 rockchip_pcie_writel_apb(rockchip,
296 PCIE_CLIENT_DBG_TRANSITION_DATA,
297 PCIE_CLIENT_DBG_FIFO_TRN_HIT_D0);
298 rockchip_pcie_writel_apb(rockchip,
299 PCIE_CLIENT_DBG_TRANSITION_DATA,
300 PCIE_CLIENT_DBG_FIFO_TRN_HIT_D1);
301 rockchip_pcie_writel_apb(rockchip,
302 PCIE_CLIENT_DBG_EN,
303 PCIE_CLIENT_DBG_FIFO_MODE_CON);
304
305 INIT_DELAYED_WORK(&rockchip->trace_work,
306 rockchip_pcie_ltssm_trace_work);
307 schedule_delayed_work(&rockchip->trace_work, 0);
308 } else {
309 rockchip_pcie_writel_apb(rockchip,
310 PCIE_CLIENT_DBG_DIS,
311 PCIE_CLIENT_DBG_FIFO_MODE_CON);
312 cancel_delayed_work_sync(&rockchip->trace_work);
313 }
314 }
315 #else
rockchip_pcie_ltssm_trace(struct rockchip_pcie * rockchip,bool enable)316 static void rockchip_pcie_ltssm_trace(struct rockchip_pcie *rockchip,
317 bool enable)
318 {
319 }
320 #endif
321
rockchip_pcie_enable_ltssm(struct rockchip_pcie * rockchip)322 static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip)
323 {
324 rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM,
325 PCIE_CLIENT_GENERAL_CON);
326 }
327
rockchip_pcie_disable_ltssm(struct rockchip_pcie * rockchip)328 static void rockchip_pcie_disable_ltssm(struct rockchip_pcie *rockchip)
329 {
330 rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_DISABLE_LTSSM,
331 PCIE_CLIENT_GENERAL_CON);
332 }
333
rockchip_pcie_link_up(struct dw_pcie * pci)334 static bool rockchip_pcie_link_up(struct dw_pcie *pci)
335 {
336 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
337 u32 val = rockchip_pcie_get_ltssm_reg(rockchip);
338
339 return FIELD_GET(PCIE_LINKUP_MASK, val) == PCIE_LINKUP;
340 }
341
342 /*
343 * See e.g. section '11.6.6.4 L1 Substate' in the RK3588 TRM V1.0 for the steps
344 * needed to support L1 substates. Currently, just enable L1 substates for RC
345 * mode if CLKREQ# is properly connected and supports-clkreq is present in DT.
346 * For EP mode, there are more things should be done to actually save power in
347 * L1 substates, so disable L1 substates until there is proper support.
348 */
rockchip_pcie_configure_l1ss(struct dw_pcie * pci)349 static void rockchip_pcie_configure_l1ss(struct dw_pcie *pci)
350 {
351 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
352
353 /* Enable L1 substates if CLKREQ# is properly connected */
354 if (rockchip->supports_clkreq) {
355 rockchip_pcie_writel_apb(rockchip, PCIE_CLKREQ_READY,
356 PCIE_CLIENT_POWER_CON);
357 pci->l1ss_support = true;
358 return;
359 }
360
361 /*
362 * Otherwise, assert CLKREQ# unconditionally. Since
363 * pci->l1ss_support is not set, the DWC core will prevent L1
364 * Substates support from being advertised.
365 */
366 rockchip_pcie_writel_apb(rockchip,
367 PCIE_CLKREQ_PULL_DOWN | PCIE_CLKREQ_NOT_READY,
368 PCIE_CLIENT_POWER_CON);
369 }
370
rockchip_pcie_enable_l0s(struct dw_pcie * pci)371 static void rockchip_pcie_enable_l0s(struct dw_pcie *pci)
372 {
373 u32 cap, lnkcap;
374
375 /* Enable L0S capability for all SoCs */
376 cap = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
377 if (cap) {
378 lnkcap = dw_pcie_readl_dbi(pci, cap + PCI_EXP_LNKCAP);
379 lnkcap |= PCI_EXP_LNKCAP_ASPM_L0S;
380 dw_pcie_dbi_ro_wr_en(pci);
381 dw_pcie_writel_dbi(pci, cap + PCI_EXP_LNKCAP, lnkcap);
382 dw_pcie_dbi_ro_wr_dis(pci);
383 }
384 }
385
rockchip_pcie_start_link(struct dw_pcie * pci)386 static int rockchip_pcie_start_link(struct dw_pcie *pci)
387 {
388 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
389
390 /* Reset device */
391 gpiod_set_value_cansleep(rockchip->rst_gpio, 0);
392
393 rockchip_pcie_enable_ltssm(rockchip);
394
395 /*
396 * PCIe requires the refclk to be stable for 100µs prior to releasing
397 * PERST. See table 2-4 in section 2.6.2 AC Specifications of the PCI
398 * Express Card Electromechanical Specification, 1.1. However, we don't
399 * know if the refclk is coming from RC's PHY or external OSC. If it's
400 * from RC, so enabling LTSSM is the just right place to release #PERST.
401 * We need more extra time as before, rather than setting just
402 * 100us as we don't know how long should the device need to reset.
403 */
404 msleep(PCIE_T_PVPERL_MS);
405
406 rockchip_pcie_ltssm_trace(rockchip, true);
407
408 gpiod_set_value_cansleep(rockchip->rst_gpio, 1);
409
410 return 0;
411 }
412
rockchip_pcie_stop_link(struct dw_pcie * pci)413 static void rockchip_pcie_stop_link(struct dw_pcie *pci)
414 {
415 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
416
417 rockchip_pcie_disable_ltssm(rockchip);
418 rockchip_pcie_ltssm_trace(rockchip, false);
419 }
420
rockchip_pcie_host_init(struct dw_pcie_rp * pp)421 static int rockchip_pcie_host_init(struct dw_pcie_rp *pp)
422 {
423 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
424
425 pci->dbi_base2 = pci->dbi_base + PCIE_TYPE0_HDR_DBI2_OFFSET;
426
427 rockchip_pcie_configure_l1ss(pci);
428 rockchip_pcie_enable_l0s(pci);
429 pp->bridge->reset_root_port = rockchip_pcie_rc_reset_root_port;
430
431 /* Disable Root Ports BAR0 and BAR1 as they report bogus size */
432 dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, 0x0);
433 dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_1, 0x0);
434
435 return 0;
436 }
437
438 static const struct dw_pcie_host_ops rockchip_pcie_host_ops = {
439 .init = rockchip_pcie_host_init,
440 };
441
442 /*
443 * ATS does not work on RK3588 when running in EP mode.
444 *
445 * After the host has enabled ATS on the EP side, it will send an IOTLB
446 * invalidation request to the EP side. However, the RK3588 will never send
447 * a completion back and eventually the host will print an IOTLB_INV_TIMEOUT
448 * error, and the EP will not be operational. If we hide the ATS capability,
449 * things work as expected.
450 */
rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep * ep)451 static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep)
452 {
453 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
454 struct device *dev = pci->dev;
455
456 /* Only hide the ATS capability for RK3588 running in EP mode. */
457 if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep"))
458 return;
459
460 dw_pcie_remove_ext_capability(pci, PCI_EXT_CAP_ID_ATS);
461 }
462
rockchip_pcie_ep_init(struct dw_pcie_ep * ep)463 static int rockchip_pcie_ep_init(struct dw_pcie_ep *ep)
464 {
465 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
466
467 rockchip_pcie_enable_l0s(pci);
468 rockchip_pcie_ep_hide_broken_ats_cap_rk3588(ep);
469
470 return 0;
471 };
472
rockchip_pcie_raise_irq(struct dw_pcie_ep * ep,u8 func_no,unsigned int type,u16 interrupt_num)473 static int rockchip_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
474 unsigned int type, u16 interrupt_num)
475 {
476 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
477
478 switch (type) {
479 case PCI_IRQ_INTX:
480 return dw_pcie_ep_raise_intx_irq(ep, func_no);
481 case PCI_IRQ_MSI:
482 return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
483 case PCI_IRQ_MSIX:
484 return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num);
485 default:
486 dev_err(pci->dev, "UNKNOWN IRQ type\n");
487 }
488
489 return 0;
490 }
491
492 static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = {
493 DWC_EPC_COMMON_FEATURES,
494 .linkup_notifier = true,
495 .msi_capable = true,
496 .msix_capable = true,
497 .align = SZ_64K,
498 .bar[BAR_0] = { .type = BAR_RESIZABLE, },
499 .bar[BAR_1] = { .type = BAR_RESIZABLE, },
500 .bar[BAR_2] = { .type = BAR_RESIZABLE, },
501 .bar[BAR_3] = { .type = BAR_RESIZABLE, },
502 .bar[BAR_4] = { .type = BAR_RESIZABLE, },
503 .bar[BAR_5] = { .type = BAR_RESIZABLE, },
504 };
505
506 static const struct pci_epc_bar_rsvd_region rk3588_bar4_rsvd[] = {
507 {
508 /* DMA_CAP (BAR4: DMA Port Logic Structure) */
509 .type = PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO,
510 .offset = 0x0,
511 .size = 0x2000,
512 },
513 };
514
515 /*
516 * BAR4 on rk3588 exposes the ATU Port Logic Structure to the host regardless of
517 * iATU settings for BAR4. This means that BAR4 cannot be used by an EPF driver,
518 * so mark it as RESERVED.
519 */
520 static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = {
521 DWC_EPC_COMMON_FEATURES,
522 .linkup_notifier = true,
523 .msi_capable = true,
524 .msix_capable = true,
525 .align = SZ_64K,
526 .bar[BAR_0] = { .type = BAR_RESIZABLE, },
527 .bar[BAR_1] = { .type = BAR_RESIZABLE, },
528 .bar[BAR_2] = { .type = BAR_RESIZABLE, },
529 .bar[BAR_3] = { .type = BAR_RESIZABLE, },
530 .bar[BAR_4] = {
531 .type = BAR_RESERVED,
532 .nr_rsvd_regions = ARRAY_SIZE(rk3588_bar4_rsvd),
533 .rsvd_regions = rk3588_bar4_rsvd,
534 },
535 .bar[BAR_5] = { .type = BAR_RESIZABLE, },
536 };
537
538 static const struct pci_epc_features *
rockchip_pcie_get_features(struct dw_pcie_ep * ep)539 rockchip_pcie_get_features(struct dw_pcie_ep *ep)
540 {
541 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
542 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
543
544 return rockchip->data->epc_features;
545 }
546
547 static const struct dw_pcie_ep_ops rockchip_pcie_ep_ops = {
548 .init = rockchip_pcie_ep_init,
549 .raise_irq = rockchip_pcie_raise_irq,
550 .get_features = rockchip_pcie_get_features,
551 };
552
rockchip_pcie_clk_init(struct rockchip_pcie * rockchip)553 static int rockchip_pcie_clk_init(struct rockchip_pcie *rockchip)
554 {
555 struct device *dev = rockchip->pci.dev;
556 int ret;
557
558 ret = devm_clk_bulk_get_all(dev, &rockchip->clks);
559 if (ret < 0)
560 return dev_err_probe(dev, ret, "failed to get clocks\n");
561
562 rockchip->clk_cnt = ret;
563
564 ret = clk_bulk_prepare_enable(rockchip->clk_cnt, rockchip->clks);
565 if (ret)
566 return dev_err_probe(dev, ret, "failed to enable clocks\n");
567
568 return 0;
569 }
570
rockchip_pcie_resource_get(struct platform_device * pdev,struct rockchip_pcie * rockchip)571 static int rockchip_pcie_resource_get(struct platform_device *pdev,
572 struct rockchip_pcie *rockchip)
573 {
574 rockchip->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb");
575 if (IS_ERR(rockchip->apb_base))
576 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->apb_base),
577 "failed to map apb registers\n");
578
579 rockchip->rst_gpio = devm_gpiod_get_optional(&pdev->dev, "reset",
580 GPIOD_OUT_LOW);
581 if (IS_ERR(rockchip->rst_gpio))
582 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst_gpio),
583 "failed to get reset gpio\n");
584
585 rockchip->rst = devm_reset_control_array_get_exclusive(&pdev->dev);
586 if (IS_ERR(rockchip->rst))
587 return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst),
588 "failed to get reset lines\n");
589
590 rockchip->supports_clkreq = of_property_read_bool(pdev->dev.of_node,
591 "supports-clkreq");
592
593 return 0;
594 }
595
rockchip_pcie_phy_init(struct rockchip_pcie * rockchip)596 static int rockchip_pcie_phy_init(struct rockchip_pcie *rockchip)
597 {
598 struct device *dev = rockchip->pci.dev;
599 int ret;
600
601 rockchip->phy = devm_phy_get(dev, "pcie-phy");
602 if (IS_ERR(rockchip->phy))
603 return dev_err_probe(dev, PTR_ERR(rockchip->phy),
604 "missing PHY\n");
605
606 ret = phy_init(rockchip->phy);
607 if (ret < 0)
608 return ret;
609
610 ret = phy_power_on(rockchip->phy);
611 if (ret)
612 phy_exit(rockchip->phy);
613
614 return ret;
615 }
616
rockchip_pcie_phy_deinit(struct rockchip_pcie * rockchip)617 static void rockchip_pcie_phy_deinit(struct rockchip_pcie *rockchip)
618 {
619 phy_power_off(rockchip->phy);
620 phy_exit(rockchip->phy);
621 }
622
623 static const struct dw_pcie_ops dw_pcie_ops = {
624 .link_up = rockchip_pcie_link_up,
625 .start_link = rockchip_pcie_start_link,
626 .stop_link = rockchip_pcie_stop_link,
627 .get_ltssm = rockchip_pcie_get_ltssm,
628 };
629
rockchip_pcie_rc_sys_irq_thread(int irq,void * arg)630 static irqreturn_t rockchip_pcie_rc_sys_irq_thread(int irq, void *arg)
631 {
632 struct rockchip_pcie *rockchip = arg;
633 struct dw_pcie *pci = &rockchip->pci;
634 struct dw_pcie_rp *pp = &pci->pp;
635 struct device *dev = pci->dev;
636 struct pci_dev *port;
637 u32 reg;
638
639 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
640 rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
641
642 dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg);
643 dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm_reg(rockchip));
644
645 if (reg & PCIE_LINK_REQ_RST_NOT_INT) {
646 dev_dbg(dev, "hot reset or link-down reset\n");
647 for_each_pci_bridge(port, pp->bridge->bus) {
648 if (pci_pcie_type(port) == PCI_EXP_TYPE_ROOT_PORT)
649 pci_host_handle_link_down(port);
650 }
651 }
652
653 return IRQ_HANDLED;
654 }
655
rockchip_pcie_ep_sys_irq_thread(int irq,void * arg)656 static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg)
657 {
658 struct rockchip_pcie *rockchip = arg;
659 struct dw_pcie *pci = &rockchip->pci;
660 struct device *dev = pci->dev;
661 u32 reg, val;
662
663 reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
664 rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
665
666 dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg);
667 dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm_reg(rockchip));
668
669 if (reg & PCIE_LINK_REQ_RST_NOT_INT) {
670 dev_dbg(dev, "hot reset or link-down reset\n");
671 dw_pcie_ep_linkdown(&pci->ep);
672 /* Stop delaying link training. */
673 val = FIELD_PREP_WM16(PCIE_LTSSM_APP_DLY2_DONE, 1);
674 rockchip_pcie_writel_apb(rockchip, val,
675 PCIE_CLIENT_HOT_RESET_CTRL);
676 }
677
678 if (reg & PCIE_RDLH_LINK_UP_CHGED) {
679 if (rockchip_pcie_link_up(pci)) {
680 dev_dbg(dev, "link up\n");
681 dw_pcie_ep_linkup(&pci->ep);
682 }
683 }
684
685 return IRQ_HANDLED;
686 }
687
rockchip_pcie_configure_rc(struct platform_device * pdev,struct rockchip_pcie * rockchip)688 static int rockchip_pcie_configure_rc(struct platform_device *pdev,
689 struct rockchip_pcie *rockchip)
690 {
691 struct device *dev = &pdev->dev;
692 struct dw_pcie_rp *pp;
693 int irq, ret;
694 u32 val;
695
696 if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_HOST))
697 return -ENODEV;
698
699 irq = platform_get_irq_byname(pdev, "sys");
700 if (irq < 0)
701 return irq;
702
703 ret = devm_request_threaded_irq(dev, irq, NULL,
704 rockchip_pcie_rc_sys_irq_thread,
705 IRQF_ONESHOT, "pcie-sys-rc", rockchip);
706 if (ret) {
707 dev_err(dev, "failed to request PCIe sys IRQ\n");
708 return ret;
709 }
710
711 /* LTSSM enable control mode */
712 val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1);
713 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
714
715 rockchip_pcie_writel_apb(rockchip,
716 PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_RC),
717 PCIE_CLIENT_GENERAL_CON);
718
719 irq = of_irq_get_byname(dev->of_node, "legacy");
720 if (irq < 0)
721 return irq;
722
723 ret = rockchip_pcie_init_irq_domain(rockchip);
724 if (ret < 0) {
725 dev_err(dev, "failed to init irq domain\n");
726 return ret;
727 }
728
729 irq_set_chained_handler_and_data(irq, rockchip_pcie_intx_handler,
730 rockchip);
731
732 pp = &rockchip->pci.pp;
733 pp->ops = &rockchip_pcie_host_ops;
734
735 ret = dw_pcie_host_init(pp);
736 if (ret) {
737 dev_err(dev, "failed to initialize host\n");
738 return ret;
739 }
740
741 /* unmask hot reset/link-down reset */
742 val = FIELD_PREP_WM16(PCIE_LINK_REQ_RST_NOT_INT, 0);
743 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC);
744
745 return ret;
746 }
747
rockchip_pcie_configure_ep(struct platform_device * pdev,struct rockchip_pcie * rockchip)748 static int rockchip_pcie_configure_ep(struct platform_device *pdev,
749 struct rockchip_pcie *rockchip)
750 {
751 struct device *dev = &pdev->dev;
752 int irq, ret;
753 u32 val;
754
755 if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_EP))
756 return -ENODEV;
757
758 irq = platform_get_irq_byname(pdev, "sys");
759 if (irq < 0)
760 return irq;
761
762 ret = devm_request_threaded_irq(dev, irq, NULL,
763 rockchip_pcie_ep_sys_irq_thread,
764 IRQF_ONESHOT, "pcie-sys-ep", rockchip);
765 if (ret) {
766 dev_err(dev, "failed to request PCIe sys IRQ\n");
767 return ret;
768 }
769
770 /*
771 * LTSSM enable control mode, and automatically delay link training on
772 * hot reset/link-down reset.
773 */
774 val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1) |
775 FIELD_PREP_WM16(PCIE_LTSSM_APP_DLY2_EN, 1);
776 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
777
778 rockchip_pcie_writel_apb(rockchip,
779 PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_EP),
780 PCIE_CLIENT_GENERAL_CON);
781
782 rockchip->pci.ep.ops = &rockchip_pcie_ep_ops;
783 rockchip->pci.ep.page_size = SZ_64K;
784
785 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
786
787 ret = dw_pcie_ep_init(&rockchip->pci.ep);
788 if (ret) {
789 dev_err(dev, "failed to initialize endpoint\n");
790 return ret;
791 }
792
793 ret = dw_pcie_ep_init_registers(&rockchip->pci.ep);
794 if (ret) {
795 dev_err(dev, "failed to initialize DWC endpoint registers\n");
796 dw_pcie_ep_deinit(&rockchip->pci.ep);
797 return ret;
798 }
799
800 pci_epc_init_notify(rockchip->pci.ep.epc);
801
802 /* unmask DLL up/down indicator and hot reset/link-down reset */
803 val = FIELD_PREP_WM16(PCIE_RDLH_LINK_UP_CHGED, 0) |
804 FIELD_PREP_WM16(PCIE_LINK_REQ_RST_NOT_INT, 0);
805 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC);
806
807 return ret;
808 }
809
rockchip_pcie_probe(struct platform_device * pdev)810 static int rockchip_pcie_probe(struct platform_device *pdev)
811 {
812 struct device *dev = &pdev->dev;
813 struct rockchip_pcie *rockchip;
814 const struct rockchip_pcie_of_data *data;
815 int ret;
816
817 data = of_device_get_match_data(dev);
818 if (!data)
819 return -EINVAL;
820
821 rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL);
822 if (!rockchip)
823 return -ENOMEM;
824
825 platform_set_drvdata(pdev, rockchip);
826
827 rockchip->pci.dev = dev;
828 rockchip->pci.ops = &dw_pcie_ops;
829 rockchip->data = data;
830
831 /* Default N_FTS value (210) is broken, override it to 255 */
832 rockchip->pci.n_fts[0] = 255; /* Gen1 */
833 rockchip->pci.n_fts[1] = 255; /* Gen2+ */
834
835 ret = rockchip_pcie_resource_get(pdev, rockchip);
836 if (ret)
837 return ret;
838
839 ret = reset_control_assert(rockchip->rst);
840 if (ret)
841 return ret;
842
843 /* DON'T MOVE ME: must be enable before PHY init */
844 ret = devm_regulator_get_enable_optional(dev, "vpcie3v3");
845 if (ret < 0 && ret != -ENODEV)
846 return dev_err_probe(dev, ret,
847 "failed to enable vpcie3v3 regulator\n");
848
849 ret = rockchip_pcie_phy_init(rockchip);
850 if (ret)
851 return dev_err_probe(dev, ret,
852 "failed to initialize the phy\n");
853
854 ret = reset_control_deassert(rockchip->rst);
855 if (ret)
856 goto deinit_phy;
857
858 ret = rockchip_pcie_clk_init(rockchip);
859 if (ret)
860 goto deinit_phy;
861
862 switch (data->mode) {
863 case DW_PCIE_RC_TYPE:
864 ret = rockchip_pcie_configure_rc(pdev, rockchip);
865 if (ret)
866 goto deinit_clk;
867 break;
868 case DW_PCIE_EP_TYPE:
869 ret = rockchip_pcie_configure_ep(pdev, rockchip);
870 if (ret)
871 goto deinit_clk;
872 break;
873 default:
874 dev_err(dev, "INVALID device type %d\n", data->mode);
875 ret = -EINVAL;
876 goto deinit_clk;
877 }
878
879 return 0;
880
881 deinit_clk:
882 clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks);
883 deinit_phy:
884 rockchip_pcie_phy_deinit(rockchip);
885
886 return ret;
887 }
888
rockchip_pcie_rc_reset_root_port(struct pci_host_bridge * bridge,struct pci_dev * pdev)889 static int rockchip_pcie_rc_reset_root_port(struct pci_host_bridge *bridge,
890 struct pci_dev *pdev)
891 {
892 struct pci_bus *bus = bridge->bus;
893 struct dw_pcie_rp *pp = bus->sysdata;
894 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
895 struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
896 struct device *dev = rockchip->pci.dev;
897 u32 val;
898 int ret;
899
900 dw_pcie_stop_link(pci);
901 clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks);
902 rockchip_pcie_phy_deinit(rockchip);
903
904 ret = reset_control_assert(rockchip->rst);
905 if (ret)
906 return ret;
907
908 ret = rockchip_pcie_phy_init(rockchip);
909 if (ret)
910 return ret;
911
912 ret = reset_control_deassert(rockchip->rst);
913 if (ret)
914 goto deinit_phy;
915
916 ret = rockchip_pcie_clk_init(rockchip);
917 if (ret)
918 goto deinit_phy;
919
920 ret = pp->ops->init(pp);
921 if (ret) {
922 dev_err(dev, "Host init failed: %d\n", ret);
923 goto deinit_clk;
924 }
925
926 /* LTSSM enable control mode */
927 val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1);
928 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
929
930 rockchip_pcie_writel_apb(rockchip,
931 PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_RC),
932 PCIE_CLIENT_GENERAL_CON);
933
934 ret = dw_pcie_setup_rc(pp);
935 if (ret) {
936 dev_err(dev, "Failed to setup RC: %d\n", ret);
937 goto deinit_clk;
938 }
939
940 /* unmask hot reset/link-down reset */
941 val = FIELD_PREP_WM16(PCIE_LINK_REQ_RST_NOT_INT, 0);
942 rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC);
943
944 ret = dw_pcie_start_link(pci);
945 if (ret)
946 goto deinit_clk;
947
948 /* Ignore errors, the link may come up later */
949 dw_pcie_wait_for_link(pci);
950 dev_dbg(dev, "Root Port reset completed\n");
951 return ret;
952
953 deinit_clk:
954 clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks);
955 deinit_phy:
956 rockchip_pcie_phy_deinit(rockchip);
957
958 return ret;
959 }
960
961 static const struct rockchip_pcie_of_data rockchip_pcie_rc_of_data_rk3568 = {
962 .mode = DW_PCIE_RC_TYPE,
963 };
964
965 static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3568 = {
966 .mode = DW_PCIE_EP_TYPE,
967 .epc_features = &rockchip_pcie_epc_features_rk3568,
968 };
969
970 static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3588 = {
971 .mode = DW_PCIE_EP_TYPE,
972 .epc_features = &rockchip_pcie_epc_features_rk3588,
973 };
974
975 static const struct of_device_id rockchip_pcie_of_match[] = {
976 {
977 .compatible = "rockchip,rk3568-pcie",
978 .data = &rockchip_pcie_rc_of_data_rk3568,
979 },
980 {
981 .compatible = "rockchip,rk3568-pcie-ep",
982 .data = &rockchip_pcie_ep_of_data_rk3568,
983 },
984 {
985 .compatible = "rockchip,rk3588-pcie-ep",
986 .data = &rockchip_pcie_ep_of_data_rk3588,
987 },
988 {},
989 };
990
991 static struct platform_driver rockchip_pcie_driver = {
992 .driver = {
993 .name = "rockchip-dw-pcie",
994 .of_match_table = rockchip_pcie_of_match,
995 .suppress_bind_attrs = true,
996 },
997 .probe = rockchip_pcie_probe,
998 };
999 builtin_platform_driver(rockchip_pcie_driver);
1000