1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * PCIe host controller driver for Freescale i.MX6 SoCs
4 *
5 * Copyright (C) 2013 Kosagi
6 * https://www.kosagi.com
7 *
8 * Author: Sean Cross <xobs@kosagi.com>
9 */
10
11 #include <linux/bitfield.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/gpio/consumer.h>
15 #include <linux/kernel.h>
16 #include <linux/mfd/syscon.h>
17 #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
18 #include <linux/mfd/syscon/imx7-iomuxc-gpr.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_address.h>
22 #include <linux/pci.h>
23 #include <linux/pci-pwrctrl.h>
24 #include <linux/platform_device.h>
25 #include <linux/regmap.h>
26 #include <linux/regulator/consumer.h>
27 #include <linux/resource.h>
28 #include <linux/signal.h>
29 #include <linux/types.h>
30 #include <linux/interrupt.h>
31 #include <linux/reset.h>
32 #include <linux/phy/pcie.h>
33 #include <linux/phy/phy.h>
34 #include <linux/pm_domain.h>
35 #include <linux/pm_runtime.h>
36
37 #include "../../pci.h"
38 #include "../pci-host-common.h"
39 #include "pcie-designware.h"
40
41 #define IMX8MQ_GPR_PCIE_REF_USE_PAD BIT(9)
42 #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN BIT(10)
43 #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE BIT(11)
44 #define IMX8MQ_GPR_PCIE_VREG_BYPASS BIT(12)
45 #define IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE GENMASK(11, 8)
46
47 #define IMX95_PCIE_PHY_GEN_CTRL 0x0
48 #define IMX95_PCIE_REF_USE_PAD BIT(17)
49
50 #define IMX95_PCIE_PHY_MPLLA_CTRL 0x10
51 #define IMX95_PCIE_PHY_MPLL_STATE BIT(30)
52
53 #define IMX95_PCIE_SS_RW_REG_0 0xf0
54 #define IMX95_PCIE_REF_CLKEN BIT(23)
55 #define IMX95_PCIE_PHY_CR_PARA_SEL BIT(9)
56 #define IMX95_PCIE_SS_RW_REG_1 0xf4
57 #define IMX95_PCIE_CLKREQ_OVERRIDE_EN BIT(8)
58 #define IMX95_PCIE_CLKREQ_OVERRIDE_VAL BIT(9)
59 #define IMX95_PCIE_SYS_AUX_PWR_DET BIT(31)
60
61 #define IMX95_PE0_GEN_CTRL_1 0x1050
62 #define IMX95_PCIE_DEVICE_TYPE GENMASK(3, 0)
63
64 #define IMX95_PE0_GEN_CTRL_3 0x1058
65 #define IMX95_PCIE_LTSSM_EN BIT(0)
66
67 #define IMX95_PE0_LUT_ACSCTRL 0x1008
68 #define IMX95_PEO_LUT_RWA BIT(16)
69 #define IMX95_PE0_LUT_ENLOC GENMASK(4, 0)
70
71 #define IMX95_PE0_LUT_DATA1 0x100c
72 #define IMX95_PE0_LUT_VLD BIT(31)
73 #define IMX95_PE0_LUT_DAC_ID GENMASK(10, 8)
74 #define IMX95_PE0_LUT_STREAM_ID GENMASK(5, 0)
75
76 #define IMX95_PE0_LUT_DATA2 0x1010
77 #define IMX95_PE0_LUT_REQID GENMASK(31, 16)
78 #define IMX95_PE0_LUT_MASK GENMASK(15, 0)
79
80 #define IMX95_SID_MASK GENMASK(5, 0)
81 #define IMX95_MAX_LUT 32
82
83 #define IMX95_PCIE_RST_CTRL 0x3010
84 #define IMX95_PCIE_COLD_RST BIT(0)
85
86 #define to_imx_pcie(x) dev_get_drvdata((x)->dev)
87
88 enum imx_pcie_variants {
89 IMX6Q,
90 IMX6SX,
91 IMX6QP,
92 IMX7D,
93 IMX8MQ,
94 IMX8MM,
95 IMX8MP,
96 IMX8Q,
97 IMX95,
98 IMX8MQ_EP,
99 IMX8MM_EP,
100 IMX8MP_EP,
101 IMX8Q_EP,
102 IMX95_EP,
103 };
104
105 #define IMX_PCIE_FLAG_IMX_PHY BIT(0)
106 #define IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND BIT(1)
107 #define IMX_PCIE_FLAG_SUPPORTS_SUSPEND BIT(2)
108 #define IMX_PCIE_FLAG_HAS_PHYDRV BIT(3)
109 #define IMX_PCIE_FLAG_HAS_APP_RESET BIT(4)
110 #define IMX_PCIE_FLAG_HAS_PHY_RESET BIT(5)
111 #define IMX_PCIE_FLAG_HAS_SERDES BIT(6)
112 #define IMX_PCIE_FLAG_SUPPORT_64BIT BIT(7)
113 #define IMX_PCIE_FLAG_CPU_ADDR_FIXUP BIT(8)
114 /*
115 * Because of ERR005723 (PCIe does not support L2 power down) we need to
116 * workaround suspend resume on some devices which are affected by this errata.
117 */
118 #define IMX_PCIE_FLAG_BROKEN_SUSPEND BIT(9)
119 #define IMX_PCIE_FLAG_HAS_LUT BIT(10)
120 #define IMX_PCIE_FLAG_8GT_ECN_ERR051586 BIT(11)
121 #define IMX_PCIE_FLAG_SKIP_L23_READY BIT(12)
122 /* Preserve MSI capability for platforms that require it */
123 #define IMX_PCIE_FLAG_KEEP_MSI_CAP BIT(13)
124 #define IMX_PCIE_FLAG_PM_RUNTIME BIT(14)
125
126 #define imx_check_flag(pci, val) (pci->drvdata->flags & val)
127
128 #define IMX_PCIE_MAX_INSTANCES 2
129
130 struct imx_pcie;
131
132 struct imx_pcie_drvdata {
133 enum imx_pcie_variants variant;
134 enum dw_pcie_device_mode mode;
135 u32 flags;
136 int dbi_length;
137 const char *gpr;
138 const u32 ltssm_off;
139 const u32 ltssm_mask;
140 const u32 mode_off[IMX_PCIE_MAX_INSTANCES];
141 const u32 mode_mask[IMX_PCIE_MAX_INSTANCES];
142 const struct pci_epc_features *epc_features;
143 int (*select_ref_clk_src)(struct imx_pcie *pcie);
144 int (*init_phy)(struct imx_pcie *pcie);
145 int (*enable_ref_clk)(struct imx_pcie *pcie, bool enable);
146 int (*core_reset)(struct imx_pcie *pcie, bool assert);
147 int (*wait_pll_lock)(struct imx_pcie *pcie);
148 void (*clr_clkreq_override)(struct imx_pcie *pcie);
149 const struct dw_pcie_host_ops *ops;
150 };
151
152 struct imx_lut_data {
153 u32 data1;
154 u32 data2;
155 };
156
157 struct imx_pcie {
158 struct dw_pcie *pci;
159 struct clk_bulk_data *clks;
160 int num_clks;
161 bool supports_clkreq;
162 bool enable_ext_refclk;
163 struct regmap *iomuxc_gpr;
164 u16 msi_ctrl;
165 u32 controller_id;
166 struct reset_control *pciephy_reset;
167 struct reset_control *apps_reset;
168 u32 tx_deemph_gen1;
169 u32 tx_deemph_gen2_3p5db;
170 u32 tx_deemph_gen2_6db;
171 u32 tx_swing_full;
172 u32 tx_swing_low;
173 struct regulator *vpcie;
174 struct regulator *vpcie_aux;
175 bool vpcie_aux_enabled;
176 struct regulator *vph;
177 void __iomem *phy_base;
178
179 /* LUT data for pcie */
180 struct imx_lut_data luts[IMX95_MAX_LUT];
181 /* power domain for pcie */
182 struct device *pd_pcie;
183 /* power domain for pcie phy */
184 struct device *pd_pcie_phy;
185 struct phy *phy;
186 const struct imx_pcie_drvdata *drvdata;
187
188 /* Ensure that only one device's LUT is configured at any given time */
189 struct mutex lock;
190 };
191
192 /* Parameters for the waiting for PCIe PHY PLL to lock on i.MX7 */
193 #define PHY_PLL_LOCK_WAIT_USLEEP_MAX 200
194 #define PHY_PLL_LOCK_WAIT_TIMEOUT (2000 * PHY_PLL_LOCK_WAIT_USLEEP_MAX)
195
196 /* PCIe Port Logic registers (memory-mapped) */
197 #define PL_OFFSET 0x700
198
199 #define PCIE_PHY_CTRL (PL_OFFSET + 0x114)
200 #define PCIE_PHY_CTRL_DATA(x) FIELD_PREP(GENMASK(15, 0), (x))
201 #define PCIE_PHY_CTRL_CAP_ADR BIT(16)
202 #define PCIE_PHY_CTRL_CAP_DAT BIT(17)
203 #define PCIE_PHY_CTRL_WR BIT(18)
204 #define PCIE_PHY_CTRL_RD BIT(19)
205
206 #define PCIE_PHY_STAT (PL_OFFSET + 0x110)
207 #define PCIE_PHY_STAT_ACK BIT(16)
208
209 /* PHY registers (not memory-mapped) */
210 #define PCIE_PHY_ATEOVRD 0x10
211 #define PCIE_PHY_ATEOVRD_EN BIT(2)
212 #define PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT 0
213 #define PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK 0x1
214
215 #define PCIE_PHY_MPLL_OVRD_IN_LO 0x11
216 #define PCIE_PHY_MPLL_MULTIPLIER_SHIFT 2
217 #define PCIE_PHY_MPLL_MULTIPLIER_MASK 0x7f
218 #define PCIE_PHY_MPLL_MULTIPLIER_OVRD BIT(9)
219
220 #define PCIE_PHY_RX_ASIC_OUT 0x100D
221 #define PCIE_PHY_RX_ASIC_OUT_VALID (1 << 0)
222
223 /* iMX7 PCIe PHY registers */
224 #define PCIE_PHY_CMN_REG4 0x14
225 /* These are probably the bits that *aren't* DCC_FB_EN */
226 #define PCIE_PHY_CMN_REG4_DCC_FB_EN 0x29
227
228 #define PCIE_PHY_CMN_REG15 0x54
229 #define PCIE_PHY_CMN_REG15_DLY_4 BIT(2)
230 #define PCIE_PHY_CMN_REG15_PLL_PD BIT(5)
231 #define PCIE_PHY_CMN_REG15_OVRD_PLL_PD BIT(7)
232
233 #define PCIE_PHY_CMN_REG24 0x90
234 #define PCIE_PHY_CMN_REG24_RX_EQ BIT(6)
235 #define PCIE_PHY_CMN_REG24_RX_EQ_SEL BIT(3)
236
237 #define PCIE_PHY_CMN_REG26 0x98
238 #define PCIE_PHY_CMN_REG26_ATT_MODE 0xBC
239
240 #define PHY_RX_OVRD_IN_LO 0x1005
241 #define PHY_RX_OVRD_IN_LO_RX_DATA_EN BIT(5)
242 #define PHY_RX_OVRD_IN_LO_RX_PLL_EN BIT(3)
243
imx_pcie_grp_offset(const struct imx_pcie * imx_pcie)244 static unsigned int imx_pcie_grp_offset(const struct imx_pcie *imx_pcie)
245 {
246 WARN_ON(imx_pcie->drvdata->variant != IMX8MQ &&
247 imx_pcie->drvdata->variant != IMX8MQ_EP &&
248 imx_pcie->drvdata->variant != IMX8MM &&
249 imx_pcie->drvdata->variant != IMX8MM_EP &&
250 imx_pcie->drvdata->variant != IMX8MP &&
251 imx_pcie->drvdata->variant != IMX8MP_EP);
252 return imx_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14;
253 }
254
imx95_pcie_select_ref_clk_src(struct imx_pcie * imx_pcie)255 static int imx95_pcie_select_ref_clk_src(struct imx_pcie *imx_pcie)
256 {
257 bool ext = imx_pcie->enable_ext_refclk;
258
259 /*
260 * Regarding the Signal Descriptions of i.MX95 PCIe PHY, ref_use_pad is
261 * used to select reference clock connected to a pair of pads.
262 *
263 * Any change in this input must be followed by phy_reset assertion.
264 */
265
266 regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_GEN_CTRL,
267 IMX95_PCIE_REF_USE_PAD,
268 ext ? IMX95_PCIE_REF_USE_PAD : 0);
269
270 return 0;
271 }
272
imx95_pcie_init_phy(struct imx_pcie * imx_pcie)273 static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie)
274 {
275 /*
276 * ERR051624: The Controller Without Vaux Cannot Exit L23 Ready
277 * Through Beacon or PERST# De-assertion
278 *
279 * When the auxiliary power is not available, the controller
280 * cannot exit from L23 Ready with beacon or PERST# de-assertion
281 * when main power is not removed.
282 *
283 * Workaround: Set SS_RW_REG_1[SYS_AUX_PWR_DET] to 1.
284 */
285 regmap_set_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1,
286 IMX95_PCIE_SYS_AUX_PWR_DET);
287
288 regmap_update_bits(imx_pcie->iomuxc_gpr,
289 IMX95_PCIE_SS_RW_REG_0,
290 IMX95_PCIE_PHY_CR_PARA_SEL,
291 IMX95_PCIE_PHY_CR_PARA_SEL);
292
293 return 0;
294 }
295
imx_pcie_configure_type(struct imx_pcie * imx_pcie)296 static void imx_pcie_configure_type(struct imx_pcie *imx_pcie)
297 {
298 const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata;
299 unsigned int mask, val, mode, id;
300
301 if (drvdata->mode == DW_PCIE_EP_TYPE)
302 mode = PCI_EXP_TYPE_ENDPOINT;
303 else
304 mode = PCI_EXP_TYPE_ROOT_PORT;
305
306 id = imx_pcie->controller_id;
307
308 /* If mode_mask is 0, generic PHY driver is used to set the mode */
309 if (!drvdata->mode_mask[0])
310 return;
311
312 /* If mode_mask[id] is 0, each controller has its individual GPR */
313 if (!drvdata->mode_mask[id])
314 id = 0;
315
316 mask = drvdata->mode_mask[id];
317 val = mode << (ffs(mask) - 1);
318
319 regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->mode_off[id], mask, val);
320 }
321
pcie_phy_poll_ack(struct imx_pcie * imx_pcie,bool exp_val)322 static int pcie_phy_poll_ack(struct imx_pcie *imx_pcie, bool exp_val)
323 {
324 struct dw_pcie *pci = imx_pcie->pci;
325 bool val;
326 u32 max_iterations = 10;
327 u32 wait_counter = 0;
328
329 do {
330 val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT) &
331 PCIE_PHY_STAT_ACK;
332 wait_counter++;
333
334 if (val == exp_val)
335 return 0;
336
337 udelay(1);
338 } while (wait_counter < max_iterations);
339
340 return -ETIMEDOUT;
341 }
342
pcie_phy_wait_ack(struct imx_pcie * imx_pcie,int addr)343 static int pcie_phy_wait_ack(struct imx_pcie *imx_pcie, int addr)
344 {
345 struct dw_pcie *pci = imx_pcie->pci;
346 u32 val;
347 int ret;
348
349 val = PCIE_PHY_CTRL_DATA(addr);
350 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
351
352 val |= PCIE_PHY_CTRL_CAP_ADR;
353 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
354
355 ret = pcie_phy_poll_ack(imx_pcie, true);
356 if (ret)
357 return ret;
358
359 val = PCIE_PHY_CTRL_DATA(addr);
360 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
361
362 return pcie_phy_poll_ack(imx_pcie, false);
363 }
364
365 /* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */
pcie_phy_read(struct imx_pcie * imx_pcie,int addr,u16 * data)366 static int pcie_phy_read(struct imx_pcie *imx_pcie, int addr, u16 *data)
367 {
368 struct dw_pcie *pci = imx_pcie->pci;
369 u32 phy_ctl;
370 int ret;
371
372 ret = pcie_phy_wait_ack(imx_pcie, addr);
373 if (ret)
374 return ret;
375
376 /* assert Read signal */
377 phy_ctl = PCIE_PHY_CTRL_RD;
378 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, phy_ctl);
379
380 ret = pcie_phy_poll_ack(imx_pcie, true);
381 if (ret)
382 return ret;
383
384 *data = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT);
385
386 /* deassert Read signal */
387 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x00);
388
389 return pcie_phy_poll_ack(imx_pcie, false);
390 }
391
pcie_phy_write(struct imx_pcie * imx_pcie,int addr,u16 data)392 static int pcie_phy_write(struct imx_pcie *imx_pcie, int addr, u16 data)
393 {
394 struct dw_pcie *pci = imx_pcie->pci;
395 u32 var;
396 int ret;
397
398 /* write addr */
399 /* cap addr */
400 ret = pcie_phy_wait_ack(imx_pcie, addr);
401 if (ret)
402 return ret;
403
404 var = PCIE_PHY_CTRL_DATA(data);
405 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
406
407 /* capture data */
408 var |= PCIE_PHY_CTRL_CAP_DAT;
409 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
410
411 ret = pcie_phy_poll_ack(imx_pcie, true);
412 if (ret)
413 return ret;
414
415 /* deassert cap data */
416 var = PCIE_PHY_CTRL_DATA(data);
417 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
418
419 /* wait for ack de-assertion */
420 ret = pcie_phy_poll_ack(imx_pcie, false);
421 if (ret)
422 return ret;
423
424 /* assert wr signal */
425 var = PCIE_PHY_CTRL_WR;
426 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
427
428 /* wait for ack */
429 ret = pcie_phy_poll_ack(imx_pcie, true);
430 if (ret)
431 return ret;
432
433 /* deassert wr signal */
434 var = PCIE_PHY_CTRL_DATA(data);
435 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
436
437 /* wait for ack de-assertion */
438 ret = pcie_phy_poll_ack(imx_pcie, false);
439 if (ret)
440 return ret;
441
442 dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x0);
443
444 return 0;
445 }
446
imx8mq_pcie_init_phy(struct imx_pcie * imx_pcie)447 static int imx8mq_pcie_init_phy(struct imx_pcie *imx_pcie)
448 {
449 /* TODO: This code assumes external oscillator is being used */
450 regmap_update_bits(imx_pcie->iomuxc_gpr,
451 imx_pcie_grp_offset(imx_pcie),
452 IMX8MQ_GPR_PCIE_REF_USE_PAD,
453 IMX8MQ_GPR_PCIE_REF_USE_PAD);
454 /*
455 * Per the datasheet, the PCIE_VPH is suggested to be 1.8V. If the
456 * PCIE_VPH is supplied by 3.3V, the VREG_BYPASS should be cleared
457 * to zero.
458 */
459 if (imx_pcie->vph && regulator_get_voltage(imx_pcie->vph) > 3000000)
460 regmap_update_bits(imx_pcie->iomuxc_gpr,
461 imx_pcie_grp_offset(imx_pcie),
462 IMX8MQ_GPR_PCIE_VREG_BYPASS,
463 0);
464
465 return 0;
466 }
467
imx_pcie_init_phy(struct imx_pcie * imx_pcie)468 static int imx_pcie_init_phy(struct imx_pcie *imx_pcie)
469 {
470 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
471 IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);
472
473 /* configure constant input signal to the pcie ctrl and phy */
474 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
475 IMX6Q_GPR12_LOS_LEVEL, 9 << 4);
476
477 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
478 IMX6Q_GPR8_TX_DEEMPH_GEN1,
479 imx_pcie->tx_deemph_gen1 << 0);
480 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
481 IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB,
482 imx_pcie->tx_deemph_gen2_3p5db << 6);
483 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
484 IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB,
485 imx_pcie->tx_deemph_gen2_6db << 12);
486 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
487 IMX6Q_GPR8_TX_SWING_FULL,
488 imx_pcie->tx_swing_full << 18);
489 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
490 IMX6Q_GPR8_TX_SWING_LOW,
491 imx_pcie->tx_swing_low << 25);
492 return 0;
493 }
494
imx6sx_pcie_init_phy(struct imx_pcie * imx_pcie)495 static int imx6sx_pcie_init_phy(struct imx_pcie *imx_pcie)
496 {
497 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
498 IMX6SX_GPR12_PCIE_RX_EQ_MASK, IMX6SX_GPR12_PCIE_RX_EQ_2);
499
500 return imx_pcie_init_phy(imx_pcie);
501 }
502
imx7d_pcie_wait_for_phy_pll_lock(struct imx_pcie * imx_pcie)503 static void imx7d_pcie_wait_for_phy_pll_lock(struct imx_pcie *imx_pcie)
504 {
505 u32 val;
506 struct device *dev = imx_pcie->pci->dev;
507
508 if (regmap_read_poll_timeout(imx_pcie->iomuxc_gpr,
509 IOMUXC_GPR22, val,
510 val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED,
511 PHY_PLL_LOCK_WAIT_USLEEP_MAX,
512 PHY_PLL_LOCK_WAIT_TIMEOUT))
513 dev_err(dev, "PCIe PLL lock timeout\n");
514 }
515
imx95_pcie_wait_for_phy_pll_lock(struct imx_pcie * imx_pcie)516 static int imx95_pcie_wait_for_phy_pll_lock(struct imx_pcie *imx_pcie)
517 {
518 u32 val;
519 struct device *dev = imx_pcie->pci->dev;
520
521 if (regmap_read_poll_timeout(imx_pcie->iomuxc_gpr,
522 IMX95_PCIE_PHY_MPLLA_CTRL, val,
523 val & IMX95_PCIE_PHY_MPLL_STATE,
524 PHY_PLL_LOCK_WAIT_USLEEP_MAX,
525 PHY_PLL_LOCK_WAIT_TIMEOUT)) {
526 dev_err(dev, "PCIe PLL lock timeout\n");
527 return -ETIMEDOUT;
528 }
529
530 return 0;
531 }
532
imx_setup_phy_mpll(struct imx_pcie * imx_pcie)533 static int imx_setup_phy_mpll(struct imx_pcie *imx_pcie)
534 {
535 unsigned long phy_rate = 0;
536 int mult, div;
537 u16 val;
538 int i;
539 struct clk_bulk_data *clks = imx_pcie->clks;
540
541 if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_IMX_PHY))
542 return 0;
543
544 for (i = 0; i < imx_pcie->num_clks; i++)
545 if (strncmp(clks[i].id, "pcie_phy", 8) == 0)
546 phy_rate = clk_get_rate(clks[i].clk);
547
548 switch (phy_rate) {
549 case 125000000:
550 /*
551 * The default settings of the MPLL are for a 125MHz input
552 * clock, so no need to reconfigure anything in that case.
553 */
554 return 0;
555 case 100000000:
556 mult = 25;
557 div = 0;
558 break;
559 case 200000000:
560 mult = 25;
561 div = 1;
562 break;
563 default:
564 dev_err(imx_pcie->pci->dev,
565 "Unsupported PHY reference clock rate %lu\n", phy_rate);
566 return -EINVAL;
567 }
568
569 pcie_phy_read(imx_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val);
570 val &= ~(PCIE_PHY_MPLL_MULTIPLIER_MASK <<
571 PCIE_PHY_MPLL_MULTIPLIER_SHIFT);
572 val |= mult << PCIE_PHY_MPLL_MULTIPLIER_SHIFT;
573 val |= PCIE_PHY_MPLL_MULTIPLIER_OVRD;
574 pcie_phy_write(imx_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val);
575
576 pcie_phy_read(imx_pcie, PCIE_PHY_ATEOVRD, &val);
577 val &= ~(PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK <<
578 PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT);
579 val |= div << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT;
580 val |= PCIE_PHY_ATEOVRD_EN;
581 pcie_phy_write(imx_pcie, PCIE_PHY_ATEOVRD, val);
582
583 return 0;
584 }
585
imx_pcie_reset_phy(struct imx_pcie * imx_pcie)586 static void imx_pcie_reset_phy(struct imx_pcie *imx_pcie)
587 {
588 u16 tmp;
589
590 if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_IMX_PHY))
591 return;
592
593 pcie_phy_read(imx_pcie, PHY_RX_OVRD_IN_LO, &tmp);
594 tmp |= (PHY_RX_OVRD_IN_LO_RX_DATA_EN |
595 PHY_RX_OVRD_IN_LO_RX_PLL_EN);
596 pcie_phy_write(imx_pcie, PHY_RX_OVRD_IN_LO, tmp);
597
598 usleep_range(2000, 3000);
599
600 pcie_phy_read(imx_pcie, PHY_RX_OVRD_IN_LO, &tmp);
601 tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN |
602 PHY_RX_OVRD_IN_LO_RX_PLL_EN);
603 pcie_phy_write(imx_pcie, PHY_RX_OVRD_IN_LO, tmp);
604 }
605
606 #ifdef CONFIG_ARM
607 /* Added for PCI abort handling */
imx6q_pcie_abort_handler(unsigned long addr,unsigned int fsr,struct pt_regs * regs)608 static int imx6q_pcie_abort_handler(unsigned long addr,
609 unsigned int fsr, struct pt_regs *regs)
610 {
611 unsigned long pc = instruction_pointer(regs);
612 unsigned long instr = *(unsigned long *)pc;
613 int reg = (instr >> 12) & 15;
614
615 /*
616 * If the instruction being executed was a read,
617 * make it look like it read all-ones.
618 */
619 if ((instr & 0x0c100000) == 0x04100000) {
620 unsigned long val;
621
622 if (instr & 0x00400000)
623 val = 255;
624 else
625 val = -1;
626
627 regs->uregs[reg] = val;
628 regs->ARM_pc += 4;
629 return 0;
630 }
631
632 if ((instr & 0x0e100090) == 0x00100090) {
633 regs->uregs[reg] = -1;
634 regs->ARM_pc += 4;
635 return 0;
636 }
637
638 return 1;
639 }
640 #endif
641
imx_pcie_attach_pd(struct device * dev)642 static int imx_pcie_attach_pd(struct device *dev)
643 {
644 struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
645 struct device_link *link;
646
647 /* Do nothing when in a single power domain */
648 if (dev->pm_domain)
649 return 0;
650
651 imx_pcie->pd_pcie = dev_pm_domain_attach_by_name(dev, "pcie");
652 if (IS_ERR(imx_pcie->pd_pcie))
653 return PTR_ERR(imx_pcie->pd_pcie);
654 /* Do nothing when power domain missing */
655 if (!imx_pcie->pd_pcie)
656 return 0;
657 link = device_link_add(dev, imx_pcie->pd_pcie,
658 DL_FLAG_STATELESS |
659 DL_FLAG_PM_RUNTIME |
660 DL_FLAG_RPM_ACTIVE);
661 if (!link) {
662 dev_err(dev, "Failed to add device_link to pcie pd\n");
663 return -EINVAL;
664 }
665
666 imx_pcie->pd_pcie_phy = dev_pm_domain_attach_by_name(dev, "pcie_phy");
667 if (IS_ERR(imx_pcie->pd_pcie_phy))
668 return PTR_ERR(imx_pcie->pd_pcie_phy);
669
670 link = device_link_add(dev, imx_pcie->pd_pcie_phy,
671 DL_FLAG_STATELESS |
672 DL_FLAG_PM_RUNTIME |
673 DL_FLAG_RPM_ACTIVE);
674 if (!link) {
675 dev_err(dev, "Failed to add device_link to pcie_phy pd\n");
676 return -EINVAL;
677 }
678
679 return 0;
680 }
681
imx6q_pcie_enable_ref_clk(struct imx_pcie * imx_pcie,bool enable)682 static int imx6q_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
683 {
684 if (enable)
685 regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1,
686 IMX6Q_GPR1_PCIE_REF_CLK_EN);
687 else
688 regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1,
689 IMX6Q_GPR1_PCIE_REF_CLK_EN);
690
691 return 0;
692 }
693
imx8mm_pcie_clkreq_override(struct imx_pcie * imx_pcie,bool enable)694 static void imx8mm_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable)
695 {
696 int offset = imx_pcie_grp_offset(imx_pcie);
697
698 regmap_update_bits(imx_pcie->iomuxc_gpr, offset,
699 IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE,
700 enable ? 0 : IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE);
701 regmap_update_bits(imx_pcie->iomuxc_gpr, offset,
702 IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN,
703 enable ? IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN : 0);
704 }
705
imx8mm_pcie_enable_ref_clk(struct imx_pcie * imx_pcie,bool enable)706 static int imx8mm_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
707 {
708 imx8mm_pcie_clkreq_override(imx_pcie, enable);
709 return 0;
710 }
711
imx7d_pcie_enable_ref_clk(struct imx_pcie * imx_pcie,bool enable)712 static int imx7d_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
713 {
714 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
715 IMX7D_GPR12_PCIE_PHY_REFCLK_SEL,
716 enable ? 0 : IMX7D_GPR12_PCIE_PHY_REFCLK_SEL);
717 return 0;
718 }
719
imx95_pcie_clkreq_override(struct imx_pcie * imx_pcie,bool enable)720 static void imx95_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable)
721 {
722 regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1,
723 IMX95_PCIE_CLKREQ_OVERRIDE_EN,
724 enable ? IMX95_PCIE_CLKREQ_OVERRIDE_EN : 0);
725 regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1,
726 IMX95_PCIE_CLKREQ_OVERRIDE_VAL,
727 enable ? IMX95_PCIE_CLKREQ_OVERRIDE_VAL : 0);
728 }
729
imx95_pcie_enable_ref_clk(struct imx_pcie * imx_pcie,bool enable)730 static int imx95_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
731 {
732 bool ext = imx_pcie->enable_ext_refclk;
733
734 imx95_pcie_clkreq_override(imx_pcie, enable);
735 /*
736 * The ref_clk_en signal must remain de-asserted until the
737 * reference clock is running at appropriate frequency, at which
738 * point this bit can be asserted. For lower power states where
739 * the reference clock to the PHY is disabled, it may also be
740 * de-asserted.
741 * +------------------- -+--------+----------------+
742 * | External clock mode | Enable | PCIE_REF_CLKEN |
743 * +---------------------+--------+----------------+
744 * | TRUE | X | 1b'0 |
745 * +---------------------+--------+----------------+
746 * | FALSE | TRUE | 1b'1 |
747 * +---------------------+--------+----------------+
748 * | FALSE | FALSE | 1b'0 |
749 * +---------------------+--------+----------------+
750 */
751 regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_0,
752 IMX95_PCIE_REF_CLKEN,
753 ext || !enable ? 0 : IMX95_PCIE_REF_CLKEN);
754
755 return 0;
756 }
757
imx8mm_pcie_clr_clkreq_override(struct imx_pcie * imx_pcie)758 static void imx8mm_pcie_clr_clkreq_override(struct imx_pcie *imx_pcie)
759 {
760 imx8mm_pcie_clkreq_override(imx_pcie, false);
761 }
762
imx95_pcie_clr_clkreq_override(struct imx_pcie * imx_pcie)763 static void imx95_pcie_clr_clkreq_override(struct imx_pcie *imx_pcie)
764 {
765 imx95_pcie_clkreq_override(imx_pcie, false);
766 }
767
imx_pcie_clk_enable(struct imx_pcie * imx_pcie)768 static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie)
769 {
770 struct dw_pcie *pci = imx_pcie->pci;
771 struct device *dev = pci->dev;
772 int ret;
773
774 ret = clk_bulk_prepare_enable(imx_pcie->num_clks, imx_pcie->clks);
775 if (ret)
776 return ret;
777
778 if (imx_pcie->drvdata->enable_ref_clk) {
779 ret = imx_pcie->drvdata->enable_ref_clk(imx_pcie, true);
780 if (ret) {
781 dev_err(dev, "Failed to enable PCIe REFCLK\n");
782 goto err_ref_clk;
783 }
784 }
785
786 /* allow the clocks to stabilize */
787 usleep_range(200, 500);
788 return 0;
789
790 err_ref_clk:
791 clk_bulk_disable_unprepare(imx_pcie->num_clks, imx_pcie->clks);
792
793 return ret;
794 }
795
imx_pcie_clk_disable(struct imx_pcie * imx_pcie)796 static void imx_pcie_clk_disable(struct imx_pcie *imx_pcie)
797 {
798 if (imx_pcie->drvdata->enable_ref_clk)
799 imx_pcie->drvdata->enable_ref_clk(imx_pcie, false);
800 clk_bulk_disable_unprepare(imx_pcie->num_clks, imx_pcie->clks);
801 }
802
imx6sx_pcie_core_reset(struct imx_pcie * imx_pcie,bool assert)803 static int imx6sx_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
804 {
805 if (assert)
806 regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
807 IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
808 else
809 regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
810 IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
811
812 /* Force PCIe PHY reset */
813 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, IMX6SX_GPR5_PCIE_BTNRST_RESET,
814 assert ? IMX6SX_GPR5_PCIE_BTNRST_RESET : 0);
815 return 0;
816 }
817
imx6qp_pcie_core_reset(struct imx_pcie * imx_pcie,bool assert)818 static int imx6qp_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
819 {
820 if (assert)
821 regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1,
822 IMX6Q_GPR1_PCIE_TEST_PD);
823 else
824 regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1,
825 IMX6Q_GPR1_PCIE_TEST_PD);
826
827 regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_SW_RST,
828 assert ? IMX6Q_GPR1_PCIE_SW_RST : 0);
829
830 if (!assert)
831 usleep_range(200, 500);
832
833 return 0;
834 }
835
imx6q_pcie_core_reset(struct imx_pcie * imx_pcie,bool assert)836 static int imx6q_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
837 {
838 if (assert)
839 regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1,
840 IMX6Q_GPR1_PCIE_TEST_PD);
841 else
842 regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1,
843 IMX6Q_GPR1_PCIE_TEST_PD);
844
845 if (!assert)
846 usleep_range(200, 500);
847
848 return 0;
849 }
850
imx7d_pcie_core_reset(struct imx_pcie * imx_pcie,bool assert)851 static int imx7d_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
852 {
853 struct dw_pcie *pci = imx_pcie->pci;
854 struct device *dev = pci->dev;
855
856 if (assert)
857 return 0;
858
859 /*
860 * Workaround for ERR010728 (IMX7DS_2N09P, Rev. 1.1, 4/2023):
861 *
862 * PCIe: PLL may fail to lock under corner conditions.
863 *
864 * Initial VCO oscillation may fail under corner conditions such as
865 * cold temperature which will cause the PCIe PLL fail to lock in the
866 * initialization phase.
867 *
868 * The Duty-cycle Corrector calibration must be disabled.
869 *
870 * 1. De-assert the G_RST signal by clearing
871 * SRC_PCIEPHY_RCR[PCIEPHY_G_RST].
872 * 2. De-assert DCC_FB_EN by writing data “0x29” to the register
873 * address 0x306d0014 (PCIE_PHY_CMN_REG4).
874 * 3. Assert RX_EQS, RX_EQ_SEL by writing data “0x48” to the register
875 * address 0x306d0090 (PCIE_PHY_CMN_REG24).
876 * 4. Assert ATT_MODE by writing data “0xbc” to the register
877 * address 0x306d0098 (PCIE_PHY_CMN_REG26).
878 * 5. De-assert the CMN_RST signal by clearing register bit
879 * SRC_PCIEPHY_RCR[PCIEPHY_BTN]
880 */
881
882 if (likely(imx_pcie->phy_base)) {
883 /* De-assert DCC_FB_EN */
884 writel(PCIE_PHY_CMN_REG4_DCC_FB_EN, imx_pcie->phy_base + PCIE_PHY_CMN_REG4);
885 /* Assert RX_EQS and RX_EQS_SEL */
886 writel(PCIE_PHY_CMN_REG24_RX_EQ_SEL | PCIE_PHY_CMN_REG24_RX_EQ,
887 imx_pcie->phy_base + PCIE_PHY_CMN_REG24);
888 /* Assert ATT_MODE */
889 writel(PCIE_PHY_CMN_REG26_ATT_MODE, imx_pcie->phy_base + PCIE_PHY_CMN_REG26);
890 } else {
891 dev_warn(dev, "Unable to apply ERR010728 workaround. DT missing fsl,imx7d-pcie-phy phandle ?\n");
892 }
893 imx7d_pcie_wait_for_phy_pll_lock(imx_pcie);
894 return 0;
895 }
896
imx95_pcie_core_reset(struct imx_pcie * imx_pcie,bool assert)897 static int imx95_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
898 {
899 u32 val;
900
901 if (assert) {
902 /*
903 * From i.MX95 PCIe PHY perspective, the COLD reset toggle
904 * should be complete after power-up by the following sequence.
905 * > 10us(at power-up)
906 * > 10ns(warm reset)
907 * |<------------>|
908 * ______________
909 * phy_reset ____/ \________________
910 * ____________
911 * ref_clk_en_______________________/
912 * Toggle COLD reset aligned with this sequence for i.MX95 PCIe.
913 */
914 regmap_set_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL,
915 IMX95_PCIE_COLD_RST);
916 /*
917 * Make sure the write to IMX95_PCIE_RST_CTRL is flushed to the
918 * hardware by doing a read. Otherwise, there is no guarantee
919 * that the write has reached the hardware before udelay().
920 */
921 regmap_read_bypassed(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL,
922 &val);
923 udelay(15);
924 regmap_clear_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL,
925 IMX95_PCIE_COLD_RST);
926 regmap_read_bypassed(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL,
927 &val);
928 udelay(10);
929 }
930
931 return 0;
932 }
933
imx_pcie_assert_core_reset(struct imx_pcie * imx_pcie)934 static void imx_pcie_assert_core_reset(struct imx_pcie *imx_pcie)
935 {
936 reset_control_assert(imx_pcie->pciephy_reset);
937
938 if (imx_pcie->drvdata->core_reset)
939 imx_pcie->drvdata->core_reset(imx_pcie, true);
940 }
941
imx_pcie_deassert_core_reset(struct imx_pcie * imx_pcie)942 static void imx_pcie_deassert_core_reset(struct imx_pcie *imx_pcie)
943 {
944 reset_control_deassert(imx_pcie->pciephy_reset);
945
946 if (imx_pcie->drvdata->core_reset)
947 imx_pcie->drvdata->core_reset(imx_pcie, false);
948 }
949
imx_pcie_wait_for_speed_change(struct imx_pcie * imx_pcie)950 static int imx_pcie_wait_for_speed_change(struct imx_pcie *imx_pcie)
951 {
952 struct dw_pcie *pci = imx_pcie->pci;
953 struct device *dev = pci->dev;
954 u32 tmp;
955 unsigned int retries;
956
957 for (retries = 0; retries < 200; retries++) {
958 tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
959 /* Test if the speed change finished. */
960 if (!(tmp & PORT_LOGIC_SPEED_CHANGE))
961 return 0;
962 usleep_range(100, 1000);
963 }
964
965 dev_err(dev, "Speed change timeout\n");
966 return -ETIMEDOUT;
967 }
968
imx_pcie_ltssm_enable(struct device * dev)969 static void imx_pcie_ltssm_enable(struct device *dev)
970 {
971 struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
972 const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata;
973 u8 offset = dw_pcie_find_capability(imx_pcie->pci, PCI_CAP_ID_EXP);
974 u32 tmp;
975
976 tmp = dw_pcie_readl_dbi(imx_pcie->pci, offset + PCI_EXP_LNKCAP);
977 phy_set_speed(imx_pcie->phy, FIELD_GET(PCI_EXP_LNKCAP_SLS, tmp));
978 if (drvdata->ltssm_mask)
979 regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off, drvdata->ltssm_mask,
980 drvdata->ltssm_mask);
981
982 reset_control_deassert(imx_pcie->apps_reset);
983 }
984
imx_pcie_ltssm_disable(struct device * dev)985 static void imx_pcie_ltssm_disable(struct device *dev)
986 {
987 struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
988 const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata;
989
990 phy_set_speed(imx_pcie->phy, 0);
991 if (drvdata->ltssm_mask)
992 regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off,
993 drvdata->ltssm_mask, 0);
994
995 reset_control_assert(imx_pcie->apps_reset);
996 }
997
imx_pcie_start_link(struct dw_pcie * pci)998 static int imx_pcie_start_link(struct dw_pcie *pci)
999 {
1000 struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1001 struct device *dev = pci->dev;
1002 u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
1003 u32 tmp;
1004 int ret;
1005
1006 if (!(imx_pcie->drvdata->flags &
1007 IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND)) {
1008 imx_pcie_ltssm_enable(dev);
1009 return 0;
1010 }
1011
1012 /*
1013 * Force Gen1 operation when starting the link. In case the link is
1014 * started in Gen2 mode, there is a possibility the devices on the
1015 * bus will not be detected at all. This happens with PCIe switches.
1016 */
1017 dw_pcie_dbi_ro_wr_en(pci);
1018 tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
1019 tmp &= ~PCI_EXP_LNKCAP_SLS;
1020 tmp |= PCI_EXP_LNKCAP_SLS_2_5GB;
1021 dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);
1022 dw_pcie_dbi_ro_wr_dis(pci);
1023
1024 /* Start LTSSM. */
1025 imx_pcie_ltssm_enable(dev);
1026
1027 if (pci->max_link_speed > 1) {
1028 ret = dw_pcie_wait_for_link(pci);
1029 if (ret)
1030 goto err_reset_phy;
1031
1032 /* Allow faster modes after the link is up */
1033 dw_pcie_dbi_ro_wr_en(pci);
1034 tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
1035 tmp &= ~PCI_EXP_LNKCAP_SLS;
1036 tmp |= pci->max_link_speed;
1037 dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);
1038
1039 /*
1040 * Start Directed Speed Change so the best possible
1041 * speed both link partners support can be negotiated.
1042 */
1043 tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
1044 tmp |= PORT_LOGIC_SPEED_CHANGE;
1045 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp);
1046 dw_pcie_dbi_ro_wr_dis(pci);
1047
1048 ret = imx_pcie_wait_for_speed_change(imx_pcie);
1049 if (ret) {
1050 dev_err(dev, "Failed to bring link up!\n");
1051 goto err_reset_phy;
1052 }
1053 } else {
1054 dev_info(dev, "Link: Only Gen1 is enabled\n");
1055 }
1056
1057 return 0;
1058
1059 err_reset_phy:
1060 dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n",
1061 dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0),
1062 dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1));
1063 imx_pcie_reset_phy(imx_pcie);
1064 return 0;
1065 }
1066
imx_pcie_stop_link(struct dw_pcie * pci)1067 static void imx_pcie_stop_link(struct dw_pcie *pci)
1068 {
1069 struct device *dev = pci->dev;
1070
1071 /* Turn off PCIe LTSSM */
1072 imx_pcie_ltssm_disable(dev);
1073 }
1074
imx_pcie_add_lut(struct imx_pcie * imx_pcie,u16 rid,u8 sid)1075 static int imx_pcie_add_lut(struct imx_pcie *imx_pcie, u16 rid, u8 sid)
1076 {
1077 struct dw_pcie *pci = imx_pcie->pci;
1078 struct device *dev = pci->dev;
1079 u32 data1, data2;
1080 int free = -1;
1081 int i;
1082
1083 if (sid >= 64) {
1084 dev_err(dev, "Invalid SID for index %d\n", sid);
1085 return -EINVAL;
1086 }
1087
1088 guard(mutex)(&imx_pcie->lock);
1089
1090 /*
1091 * Iterate through all LUT entries to check for duplicate RID and
1092 * identify the first available entry. Configure this available entry
1093 * immediately after verification to avoid rescanning it.
1094 */
1095 for (i = 0; i < IMX95_MAX_LUT; i++) {
1096 regmap_write(imx_pcie->iomuxc_gpr,
1097 IMX95_PE0_LUT_ACSCTRL, IMX95_PEO_LUT_RWA | i);
1098 regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, &data1);
1099
1100 if (!(data1 & IMX95_PE0_LUT_VLD)) {
1101 if (free < 0)
1102 free = i;
1103 continue;
1104 }
1105
1106 regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2);
1107
1108 /* Do not add duplicate RID */
1109 if (rid == FIELD_GET(IMX95_PE0_LUT_REQID, data2)) {
1110 dev_warn(dev, "Existing LUT entry available for RID (%d)", rid);
1111 return 0;
1112 }
1113 }
1114
1115 if (free < 0) {
1116 dev_err(dev, "LUT entry is not available\n");
1117 return -ENOSPC;
1118 }
1119
1120 data1 = FIELD_PREP(IMX95_PE0_LUT_DAC_ID, 0);
1121 data1 |= FIELD_PREP(IMX95_PE0_LUT_STREAM_ID, sid);
1122 data1 |= IMX95_PE0_LUT_VLD;
1123 regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, data1);
1124
1125 if (imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE)
1126 data2 = 0x7; /* In the EP mode, only 'Device ID' is required */
1127 else
1128 data2 = IMX95_PE0_LUT_MASK; /* Match all bits of RID */
1129 data2 |= FIELD_PREP(IMX95_PE0_LUT_REQID, rid);
1130 regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, data2);
1131
1132 regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL, free);
1133
1134 return 0;
1135 }
1136
imx_pcie_remove_lut(struct imx_pcie * imx_pcie,u16 rid)1137 static void imx_pcie_remove_lut(struct imx_pcie *imx_pcie, u16 rid)
1138 {
1139 u32 data2;
1140 int i;
1141
1142 guard(mutex)(&imx_pcie->lock);
1143
1144 for (i = 0; i < IMX95_MAX_LUT; i++) {
1145 regmap_write(imx_pcie->iomuxc_gpr,
1146 IMX95_PE0_LUT_ACSCTRL, IMX95_PEO_LUT_RWA | i);
1147 regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2);
1148 if (FIELD_GET(IMX95_PE0_LUT_REQID, data2) == rid) {
1149 regmap_write(imx_pcie->iomuxc_gpr,
1150 IMX95_PE0_LUT_DATA1, 0);
1151 regmap_write(imx_pcie->iomuxc_gpr,
1152 IMX95_PE0_LUT_DATA2, 0);
1153 regmap_write(imx_pcie->iomuxc_gpr,
1154 IMX95_PE0_LUT_ACSCTRL, i);
1155
1156 break;
1157 }
1158 }
1159 }
1160
imx_pcie_add_lut_by_rid(struct imx_pcie * imx_pcie,u32 rid)1161 static int imx_pcie_add_lut_by_rid(struct imx_pcie *imx_pcie, u32 rid)
1162 {
1163 struct of_phandle_args iommu_spec = {};
1164 struct of_phandle_args msi_spec = {};
1165 struct device *dev = imx_pcie->pci->dev;
1166 struct device_node *msi_filter = NULL;
1167 u32 sid_i, sid_m;
1168 int err_i, err_m;
1169 u32 sid = 0;
1170
1171 err_i = of_map_iommu_id(dev->of_node, rid, &iommu_spec);
1172 if (!err_i) {
1173 if (!iommu_spec.np)
1174 /*
1175 * "iommu_spec.np == NULL && err_i == 0" means RID out of map
1176 * range. Use 1:1 map RID to streamID. Hardware can't support
1177 * this because the streamID is only 6 bits.
1178 */
1179 err_i = -EINVAL;
1180 else
1181 sid_i = iommu_spec.args[0];
1182 }
1183 of_node_put(iommu_spec.np);
1184
1185 err_m = of_map_msi_id(dev->of_node, rid, &msi_filter, &msi_spec);
1186 /*
1187 * err_m msi_spec.np
1188 * 0 != NULL Got correct streamID from RID via msi-map
1189 * 0 NULL msi-map present but RID out of range
1190 * -ENODEV NULL msi-map absent, use built-in MSI controller
1191 */
1192 if (!err_m) {
1193 if (!msi_spec.np)
1194 /* msi-map present but RID out of range */
1195 return -EINVAL;
1196 sid_m = msi_spec.args[0];
1197 }
1198 of_node_put(msi_spec.np);
1199
1200 /*
1201 * msi-map iommu-map
1202 * N N DWC MSI Ctrl
1203 * Y Y ITS + SMMU, require the same SID
1204 * Y N ITS
1205 * N Y DWC MSI Ctrl + SMMU
1206 */
1207 if (err_i && err_m)
1208 return 0;
1209
1210 if (!err_i && !err_m) {
1211 /*
1212 * Glue Layer
1213 * <==========>
1214 * ┌─────┐ ┌──────────┐
1215 * │ LUT │ 6-bit streamID │ │
1216 * │ │─────────────────►│ MSI │
1217 * └─────┘ 2-bit ctrl ID │ │
1218 * ┌───────────►│ │
1219 * (i.MX95) │ │ │
1220 * 00 PCIe0 │ │ │
1221 * 01 ENETC │ │ │
1222 * 10 PCIe1 │ │ │
1223 * │ └──────────┘
1224 * The MSI glue layer auto adds 2 bits controller ID ahead of
1225 * streamID, so mask these 2 bits to get streamID. The
1226 * IOMMU glue layer doesn't do that.
1227 */
1228 if (sid_i != (sid_m & IMX95_SID_MASK)) {
1229 dev_err(dev, "iommu-map and msi-map entries mismatch!\n");
1230 return -EINVAL;
1231 }
1232 }
1233
1234 if (!err_i)
1235 sid = sid_i;
1236 else if (!err_m)
1237 sid = sid_m & IMX95_SID_MASK;
1238
1239 return imx_pcie_add_lut(imx_pcie, rid, sid);
1240 }
1241
imx_pcie_enable_device(struct pci_host_bridge * bridge,struct pci_dev * pdev)1242 static int imx_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev)
1243 {
1244 struct imx_pcie *imx_pcie = to_imx_pcie(to_dw_pcie_from_pp(bridge->sysdata));
1245
1246 return imx_pcie_add_lut_by_rid(imx_pcie, pci_dev_id(pdev));
1247 }
1248
imx_pcie_disable_device(struct pci_host_bridge * bridge,struct pci_dev * pdev)1249 static void imx_pcie_disable_device(struct pci_host_bridge *bridge,
1250 struct pci_dev *pdev)
1251 {
1252 struct imx_pcie *imx_pcie;
1253
1254 imx_pcie = to_imx_pcie(to_dw_pcie_from_pp(bridge->sysdata));
1255 imx_pcie_remove_lut(imx_pcie, pci_dev_id(pdev));
1256 }
1257
imx_pcie_parse_legacy_binding(struct imx_pcie * pcie)1258 static int imx_pcie_parse_legacy_binding(struct imx_pcie *pcie)
1259 {
1260 struct device *dev = pcie->pci->dev;
1261 struct pci_host_bridge *bridge = pcie->pci->pp.bridge;
1262 struct pci_host_port *port;
1263 struct pci_host_perst *perst;
1264 struct gpio_desc *reset;
1265
1266 reset = devm_gpiod_get_optional(dev, "reset", GPIOD_ASIS);
1267 if (IS_ERR(reset))
1268 return PTR_ERR(reset);
1269
1270 if (!reset)
1271 return 0;
1272
1273 port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
1274 if (!port)
1275 return -ENOMEM;
1276
1277 perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL);
1278 if (!perst)
1279 return -ENOMEM;
1280
1281 INIT_LIST_HEAD(&port->perst);
1282 perst->desc = reset;
1283 INIT_LIST_HEAD(&perst->list);
1284 list_add_tail(&perst->list, &port->perst);
1285
1286 INIT_LIST_HEAD(&port->list);
1287 list_add_tail(&port->list, &bridge->ports);
1288
1289 return devm_add_action_or_reset(dev, pci_host_common_delete_ports,
1290 &bridge->ports);
1291 }
1292
imx_pcie_vpcie_aux_disable(void * data)1293 static void imx_pcie_vpcie_aux_disable(void *data)
1294 {
1295 struct regulator *vpcie_aux = data;
1296
1297 regulator_disable(vpcie_aux);
1298 }
1299
imx_pcie_assert_perst(struct imx_pcie * imx_pcie,bool assert)1300 static void imx_pcie_assert_perst(struct imx_pcie *imx_pcie, bool assert)
1301 {
1302 struct dw_pcie *pci = imx_pcie->pci;
1303 struct pci_host_bridge *bridge = pci->pp.bridge;
1304 struct pci_host_perst *perst;
1305 struct pci_host_port *port;
1306
1307 if (!bridge || list_empty(&bridge->ports))
1308 return;
1309
1310 if (assert) {
1311 list_for_each_entry(port, &bridge->ports, list) {
1312 list_for_each_entry(perst, &port->perst, list)
1313 gpiod_direction_output(perst->desc, 1);
1314 }
1315 } else {
1316 mdelay(PCIE_T_PVPERL_MS);
1317 list_for_each_entry(port, &bridge->ports, list) {
1318 list_for_each_entry(perst, &port->perst, list)
1319 gpiod_direction_output(perst->desc, 0);
1320 }
1321 mdelay(PCIE_RESET_CONFIG_WAIT_MS);
1322 }
1323 }
1324
imx_pcie_perst_found(struct pci_host_bridge * bridge)1325 static bool imx_pcie_perst_found(struct pci_host_bridge *bridge)
1326 {
1327 struct pci_host_port *port;
1328
1329 list_for_each_entry(port, &bridge->ports, list) {
1330 if (!list_empty(&port->perst))
1331 return true;
1332 }
1333
1334 return false;
1335 }
1336
imx_pcie_host_init(struct dw_pcie_rp * pp)1337 static int imx_pcie_host_init(struct dw_pcie_rp *pp)
1338 {
1339 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1340 struct device *dev = pci->dev;
1341 struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1342 struct pci_host_bridge *bridge = pp->bridge;
1343 int ret;
1344
1345 if (bridge && list_empty(&bridge->ports)) {
1346 /* Parse Root Port nodes if present */
1347 ret = pci_host_common_parse_ports(dev, bridge);
1348 if (ret) {
1349 dev_err(dev, "Failed to parse Root Port nodes: %d\n", ret);
1350 return ret;
1351 }
1352
1353 /* Fall back to legacy binding for DT backwards compatibility */
1354 if (!imx_pcie_perst_found(bridge)) {
1355 ret = imx_pcie_parse_legacy_binding(imx_pcie);
1356 if (ret)
1357 return ret;
1358 }
1359 }
1360
1361 imx_pcie_assert_perst(imx_pcie, true);
1362
1363 /* Keep 3.3Vaux supply enabled for entire PCIe controller lifecycle */
1364 if (imx_pcie->vpcie_aux && !imx_pcie->vpcie_aux_enabled) {
1365 ret = regulator_enable(imx_pcie->vpcie_aux);
1366 if (ret) {
1367 dev_err(dev, "failed to enable vpcie_aux regulator: %d\n",
1368 ret);
1369 return ret;
1370 }
1371 imx_pcie->vpcie_aux_enabled = true;
1372
1373 ret = devm_add_action_or_reset(dev, imx_pcie_vpcie_aux_disable,
1374 imx_pcie->vpcie_aux);
1375 if (ret)
1376 return ret;
1377 }
1378
1379 /* Legacy regulator handling for DT backward compatibility. */
1380 if (imx_pcie->vpcie) {
1381 ret = regulator_enable(imx_pcie->vpcie);
1382 if (ret) {
1383 dev_err(dev, "failed to enable vpcie regulator: %d\n",
1384 ret);
1385 return ret;
1386 }
1387 }
1388
1389 if (!pp->skip_pwrctrl_off) {
1390 ret = pci_pwrctrl_power_on_devices(dev);
1391 if (ret) {
1392 dev_err(dev, "failed to power on pwrctrl devices\n");
1393 goto err_reg_disable;
1394 }
1395 }
1396
1397 if (pp->bridge && imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT)) {
1398 pp->bridge->enable_device = imx_pcie_enable_device;
1399 pp->bridge->disable_device = imx_pcie_disable_device;
1400 }
1401
1402 if (imx_pcie->drvdata->select_ref_clk_src)
1403 imx_pcie->drvdata->select_ref_clk_src(imx_pcie);
1404
1405 imx_pcie_assert_core_reset(imx_pcie);
1406
1407 if (imx_pcie->drvdata->init_phy)
1408 imx_pcie->drvdata->init_phy(imx_pcie);
1409
1410 imx_pcie_configure_type(imx_pcie);
1411
1412 ret = imx_pcie_clk_enable(imx_pcie);
1413 if (ret) {
1414 dev_err(dev, "unable to enable pcie clocks: %d\n", ret);
1415 goto err_pwrctrl_power_off;
1416 }
1417
1418 if (imx_pcie->phy) {
1419 ret = phy_init(imx_pcie->phy);
1420 if (ret) {
1421 dev_err(dev, "pcie PHY power up failed\n");
1422 goto err_clk_disable;
1423 }
1424
1425 ret = phy_set_mode_ext(imx_pcie->phy, PHY_MODE_PCIE,
1426 imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE ?
1427 PHY_MODE_PCIE_EP : PHY_MODE_PCIE_RC);
1428 if (ret) {
1429 dev_err(dev, "unable to set PCIe PHY mode\n");
1430 goto err_phy_exit;
1431 }
1432
1433 ret = phy_power_on(imx_pcie->phy);
1434 if (ret) {
1435 dev_err(dev, "waiting for PHY ready timeout!\n");
1436 goto err_phy_exit;
1437 }
1438 }
1439
1440 /* Make sure that PCIe LTSSM is cleared */
1441 imx_pcie_ltssm_disable(dev);
1442
1443 imx_pcie_deassert_core_reset(imx_pcie);
1444 imx_pcie_assert_perst(imx_pcie, false);
1445
1446 if (imx_pcie->drvdata->wait_pll_lock) {
1447 ret = imx_pcie->drvdata->wait_pll_lock(imx_pcie);
1448 if (ret < 0)
1449 goto err_phy_off;
1450 }
1451
1452 imx_setup_phy_mpll(imx_pcie);
1453
1454 return 0;
1455
1456 err_phy_off:
1457 imx_pcie_assert_core_reset(imx_pcie);
1458 phy_power_off(imx_pcie->phy);
1459 err_phy_exit:
1460 phy_exit(imx_pcie->phy);
1461 err_clk_disable:
1462 imx_pcie_clk_disable(imx_pcie);
1463 err_pwrctrl_power_off:
1464 if (!pp->skip_pwrctrl_off)
1465 pci_pwrctrl_power_off_devices(dev);
1466 err_reg_disable:
1467 if (imx_pcie->vpcie)
1468 regulator_disable(imx_pcie->vpcie);
1469 return ret;
1470 }
1471
imx_pcie_host_exit(struct dw_pcie_rp * pp)1472 static void imx_pcie_host_exit(struct dw_pcie_rp *pp)
1473 {
1474 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1475 struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1476
1477 imx_pcie_assert_core_reset(imx_pcie);
1478 if (imx_pcie->phy) {
1479 if (phy_power_off(imx_pcie->phy))
1480 dev_err(pci->dev, "unable to power off PHY\n");
1481 phy_exit(imx_pcie->phy);
1482 }
1483 imx_pcie_clk_disable(imx_pcie);
1484
1485 if (!pci->pp.skip_pwrctrl_off)
1486 pci_pwrctrl_power_off_devices(pci->dev);
1487 if (imx_pcie->vpcie)
1488 regulator_disable(imx_pcie->vpcie);
1489 }
1490
imx_pcie_host_post_init(struct dw_pcie_rp * pp)1491 static void imx_pcie_host_post_init(struct dw_pcie_rp *pp)
1492 {
1493 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1494 struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1495 u32 val;
1496
1497 if (imx_pcie->drvdata->flags & IMX_PCIE_FLAG_8GT_ECN_ERR051586) {
1498 /*
1499 * ERR051586: Compliance with 8GT/s Receiver Impedance ECN
1500 *
1501 * The default value of GEN3_RELATED_OFF[GEN3_ZRXDC_NONCOMPL]
1502 * is 1 which makes receiver non-compliant with the ZRX-DC
1503 * parameter for 2.5 GT/s when operating at 8 GT/s or higher.
1504 * It causes unnecessary timeout in L1.
1505 *
1506 * Workaround: Program GEN3_RELATED_OFF[GEN3_ZRXDC_NONCOMPL]
1507 * to 0.
1508 */
1509 dw_pcie_dbi_ro_wr_en(pci);
1510 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1511 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1512 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1513 dw_pcie_dbi_ro_wr_dis(pci);
1514 }
1515
1516 /* Clear CLKREQ# override if supports_clkreq is true and link is up */
1517 if (dw_pcie_link_up(pci) && imx_pcie->supports_clkreq) {
1518 if (imx_pcie->drvdata->clr_clkreq_override)
1519 imx_pcie->drvdata->clr_clkreq_override(imx_pcie);
1520 }
1521 }
1522
1523 /*
1524 * In old DWC implementations, PCIE_ATU_INHIBIT_PAYLOAD in iATU Ctrl2
1525 * register is reserved, so the generic DWC implementation of sending the
1526 * PME_Turn_Off message using a dummy MMIO write cannot be used.
1527 */
imx_pcie_pme_turn_off(struct dw_pcie_rp * pp)1528 static void imx_pcie_pme_turn_off(struct dw_pcie_rp *pp)
1529 {
1530 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1531 struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1532
1533 regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_PM_TURN_OFF);
1534 regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_PM_TURN_OFF);
1535
1536 usleep_range(PCIE_PME_TO_L2_TIMEOUT_US/10, PCIE_PME_TO_L2_TIMEOUT_US);
1537 }
1538
1539 static const struct dw_pcie_host_ops imx_pcie_host_ops = {
1540 .init = imx_pcie_host_init,
1541 .deinit = imx_pcie_host_exit,
1542 .pme_turn_off = imx_pcie_pme_turn_off,
1543 };
1544
1545 static const struct dw_pcie_host_ops imx_pcie_host_dw_pme_ops = {
1546 .init = imx_pcie_host_init,
1547 .deinit = imx_pcie_host_exit,
1548 .post_init = imx_pcie_host_post_init,
1549 };
1550
1551 static const struct dw_pcie_ops dw_pcie_ops = {
1552 .start_link = imx_pcie_start_link,
1553 .stop_link = imx_pcie_stop_link,
1554 };
1555
imx_pcie_ep_raise_irq(struct dw_pcie_ep * ep,u8 func_no,unsigned int type,u16 interrupt_num)1556 static int imx_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1557 unsigned int type, u16 interrupt_num)
1558 {
1559 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1560
1561 switch (type) {
1562 case PCI_IRQ_INTX:
1563 return dw_pcie_ep_raise_intx_irq(ep, func_no);
1564 case PCI_IRQ_MSI:
1565 return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
1566 case PCI_IRQ_MSIX:
1567 return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num);
1568 default:
1569 dev_err(pci->dev, "UNKNOWN IRQ type\n");
1570 return -EINVAL;
1571 }
1572
1573 return 0;
1574 }
1575
1576 static const struct pci_epc_features imx8m_pcie_epc_features = {
1577 DWC_EPC_COMMON_FEATURES,
1578 .msi_capable = true,
1579 .bar[BAR_1] = { .type = BAR_DISABLED, },
1580 .bar[BAR_3] = { .type = BAR_DISABLED, },
1581 .bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = SZ_256, },
1582 .bar[BAR_5] = { .type = BAR_DISABLED, },
1583 .align = SZ_64K,
1584 };
1585
1586 static const struct pci_epc_features imx8q_pcie_epc_features = {
1587 DWC_EPC_COMMON_FEATURES,
1588 .msi_capable = true,
1589 .bar[BAR_1] = { .type = BAR_DISABLED, },
1590 .bar[BAR_3] = { .type = BAR_DISABLED, },
1591 .bar[BAR_5] = { .type = BAR_DISABLED, },
1592 .align = SZ_64K,
1593 };
1594
1595 /*
1596 * | Default | Default | Default | BAR Sizing
1597 * BAR# | Enable? | Type | Size | Scheme
1598 * =======================================================
1599 * BAR0 | Enable | 64-bit | 1 MB | Programmable Size
1600 * BAR1 | Disable | 32-bit | 64 KB | Fixed Size
1601 * (BAR1 should be disabled if BAR0 is 64-bit)
1602 * BAR2 | Enable | 32-bit | 1 MB | Programmable Size
1603 * BAR3 | Enable | 32-bit | 64 KB | Programmable Size
1604 * BAR4 | Enable | 32-bit | 1 MB | Programmable Size
1605 * BAR5 | Enable | 32-bit | 64 KB | Programmable Size
1606 */
1607 static const struct pci_epc_features imx95_pcie_epc_features = {
1608 DWC_EPC_COMMON_FEATURES,
1609 .msi_capable = true,
1610 .bar[BAR_1] = { .type = BAR_FIXED, .fixed_size = SZ_64K, },
1611 .align = SZ_4K,
1612 };
1613
1614 static const struct pci_epc_features*
imx_pcie_ep_get_features(struct dw_pcie_ep * ep)1615 imx_pcie_ep_get_features(struct dw_pcie_ep *ep)
1616 {
1617 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1618 struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1619
1620 return imx_pcie->drvdata->epc_features;
1621 }
1622
1623 static const struct dw_pcie_ep_ops pcie_ep_ops = {
1624 .raise_irq = imx_pcie_ep_raise_irq,
1625 .get_features = imx_pcie_ep_get_features,
1626 };
1627
imx_add_pcie_ep(struct imx_pcie * imx_pcie,struct platform_device * pdev)1628 static int imx_add_pcie_ep(struct imx_pcie *imx_pcie,
1629 struct platform_device *pdev)
1630 {
1631 int ret;
1632 struct dw_pcie_ep *ep;
1633 struct dw_pcie *pci = imx_pcie->pci;
1634 struct dw_pcie_rp *pp = &pci->pp;
1635 struct device *dev = pci->dev;
1636
1637 imx_pcie_host_init(pp);
1638 ep = &pci->ep;
1639 ep->ops = &pcie_ep_ops;
1640
1641 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SUPPORT_64BIT))
1642 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1643
1644 ep->page_size = imx_pcie->drvdata->epc_features->align;
1645
1646 ret = dw_pcie_ep_init(ep);
1647 if (ret) {
1648 dev_err(dev, "failed to initialize endpoint\n");
1649 return ret;
1650 }
1651 imx_pcie_host_post_init(pp);
1652
1653 ret = dw_pcie_ep_init_registers(ep);
1654 if (ret) {
1655 dev_err(dev, "Failed to initialize DWC endpoint registers\n");
1656 dw_pcie_ep_deinit(ep);
1657 return ret;
1658 }
1659
1660 pci_epc_init_notify(ep->epc);
1661
1662 return 0;
1663 }
1664
imx_pcie_msi_save_restore(struct imx_pcie * imx_pcie,bool save)1665 static void imx_pcie_msi_save_restore(struct imx_pcie *imx_pcie, bool save)
1666 {
1667 u8 offset;
1668 u16 val;
1669 struct dw_pcie *pci = imx_pcie->pci;
1670
1671 if (pci_msi_enabled()) {
1672 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI);
1673 if (save) {
1674 val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS);
1675 imx_pcie->msi_ctrl = val;
1676 } else {
1677 dw_pcie_dbi_ro_wr_en(pci);
1678 val = imx_pcie->msi_ctrl;
1679 dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val);
1680 dw_pcie_dbi_ro_wr_dis(pci);
1681 }
1682 }
1683 }
1684
imx_pcie_lut_save(struct imx_pcie * imx_pcie)1685 static void imx_pcie_lut_save(struct imx_pcie *imx_pcie)
1686 {
1687 u32 data1, data2;
1688 int i;
1689
1690 for (i = 0; i < IMX95_MAX_LUT; i++) {
1691 regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL,
1692 IMX95_PEO_LUT_RWA | i);
1693 regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, &data1);
1694 regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2);
1695 if (data1 & IMX95_PE0_LUT_VLD) {
1696 imx_pcie->luts[i].data1 = data1;
1697 imx_pcie->luts[i].data2 = data2;
1698 } else {
1699 imx_pcie->luts[i].data1 = 0;
1700 imx_pcie->luts[i].data2 = 0;
1701 }
1702 }
1703 }
1704
imx_pcie_lut_restore(struct imx_pcie * imx_pcie)1705 static void imx_pcie_lut_restore(struct imx_pcie *imx_pcie)
1706 {
1707 int i;
1708
1709 for (i = 0; i < IMX95_MAX_LUT; i++) {
1710 if ((imx_pcie->luts[i].data1 & IMX95_PE0_LUT_VLD) == 0)
1711 continue;
1712
1713 regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1,
1714 imx_pcie->luts[i].data1);
1715 regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2,
1716 imx_pcie->luts[i].data2);
1717 regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL, i);
1718 }
1719 }
1720
imx_pcie_suspend_noirq(struct device * dev)1721 static int imx_pcie_suspend_noirq(struct device *dev)
1722 {
1723 struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
1724
1725 if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND))
1726 return 0;
1727
1728 imx_pcie_msi_save_restore(imx_pcie, true);
1729 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT))
1730 imx_pcie_lut_save(imx_pcie);
1731 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_BROKEN_SUSPEND)) {
1732 /*
1733 * The minimum for a workaround would be to set PERST# and to
1734 * set the PCIE_TEST_PD flag. However, we can also disable the
1735 * clock which saves some power.
1736 */
1737 imx_pcie_assert_core_reset(imx_pcie);
1738 imx_pcie_assert_perst(imx_pcie, true);
1739 imx_pcie->drvdata->enable_ref_clk(imx_pcie, false);
1740 } else {
1741 return dw_pcie_suspend_noirq(imx_pcie->pci);
1742 }
1743
1744 return 0;
1745 }
1746
imx_pcie_resume_noirq(struct device * dev)1747 static int imx_pcie_resume_noirq(struct device *dev)
1748 {
1749 int ret;
1750 struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
1751
1752 if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND))
1753 return 0;
1754
1755 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_BROKEN_SUSPEND)) {
1756 ret = imx_pcie->drvdata->enable_ref_clk(imx_pcie, true);
1757 if (ret)
1758 return ret;
1759 imx_pcie_deassert_core_reset(imx_pcie);
1760 imx_pcie_assert_perst(imx_pcie, false);
1761
1762 /*
1763 * Using PCIE_TEST_PD seems to disable MSI and powers down the
1764 * root complex. This is why we have to setup the rc again and
1765 * why we have to restore the MSI register.
1766 */
1767 ret = dw_pcie_setup_rc(&imx_pcie->pci->pp);
1768 if (ret)
1769 return ret;
1770 } else {
1771 ret = dw_pcie_resume_noirq(imx_pcie->pci);
1772 if (ret)
1773 return ret;
1774 }
1775 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT))
1776 imx_pcie_lut_restore(imx_pcie);
1777 imx_pcie_msi_save_restore(imx_pcie, false);
1778
1779 return 0;
1780 }
1781
1782 static const struct dev_pm_ops imx_pcie_pm_ops = {
1783 NOIRQ_SYSTEM_SLEEP_PM_OPS(imx_pcie_suspend_noirq,
1784 imx_pcie_resume_noirq)
1785 };
1786
imx_pcie_probe(struct platform_device * pdev)1787 static int imx_pcie_probe(struct platform_device *pdev)
1788 {
1789 struct device *dev = &pdev->dev;
1790 struct dw_pcie *pci;
1791 struct imx_pcie *imx_pcie;
1792 struct device_node *node = dev->of_node;
1793 int i, ret, domain;
1794 u16 val;
1795
1796 imx_pcie = devm_kzalloc(dev, sizeof(*imx_pcie), GFP_KERNEL);
1797 if (!imx_pcie)
1798 return -ENOMEM;
1799
1800 pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
1801 if (!pci)
1802 return -ENOMEM;
1803
1804 pci->dev = dev;
1805 pci->ops = &dw_pcie_ops;
1806
1807 imx_pcie->pci = pci;
1808 imx_pcie->drvdata = of_device_get_match_data(dev);
1809
1810 mutex_init(&imx_pcie->lock);
1811
1812 if (imx_pcie->drvdata->ops)
1813 pci->pp.ops = imx_pcie->drvdata->ops;
1814 else
1815 pci->pp.ops = &imx_pcie_host_dw_pme_ops;
1816
1817 /* Find the PHY if one is defined, only imx7d uses it */
1818 struct device_node *np __free(device_node) =
1819 of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0);
1820 if (np) {
1821 struct resource res;
1822
1823 ret = of_address_to_resource(np, 0, &res);
1824 if (ret) {
1825 dev_err(dev, "Unable to map PCIe PHY\n");
1826 return ret;
1827 }
1828 imx_pcie->phy_base = devm_ioremap_resource(dev, &res);
1829 if (IS_ERR(imx_pcie->phy_base))
1830 return PTR_ERR(imx_pcie->phy_base);
1831 }
1832
1833 /* Fetch clocks */
1834 imx_pcie->num_clks = devm_clk_bulk_get_all(dev, &imx_pcie->clks);
1835 if (imx_pcie->num_clks < 0)
1836 return dev_err_probe(dev, imx_pcie->num_clks,
1837 "failed to get clocks\n");
1838 for (i = 0; i < imx_pcie->num_clks; i++)
1839 if (strncmp(imx_pcie->clks[i].id, "extref", 6) == 0)
1840 imx_pcie->enable_ext_refclk = true;
1841
1842 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_PHYDRV)) {
1843 imx_pcie->phy = devm_phy_get(dev, "pcie-phy");
1844 if (IS_ERR(imx_pcie->phy))
1845 return dev_err_probe(dev, PTR_ERR(imx_pcie->phy),
1846 "failed to get pcie phy\n");
1847 }
1848
1849 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_APP_RESET)) {
1850 imx_pcie->apps_reset = devm_reset_control_get_exclusive(dev, "apps");
1851 if (IS_ERR(imx_pcie->apps_reset))
1852 return dev_err_probe(dev, PTR_ERR(imx_pcie->apps_reset),
1853 "failed to get pcie apps reset control\n");
1854 }
1855
1856 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_PHY_RESET)) {
1857 imx_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev, "pciephy");
1858 if (IS_ERR(imx_pcie->pciephy_reset))
1859 return dev_err_probe(dev, PTR_ERR(imx_pcie->pciephy_reset),
1860 "Failed to get PCIEPHY reset control\n");
1861 }
1862
1863 switch (imx_pcie->drvdata->variant) {
1864 case IMX8MQ:
1865 case IMX8MQ_EP:
1866 domain = of_get_pci_domain_nr(node);
1867 if (domain < 0 || domain > 1)
1868 return dev_err_probe(dev, -ENODEV, "no \"linux,pci-domain\" property in devicetree\n");
1869
1870 imx_pcie->controller_id = domain;
1871 break;
1872 default:
1873 break;
1874 }
1875
1876 if (imx_pcie->drvdata->gpr) {
1877 /* Grab GPR config register range */
1878 imx_pcie->iomuxc_gpr =
1879 syscon_regmap_lookup_by_compatible(imx_pcie->drvdata->gpr);
1880 if (IS_ERR(imx_pcie->iomuxc_gpr))
1881 return dev_err_probe(dev, PTR_ERR(imx_pcie->iomuxc_gpr),
1882 "unable to find iomuxc registers\n");
1883 }
1884
1885 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_SERDES)) {
1886 void __iomem *off = devm_platform_ioremap_resource_byname(pdev, "app");
1887
1888 if (IS_ERR(off))
1889 return dev_err_probe(dev, PTR_ERR(off),
1890 "unable to find serdes registers\n");
1891
1892 static const struct regmap_config regmap_config = {
1893 .reg_bits = 32,
1894 .val_bits = 32,
1895 .reg_stride = 4,
1896 };
1897
1898 imx_pcie->iomuxc_gpr = devm_regmap_init_mmio(dev, off, ®map_config);
1899 if (IS_ERR(imx_pcie->iomuxc_gpr))
1900 return dev_err_probe(dev, PTR_ERR(imx_pcie->iomuxc_gpr),
1901 "unable to find iomuxc registers\n");
1902 }
1903
1904 /* Grab PCIe PHY Tx Settings */
1905 if (of_property_read_u32(node, "fsl,tx-deemph-gen1",
1906 &imx_pcie->tx_deemph_gen1))
1907 imx_pcie->tx_deemph_gen1 = 0;
1908
1909 if (of_property_read_u32(node, "fsl,tx-deemph-gen2-3p5db",
1910 &imx_pcie->tx_deemph_gen2_3p5db))
1911 imx_pcie->tx_deemph_gen2_3p5db = 0;
1912
1913 if (of_property_read_u32(node, "fsl,tx-deemph-gen2-6db",
1914 &imx_pcie->tx_deemph_gen2_6db))
1915 imx_pcie->tx_deemph_gen2_6db = 20;
1916
1917 if (of_property_read_u32(node, "fsl,tx-swing-full",
1918 &imx_pcie->tx_swing_full))
1919 imx_pcie->tx_swing_full = 127;
1920
1921 if (of_property_read_u32(node, "fsl,tx-swing-low",
1922 &imx_pcie->tx_swing_low))
1923 imx_pcie->tx_swing_low = 127;
1924
1925 /* Limit link speed */
1926 pci->max_link_speed = 1;
1927 of_property_read_u32(node, "fsl,max-link-speed", &pci->max_link_speed);
1928 imx_pcie->supports_clkreq = of_property_read_bool(node, "supports-clkreq");
1929
1930 imx_pcie->vpcie_aux = devm_regulator_get_optional(&pdev->dev,
1931 "vpcie3v3aux");
1932 if (IS_ERR(imx_pcie->vpcie_aux)) {
1933 if (PTR_ERR(imx_pcie->vpcie_aux) != -ENODEV)
1934 return PTR_ERR(imx_pcie->vpcie_aux);
1935 imx_pcie->vpcie_aux = NULL;
1936 }
1937
1938 imx_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie");
1939 if (IS_ERR(imx_pcie->vpcie)) {
1940 if (PTR_ERR(imx_pcie->vpcie) != -ENODEV)
1941 return PTR_ERR(imx_pcie->vpcie);
1942 imx_pcie->vpcie = NULL;
1943 }
1944
1945 imx_pcie->vph = devm_regulator_get_optional(&pdev->dev, "vph");
1946 if (IS_ERR(imx_pcie->vph)) {
1947 if (PTR_ERR(imx_pcie->vph) != -ENODEV)
1948 return PTR_ERR(imx_pcie->vph);
1949 imx_pcie->vph = NULL;
1950 }
1951
1952 platform_set_drvdata(pdev, imx_pcie);
1953
1954 ret = imx_pcie_attach_pd(dev);
1955 if (ret)
1956 return ret;
1957
1958 ret = pci_pwrctrl_create_devices(dev);
1959 if (ret)
1960 return dev_err_probe(dev, ret, "failed to create pwrctrl devices\n");
1961
1962 pci->use_parent_dt_ranges = true;
1963 if (imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE) {
1964 ret = imx_add_pcie_ep(imx_pcie, pdev);
1965 if (ret < 0)
1966 goto err_pwrctrl_destroy;
1967
1968 /*
1969 * FIXME: Only single Device (EPF) is supported due to the
1970 * Endpoint framework limitation.
1971 */
1972 imx_pcie_add_lut_by_rid(imx_pcie, 0);
1973 } else {
1974 if (imx_pcie->drvdata->flags & IMX_PCIE_FLAG_PM_RUNTIME) {
1975 pm_runtime_no_callbacks(dev);
1976 ret = devm_pm_runtime_set_active_enabled(dev);
1977 if (ret < 0)
1978 return ret;
1979 }
1980
1981 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SKIP_L23_READY))
1982 pci->pp.skip_l23_ready = true;
1983 if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_KEEP_MSI_CAP))
1984 pci->pp.keep_rp_msi_en = true;
1985 pci->pp.use_atu_msg = true;
1986 ret = dw_pcie_host_init(&pci->pp);
1987 if (ret < 0)
1988 goto err_pwrctrl_destroy;
1989
1990 if (pci_msi_enabled()) {
1991 u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI);
1992
1993 val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS);
1994 val |= PCI_MSI_FLAGS_ENABLE;
1995 dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val);
1996 }
1997 }
1998
1999 return 0;
2000
2001 err_pwrctrl_destroy:
2002 if (ret != -EPROBE_DEFER)
2003 pci_pwrctrl_destroy_devices(dev);
2004 return ret;
2005 }
2006
imx_pcie_shutdown(struct platform_device * pdev)2007 static void imx_pcie_shutdown(struct platform_device *pdev)
2008 {
2009 struct imx_pcie *imx_pcie = platform_get_drvdata(pdev);
2010
2011 /* bring down link, so bootloader gets clean state in case of reboot */
2012 imx_pcie_assert_core_reset(imx_pcie);
2013 imx_pcie_assert_perst(imx_pcie, true);
2014 pci_pwrctrl_power_off_devices(&pdev->dev);
2015 pci_pwrctrl_destroy_devices(&pdev->dev);
2016 }
2017
2018 static const struct imx_pcie_drvdata drvdata[] = {
2019 [IMX6Q] = {
2020 .variant = IMX6Q,
2021 .flags = IMX_PCIE_FLAG_IMX_PHY |
2022 IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND |
2023 IMX_PCIE_FLAG_BROKEN_SUSPEND |
2024 IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
2025 IMX_PCIE_FLAG_KEEP_MSI_CAP,
2026 .dbi_length = 0x200,
2027 .gpr = "fsl,imx6q-iomuxc-gpr",
2028 .ltssm_off = IOMUXC_GPR12,
2029 .ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2,
2030 .mode_off[0] = IOMUXC_GPR12,
2031 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2032 .init_phy = imx_pcie_init_phy,
2033 .enable_ref_clk = imx6q_pcie_enable_ref_clk,
2034 .core_reset = imx6q_pcie_core_reset,
2035 },
2036 [IMX6SX] = {
2037 .variant = IMX6SX,
2038 .flags = IMX_PCIE_FLAG_IMX_PHY |
2039 IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND |
2040 IMX_PCIE_FLAG_SKIP_L23_READY |
2041 IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
2042 IMX_PCIE_FLAG_KEEP_MSI_CAP,
2043 .gpr = "fsl,imx6q-iomuxc-gpr",
2044 .ltssm_off = IOMUXC_GPR12,
2045 .ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2,
2046 .mode_off[0] = IOMUXC_GPR12,
2047 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2048 .init_phy = imx6sx_pcie_init_phy,
2049 .core_reset = imx6sx_pcie_core_reset,
2050 .ops = &imx_pcie_host_ops,
2051 },
2052 [IMX6QP] = {
2053 .variant = IMX6QP,
2054 .flags = IMX_PCIE_FLAG_IMX_PHY |
2055 IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND |
2056 IMX_PCIE_FLAG_SKIP_L23_READY |
2057 IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
2058 IMX_PCIE_FLAG_KEEP_MSI_CAP,
2059 .dbi_length = 0x200,
2060 .gpr = "fsl,imx6q-iomuxc-gpr",
2061 .ltssm_off = IOMUXC_GPR12,
2062 .ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2,
2063 .mode_off[0] = IOMUXC_GPR12,
2064 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2065 .init_phy = imx_pcie_init_phy,
2066 .enable_ref_clk = imx6q_pcie_enable_ref_clk,
2067 .core_reset = imx6qp_pcie_core_reset,
2068 .ops = &imx_pcie_host_ops,
2069 },
2070 [IMX7D] = {
2071 .variant = IMX7D,
2072 .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
2073 IMX_PCIE_FLAG_KEEP_MSI_CAP |
2074 IMX_PCIE_FLAG_HAS_APP_RESET |
2075 IMX_PCIE_FLAG_SKIP_L23_READY |
2076 IMX_PCIE_FLAG_HAS_PHY_RESET,
2077 .gpr = "fsl,imx7d-iomuxc-gpr",
2078 .mode_off[0] = IOMUXC_GPR12,
2079 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2080 .enable_ref_clk = imx7d_pcie_enable_ref_clk,
2081 .core_reset = imx7d_pcie_core_reset,
2082 },
2083 [IMX8MQ] = {
2084 .variant = IMX8MQ,
2085 .flags = IMX_PCIE_FLAG_HAS_APP_RESET |
2086 IMX_PCIE_FLAG_KEEP_MSI_CAP |
2087 IMX_PCIE_FLAG_HAS_PHY_RESET |
2088 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
2089 .gpr = "fsl,imx8mq-iomuxc-gpr",
2090 .mode_off[0] = IOMUXC_GPR12,
2091 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2092 .mode_off[1] = IOMUXC_GPR12,
2093 .mode_mask[1] = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE,
2094 .init_phy = imx8mq_pcie_init_phy,
2095 .enable_ref_clk = imx8mm_pcie_enable_ref_clk,
2096 .clr_clkreq_override = imx8mm_pcie_clr_clkreq_override,
2097 },
2098 [IMX8MM] = {
2099 .variant = IMX8MM,
2100 .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
2101 IMX_PCIE_FLAG_KEEP_MSI_CAP |
2102 IMX_PCIE_FLAG_HAS_PHYDRV |
2103 IMX_PCIE_FLAG_HAS_APP_RESET,
2104 .gpr = "fsl,imx8mm-iomuxc-gpr",
2105 .mode_off[0] = IOMUXC_GPR12,
2106 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2107 .enable_ref_clk = imx8mm_pcie_enable_ref_clk,
2108 .clr_clkreq_override = imx8mm_pcie_clr_clkreq_override,
2109 },
2110 [IMX8MP] = {
2111 .variant = IMX8MP,
2112 .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
2113 IMX_PCIE_FLAG_HAS_PHYDRV |
2114 IMX_PCIE_FLAG_HAS_APP_RESET,
2115 .gpr = "fsl,imx8mp-iomuxc-gpr",
2116 .mode_off[0] = IOMUXC_GPR12,
2117 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2118 .enable_ref_clk = imx8mm_pcie_enable_ref_clk,
2119 .clr_clkreq_override = imx8mm_pcie_clr_clkreq_override,
2120 },
2121 [IMX8Q] = {
2122 .variant = IMX8Q,
2123 .flags = IMX_PCIE_FLAG_HAS_PHYDRV |
2124 IMX_PCIE_FLAG_CPU_ADDR_FIXUP |
2125 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
2126 },
2127 [IMX95] = {
2128 .variant = IMX95,
2129 .flags = IMX_PCIE_FLAG_HAS_SERDES |
2130 IMX_PCIE_FLAG_HAS_LUT |
2131 IMX_PCIE_FLAG_8GT_ECN_ERR051586 |
2132 IMX_PCIE_FLAG_PM_RUNTIME |
2133 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
2134 .ltssm_off = IMX95_PE0_GEN_CTRL_3,
2135 .ltssm_mask = IMX95_PCIE_LTSSM_EN,
2136 .mode_off[0] = IMX95_PE0_GEN_CTRL_1,
2137 .mode_mask[0] = IMX95_PCIE_DEVICE_TYPE,
2138 .core_reset = imx95_pcie_core_reset,
2139 .init_phy = imx95_pcie_init_phy,
2140 .select_ref_clk_src = imx95_pcie_select_ref_clk_src,
2141 .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock,
2142 .enable_ref_clk = imx95_pcie_enable_ref_clk,
2143 .clr_clkreq_override = imx95_pcie_clr_clkreq_override,
2144 },
2145 [IMX8MQ_EP] = {
2146 .variant = IMX8MQ_EP,
2147 .flags = IMX_PCIE_FLAG_HAS_APP_RESET |
2148 IMX_PCIE_FLAG_HAS_PHY_RESET,
2149 .mode = DW_PCIE_EP_TYPE,
2150 .gpr = "fsl,imx8mq-iomuxc-gpr",
2151 .mode_off[0] = IOMUXC_GPR12,
2152 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2153 .mode_off[1] = IOMUXC_GPR12,
2154 .mode_mask[1] = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE,
2155 .epc_features = &imx8q_pcie_epc_features,
2156 .init_phy = imx8mq_pcie_init_phy,
2157 .enable_ref_clk = imx8mm_pcie_enable_ref_clk,
2158 },
2159 [IMX8MM_EP] = {
2160 .variant = IMX8MM_EP,
2161 .flags = IMX_PCIE_FLAG_HAS_APP_RESET |
2162 IMX_PCIE_FLAG_HAS_PHYDRV,
2163 .mode = DW_PCIE_EP_TYPE,
2164 .gpr = "fsl,imx8mm-iomuxc-gpr",
2165 .mode_off[0] = IOMUXC_GPR12,
2166 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2167 .epc_features = &imx8m_pcie_epc_features,
2168 .enable_ref_clk = imx8mm_pcie_enable_ref_clk,
2169 },
2170 [IMX8MP_EP] = {
2171 .variant = IMX8MP_EP,
2172 .flags = IMX_PCIE_FLAG_HAS_APP_RESET |
2173 IMX_PCIE_FLAG_HAS_PHYDRV,
2174 .mode = DW_PCIE_EP_TYPE,
2175 .gpr = "fsl,imx8mp-iomuxc-gpr",
2176 .mode_off[0] = IOMUXC_GPR12,
2177 .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
2178 .epc_features = &imx8m_pcie_epc_features,
2179 .enable_ref_clk = imx8mm_pcie_enable_ref_clk,
2180 },
2181 [IMX8Q_EP] = {
2182 .variant = IMX8Q_EP,
2183 .flags = IMX_PCIE_FLAG_HAS_PHYDRV,
2184 .mode = DW_PCIE_EP_TYPE,
2185 .epc_features = &imx8q_pcie_epc_features,
2186 },
2187 [IMX95_EP] = {
2188 .variant = IMX95_EP,
2189 .flags = IMX_PCIE_FLAG_HAS_SERDES |
2190 IMX_PCIE_FLAG_8GT_ECN_ERR051586 |
2191 IMX_PCIE_FLAG_SUPPORT_64BIT,
2192 .ltssm_off = IMX95_PE0_GEN_CTRL_3,
2193 .ltssm_mask = IMX95_PCIE_LTSSM_EN,
2194 .mode_off[0] = IMX95_PE0_GEN_CTRL_1,
2195 .mode_mask[0] = IMX95_PCIE_DEVICE_TYPE,
2196 .select_ref_clk_src = imx95_pcie_select_ref_clk_src,
2197 .init_phy = imx95_pcie_init_phy,
2198 .core_reset = imx95_pcie_core_reset,
2199 .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock,
2200 .epc_features = &imx95_pcie_epc_features,
2201 .enable_ref_clk = imx95_pcie_enable_ref_clk,
2202 .mode = DW_PCIE_EP_TYPE,
2203 },
2204 };
2205
2206 static const struct of_device_id imx_pcie_of_match[] = {
2207 { .compatible = "fsl,imx6q-pcie", .data = &drvdata[IMX6Q], },
2208 { .compatible = "fsl,imx6sx-pcie", .data = &drvdata[IMX6SX], },
2209 { .compatible = "fsl,imx6qp-pcie", .data = &drvdata[IMX6QP], },
2210 { .compatible = "fsl,imx7d-pcie", .data = &drvdata[IMX7D], },
2211 { .compatible = "fsl,imx8mq-pcie", .data = &drvdata[IMX8MQ], },
2212 { .compatible = "fsl,imx8mm-pcie", .data = &drvdata[IMX8MM], },
2213 { .compatible = "fsl,imx8mp-pcie", .data = &drvdata[IMX8MP], },
2214 { .compatible = "fsl,imx8q-pcie", .data = &drvdata[IMX8Q], },
2215 { .compatible = "fsl,imx95-pcie", .data = &drvdata[IMX95], },
2216 { .compatible = "fsl,imx8mq-pcie-ep", .data = &drvdata[IMX8MQ_EP], },
2217 { .compatible = "fsl,imx8mm-pcie-ep", .data = &drvdata[IMX8MM_EP], },
2218 { .compatible = "fsl,imx8mp-pcie-ep", .data = &drvdata[IMX8MP_EP], },
2219 { .compatible = "fsl,imx8q-pcie-ep", .data = &drvdata[IMX8Q_EP], },
2220 { .compatible = "fsl,imx95-pcie-ep", .data = &drvdata[IMX95_EP], },
2221 {},
2222 };
2223
2224 static struct platform_driver imx_pcie_driver = {
2225 .driver = {
2226 .name = "imx6q-pcie",
2227 .of_match_table = imx_pcie_of_match,
2228 .suppress_bind_attrs = true,
2229 .pm = &imx_pcie_pm_ops,
2230 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
2231 },
2232 .probe = imx_pcie_probe,
2233 .shutdown = imx_pcie_shutdown,
2234 };
2235
imx_pcie_quirk(struct pci_dev * dev)2236 static void imx_pcie_quirk(struct pci_dev *dev)
2237 {
2238 struct pci_bus *bus = dev->bus;
2239 struct dw_pcie_rp *pp = bus->sysdata;
2240
2241 /* Bus parent is the PCI bridge, its parent is this platform driver */
2242 if (!bus->dev.parent || !bus->dev.parent->parent)
2243 return;
2244
2245 /* Make sure we only quirk devices associated with this driver */
2246 if (bus->dev.parent->parent->driver != &imx_pcie_driver.driver)
2247 return;
2248
2249 if (pci_is_root_bus(bus)) {
2250 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
2251 struct imx_pcie *imx_pcie = to_imx_pcie(pci);
2252
2253 /*
2254 * Limit config length to avoid the kernel reading beyond
2255 * the register set and causing an abort on i.MX 6Quad
2256 */
2257 if (imx_pcie->drvdata->dbi_length) {
2258 dev->cfg_size = imx_pcie->drvdata->dbi_length;
2259 dev_info(&dev->dev, "Limiting cfg_size to %d\n",
2260 dev->cfg_size);
2261 }
2262 }
2263 }
2264 DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_SYNOPSYS, 0xabcd,
2265 PCI_CLASS_BRIDGE_PCI, 8, imx_pcie_quirk);
2266
imx_pcie_init(void)2267 static int __init imx_pcie_init(void)
2268 {
2269 #ifdef CONFIG_ARM
2270 struct device_node *np;
2271
2272 np = of_find_matching_node(NULL, imx_pcie_of_match);
2273 if (!np)
2274 return -ENODEV;
2275 of_node_put(np);
2276
2277 /*
2278 * Since probe() can be deferred we need to make sure that
2279 * hook_fault_code is not called after __init memory is freed
2280 * by kernel and since imx6q_pcie_abort_handler() is a no-op,
2281 * we can install the handler here without risking it
2282 * accessing some uninitialized driver state.
2283 */
2284 hook_fault_code(8, imx6q_pcie_abort_handler, SIGBUS, 0,
2285 "external abort on non-linefetch");
2286 #endif
2287
2288 return platform_driver_register(&imx_pcie_driver);
2289 }
2290 device_initcall(imx_pcie_init);
2291