1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Rockchip AXI PCIe host controller driver
4 *
5 * Copyright (c) 2016 Rockchip, Inc.
6 *
7 * Author: Shawn Lin <shawn.lin@rock-chips.com>
8 * Wenrui Li <wenrui.li@rock-chips.com>
9 *
10 * Bits taken from Synopsys DesignWare Host controller driver and
11 * ARM PCI Host generic driver.
12 */
13
14 #include <linux/bitfield.h>
15 #include <linux/bitrev.h>
16 #include <linux/gpio/consumer.h>
17 #include <linux/interrupt.h>
18 #include <linux/iopoll.h>
19 #include <linux/irq.h>
20 #include <linux/irqchip/chained_irq.h>
21 #include <linux/irqdomain.h>
22 #include <linux/module.h>
23 #include <linux/of.h>
24 #include <linux/of_pci.h>
25 #include <linux/phy/phy.h>
26 #include <linux/platform_device.h>
27
28 #include "../pci.h"
29 #include "pcie-rockchip.h"
30
rockchip_pcie_enable_bw_int(struct rockchip_pcie * rockchip)31 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
32 {
33 u32 status;
34
35 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
36 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
37 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
38 }
39
rockchip_pcie_clr_bw_int(struct rockchip_pcie * rockchip)40 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
41 {
42 u32 status;
43
44 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
45 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
46 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
47 }
48
rockchip_pcie_update_txcredit_mui(struct rockchip_pcie * rockchip)49 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
50 {
51 u32 val;
52
53 /* Update Tx credit maximum update interval */
54 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
55 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
56 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */
57 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
58 }
59
rockchip_pcie_valid_device(struct rockchip_pcie * rockchip,struct pci_bus * bus,int dev)60 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
61 struct pci_bus *bus, int dev)
62 {
63 /*
64 * Access only one slot on each root port.
65 * Do not read more than one device on the bus directly attached
66 * to RC's downstream side.
67 */
68 if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent))
69 return dev == 0;
70
71 return 1;
72 }
73
rockchip_pcie_lane_map(struct rockchip_pcie * rockchip)74 static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
75 {
76 u32 val;
77 u8 map;
78
79 if (rockchip->legacy_phy)
80 return GENMASK(MAX_LANE_NUM - 1, 0);
81
82 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
83 map = val & PCIE_CORE_LANE_MAP_MASK;
84
85 /* The link may be using a reverse-indexed mapping. */
86 if (val & PCIE_CORE_LANE_MAP_REVERSE)
87 map = bitrev8(map) >> 4;
88
89 return map;
90 }
91
rockchip_pcie_rd_own_conf(struct rockchip_pcie * rockchip,int where,int size,u32 * val)92 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
93 int where, int size, u32 *val)
94 {
95 void __iomem *addr;
96
97 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
98
99 if (!IS_ALIGNED((uintptr_t)addr, size)) {
100 *val = 0;
101 return PCIBIOS_BAD_REGISTER_NUMBER;
102 }
103
104 if (size == 4) {
105 *val = readl(addr);
106 } else if (size == 2) {
107 *val = readw(addr);
108 } else if (size == 1) {
109 *val = readb(addr);
110 } else {
111 *val = 0;
112 return PCIBIOS_BAD_REGISTER_NUMBER;
113 }
114 return PCIBIOS_SUCCESSFUL;
115 }
116
rockchip_pcie_wr_own_conf(struct rockchip_pcie * rockchip,int where,int size,u32 val)117 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
118 int where, int size, u32 val)
119 {
120 u32 mask, tmp, offset;
121 void __iomem *addr;
122
123 offset = where & ~0x3;
124 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
125
126 if (size == 4) {
127 writel(val, addr);
128 return PCIBIOS_SUCCESSFUL;
129 }
130
131 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
132
133 /*
134 * N.B. This read/modify/write isn't safe in general because it can
135 * corrupt RW1C bits in adjacent registers. But the hardware
136 * doesn't support smaller writes.
137 */
138 tmp = readl(addr) & mask;
139 tmp |= val << ((where & 0x3) * 8);
140 writel(tmp, addr);
141
142 return PCIBIOS_SUCCESSFUL;
143 }
144
rockchip_pcie_rd_other_conf(struct rockchip_pcie * rockchip,struct pci_bus * bus,u32 devfn,int where,int size,u32 * val)145 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
146 struct pci_bus *bus, u32 devfn,
147 int where, int size, u32 *val)
148 {
149 void __iomem *addr;
150
151 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where);
152
153 if (!IS_ALIGNED((uintptr_t)addr, size)) {
154 *val = 0;
155 return PCIBIOS_BAD_REGISTER_NUMBER;
156 }
157
158 if (pci_is_root_bus(bus->parent))
159 rockchip_pcie_cfg_configuration_accesses(rockchip,
160 AXI_WRAPPER_TYPE0_CFG);
161 else
162 rockchip_pcie_cfg_configuration_accesses(rockchip,
163 AXI_WRAPPER_TYPE1_CFG);
164
165 if (size == 4) {
166 *val = readl(addr);
167 } else if (size == 2) {
168 *val = readw(addr);
169 } else if (size == 1) {
170 *val = readb(addr);
171 } else {
172 *val = 0;
173 return PCIBIOS_BAD_REGISTER_NUMBER;
174 }
175 return PCIBIOS_SUCCESSFUL;
176 }
177
rockchip_pcie_wr_other_conf(struct rockchip_pcie * rockchip,struct pci_bus * bus,u32 devfn,int where,int size,u32 val)178 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
179 struct pci_bus *bus, u32 devfn,
180 int where, int size, u32 val)
181 {
182 void __iomem *addr;
183
184 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where);
185
186 if (!IS_ALIGNED((uintptr_t)addr, size))
187 return PCIBIOS_BAD_REGISTER_NUMBER;
188
189 if (pci_is_root_bus(bus->parent))
190 rockchip_pcie_cfg_configuration_accesses(rockchip,
191 AXI_WRAPPER_TYPE0_CFG);
192 else
193 rockchip_pcie_cfg_configuration_accesses(rockchip,
194 AXI_WRAPPER_TYPE1_CFG);
195
196 if (size == 4)
197 writel(val, addr);
198 else if (size == 2)
199 writew(val, addr);
200 else if (size == 1)
201 writeb(val, addr);
202 else
203 return PCIBIOS_BAD_REGISTER_NUMBER;
204
205 return PCIBIOS_SUCCESSFUL;
206 }
207
rockchip_pcie_rd_conf(struct pci_bus * bus,u32 devfn,int where,int size,u32 * val)208 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
209 int size, u32 *val)
210 {
211 struct rockchip_pcie *rockchip = bus->sysdata;
212
213 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
214 return PCIBIOS_DEVICE_NOT_FOUND;
215
216 if (pci_is_root_bus(bus))
217 return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
218
219 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size,
220 val);
221 }
222
rockchip_pcie_wr_conf(struct pci_bus * bus,u32 devfn,int where,int size,u32 val)223 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
224 int where, int size, u32 val)
225 {
226 struct rockchip_pcie *rockchip = bus->sysdata;
227
228 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
229 return PCIBIOS_DEVICE_NOT_FOUND;
230
231 if (pci_is_root_bus(bus))
232 return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
233
234 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size,
235 val);
236 }
237
238 static struct pci_ops rockchip_pcie_ops = {
239 .read = rockchip_pcie_rd_conf,
240 .write = rockchip_pcie_wr_conf,
241 };
242
rockchip_pcie_set_power_limit(struct rockchip_pcie * rockchip)243 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
244 {
245 int curr;
246 u32 status, scale, power;
247
248 if (IS_ERR(rockchip->vpcie3v3))
249 return;
250
251 /*
252 * Set RC's captured slot power limit and scale if
253 * vpcie3v3 available. The default values are both zero
254 * which means the software should set these two according
255 * to the actual power supply.
256 */
257 curr = regulator_get_current_limit(rockchip->vpcie3v3);
258 if (curr <= 0)
259 return;
260
261 scale = 3; /* 0.001x */
262 curr = curr / 1000; /* convert to mA */
263 power = (curr * 3300) / 1000; /* milliwatt */
264 while (power > FIELD_MAX(PCI_EXP_DEVCAP_PWR_VAL)) {
265 if (!scale) {
266 dev_warn(rockchip->dev, "invalid power supply\n");
267 return;
268 }
269 scale--;
270 power = power / 10;
271 }
272
273 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP);
274 status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_VAL, power);
275 status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_SCL, scale);
276 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP);
277 }
278
279 /**
280 * rockchip_pcie_host_init_port - Initialize hardware
281 * @rockchip: PCIe port information
282 */
rockchip_pcie_host_init_port(struct rockchip_pcie * rockchip)283 static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip)
284 {
285 struct device *dev = rockchip->dev;
286 int err, i = MAX_LANE_NUM;
287 u32 status;
288
289 gpiod_set_value_cansleep(rockchip->perst_gpio, 0);
290
291 err = rockchip_pcie_init_port(rockchip);
292 if (err)
293 return err;
294
295 /* Fix the transmitted FTS count desired to exit from L0s. */
296 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
297 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
298 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
299 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
300
301 rockchip_pcie_set_power_limit(rockchip);
302
303 /* Set RC's clock architecture as common clock */
304 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
305 status |= PCI_EXP_LNKSTA_SLC << 16;
306 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
307
308 /* Set RC's RCB to 128 */
309 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
310 status |= PCI_EXP_LNKCTL_RCB;
311 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
312
313 /* Enable Gen1 training */
314 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
315 PCIE_CLIENT_CONFIG);
316
317 msleep(PCIE_T_PVPERL_MS);
318 gpiod_set_value_cansleep(rockchip->perst_gpio, 1);
319
320 msleep(PCIE_RESET_CONFIG_WAIT_MS);
321
322 /* 500ms timeout value should be enough for Gen1/2 training */
323 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
324 status, PCIE_LINK_UP(status), 20,
325 500 * USEC_PER_MSEC);
326 if (err) {
327 dev_err(dev, "PCIe link training gen1 timeout!\n");
328 goto err_power_off_phy;
329 }
330
331 if (rockchip->link_gen == 2) {
332 /*
333 * Enable retrain for gen2. This should be configured only after
334 * gen1 finished.
335 */
336 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL2);
337 status &= ~PCI_EXP_LNKCTL2_TLS;
338 status |= PCI_EXP_LNKCTL2_TLS_5_0GT;
339 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL2);
340 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
341 status |= PCI_EXP_LNKCTL_RL;
342 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
343
344 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
345 status, PCIE_LINK_IS_GEN2(status), 20,
346 500 * USEC_PER_MSEC);
347 if (err)
348 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
349 }
350
351 /* Check the final link width from negotiated lane counter from MGMT */
352 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
353 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
354 PCIE_CORE_PL_CONF_LANE_SHIFT);
355 dev_dbg(dev, "current link width is x%d\n", status);
356
357 /* Power off unused lane(s) */
358 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
359 for (i = 0; i < MAX_LANE_NUM; i++) {
360 if (!(rockchip->lanes_map & BIT(i))) {
361 dev_dbg(dev, "idling lane %d\n", i);
362 phy_power_off(rockchip->phys[i]);
363 }
364 }
365
366 rockchip_pcie_write(rockchip, PCI_VENDOR_ID_ROCKCHIP,
367 PCIE_CORE_CONFIG_VENDOR);
368 rockchip_pcie_write(rockchip,
369 PCI_CLASS_BRIDGE_PCI_NORMAL << 8,
370 PCIE_RC_CONFIG_RID_CCR);
371
372 /* Clear THP cap's next cap pointer to remove L1 substate cap */
373 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
374 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
375 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
376
377 /* Clear L0s from RC's link cap */
378 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
379 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP);
380 status &= ~PCI_EXP_LNKCAP_ASPM_L0S;
381 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP);
382 }
383
384 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL);
385 status &= ~PCI_EXP_DEVCTL_PAYLOAD;
386 status |= PCI_EXP_DEVCTL_PAYLOAD_256B;
387 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL);
388
389 return 0;
390 err_power_off_phy:
391 while (i--)
392 phy_power_off(rockchip->phys[i]);
393 i = MAX_LANE_NUM;
394 while (i--)
395 phy_exit(rockchip->phys[i]);
396 return err;
397 }
398
rockchip_pcie_subsys_irq_handler(int irq,void * arg)399 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
400 {
401 struct rockchip_pcie *rockchip = arg;
402 struct device *dev = rockchip->dev;
403 u32 reg;
404 u32 sub_reg;
405
406 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
407 if (reg & PCIE_CLIENT_INT_LOCAL) {
408 dev_dbg(dev, "local interrupt received\n");
409 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
410 if (sub_reg & PCIE_CORE_INT_PRFPE)
411 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
412
413 if (sub_reg & PCIE_CORE_INT_CRFPE)
414 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
415
416 if (sub_reg & PCIE_CORE_INT_RRPE)
417 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
418
419 if (sub_reg & PCIE_CORE_INT_PRFO)
420 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
421
422 if (sub_reg & PCIE_CORE_INT_CRFO)
423 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
424
425 if (sub_reg & PCIE_CORE_INT_RT)
426 dev_dbg(dev, "replay timer timed out\n");
427
428 if (sub_reg & PCIE_CORE_INT_RTR)
429 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
430
431 if (sub_reg & PCIE_CORE_INT_PE)
432 dev_dbg(dev, "phy error detected on receive side\n");
433
434 if (sub_reg & PCIE_CORE_INT_MTR)
435 dev_dbg(dev, "malformed TLP received from the link\n");
436
437 if (sub_reg & PCIE_CORE_INT_UCR)
438 dev_dbg(dev, "Unexpected Completion received from the link\n");
439
440 if (sub_reg & PCIE_CORE_INT_FCE)
441 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
442
443 if (sub_reg & PCIE_CORE_INT_CT)
444 dev_dbg(dev, "a request timed out waiting for completion\n");
445
446 if (sub_reg & PCIE_CORE_INT_UTC)
447 dev_dbg(dev, "unmapped TC error\n");
448
449 if (sub_reg & PCIE_CORE_INT_MMVC)
450 dev_dbg(dev, "MSI mask register changes\n");
451
452 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
453 } else if (reg & PCIE_CLIENT_INT_PHY) {
454 dev_dbg(dev, "phy link changes\n");
455 rockchip_pcie_update_txcredit_mui(rockchip);
456 rockchip_pcie_clr_bw_int(rockchip);
457 }
458
459 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
460 PCIE_CLIENT_INT_STATUS);
461
462 return IRQ_HANDLED;
463 }
464
rockchip_pcie_client_irq_handler(int irq,void * arg)465 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
466 {
467 struct rockchip_pcie *rockchip = arg;
468 struct device *dev = rockchip->dev;
469 u32 reg;
470
471 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
472 if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
473 dev_dbg(dev, "legacy done interrupt received\n");
474
475 if (reg & PCIE_CLIENT_INT_MSG)
476 dev_dbg(dev, "message done interrupt received\n");
477
478 if (reg & PCIE_CLIENT_INT_HOT_RST)
479 dev_dbg(dev, "hot reset interrupt received\n");
480
481 if (reg & PCIE_CLIENT_INT_DPA)
482 dev_dbg(dev, "dpa interrupt received\n");
483
484 if (reg & PCIE_CLIENT_INT_FATAL_ERR)
485 dev_dbg(dev, "fatal error interrupt received\n");
486
487 if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
488 dev_dbg(dev, "non fatal error interrupt received\n");
489
490 if (reg & PCIE_CLIENT_INT_CORR_ERR)
491 dev_dbg(dev, "correctable error interrupt received\n");
492
493 if (reg & PCIE_CLIENT_INT_PHY)
494 dev_dbg(dev, "phy interrupt received\n");
495
496 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
497 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
498 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
499 PCIE_CLIENT_INT_NFATAL_ERR |
500 PCIE_CLIENT_INT_CORR_ERR |
501 PCIE_CLIENT_INT_PHY),
502 PCIE_CLIENT_INT_STATUS);
503
504 return IRQ_HANDLED;
505 }
506
rockchip_pcie_intx_handler(struct irq_desc * desc)507 static void rockchip_pcie_intx_handler(struct irq_desc *desc)
508 {
509 struct irq_chip *chip = irq_desc_get_chip(desc);
510 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
511 struct device *dev = rockchip->dev;
512 u32 reg;
513 u32 hwirq;
514 int ret;
515
516 chained_irq_enter(chip, desc);
517
518 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
519 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
520
521 while (reg) {
522 hwirq = ffs(reg) - 1;
523 reg &= ~BIT(hwirq);
524
525 ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq);
526 if (ret)
527 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
528 }
529
530 chained_irq_exit(chip, desc);
531 }
532
rockchip_pcie_setup_irq(struct rockchip_pcie * rockchip)533 static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
534 {
535 int irq, err;
536 struct device *dev = rockchip->dev;
537 struct platform_device *pdev = to_platform_device(dev);
538
539 irq = platform_get_irq_byname(pdev, "sys");
540 if (irq < 0)
541 return irq;
542
543 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
544 IRQF_SHARED, "pcie-sys", rockchip);
545 if (err) {
546 dev_err(dev, "failed to request PCIe subsystem IRQ\n");
547 return err;
548 }
549
550 irq = platform_get_irq_byname(pdev, "legacy");
551 if (irq < 0)
552 return irq;
553
554 irq_set_chained_handler_and_data(irq,
555 rockchip_pcie_intx_handler,
556 rockchip);
557
558 irq = platform_get_irq_byname(pdev, "client");
559 if (irq < 0)
560 return irq;
561
562 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
563 IRQF_SHARED, "pcie-client", rockchip);
564 if (err) {
565 dev_err(dev, "failed to request PCIe client IRQ\n");
566 return err;
567 }
568
569 return 0;
570 }
571
572 /**
573 * rockchip_pcie_parse_host_dt - Parse Device Tree
574 * @rockchip: PCIe port information
575 *
576 * Return: '0' on success and error value on failure
577 */
rockchip_pcie_parse_host_dt(struct rockchip_pcie * rockchip)578 static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip)
579 {
580 struct device *dev = rockchip->dev;
581 int err;
582
583 err = rockchip_pcie_parse_dt(rockchip);
584 if (err)
585 return err;
586
587 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
588 if (IS_ERR(rockchip->vpcie12v)) {
589 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV)
590 return PTR_ERR(rockchip->vpcie12v);
591 dev_info(dev, "no vpcie12v regulator found\n");
592 }
593
594 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
595 if (IS_ERR(rockchip->vpcie3v3)) {
596 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV)
597 return PTR_ERR(rockchip->vpcie3v3);
598 dev_info(dev, "no vpcie3v3 regulator found\n");
599 }
600
601 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8");
602 if (IS_ERR(rockchip->vpcie1v8))
603 return PTR_ERR(rockchip->vpcie1v8);
604
605 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9");
606 if (IS_ERR(rockchip->vpcie0v9))
607 return PTR_ERR(rockchip->vpcie0v9);
608
609 return 0;
610 }
611
rockchip_pcie_set_vpcie(struct rockchip_pcie * rockchip)612 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
613 {
614 struct device *dev = rockchip->dev;
615 int err;
616
617 if (!IS_ERR(rockchip->vpcie12v)) {
618 err = regulator_enable(rockchip->vpcie12v);
619 if (err) {
620 dev_err(dev, "fail to enable vpcie12v regulator\n");
621 goto err_out;
622 }
623 }
624
625 if (!IS_ERR(rockchip->vpcie3v3)) {
626 err = regulator_enable(rockchip->vpcie3v3);
627 if (err) {
628 dev_err(dev, "fail to enable vpcie3v3 regulator\n");
629 goto err_disable_12v;
630 }
631 }
632
633 err = regulator_enable(rockchip->vpcie1v8);
634 if (err) {
635 dev_err(dev, "fail to enable vpcie1v8 regulator\n");
636 goto err_disable_3v3;
637 }
638
639 err = regulator_enable(rockchip->vpcie0v9);
640 if (err) {
641 dev_err(dev, "fail to enable vpcie0v9 regulator\n");
642 goto err_disable_1v8;
643 }
644
645 return 0;
646
647 err_disable_1v8:
648 regulator_disable(rockchip->vpcie1v8);
649 err_disable_3v3:
650 if (!IS_ERR(rockchip->vpcie3v3))
651 regulator_disable(rockchip->vpcie3v3);
652 err_disable_12v:
653 if (!IS_ERR(rockchip->vpcie12v))
654 regulator_disable(rockchip->vpcie12v);
655 err_out:
656 return err;
657 }
658
rockchip_pcie_enable_interrupts(struct rockchip_pcie * rockchip)659 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
660 {
661 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
662 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
663 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
664 PCIE_CORE_INT_MASK);
665
666 rockchip_pcie_enable_bw_int(rockchip);
667 }
668
rockchip_pcie_intx_map(struct irq_domain * domain,unsigned int irq,irq_hw_number_t hwirq)669 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
670 irq_hw_number_t hwirq)
671 {
672 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
673 irq_set_chip_data(irq, domain->host_data);
674
675 return 0;
676 }
677
678 static const struct irq_domain_ops intx_domain_ops = {
679 .map = rockchip_pcie_intx_map,
680 };
681
rockchip_pcie_init_irq_domain(struct rockchip_pcie * rockchip)682 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
683 {
684 struct device *dev = rockchip->dev;
685 struct device_node *intc = of_get_next_child(dev->of_node, NULL);
686
687 if (!intc) {
688 dev_err(dev, "missing child interrupt-controller node\n");
689 return -EINVAL;
690 }
691
692 rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX,
693 &intx_domain_ops, rockchip);
694 of_node_put(intc);
695 if (!rockchip->irq_domain) {
696 dev_err(dev, "failed to get a INTx IRQ domain\n");
697 return -EINVAL;
698 }
699
700 return 0;
701 }
702
rockchip_pcie_prog_ob_atu(struct rockchip_pcie * rockchip,int region_no,int type,u8 num_pass_bits,u32 lower_addr,u32 upper_addr)703 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
704 int region_no, int type, u8 num_pass_bits,
705 u32 lower_addr, u32 upper_addr)
706 {
707 u32 ob_addr_0;
708 u32 ob_addr_1;
709 u32 ob_desc_0;
710 u32 aw_offset;
711
712 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
713 return -EINVAL;
714 if (num_pass_bits + 1 < 8)
715 return -EINVAL;
716 if (num_pass_bits > 63)
717 return -EINVAL;
718 if (region_no == 0) {
719 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
720 return -EINVAL;
721 }
722 if (region_no != 0) {
723 if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
724 return -EINVAL;
725 }
726
727 aw_offset = (region_no << OB_REG_SIZE_SHIFT);
728
729 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
730 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
731 ob_addr_1 = upper_addr;
732 ob_desc_0 = (1 << 23 | type);
733
734 rockchip_pcie_write(rockchip, ob_addr_0,
735 PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
736 rockchip_pcie_write(rockchip, ob_addr_1,
737 PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
738 rockchip_pcie_write(rockchip, ob_desc_0,
739 PCIE_CORE_OB_REGION_DESC0 + aw_offset);
740 rockchip_pcie_write(rockchip, 0,
741 PCIE_CORE_OB_REGION_DESC1 + aw_offset);
742
743 return 0;
744 }
745
rockchip_pcie_prog_ib_atu(struct rockchip_pcie * rockchip,int region_no,u8 num_pass_bits,u32 lower_addr,u32 upper_addr)746 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
747 int region_no, u8 num_pass_bits,
748 u32 lower_addr, u32 upper_addr)
749 {
750 u32 ib_addr_0;
751 u32 ib_addr_1;
752 u32 aw_offset;
753
754 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
755 return -EINVAL;
756 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
757 return -EINVAL;
758 if (num_pass_bits > 63)
759 return -EINVAL;
760
761 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
762
763 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
764 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
765 ib_addr_1 = upper_addr;
766
767 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
768 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
769
770 return 0;
771 }
772
rockchip_pcie_cfg_atu(struct rockchip_pcie * rockchip)773 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
774 {
775 struct device *dev = rockchip->dev;
776 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);
777 struct resource_entry *entry;
778 u64 pci_addr, size;
779 int offset;
780 int err;
781 int reg_no;
782
783 rockchip_pcie_cfg_configuration_accesses(rockchip,
784 AXI_WRAPPER_TYPE0_CFG);
785 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM);
786 if (!entry)
787 return -ENODEV;
788
789 size = resource_size(entry->res);
790 pci_addr = entry->res->start - entry->offset;
791 rockchip->msg_bus_addr = pci_addr;
792
793 for (reg_no = 0; reg_no < (size >> 20); reg_no++) {
794 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
795 AXI_WRAPPER_MEM_WRITE,
796 20 - 1,
797 pci_addr + (reg_no << 20),
798 0);
799 if (err) {
800 dev_err(dev, "program RC mem outbound ATU failed\n");
801 return err;
802 }
803 }
804
805 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
806 if (err) {
807 dev_err(dev, "program RC mem inbound ATU failed\n");
808 return err;
809 }
810
811 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO);
812 if (!entry)
813 return -ENODEV;
814
815 /* store the register number offset to program RC io outbound ATU */
816 offset = size >> 20;
817
818 size = resource_size(entry->res);
819 pci_addr = entry->res->start - entry->offset;
820
821 for (reg_no = 0; reg_no < (size >> 20); reg_no++) {
822 err = rockchip_pcie_prog_ob_atu(rockchip,
823 reg_no + 1 + offset,
824 AXI_WRAPPER_IO_WRITE,
825 20 - 1,
826 pci_addr + (reg_no << 20),
827 0);
828 if (err) {
829 dev_err(dev, "program RC io outbound ATU failed\n");
830 return err;
831 }
832 }
833
834 /* assign message regions */
835 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
836 AXI_WRAPPER_NOR_MSG,
837 20 - 1, 0, 0);
838
839 rockchip->msg_bus_addr += ((reg_no + offset) << 20);
840 return err;
841 }
842
rockchip_pcie_wait_l2(struct rockchip_pcie * rockchip)843 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
844 {
845 u32 value;
846 int err;
847
848 /* send PME_TURN_OFF message */
849 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
850
851 /* read LTSSM and wait for falling into L2 link state */
852 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
853 value, PCIE_LINK_IS_L2(value), 20,
854 jiffies_to_usecs(5 * HZ));
855 if (err) {
856 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
857 return err;
858 }
859
860 return 0;
861 }
862
rockchip_pcie_suspend_noirq(struct device * dev)863 static int rockchip_pcie_suspend_noirq(struct device *dev)
864 {
865 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
866 int ret;
867
868 /* disable core and cli int since we don't need to ack PME_ACK */
869 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
870 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
871 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
872
873 ret = rockchip_pcie_wait_l2(rockchip);
874 if (ret) {
875 rockchip_pcie_enable_interrupts(rockchip);
876 return ret;
877 }
878
879 rockchip_pcie_deinit_phys(rockchip);
880
881 rockchip_pcie_disable_clocks(rockchip);
882
883 regulator_disable(rockchip->vpcie0v9);
884
885 return ret;
886 }
887
rockchip_pcie_resume_noirq(struct device * dev)888 static int rockchip_pcie_resume_noirq(struct device *dev)
889 {
890 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
891 int err;
892
893 err = regulator_enable(rockchip->vpcie0v9);
894 if (err) {
895 dev_err(dev, "fail to enable vpcie0v9 regulator\n");
896 return err;
897 }
898
899 err = rockchip_pcie_enable_clocks(rockchip);
900 if (err)
901 goto err_disable_0v9;
902
903 err = rockchip_pcie_host_init_port(rockchip);
904 if (err)
905 goto err_pcie_resume;
906
907 err = rockchip_pcie_cfg_atu(rockchip);
908 if (err)
909 goto err_err_deinit_port;
910
911 /* Need this to enter L1 again */
912 rockchip_pcie_update_txcredit_mui(rockchip);
913 rockchip_pcie_enable_interrupts(rockchip);
914
915 return 0;
916
917 err_err_deinit_port:
918 rockchip_pcie_deinit_phys(rockchip);
919 err_pcie_resume:
920 rockchip_pcie_disable_clocks(rockchip);
921 err_disable_0v9:
922 regulator_disable(rockchip->vpcie0v9);
923 return err;
924 }
925
rockchip_pcie_probe(struct platform_device * pdev)926 static int rockchip_pcie_probe(struct platform_device *pdev)
927 {
928 struct rockchip_pcie *rockchip;
929 struct device *dev = &pdev->dev;
930 struct pci_host_bridge *bridge;
931 int err;
932
933 if (!dev->of_node)
934 return -ENODEV;
935
936 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
937 if (!bridge)
938 return -ENOMEM;
939
940 rockchip = pci_host_bridge_priv(bridge);
941
942 platform_set_drvdata(pdev, rockchip);
943 rockchip->dev = dev;
944 rockchip->is_rc = true;
945
946 err = rockchip_pcie_parse_host_dt(rockchip);
947 if (err)
948 return err;
949
950 err = rockchip_pcie_enable_clocks(rockchip);
951 if (err)
952 return err;
953
954 err = rockchip_pcie_set_vpcie(rockchip);
955 if (err) {
956 dev_err(dev, "failed to set vpcie regulator\n");
957 goto err_set_vpcie;
958 }
959
960 err = rockchip_pcie_host_init_port(rockchip);
961 if (err)
962 goto err_vpcie;
963
964 err = rockchip_pcie_init_irq_domain(rockchip);
965 if (err < 0)
966 goto err_deinit_port;
967
968 err = rockchip_pcie_cfg_atu(rockchip);
969 if (err)
970 goto err_remove_irq_domain;
971
972 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
973 if (!rockchip->msg_region) {
974 err = -ENOMEM;
975 goto err_remove_irq_domain;
976 }
977
978 bridge->sysdata = rockchip;
979 bridge->ops = &rockchip_pcie_ops;
980
981 err = rockchip_pcie_setup_irq(rockchip);
982 if (err)
983 goto err_remove_irq_domain;
984
985 rockchip_pcie_enable_interrupts(rockchip);
986
987 err = pci_host_probe(bridge);
988 if (err < 0)
989 goto err_remove_irq_domain;
990
991 return 0;
992
993 err_remove_irq_domain:
994 irq_domain_remove(rockchip->irq_domain);
995 err_deinit_port:
996 rockchip_pcie_deinit_phys(rockchip);
997 err_vpcie:
998 if (!IS_ERR(rockchip->vpcie12v))
999 regulator_disable(rockchip->vpcie12v);
1000 if (!IS_ERR(rockchip->vpcie3v3))
1001 regulator_disable(rockchip->vpcie3v3);
1002 regulator_disable(rockchip->vpcie1v8);
1003 regulator_disable(rockchip->vpcie0v9);
1004 err_set_vpcie:
1005 rockchip_pcie_disable_clocks(rockchip);
1006 return err;
1007 }
1008
rockchip_pcie_remove(struct platform_device * pdev)1009 static void rockchip_pcie_remove(struct platform_device *pdev)
1010 {
1011 struct device *dev = &pdev->dev;
1012 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1013 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);
1014
1015 pci_stop_root_bus(bridge->bus);
1016 pci_remove_root_bus(bridge->bus);
1017 irq_domain_remove(rockchip->irq_domain);
1018
1019 rockchip_pcie_deinit_phys(rockchip);
1020
1021 rockchip_pcie_disable_clocks(rockchip);
1022
1023 if (!IS_ERR(rockchip->vpcie12v))
1024 regulator_disable(rockchip->vpcie12v);
1025 if (!IS_ERR(rockchip->vpcie3v3))
1026 regulator_disable(rockchip->vpcie3v3);
1027 regulator_disable(rockchip->vpcie1v8);
1028 regulator_disable(rockchip->vpcie0v9);
1029 }
1030
1031 static const struct dev_pm_ops rockchip_pcie_pm_ops = {
1032 NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
1033 rockchip_pcie_resume_noirq)
1034 };
1035
1036 static const struct of_device_id rockchip_pcie_of_match[] = {
1037 { .compatible = "rockchip,rk3399-pcie", },
1038 {}
1039 };
1040 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
1041
1042 static struct platform_driver rockchip_pcie_driver = {
1043 .driver = {
1044 .name = "rockchip-pcie",
1045 .of_match_table = rockchip_pcie_of_match,
1046 .pm = &rockchip_pcie_pm_ops,
1047 },
1048 .probe = rockchip_pcie_probe,
1049 .remove = rockchip_pcie_remove,
1050 };
1051 module_platform_driver(rockchip_pcie_driver);
1052
1053 MODULE_AUTHOR("Rockchip Inc");
1054 MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
1055 MODULE_LICENSE("GPL v2");
1056