xref: /freebsd/sys/arm64/rockchip/rk_pcie.c (revision 6966ac055c3b7a39266fb982493330df7a097997)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2019 Michal Meloun <mmel@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  *
27  */
28 
29 /* Rockchip PCIe controller driver */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/bus.h>
37 #include <sys/gpio.h>
38 #include <sys/proc.h>
39 #include <sys/kernel.h>
40 #include <sys/lock.h>
41 #include <sys/malloc.h>
42 #include <sys/module.h>
43 #include <sys/mutex.h>
44 #include <sys/rman.h>
45 
46 #include <machine/bus.h>
47 #include <machine/intr.h>
48 #include <machine/resource.h>
49 
50 #include <dev/extres/clk/clk.h>
51 #include <dev/extres/hwreset/hwreset.h>
52 #include <dev/extres/phy/phy.h>
53 #include <dev/extres/regulator/regulator.h>
54 #include <dev/gpio/gpiobusvar.h>
55 #include <dev/ofw/ofw_bus.h>
56 #include <dev/ofw/ofw_bus_subr.h>
57 #include <dev/ofw/ofw_pci.h>
58 #include <dev/ofw/ofwpci.h>
59 #include <dev/pci/pcivar.h>
60 #include <dev/pci/pcireg.h>
61 #include <dev/pci/pcib_private.h>
62 
63 #include <dev/ofw/ofw_bus.h>
64 
65 #include "pcib_if.h"
66 
67 #define ATU_CFG_BUS(x)		(((x) & 0x0ff) << 20)
68 #define ATU_CFG_SLOT(x)		(((x) & 0x01f) << 15)
69 #define ATU_CFG_FUNC(x)		(((x) & 0x007) << 12)
70 #define ATU_CFG_REG(x)		(((x) & 0xfff) << 0)
71 
72 #define ATU_TYPE_MEM		0x2
73 #define ATU_TYPE_IO		0x6
74 #define ATU_TYPE_CFG0		0xA
75 #define ATU_TYPE_CFG1		0xB
76 #define ATY_TYPE_NOR_MSG	0xC
77 
78 #define ATU_OB_REGIONS		33
79 #define	ATU_OB_REGION_SHIFT	20
80 #define ATU_OB_REGION_SIZE	(1 << ATU_OB_REGION_SHIFT)
81 #define ATU_OB_REGION_0_SIZE	(( ATU_OB_REGIONS - 1) * ATU_OB_REGION_SIZE)
82 
83 #define ATU_IB_REGIONS		3
84 
85 #define	PCIE_CLIENT_BASIC_STRAP_CONF		0x000000
86 #define	 STRAP_CONF_GEN_2				(1 << 7)
87 #define	 STRAP_CONF_MODE_RC				(1 << 6)
88 #define	 STRAP_CONF_LANES(n)				((((n) / 2) & 0x3) << 4)
89 #define	 STRAP_CONF_ARI_EN				(1 << 3)
90 #define	 STRAP_CONF_SR_IOV_EN				(1 << 2)
91 #define	 STRAP_CONF_LINK_TRAIN_EN			(1 << 1)
92 #define	 STRAP_CONF_CONF_EN				(1 << 0)
93 #define	PCIE_CLIENT_HOT_RESET_CTRL		0x000018
94 #define	 HOT_RESET_CTRL_LINK_DOWN_RESET			(1 << 1)
95 #define	 HOT_RESET_CTRL_HOT_RESET_IN			(1 << 0)
96 #define	PCIE_CLIENT_BASIC_STATUS0		0x000044
97 #define	PCIE_CLIENT_BASIC_STATUS1		0x000048
98 #define	 STATUS1_LINK_ST_GET(x)				(((x) >> 20) & 0x3)
99 #define	  STATUS1_LINK_ST_UP				3
100 #define	PCIE_CLIENT_INT_MASK			0x00004C
101 #define	PCIE_CLIENT_INT_STATUS			0x000050
102 #define	 PCIE_CLIENT_INT_LEGACY_DONE			(1 << 15)
103 #define	 PCIE_CLIENT_INT_MSG				(1 << 14)
104 #define	 PCIE_CLIENT_INT_HOT_RST			(1 << 13)
105 #define	 PCIE_CLIENT_INT_DPA				(1 << 12)
106 #define	 PCIE_CLIENT_INT_FATAL_ERR			(1 << 11)
107 #define	 PCIE_CLIENT_INT_NFATAL_ERR			(1 << 10)
108 #define	 PCIE_CLIENT_INT_CORR_ERR			(1 << 9)
109 #define	 PCIE_CLIENT_INT_INTD				(1 << 8)
110 #define	 PCIE_CLIENT_INT_INTC				(1 << 7)
111 #define	 PCIE_CLIENT_INT_INTB				(1 << 6)
112 #define	 PCIE_CLIENT_INT_INTA				(1 << 5)
113 #define	 PCIE_CLIENT_INT_LOCAL				(1 << 4)
114 #define	 PCIE_CLIENT_INT_UDMA				(1 << 3)
115 #define	 PCIE_CLIENT_INT_PHY				(1 << 2)
116 #define	 PCIE_CLIENT_INT_HOT_PLUG			(1 << 1)
117 #define	 PCIE_CLIENT_INT_PWR_STCG			(1 << 0)
118 #define	 PCIE_CLIENT_INT_LEGACY			(PCIE_CLIENT_INT_INTA | \
119 						PCIE_CLIENT_INT_INTB | \
120 						PCIE_CLIENT_INT_INTC | \
121 						PCIE_CLIENT_INT_INTD)
122 
123 #define	PCIE_CORE_CTRL0				0x900000
124 #define	 CORE_CTRL_LANES_GET(x)				(((x) >> 20) & 0x3)
125 #define	PCIE_CORE_CTRL1				0x900004
126 #define	PCIE_CORE_CONFIG_VENDOR			0x900044
127 #define	PCIE_CORE_INT_STATUS			0x90020c
128 #define	 PCIE_CORE_INT_PRFPE				(1 << 0)
129 #define	 PCIE_CORE_INT_CRFPE				(1 << 1)
130 #define	 PCIE_CORE_INT_RRPE				(1 << 2)
131 #define	 PCIE_CORE_INT_PRFO				(1 << 3)
132 #define	 PCIE_CORE_INT_CRFO				(1 << 4)
133 #define	 PCIE_CORE_INT_RT				(1 << 5)
134 #define	 PCIE_CORE_INT_RTR				(1 << 6)
135 #define	 PCIE_CORE_INT_PE				(1 << 7)
136 #define	 PCIE_CORE_INT_MTR				(1 << 8)
137 #define	 PCIE_CORE_INT_UCR				(1 << 9)
138 #define	 PCIE_CORE_INT_FCE				(1 << 10)
139 #define	 PCIE_CORE_INT_CT				(1 << 11)
140 #define	 PCIE_CORE_INT_UTC				(1 << 18)
141 #define	 PCIE_CORE_INT_MMVC				(1 << 19)
142 #define	PCIE_CORE_INT_MASK			0x900210
143 #define	PCIE_CORE_PHY_FUNC_CONF			0x9002C0
144 #define	PCIE_CORE_RC_BAR_CONF			0x900300
145 
146 #define PCIE_RC_CONFIG_STD_BASE			0x800000
147 #define PCIE_RC_CONFIG_PRIV_BASE		0xA00000
148 #define	PCIE_RC_CONFIG_DCSR			0xA000C8
149 #define	 PCIE_RC_CONFIG_DCSR_MPS_MASK			(0x7 << 5)
150 #define	 PCIE_RC_CONFIG_DCSR_MPS_128			(0 << 5)
151 #define	 PCIE_RC_CONFIG_DCSR_MPS_256			(1 << 5)
152 #define	 PCIE_RC_CONFIG_LINK_CAP		0xA00CC
153 #define   PCIE_RC_CONFIG_LINK_CAP_L0S			(1 << 10)
154 
155 #define	PCIE_RC_CONFIG_LCS			0xA000D0
156 #define	PCIE_RC_CONFIG_THP_CAP			0xA00274
157 #define	 PCIE_RC_CONFIG_THP_CAP_NEXT_MASK		0xFFF00000
158 
159 #define	PCIE_CORE_OB_ADDR0(n)			(0xC00000 + 0x20 * (n) + 0x00)
160 #define	PCIE_CORE_OB_ADDR1(n)			(0xC00000 + 0x20 * (n) + 0x04)
161 #define	PCIE_CORE_OB_DESC0(n)			(0xC00000 + 0x20 * (n) + 0x08)
162 #define	PCIE_CORE_OB_DESC1(n)			(0xC00000 + 0x20 * (n) + 0x0C)
163 #define	PCIE_CORE_OB_DESC2(n)			(0xC00000 + 0x20 * (n) + 0x10)
164 #define	PCIE_CORE_OB_DESC3(n)			(0xC00000 + 0x20 * (n) + 0x14)
165 
166 #define	PCIE_CORE_IB_ADDR0(n)			(0xC00800 + 0x8 * (n) + 0x00)
167 #define	PCIE_CORE_IB_ADDR1(n)			(0xC00800 + 0x8 * (n) + 0x04)
168 
169 #define	PRIV_CFG_RD4(sc, reg)						\
170     (uint32_t)rk_pcie_local_cfg_read(sc, true, reg, 4)
171 #define	PRIV_CFG_RD2(sc, reg)						\
172     (uint16_t)rk_pcie_local_cfg_read(sc, true, reg, 2)
173 #define	PRIV_CFG_RD1(sc, reg)						\
174     (uint8_t)rk_pcie_local_cfg_read(sc, true, reg, 1)
175 #define	PRIV_CFG_WR4(sc, reg, val)					\
176     rk_pcie_local_cfg_write(sc, true, reg, val, 4)
177 #define	PRIV_CFG_WR2(sc, reg, val)					\
178     rk_pcie_local_cfg_write(sc, true, reg, val, 2)
179 #define	PRIV_CFG_WR1(sc, reg, val)					\
180     rk_pcie_local_cfg_write(sc, true, reg, val, 1)
181 
182 #define APB_WR4(_sc, _r, _v)	bus_write_4((_sc)->apb_mem_res, (_r), (_v))
183 #define	APB_RD4(_sc, _r)	bus_read_4((_sc)->apb_mem_res, (_r))
184 
185 #define	MAX_LANES	4
186 
187 #define RK_PCIE_ENABLE_MSI
188 #define RK_PCIE_ENABLE_MSIX
189 
190 struct rk_pcie_softc {
191 	struct ofw_pci_softc	ofw_pci;	/* Must be first */
192 
193 	struct resource		*axi_mem_res;
194 	struct resource		*apb_mem_res;
195 	struct resource		*client_irq_res;
196 	struct resource		*legacy_irq_res;
197 	struct resource		*sys_irq_res;
198 	void			*client_irq_cookie;
199 	void			*legacy_irq_cookie;
200 	void			*sys_irq_cookie;
201 
202 	device_t		dev;
203 	phandle_t		node;
204 	struct mtx		mtx;
205 
206 
207 	struct ofw_pci_range	mem_range;
208 	struct ofw_pci_range	pref_mem_range;
209 	struct ofw_pci_range	io_range;
210 
211 	bool			coherent;
212 	bus_dma_tag_t		dmat;
213 
214 	int			num_lanes;
215 	bool			link_is_gen2;
216 	bool			no_l0s;
217 
218 	u_int 			bus_start;
219 	u_int 			bus_end;
220 	u_int 			root_bus;
221 	u_int 			sub_bus;
222 
223 	regulator_t		supply_12v;
224 	regulator_t		supply_3v3;
225 	regulator_t		supply_1v8;
226 	regulator_t		supply_0v9;
227 	hwreset_t		hwreset_core;
228 	hwreset_t		hwreset_mgmt;
229 	hwreset_t		hwreset_mgmt_sticky;
230 	hwreset_t		hwreset_pipe;
231 	hwreset_t		hwreset_pm;
232 	hwreset_t		hwreset_aclk;
233 	hwreset_t		hwreset_pclk;
234 	clk_t			clk_aclk;
235 	clk_t			clk_aclk_perf;
236 	clk_t			clk_hclk;
237 	clk_t			clk_pm;
238 	phy_t 			phys[MAX_LANES];
239 	gpio_pin_t		gpio_ep;
240 };
241 
242 /* Compatible devices. */
243 static struct ofw_compat_data compat_data[] = {
244 	{"rockchip,rk3399-pcie", 1},
245 	{NULL,		 	 0},
246 };
247 
248 
249 static uint32_t
250 rk_pcie_local_cfg_read(struct rk_pcie_softc *sc, bool priv, u_int reg,
251     int bytes)
252 {
253 	uint32_t val;
254 	bus_addr_t base;
255 
256 	if (priv)
257 		base = PCIE_RC_CONFIG_PRIV_BASE;
258 	else
259 		base = PCIE_RC_CONFIG_STD_BASE;
260 
261 	switch (bytes) {
262 	case 4:
263 		val = bus_read_4(sc->apb_mem_res, base + reg);
264 		break;
265 	case 2:
266 		val = bus_read_2(sc->apb_mem_res, base + reg);
267 		break;
268 	case 1:
269 		val = bus_read_1(sc->apb_mem_res, base + reg);
270 		break;
271 	default:
272 		val = 0xFFFFFFFF;
273 	}
274 	return (val);
275 }
276 
277 static void
278 rk_pcie_local_cfg_write(struct rk_pcie_softc *sc, bool priv, u_int reg,
279     uint32_t val, int bytes)
280 {
281 	uint32_t val2;
282 	bus_addr_t base;
283 
284 	if (priv)
285 		base = PCIE_RC_CONFIG_PRIV_BASE;
286 	else
287 		base = PCIE_RC_CONFIG_STD_BASE;
288 
289 	switch (bytes) {
290 	case 4:
291 		bus_write_4(sc->apb_mem_res, base + reg, val);
292 		break;
293 	case 2:
294 		val2 = bus_read_4(sc->apb_mem_res, base + (reg & ~3));
295 		val2 &= ~(0xffff << ((reg & 3) << 3));
296 		val2 |= ((val & 0xffff) << ((reg & 3) << 3));
297 		bus_write_4(sc->apb_mem_res, base + (reg & ~3), val2);
298 		break;
299 	case 1:
300 		val2 = bus_read_4(sc->apb_mem_res, base + (reg & ~3));
301 		val2 &= ~(0xff << ((reg & 3) << 3));
302 		val2 |= ((val & 0xff) << ((reg & 3) << 3));
303 		bus_write_4(sc->apb_mem_res, base + (reg & ~3), val2);
304 		break;
305 	}
306 }
307 
308 
309 static bool
310 rk_pcie_check_dev(struct rk_pcie_softc *sc, u_int bus, u_int slot, u_int func,
311     u_int reg)
312 {
313 	uint32_t val;
314 
315 	if (bus < sc->bus_start || bus > sc->bus_end || slot > PCI_SLOTMAX ||
316 	    func > PCI_FUNCMAX || reg > PCI_REGMAX)
317 		return (false);
318 
319 	if (bus == sc->root_bus) {
320 		/* we have only 1 device with 1 function root port */
321 		if (slot > 0 || func > 0)
322 			return (false);
323 		return (true);
324 	}
325 
326 	/* link is needed for accessing non-root busses */
327 	val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
328 	if (STATUS1_LINK_ST_GET(val) != STATUS1_LINK_ST_UP)
329 		return (false);
330 
331 	/* only one device is on first subordinate bus */
332 	if (bus == sc->sub_bus  && slot)
333 		return (false);
334 	return (true);
335 }
336 
337 
338 static void
339 rk_pcie_map_out_atu(struct rk_pcie_softc *sc, int idx, int type,
340    int num_bits, uint64_t pa)
341 {
342 	uint32_t addr0;
343 	uint64_t max_size;
344 
345 
346 	/* Check HW constrains */
347 	max_size = idx == 0 ? ATU_OB_REGION_0_SIZE: ATU_OB_REGION_SIZE;
348 	KASSERT(idx <  ATU_OB_REGIONS, ("Invalid region index: %d\n", idx));
349 	KASSERT(num_bits  >= 7 &&  num_bits <= 63,
350 	    ("Bit width of region is invalid: %d\n", num_bits));
351 	KASSERT(max_size <= (1ULL << (num_bits + 1)),
352 	    ("Bit width is invalid for given region[%d]: %d\n",	idx, num_bits));
353 
354 	addr0 = (uint32_t)pa & 0xFFFFFF00;
355 	addr0 |= num_bits;
356 	APB_WR4(sc, PCIE_CORE_OB_ADDR0(idx), addr0);
357 	APB_WR4(sc, PCIE_CORE_OB_ADDR1(idx), (uint32_t)(pa >> 32));
358 	APB_WR4(sc, PCIE_CORE_OB_DESC0(idx), 1 << 23 | type);
359 	APB_WR4(sc, PCIE_CORE_OB_DESC1(idx), sc->root_bus);
360 
361 	/* Readback for sync */
362 	APB_RD4(sc, PCIE_CORE_OB_DESC1(idx));
363 }
364 
365 static void
366 rk_pcie_map_cfg_atu(struct rk_pcie_softc *sc, int idx, int type)
367 {
368 
369 	/* Check HW constrains */
370 	KASSERT(idx <  ATU_OB_REGIONS, ("Invalid region index: %d\n", idx));
371 
372 	/*
373 	 * Config window is only 25 bits width, so we cannot encode full bus
374 	 * range into it. Remaining bits of bus number should be taken from
375 	 * DESC1 field.
376 	 */
377 	APB_WR4(sc, PCIE_CORE_OB_ADDR0(idx), 25 - 1);
378 	APB_WR4(sc, PCIE_CORE_OB_ADDR1(idx), 0);
379 	APB_WR4(sc, PCIE_CORE_OB_DESC0(idx), 1 << 23 |  type);
380 	APB_WR4(sc, PCIE_CORE_OB_DESC1(idx), sc->root_bus);
381 
382 	/* Readback for sync */
383 	APB_RD4(sc, PCIE_CORE_OB_DESC1(idx));
384 
385 }
386 
387 static void
388 rk_pcie_map_in_atu(struct rk_pcie_softc *sc, int idx, int num_bits, uint64_t pa)
389 {
390 	uint32_t addr0;
391 
392 	/* Check HW constrains */
393 	KASSERT(idx <  ATU_IB_REGIONS, ("Invalid region index: %d\n", idx));
394 	KASSERT(num_bits  >= 7 &&  num_bits <= 63,
395 	    ("Bit width of region is invalid: %d\n", num_bits));
396 
397 	addr0 = (uint32_t)pa & 0xFFFFFF00;
398 	addr0 |= num_bits;
399 	APB_WR4(sc, PCIE_CORE_IB_ADDR0(idx), addr0);
400 	APB_WR4(sc, PCIE_CORE_IB_ADDR1(idx), (uint32_t)(pa >> 32));
401 
402 	/* Readback for sync */
403 	APB_RD4(sc, PCIE_CORE_IB_ADDR1(idx));
404 }
405 
406 static int
407 rk_pcie_decode_ranges(struct rk_pcie_softc *sc, struct ofw_pci_range *ranges,
408      int nranges)
409 {
410 	int i;
411 
412 	for (i = 0; i < nranges; i++) {
413 		if ((ranges[i].pci_hi & OFW_PCI_PHYS_HI_SPACEMASK)  ==
414 		    OFW_PCI_PHYS_HI_SPACE_IO) {
415 			if (sc->io_range.size != 0) {
416 				device_printf(sc->dev,
417 				    "Duplicated IO range found in DT\n");
418 				return (ENXIO);
419 			}
420 			sc->io_range = ranges[i];
421 		}
422 		if (((ranges[i].pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) ==
423 		    OFW_PCI_PHYS_HI_SPACE_MEM64))  {
424 			if (ranges[i].pci_hi & OFW_PCI_PHYS_HI_PREFETCHABLE) {
425 				if (sc->pref_mem_range.size != 0) {
426 					device_printf(sc->dev,
427 					    "Duplicated memory range found "
428 					    "in DT\n");
429 					return (ENXIO);
430 				}
431 				sc->pref_mem_range = ranges[i];
432 			} else {
433 				if (sc->mem_range.size != 0) {
434 					device_printf(sc->dev,
435 					    "Duplicated memory range found "
436 					    "in DT\n");
437 					return (ENXIO);
438 				}
439 				sc->mem_range = ranges[i];
440 			}
441 		}
442 	}
443 	if (sc->mem_range.size == 0) {
444 		device_printf(sc->dev,
445 		    " At least memory range should be defined in DT.\n");
446 		return (ENXIO);
447 	}
448 	return (0);
449 }
450 
451 /*-----------------------------------------------------------------------------
452  *
453  *  P C I B   I N T E R F A C E
454  */
455 static uint32_t
456 rk_pcie_read_config(device_t dev, u_int bus, u_int slot,
457     u_int func, u_int reg, int bytes)
458 {
459 	struct rk_pcie_softc *sc;
460 	uint32_t data;
461 	uint64_t addr;
462 	int type;
463 
464 	sc = device_get_softc(dev);
465 
466 	if (!rk_pcie_check_dev(sc, bus, slot, func, reg))
467 		return (0xFFFFFFFFU);
468 
469 	if (bus == sc->root_bus)
470 		return (rk_pcie_local_cfg_read(sc, false, reg, bytes));
471 
472 	addr = ATU_CFG_BUS(bus) | ATU_CFG_SLOT(slot) | ATU_CFG_FUNC(func) |
473 	    ATU_CFG_REG(reg);
474 	if (bus == sc->sub_bus) {
475 		type = ATU_TYPE_CFG0;
476 	} else {
477 		type = ATU_TYPE_CFG1;
478 		/*
479 		* XXX FIXME: any attempt to generate type1 configuration
480 		* access causes external data abort
481 		*/
482 		return (0xFFFFFFFFU);
483 	}
484 	rk_pcie_map_cfg_atu(sc, 0, type);
485 
486 	switch (bytes) {
487 	case 1:
488 		data = bus_read_1(sc->axi_mem_res, addr);
489 		break;
490 	case 2:
491 		data = bus_read_2(sc->axi_mem_res, addr);
492 		break;
493 	case 4:
494 		data = bus_read_4(sc->axi_mem_res, addr);
495 		break;
496 	default:
497 		data =  0xFFFFFFFFU;
498 	}
499 	return (data);
500 }
501 
502 static void
503 rk_pcie_write_config(device_t dev, u_int bus, u_int slot,
504     u_int func, u_int reg, uint32_t val, int bytes)
505 {
506 	struct rk_pcie_softc *sc;
507 	uint64_t addr;
508 	int type;
509 
510 	sc = device_get_softc(dev);
511 
512 	if (!rk_pcie_check_dev(sc, bus, slot, func, reg))
513 		return;
514 
515 	if (bus == sc->root_bus)
516 		return (rk_pcie_local_cfg_write(sc, false,  reg, val, bytes));
517 
518 	addr = ATU_CFG_BUS(bus) | ATU_CFG_SLOT(slot) | ATU_CFG_FUNC(func) |
519 	    ATU_CFG_REG(reg);
520 	if (bus == sc->sub_bus){
521 		type = ATU_TYPE_CFG0;
522 	} else {
523 		type = ATU_TYPE_CFG1;
524 		/*
525 		* XXX FIXME: any attempt to generate type1 configuration
526 		* access causes external data abort
527 		*/
528 		return;
529 	}
530 	rk_pcie_map_cfg_atu(sc, 0, type);
531 
532 	switch (bytes) {
533 	case 1:
534 		bus_write_1(sc->axi_mem_res, addr, val);
535 		break;
536 	case 2:
537 		bus_write_2(sc->axi_mem_res, addr, val);
538 		break;
539 	case 4:
540 		bus_write_4(sc->axi_mem_res, addr, val);
541 		break;
542 	default:
543 		break;
544 	}
545 }
546 
547 #ifdef RK_PCIE_ENABLE_MSI
548 static int
549 rk_pcie_alloc_msi(device_t pci, device_t child, int count,
550     int maxcount, int *irqs)
551 {
552 	phandle_t msi_parent;
553 	int rv;
554 
555 	rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
556 	    &msi_parent, NULL);
557 	if (rv != 0)
558 		return (rv);
559 
560 	rv = intr_alloc_msi(pci, child, msi_parent, count, maxcount,irqs);
561 	return (rv);
562 }
563 
564 static int
565 rk_pcie_release_msi(device_t pci, device_t child, int count, int *irqs)
566 {
567 	phandle_t msi_parent;
568 	int rv;
569 
570 	rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
571 	    &msi_parent, NULL);
572 	if (rv != 0)
573 		return (rv);
574 	rv = intr_release_msi(pci, child, msi_parent, count, irqs);
575 	return (rv);
576 }
577 #endif
578 
579 static int
580 rk_pcie_map_msi(device_t pci, device_t child, int irq, uint64_t *addr,
581     uint32_t *data)
582 {
583 	phandle_t msi_parent;
584 	int rv;
585 
586 	rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
587 	    &msi_parent, NULL);
588 	if (rv != 0)
589 		return (rv);
590 	rv = intr_map_msi(pci, child, msi_parent, irq, addr, data);
591 	return (rv);
592 }
593 
594 #ifdef RK_PCIE_ENABLE_MSIX
595 static int
596 rk_pcie_alloc_msix(device_t pci, device_t child, int *irq)
597 {
598 	phandle_t msi_parent;
599 	int rv;
600 
601 	rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
602 	    &msi_parent, NULL);
603 	if (rv != 0)
604 		return (rv);
605 	rv = intr_alloc_msix(pci, child, msi_parent, irq);
606 	return (rv);
607 }
608 
609 static int
610 rk_pcie_release_msix(device_t pci, device_t child, int irq)
611 {
612 	phandle_t msi_parent;
613 	int rv;
614 
615 	rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
616 	    &msi_parent, NULL);
617 	if (rv != 0)
618 		return (rv);
619 	rv = intr_release_msix(pci, child, msi_parent, irq);
620 	return (rv);
621 }
622 #endif
623 
624 static int
625 rk_pcie_get_id(device_t pci, device_t child, enum pci_id_type type,
626     uintptr_t *id)
627 {
628 	phandle_t node;
629 	int rv;
630 	uint32_t rid;
631 	uint16_t pci_rid;
632 
633 	if (type != PCI_ID_MSI)
634 		return (pcib_get_id(pci, child, type, id));
635 
636 	node = ofw_bus_get_node(pci);
637 	pci_rid = pci_get_rid(child);
638 
639 	rv = ofw_bus_msimap(node, pci_rid, NULL, &rid);
640 	if (rv != 0)
641 		return (rv);
642 
643 	*id = rid;
644 	return (0);
645 }
646 
647 static int
648 rk_pcie_route_interrupt(device_t bus, device_t dev, int pin)
649 {
650 	struct rk_pcie_softc *sc;
651 	u_int irq;
652 
653 	sc = device_get_softc(bus);
654 	irq = intr_map_clone_irq(rman_get_start(sc->legacy_irq_res));
655 	device_printf(bus, "route pin %d for device %d.%d to %u\n",
656 		    pin, pci_get_slot(dev), pci_get_function(dev), irq);
657 
658 	return (irq);
659 }
660 
661 /*-----------------------------------------------------------------------------
662  *
663  *  B U S  / D E V I C E   I N T E R F A C E
664  */
665 static int
666 rk_pcie_parse_fdt_resources(struct rk_pcie_softc *sc)
667 {
668 	int i, rv;
669 	char buf[16];
670 
671 	/* Regulators. All are optional. */
672 	rv = regulator_get_by_ofw_property(sc->dev, 0,
673 	    "vpcie12v-supply", &sc->supply_12v);
674 	if (rv != 0 && rv != ENOENT) {
675 		device_printf(sc->dev,"Cannot get 'vpcie12' regulator\n");
676 		return (ENXIO);
677 	}
678 	rv = regulator_get_by_ofw_property(sc->dev, 0,
679 	    "vpcie3v3-supply", &sc->supply_3v3);
680 	if (rv != 0 && rv != ENOENT) {
681 		device_printf(sc->dev,"Cannot get 'vpcie3v3' regulator\n");
682 		return (ENXIO);
683 	}
684 	rv = regulator_get_by_ofw_property(sc->dev, 0,
685 	    "vpcie1v8-supply", &sc->supply_1v8);
686 	if (rv != 0 && rv != ENOENT) {
687 		device_printf(sc->dev,"Cannot get 'vpcie1v8' regulator\n");
688 		return (ENXIO);
689 	}
690 	rv = regulator_get_by_ofw_property(sc->dev, 0,
691 	    "vpcie0v9-supply", &sc->supply_0v9);
692 	if (rv != 0 && rv != ENOENT) {
693 		device_printf(sc->dev,"Cannot get 'vpcie0v9' regulator\n");
694 		return (ENXIO);
695 	}
696 
697 	/* Resets. */
698 	rv = hwreset_get_by_ofw_name(sc->dev, 0, "core", &sc->hwreset_core);
699 	if (rv != 0) {
700 		device_printf(sc->dev, "Cannot get 'core' reset\n");
701 		return (ENXIO);
702 	}
703 	rv = hwreset_get_by_ofw_name(sc->dev, 0, "mgmt", &sc->hwreset_mgmt);
704 	if (rv != 0) {
705 		device_printf(sc->dev, "Cannot get 'mgmt' reset\n");
706 		return (ENXIO);
707 	}
708 	rv = hwreset_get_by_ofw_name(sc->dev, 0, "mgmt-sticky",
709 	    &sc->hwreset_mgmt_sticky);
710 	if (rv != 0) {
711 		device_printf(sc->dev, "Cannot get 'mgmt-sticky' reset\n");
712 		return (ENXIO);
713 	}
714 	rv = hwreset_get_by_ofw_name(sc->dev, 0, "pipe", &sc->hwreset_pipe);
715 	if (rv != 0) {
716 		device_printf(sc->dev, "Cannot get 'pipe' reset\n");
717 		return (ENXIO);
718 	}
719 	rv = hwreset_get_by_ofw_name(sc->dev, 0, "pm", &sc->hwreset_pm);
720 	if (rv != 0) {
721 		device_printf(sc->dev, "Cannot get 'pm' reset\n");
722 		return (ENXIO);
723 	}
724 	rv = hwreset_get_by_ofw_name(sc->dev, 0, "aclk", &sc->hwreset_aclk);
725 	if (rv != 0) {
726 		device_printf(sc->dev, "Cannot get 'aclk' reset\n");
727 		return (ENXIO);
728 	}
729 	rv = hwreset_get_by_ofw_name(sc->dev, 0, "pclk", &sc->hwreset_pclk);
730 	if (rv != 0) {
731 		device_printf(sc->dev, "Cannot get 'pclk' reset\n");
732 		return (ENXIO);
733 	}
734 
735 	/* Clocks. */
736 	rv = clk_get_by_ofw_name(sc->dev, 0, "aclk", &sc->clk_aclk);
737 	if (rv != 0) {
738 		device_printf(sc->dev, "Cannot get 'aclk' clock\n");
739 		return (ENXIO);
740 	}
741 	rv = clk_get_by_ofw_name(sc->dev, 0, "aclk-perf", &sc->clk_aclk_perf);
742 	if (rv != 0) {
743 		device_printf(sc->dev, "Cannot get 'aclk-perf' clock\n");
744 		return (ENXIO);
745 	}
746 	rv = clk_get_by_ofw_name(sc->dev, 0, "hclk", &sc->clk_hclk);
747 	if (rv != 0) {
748 		device_printf(sc->dev, "Cannot get 'hclk' clock\n");
749 		return (ENXIO);
750 	}
751 	rv = clk_get_by_ofw_name(sc->dev, 0, "pm", &sc->clk_pm);
752 	if (rv != 0) {
753 		device_printf(sc->dev, "Cannot get 'pm' clock\n");
754 		return (ENXIO);
755 	}
756 
757 	/* Phys. */
758 	for (i = 0; i < MAX_LANES; i++ ) {
759 		sprintf (buf, "pcie-phy-%d", i);
760 		rv = phy_get_by_ofw_name(sc->dev, 0, buf, sc->phys + i);
761 		if (rv != 0) {
762 			device_printf(sc->dev, "Cannot get '%s' phy\n", buf);
763 			return (ENXIO);
764 		}
765 	}
766 
767 	/* GPIO for PERST#. Optional */
768 	rv = gpio_pin_get_by_ofw_property(sc->dev, sc->node, "ep-gpios",
769 	    &sc->gpio_ep);
770 	if (rv != 0 && rv != ENOENT) {
771 		device_printf(sc->dev, "Cannot get 'ep-gpios' gpio\n");
772 		return (ENXIO);
773 	}
774 
775 	return (0);
776 }
777 
778 static int
779 rk_pcie_enable_resources(struct rk_pcie_softc *sc)
780 {
781 	int i, rv;
782 	uint32_t val;
783 
784 	/* Assert all resets */
785 	rv = hwreset_assert(sc->hwreset_pclk);
786 	if (rv != 0) {
787 		device_printf(sc->dev, "Cannot assert 'pclk' reset\n");
788 		return (rv);
789 	}
790 	rv = hwreset_assert(sc->hwreset_aclk);
791 	if (rv != 0) {
792 		device_printf(sc->dev, "Cannot assert 'aclk' reset\n");
793 		return (rv);
794 	}
795 	rv = hwreset_assert(sc->hwreset_pm);
796 	if (rv != 0) {
797 		device_printf(sc->dev, "Cannot assert 'pm' reset\n");
798 		return (rv);
799 	}
800 	rv = hwreset_assert(sc->hwreset_pipe);
801 	if (rv != 0) {
802 		device_printf(sc->dev, "Cannot assert 'pipe' reset\n");
803 		return (rv);
804 	}
805 	rv = hwreset_assert(sc->hwreset_mgmt_sticky);
806 	if (rv != 0) {
807 		device_printf(sc->dev, "Cannot assert 'mgmt_sticky' reset\n");
808 		return (rv);
809 	}
810 	rv = hwreset_assert(sc->hwreset_mgmt);
811 	if (rv != 0) {
812 		device_printf(sc->dev, "Cannot assert 'hmgmt' reset\n");
813 		return (rv);
814 	}
815 	rv = hwreset_assert(sc->hwreset_core);
816 	if (rv != 0) {
817 		device_printf(sc->dev, "Cannot assert 'hcore' reset\n");
818 		return (rv);
819 	}
820 	DELAY(10000);
821 
822 	/* Enable clockls */
823 	rv = clk_enable(sc->clk_aclk);
824 	if (rv != 0) {
825 		device_printf(sc->dev, "Cannot enable 'aclk' clock\n");
826 		return (rv);
827 	}
828 	rv = clk_enable(sc->clk_aclk_perf);
829 	if (rv != 0) {
830 		device_printf(sc->dev, "Cannot enable 'aclk_perf' clock\n");
831 		return (rv);
832 	}
833 	rv = clk_enable(sc->clk_hclk);
834 	if (rv != 0) {
835 		device_printf(sc->dev, "Cannot enable 'hclk' clock\n");
836 		return (rv);
837 	}
838 	rv = clk_enable(sc->clk_pm);
839 	if (rv != 0) {
840 		device_printf(sc->dev, "Cannot enable 'pm' clock\n");
841 		return (rv);
842 	}
843 
844 	/* Power up regulators */
845 	if (sc->supply_12v != NULL) {
846 		rv = regulator_enable(sc->supply_12v);
847 		if (rv != 0) {
848 			device_printf(sc->dev,
849 			    "Cannot enable 'vpcie12' regulator\n");
850 			return (rv);
851 		}
852 	}
853 	if (sc->supply_3v3 != NULL) {
854 		rv = regulator_enable(sc->supply_3v3);
855 		if (rv != 0) {
856 			device_printf(sc->dev,
857 			    "Cannot enable 'vpcie3v3' regulator\n");
858 			return (rv);
859 		}
860 	}
861 	if (sc->supply_1v8 != NULL) {
862 		rv = regulator_enable(sc->supply_1v8);
863 		if (rv != 0) {
864 			device_printf(sc->dev,
865 			    "Cannot enable 'vpcie1v8' regulator\n");
866 			return (rv);
867 		}
868 	}
869 	if (sc->supply_0v9 != NULL) {
870 		rv = regulator_enable(sc->supply_0v9);
871 		if (rv != 0) {
872 			device_printf(sc->dev,
873 			    "Cannot enable 'vpcie1v8' regulator\n");
874 			return (rv);
875 		}
876 	}
877 	DELAY(1000);
878 
879 	/* Deassert basic resets*/
880 	rv = hwreset_deassert(sc->hwreset_pm);
881 	if (rv != 0) {
882 		device_printf(sc->dev, "Cannot deassert 'pm' reset\n");
883 		return (rv);
884 	}
885 	rv = hwreset_deassert(sc->hwreset_aclk);
886 	if (rv != 0) {
887 		device_printf(sc->dev, "Cannot deassert 'aclk' reset\n");
888 		return (rv);
889 	}
890 	rv = hwreset_deassert(sc->hwreset_pclk);
891 	if (rv != 0) {
892 		device_printf(sc->dev, "Cannot deassert 'pclk' reset\n");
893 		return (rv);
894 	}
895 
896 	/* Set basic PCIe core mode (RC, lanes, gen1 or 2) */
897 	val  = STRAP_CONF_GEN_2 << 16 |
898 	    sc->link_is_gen2 ? STRAP_CONF_GEN_2: 0;
899 	val |= STRAP_CONF_MODE_RC << 16 | STRAP_CONF_MODE_RC;
900 	val |= STRAP_CONF_LANES(~0) << 16 | STRAP_CONF_LANES(sc->num_lanes);
901 	val |= STRAP_CONF_ARI_EN << 16 | STRAP_CONF_ARI_EN;
902 	val |= STRAP_CONF_CONF_EN << 16 | STRAP_CONF_CONF_EN;
903 	APB_WR4(sc, PCIE_CLIENT_BASIC_STRAP_CONF, val);
904 
905 	for (i = 0; i < MAX_LANES; i++) {
906 		rv = phy_enable(sc->phys[i]);
907 		if (rv != 0) {
908 			device_printf(sc->dev, "Cannot enable phy %d\n", i);
909 			return (rv);
910 		}
911 	}
912 
913 	/* Deassert rest of resets - order is important ! */
914 	rv = hwreset_deassert(sc->hwreset_mgmt_sticky);
915 	if (rv != 0) {
916 		device_printf(sc->dev, "Cannot deassert 'mgmt_sticky' reset\n");
917 		return (rv);
918 	}
919 	rv = hwreset_deassert(sc->hwreset_core);
920 	if (rv != 0) {
921 		device_printf(sc->dev, "Cannot deassert 'core' reset\n");
922 		return (rv);
923 	}
924 	rv = hwreset_deassert(sc->hwreset_mgmt);
925 	if (rv != 0) {
926 		device_printf(sc->dev, "Cannot deassert 'mgmt' reset\n");
927 		return (rv);
928 	}
929 	rv = hwreset_deassert(sc->hwreset_pipe);
930 	if (rv != 0) {
931 		device_printf(sc->dev, "Cannot deassert 'pipe' reset\n");
932 		return (rv);
933 	}
934 	return (0);
935 }
936 
937 static int
938 rk_pcie_setup_hw(struct rk_pcie_softc *sc)
939 {
940 	uint32_t val;
941 	int i, rv;
942 
943 	/* Assert PERST# if defined */
944 	if (sc->gpio_ep != NULL) {
945 		rv = gpio_pin_set_active(sc->gpio_ep, 0);
946 		if (rv != 0) {
947 			device_printf(sc->dev,
948 			    "Cannot clear 'gpio-ep' gpio\n");
949 			return (rv);
950 		}
951 	}
952 
953 	rv = rk_pcie_enable_resources(sc);
954 	if (rv != 0)
955 		return(rv);
956 
957 	/* Fix wrong default value for transmited FTS for L0s exit */
958 	val = APB_RD4(sc, PCIE_CORE_CTRL1);
959 	val |= 0xFFFF << 8;
960 	APB_WR4(sc, PCIE_CORE_CTRL1, val);
961 
962 	/* Setup PCIE Link Status & Control register */
963 	val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
964 	val |= PCIEM_LINK_CTL_COMMON_CLOCK;
965 	APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
966 	val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
967 	val |= PCIEM_LINK_CTL_RCB;
968 	APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
969 
970 	/* Enable training for GEN1 */
971 	APB_WR4(sc, PCIE_CLIENT_BASIC_STRAP_CONF,
972 	    STRAP_CONF_LINK_TRAIN_EN << 16 | STRAP_CONF_LINK_TRAIN_EN);
973 
974 	/* Deassert PERST# if defined */
975 	if (sc->gpio_ep != NULL) {
976 		rv = gpio_pin_set_active(sc->gpio_ep, 1);
977 		if (rv != 0) {
978 			device_printf(sc->dev, "Cannot set 'gpio-ep' gpio\n");
979 			return (rv);
980 		}
981 	}
982 
983 	/* Wait for link */
984 	for (i = 500; i > 0; i--) {
985 		val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
986 		if (STATUS1_LINK_ST_GET(val) == STATUS1_LINK_ST_UP)
987 			break;
988 		DELAY(1000);
989 	}
990 	if (i <= 0) {
991 		device_printf(sc->dev,
992 		    "Gen1 link training timeouted: 0x%08X.\n", val);
993 		return (0);
994 	}
995 
996 	if (sc->link_is_gen2) {
997 			val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
998 			val |= PCIEM_LINK_CTL_RETRAIN_LINK;
999 			APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
1000 
1001 			/* Wait for link */
1002 			for (i = 500; i > 0; i--) {
1003 				val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
1004 				if (STATUS1_LINK_ST_GET(val) ==
1005 				    STATUS1_LINK_ST_UP)
1006 					break;
1007 				DELAY(1000);
1008 			}
1009 			if (i <= 0)
1010 				device_printf(sc->dev, "Gen2 link training "
1011 				    "timeouted: 0x%08X.\n", val);
1012 	}
1013 
1014 	val = APB_RD4(sc, PCIE_CORE_CTRL0);
1015 	val = CORE_CTRL_LANES_GET(val);
1016 	if (bootverbose)
1017 		device_printf(sc->dev, "Link width: %d\n", 1 << val);
1018 
1019 	return (0);
1020 }
1021 
1022 static int
1023 rk_pcie_setup_sw(struct rk_pcie_softc *sc)
1024 {
1025 	uint32_t val;
1026 	int i, region;
1027 
1028 	pcib_bridge_init(sc->dev);
1029 
1030 
1031 	/* Setup config registers */
1032 	APB_WR4(sc, PCIE_CORE_CONFIG_VENDOR, 0x1D87); /* Rockchip vendor ID*/
1033 	PRIV_CFG_WR1(sc, PCIR_CLASS, PCIC_BRIDGE);
1034 	PRIV_CFG_WR1(sc, PCIR_SUBCLASS, PCIS_BRIDGE_PCI);
1035 	PRIV_CFG_WR1(sc, PCIR_PRIBUS_1, sc->root_bus);
1036 	PRIV_CFG_WR1(sc, PCIR_SECBUS_1, sc->sub_bus);
1037 	PRIV_CFG_WR1(sc, PCIR_SUBBUS_1, sc->bus_end);
1038 	PRIV_CFG_WR2(sc, PCIR_COMMAND, PCIM_CMD_MEMEN |
1039 	   PCIM_CMD_BUSMASTEREN | PCIM_CMD_SERRESPEN);
1040 
1041 	/* Don't advertise L1 power substate */
1042 	val = APB_RD4(sc, PCIE_RC_CONFIG_THP_CAP);
1043 	val &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
1044 	APB_WR4(sc, PCIE_RC_CONFIG_THP_CAP, val);
1045 
1046 	/* Don't advertise L0s */
1047 	if (sc->no_l0s) {
1048 		val = APB_RD4(sc, PCIE_RC_CONFIG_LINK_CAP);
1049 		val &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
1050 		APB_WR4(sc, PCIE_RC_CONFIG_LINK_CAP_L0S, val);
1051 	}
1052 
1053 	/*Adjust maximum payload size*/
1054 	val = APB_RD4(sc, PCIE_RC_CONFIG_DCSR);
1055 	val &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
1056 	val |= PCIE_RC_CONFIG_DCSR_MPS_128;
1057 	APB_WR4(sc, PCIE_RC_CONFIG_DCSR, val);
1058 
1059 	/*
1060 	 * Prepare IB ATU
1061 	 * map whole address range in 1:1 mappings
1062 	 */
1063 	rk_pcie_map_in_atu(sc, 2, 64 - 1, 0);
1064 
1065 	/* Prepare OB ATU */
1066 	/* - region 0 (32 MB) is used for config access */
1067 	region = 0;
1068 	rk_pcie_map_out_atu(sc, region++, ATU_TYPE_CFG0, 25 - 1, 0);
1069 
1070 	/* - then map memory (by using 1MB regions */
1071 	for (i = 0; i  < sc->mem_range.size / ATU_OB_REGION_SIZE; i++) {
1072 		rk_pcie_map_out_atu(sc,  region++, ATU_TYPE_MEM,
1073 		    ATU_OB_REGION_SHIFT - 1,
1074 		    sc->mem_range.pci + ATU_OB_REGION_SIZE * i);
1075 	}
1076 
1077 	/* - IO space is next, one region typically*/
1078 	for (i = 0; i  < sc->io_range.size / ATU_OB_REGION_SIZE; i++) {
1079 		rk_pcie_map_out_atu(sc, region++, ATU_TYPE_IO,
1080 		    ATU_OB_REGION_SHIFT - 1,
1081 		    sc->io_range.pci + ATU_OB_REGION_SIZE * i);
1082 	}
1083 	APB_WR4(sc, PCIE_CORE_RC_BAR_CONF, 0);
1084 	return (0);
1085 }
1086 
1087 static int
1088 rk_pcie_sys_irq(void *arg)
1089 {
1090 	struct rk_pcie_softc *sc;
1091 	uint32_t irq;
1092 
1093 	sc = (struct rk_pcie_softc *)arg;
1094 	irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
1095 	if (irq & PCIE_CLIENT_INT_LOCAL) {
1096 		irq = APB_RD4(sc, PCIE_CORE_INT_STATUS);
1097 		APB_WR4(sc, PCIE_CORE_INT_STATUS, irq);
1098 		APB_WR4(sc, PCIE_CLIENT_INT_STATUS, PCIE_CLIENT_INT_LOCAL);
1099 
1100 		device_printf(sc->dev, "'sys' interrupt received: 0x%04X\n",
1101 		    irq);
1102 	}
1103 
1104 	return (FILTER_HANDLED);
1105 }
1106 
1107 static int
1108 rk_pcie_client_irq(void *arg)
1109 {
1110 	struct rk_pcie_softc *sc;
1111 	uint32_t irq;
1112 
1113 	sc = (struct rk_pcie_softc *)arg;
1114 	irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
1115 	/* Clear causes handled by other interrups */
1116 	irq &= ~PCIE_CLIENT_INT_LOCAL;
1117 	irq &= ~PCIE_CLIENT_INT_LEGACY;
1118 	APB_WR4(sc, PCIE_CLIENT_INT_STATUS, irq);
1119 
1120 	device_printf(sc->dev, "'client' interrupt received: 0x%04X\n", irq);
1121 
1122 	return (FILTER_HANDLED);
1123 }
1124 
1125 static int
1126 rk_pcie_legacy_irq(void *arg)
1127 {
1128 	struct rk_pcie_softc *sc;
1129 	uint32_t irq;
1130 
1131 	sc = (struct rk_pcie_softc *)arg;
1132 	irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
1133 	irq &= PCIE_CLIENT_INT_LEGACY;
1134 	APB_WR4(sc, PCIE_CLIENT_INT_STATUS, irq);
1135 
1136 	/* all legacy interrupt are shared, do nothing */
1137 	return (FILTER_STRAY);
1138 }
1139 
1140 
1141 static bus_dma_tag_t
1142 rk_pcie_get_dma_tag(device_t dev, device_t child)
1143 {
1144 	struct rk_pcie_softc *sc;
1145 
1146 	sc = device_get_softc(dev);
1147 	return (sc->dmat);
1148 }
1149 
1150 
1151 static int
1152 rk_pcie_probe(device_t dev)
1153 {
1154 
1155 	if (!ofw_bus_status_okay(dev))
1156 		return (ENXIO);
1157 
1158 	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
1159 		return (ENXIO);
1160 
1161 	device_set_desc(dev, "Rockchip PCIe controller");
1162 	return (BUS_PROBE_DEFAULT);
1163 }
1164 
1165 static int
1166 rk_pcie_attach(device_t dev)
1167 {	struct rk_pcie_softc *sc;
1168 	uint32_t val;
1169 	int rv, rid, max_speed;
1170 
1171 	sc = device_get_softc(dev);
1172 	sc->dev = dev;
1173 	sc->node = ofw_bus_get_node(dev);
1174 
1175 	mtx_init(&sc->mtx, "rk_pcie_mtx", NULL, MTX_DEF);
1176 
1177 	/* XXX Should not be this configurable ? */
1178 	sc->bus_start = 0;
1179 	sc->bus_end =  0x1F;
1180 	sc->root_bus = sc->bus_start;
1181 	sc->sub_bus = 1;
1182 
1183 	/* Read FDT properties */
1184 	rv = rk_pcie_parse_fdt_resources(sc);
1185 	if (rv != 0)
1186 		return (rv);
1187 
1188 	sc->coherent = OF_hasprop(sc->node, "dma-coherent");
1189 	sc->no_l0s = OF_hasprop(sc->node, "aspm-no-l0s");
1190 	rv = OF_getencprop(sc->node, "num-lanes", &sc->num_lanes,
1191 	    sizeof(sc->num_lanes));
1192 	if (rv != sizeof(sc->num_lanes))
1193 		sc->num_lanes = 1;
1194 	if (sc->num_lanes != 1 && sc->num_lanes != 2 && sc->num_lanes != 4) {
1195 		device_printf(dev,
1196 		    "invalid number of lanes: %d\n",sc->num_lanes);
1197 		sc->num_lanes = 0;
1198 		rv = ENXIO;
1199 		goto out;
1200 	}
1201 
1202 	rv = OF_getencprop(sc->node, "max-link-speed", &max_speed,
1203 	    sizeof(max_speed));
1204 	if (rv != sizeof(max_speed) || max_speed != 1)
1205 		sc->link_is_gen2 = true;
1206 	else
1207 		sc->link_is_gen2 = false;
1208 
1209 	rv = ofw_bus_find_string_index(sc->node, "reg-names", "axi-base", &rid);
1210 	if (rv != 0) {
1211 		device_printf(dev, "Cannot get 'axi-base' memory\n");
1212 		rv = ENXIO;
1213 		goto out;
1214 	}
1215 	sc->axi_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1216 	    RF_ACTIVE);
1217 	if (sc->axi_mem_res == NULL) {
1218 		device_printf(dev, "Cannot allocate 'axi-base' (rid: %d)\n",
1219 		    rid);
1220 		rv = ENXIO;
1221 		goto out;
1222 	}
1223 	rv = ofw_bus_find_string_index(sc->node, "reg-names", "apb-base", &rid);
1224 	if (rv != 0) {
1225 		device_printf(dev, "Cannot get 'apb-base' memory\n");
1226 		rv = ENXIO;
1227 		goto out;
1228 	}
1229 	sc->apb_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1230 	    RF_ACTIVE);
1231 	if (sc->apb_mem_res == NULL) {
1232 		device_printf(dev, "Cannot allocate 'apb-base' (rid: %d)\n",
1233 		    rid);
1234 		rv = ENXIO;
1235 		goto out;
1236 	}
1237 
1238 	rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
1239 	    "client", &rid);
1240 	if (rv != 0) {
1241 		device_printf(dev, "Cannot get 'client' IRQ\n");
1242 		rv = ENXIO;
1243 		goto out;
1244 	}
1245 	sc->client_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1246 	    RF_ACTIVE | RF_SHAREABLE);
1247 	if (sc->client_irq_res == NULL) {
1248 		device_printf(dev, "Cannot allocate 'client' IRQ resource\n");
1249 		rv = ENXIO;
1250 		goto out;
1251 	}
1252 
1253 	rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
1254 	    "legacy", &rid);
1255 	if (rv != 0) {
1256 		device_printf(dev, "Cannot get 'legacy' IRQ\n");
1257 		rv = ENXIO;
1258 		goto out;
1259 	}
1260 	sc->legacy_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1261 	    RF_ACTIVE | RF_SHAREABLE);
1262 	if (sc->legacy_irq_res == NULL) {
1263 		device_printf(dev, "Cannot allocate 'legacy' IRQ resource\n");
1264 		rv = ENXIO;
1265 		goto out;
1266 	}
1267 
1268 	rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
1269 	    "sys", &rid);
1270 	if (rv != 0) {
1271 		device_printf(dev, "Cannot get 'sys' IRQ\n");
1272 		rv = ENXIO;
1273 		goto out;
1274 	}
1275 	sc->sys_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1276 	    RF_ACTIVE | RF_SHAREABLE);
1277 	if (sc->sys_irq_res == NULL) {
1278 		device_printf(dev, "Cannot allocate 'sys' IRQ resource\n");
1279 		rv = ENXIO;
1280 		goto out;
1281 	}
1282 
1283 	if (bootverbose)
1284 		device_printf(dev, "Bus is%s cache-coherent\n",
1285 		    sc->coherent ? "" : " not");
1286 	rv = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1287 	    1, 0,				/* alignment, bounds */
1288 	    BUS_SPACE_MAXADDR,			/* lowaddr */
1289 	    BUS_SPACE_MAXADDR,			/* highaddr */
1290 	    NULL, NULL,				/* filter, filterarg */
1291 	    BUS_SPACE_MAXSIZE,			/* maxsize */
1292 	    BUS_SPACE_UNRESTRICTED,		/* nsegments */
1293 	    BUS_SPACE_MAXSIZE,			/* maxsegsize */
1294 	    sc->coherent ? BUS_DMA_COHERENT : 0, /* flags */
1295 	    NULL, NULL,				/* lockfunc, lockarg */
1296 	    &sc->dmat);
1297 	if (rv != 0)
1298 		goto out;
1299 
1300 	rv = ofw_pci_init(dev);
1301 	if (rv != 0)
1302 		goto out;
1303 
1304 	rv = rk_pcie_decode_ranges(sc, sc->ofw_pci.sc_range,
1305 	    sc->ofw_pci.sc_nrange);
1306 	if (rv != 0)
1307 		goto out;
1308 	rv = rk_pcie_setup_hw(sc);
1309 	if (rv != 0)
1310 		goto out;
1311 
1312 	rv = rk_pcie_setup_sw(sc);
1313 	if (rv != 0)
1314 		goto out;
1315 
1316 	rv = bus_setup_intr(dev, sc->client_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
1317 	   rk_pcie_client_irq, NULL, sc, &sc->client_irq_cookie);
1318 	if (rv != 0) {
1319 		device_printf(dev, "cannot setup client interrupt handler\n");
1320 		rv = ENXIO;
1321 		goto out;
1322 	}
1323 
1324 	rv = bus_setup_intr(dev, sc->legacy_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
1325 	   rk_pcie_legacy_irq, NULL, sc, &sc->legacy_irq_cookie);
1326 	if (rv != 0) {
1327 		device_printf(dev, "cannot setup client interrupt handler\n");
1328 		rv = ENXIO;
1329 		goto out;
1330 	}
1331 
1332 	rv = bus_setup_intr(dev, sc->sys_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
1333 	   rk_pcie_sys_irq, NULL, sc, &sc->sys_irq_cookie);
1334 	if (rv != 0) {
1335 		device_printf(dev, "cannot setup client interrupt handler\n");
1336 		rv = ENXIO;
1337 		goto out;
1338 	}
1339 
1340 	/* Enable interrupts */
1341 	val =
1342 	    PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR |
1343 	    PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA |
1344 	    PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG |
1345 	    PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_INTA |
1346 	    PCIE_CLIENT_INT_INTB | PCIE_CLIENT_INT_INTC |
1347 	    PCIE_CLIENT_INT_INTD | PCIE_CLIENT_INT_PHY;
1348 
1349 	APB_WR4(sc, PCIE_CLIENT_INT_MASK, (val << 16) &  ~val);
1350 
1351 	val =
1352 	    PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE |
1353 	    PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO |
1354 	    PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR |
1355 	    PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR |
1356 	    PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE |
1357 	    PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC |
1358 	    PCIE_CORE_INT_MMVC;
1359 	APB_WR4(sc, PCIE_CORE_INT_MASK, ~(val));
1360 
1361 	val  = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
1362 	val |= PCIEM_LINK_CTL_LBMIE | PCIEM_LINK_CTL_LABIE;
1363 	APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
1364 
1365 	DELAY(250000);
1366 	device_add_child(dev, "pci", -1);
1367 	return (bus_generic_attach(dev));
1368 out:
1369 	/* XXX Cleanup */
1370 	return (rv);
1371 }
1372 
1373 
1374 static device_method_t rk_pcie_methods[] = {
1375 	/* Device interface */
1376 	DEVMETHOD(device_probe,		rk_pcie_probe),
1377 	DEVMETHOD(device_attach,	rk_pcie_attach),
1378 
1379 	/* Bus interface */
1380 	DEVMETHOD(bus_get_dma_tag,	rk_pcie_get_dma_tag),
1381 
1382 	/* pcib interface */
1383 	DEVMETHOD(pcib_read_config,	rk_pcie_read_config),
1384 	DEVMETHOD(pcib_write_config,	rk_pcie_write_config),
1385 	DEVMETHOD(pcib_route_interrupt,	rk_pcie_route_interrupt),
1386 #ifdef RK_PCIE_ENABLE_MSI
1387 	DEVMETHOD(pcib_alloc_msi,	rk_pcie_alloc_msi),
1388 	DEVMETHOD(pcib_release_msi,	rk_pcie_release_msi),
1389 #endif
1390 #ifdef RK_PCIE_ENABLE_MSIX
1391 	DEVMETHOD(pcib_alloc_msix,	rk_pcie_alloc_msix),
1392 	DEVMETHOD(pcib_release_msix,	rk_pcie_release_msix),
1393 #endif
1394 	DEVMETHOD(pcib_map_msi,		rk_pcie_map_msi),
1395 	DEVMETHOD(pcib_get_id,		rk_pcie_get_id),
1396 
1397 	/* OFW bus interface */
1398 	DEVMETHOD(ofw_bus_get_compat,	ofw_bus_gen_get_compat),
1399 	DEVMETHOD(ofw_bus_get_model,	ofw_bus_gen_get_model),
1400 	DEVMETHOD(ofw_bus_get_name,	ofw_bus_gen_get_name),
1401 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
1402 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
1403 
1404 	DEVMETHOD_END
1405 };
1406 
1407 DEFINE_CLASS_1(pcib, rk_pcie_driver, rk_pcie_methods,
1408     sizeof(struct rk_pcie_softc), ofw_pci_driver);
1409 static devclass_t rk_pcie_devclass;
1410 DRIVER_MODULE( rk_pcie, simplebus, rk_pcie_driver, rk_pcie_devclass,
1411     NULL, NULL);