1 /*-
2 * Copyright (c) 2016 Michal Meloun <mmel@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 /*
29 * Nvidia Integrated PCI/PCI-Express controller driver.
30 */
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/proc.h>
36 #include <sys/kernel.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/mutex.h>
40 #include <sys/rman.h>
41
42 #include <machine/intr.h>
43
44 #include <vm/vm.h>
45 #include <vm/vm_extern.h>
46 #include <vm/vm_kern.h>
47 #include <vm/pmap.h>
48
49 #include <dev/clk/clk.h>
50 #include <dev/hwreset/hwreset.h>
51 #include <dev/phy/phy.h>
52 #include <dev/regulator/regulator.h>
53 #include <dev/ofw/ofw_bus.h>
54 #include <dev/ofw/ofw_bus_subr.h>
55 #include <dev/ofw/ofw_pci.h>
56 #include <dev/ofw/ofwpci.h>
57 #include <dev/pci/pcivar.h>
58 #include <dev/pci/pcireg.h>
59 #include <dev/pci/pcib_private.h>
60
61 #include <machine/resource.h>
62 #include <machine/bus.h>
63
64 #include <arm/nvidia/tegra_pmc.h>
65
66 #include "ofw_bus_if.h"
67 #include "msi_if.h"
68 #include "pcib_if.h"
69 #include "pic_if.h"
70
71 #define AFI_AXI_BAR0_SZ 0x000
72 #define AFI_AXI_BAR1_SZ 0x004
73 #define AFI_AXI_BAR2_SZ 0x008
74 #define AFI_AXI_BAR3_SZ 0x00c
75 #define AFI_AXI_BAR4_SZ 0x010
76 #define AFI_AXI_BAR5_SZ 0x014
77 #define AFI_AXI_BAR0_START 0x018
78 #define AFI_AXI_BAR1_START 0x01c
79 #define AFI_AXI_BAR2_START 0x020
80 #define AFI_AXI_BAR3_START 0x024
81 #define AFI_AXI_BAR4_START 0x028
82 #define AFI_AXI_BAR5_START 0x02c
83 #define AFI_FPCI_BAR0 0x030
84 #define AFI_FPCI_BAR1 0x034
85 #define AFI_FPCI_BAR2 0x038
86 #define AFI_FPCI_BAR3 0x03c
87 #define AFI_FPCI_BAR4 0x040
88 #define AFI_FPCI_BAR5 0x044
89 #define AFI_MSI_BAR_SZ 0x060
90 #define AFI_MSI_FPCI_BAR_ST 0x064
91 #define AFI_MSI_AXI_BAR_ST 0x068
92 #define AFI_MSI_VEC(x) (0x06c + 4 * (x))
93 #define AFI_MSI_EN_VEC(x) (0x08c + 4 * (x))
94 #define AFI_MSI_INTR_IN_REG 32
95 #define AFI_MSI_REGS 8
96
97 #define AFI_CONFIGURATION 0x0ac
98 #define AFI_CONFIGURATION_EN_FPCI (1 << 0)
99
100 #define AFI_FPCI_ERROR_MASKS 0x0b0
101 #define AFI_INTR_MASK 0x0b4
102 #define AFI_INTR_MASK_MSI_MASK (1 << 8)
103 #define AFI_INTR_MASK_INT_MASK (1 << 0)
104
105 #define AFI_INTR_CODE 0x0b8
106 #define AFI_INTR_CODE_MASK 0xf
107 #define AFI_INTR_CODE_INT_CODE_INI_SLVERR 1
108 #define AFI_INTR_CODE_INT_CODE_INI_DECERR 2
109 #define AFI_INTR_CODE_INT_CODE_TGT_SLVERR 3
110 #define AFI_INTR_CODE_INT_CODE_TGT_DECERR 4
111 #define AFI_INTR_CODE_INT_CODE_TGT_WRERR 5
112 #define AFI_INTR_CODE_INT_CODE_SM_MSG 6
113 #define AFI_INTR_CODE_INT_CODE_DFPCI_DECERR 7
114 #define AFI_INTR_CODE_INT_CODE_AXI_DECERR 8
115 #define AFI_INTR_CODE_INT_CODE_FPCI_TIMEOUT 9
116 #define AFI_INTR_CODE_INT_CODE_PE_PRSNT_SENSE 10
117 #define AFI_INTR_CODE_INT_CODE_PE_CLKREQ_SENSE 11
118 #define AFI_INTR_CODE_INT_CODE_CLKCLAMP_SENSE 12
119 #define AFI_INTR_CODE_INT_CODE_RDY4PD_SENSE 13
120 #define AFI_INTR_CODE_INT_CODE_P2P_ERROR 14
121
122 #define AFI_INTR_SIGNATURE 0x0bc
123 #define AFI_UPPER_FPCI_ADDRESS 0x0c0
124 #define AFI_SM_INTR_ENABLE 0x0c4
125 #define AFI_SM_INTR_RP_DEASSERT (1 << 14)
126 #define AFI_SM_INTR_RP_ASSERT (1 << 13)
127 #define AFI_SM_INTR_HOTPLUG (1 << 12)
128 #define AFI_SM_INTR_PME (1 << 11)
129 #define AFI_SM_INTR_FATAL_ERROR (1 << 10)
130 #define AFI_SM_INTR_UNCORR_ERROR (1 << 9)
131 #define AFI_SM_INTR_CORR_ERROR (1 << 8)
132 #define AFI_SM_INTR_INTD_DEASSERT (1 << 7)
133 #define AFI_SM_INTR_INTC_DEASSERT (1 << 6)
134 #define AFI_SM_INTR_INTB_DEASSERT (1 << 5)
135 #define AFI_SM_INTR_INTA_DEASSERT (1 << 4)
136 #define AFI_SM_INTR_INTD_ASSERT (1 << 3)
137 #define AFI_SM_INTR_INTC_ASSERT (1 << 2)
138 #define AFI_SM_INTR_INTB_ASSERT (1 << 1)
139 #define AFI_SM_INTR_INTA_ASSERT (1 << 0)
140
141 #define AFI_AFI_INTR_ENABLE 0x0c8
142 #define AFI_AFI_INTR_ENABLE_CODE(code) (1 << (code))
143
144 #define AFI_PCIE_CONFIG 0x0f8
145 #define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1))
146 #define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0x6
147 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20)
148 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_XBAR2_1 (0x0 << 20)
149 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_XBAR4_1 (0x1 << 20)
150
151 #define AFI_FUSE 0x104
152 #define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2)
153
154 #define AFI_PEX0_CTRL 0x110
155 #define AFI_PEX1_CTRL 0x118
156 #define AFI_PEX2_CTRL 0x128
157 #define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4)
158 #define AFI_PEX_CTRL_REFCLK_EN (1 << 3)
159 #define AFI_PEX_CTRL_CLKREQ_EN (1 << 1)
160 #define AFI_PEX_CTRL_RST_L (1 << 0)
161
162 #define AFI_AXI_BAR6_SZ 0x134
163 #define AFI_AXI_BAR7_SZ 0x138
164 #define AFI_AXI_BAR8_SZ 0x13c
165 #define AFI_AXI_BAR6_START 0x140
166 #define AFI_AXI_BAR7_START 0x144
167 #define AFI_AXI_BAR8_START 0x148
168 #define AFI_FPCI_BAR6 0x14c
169 #define AFI_FPCI_BAR7 0x150
170 #define AFI_FPCI_BAR8 0x154
171 #define AFI_PLLE_CONTROL 0x160
172 #define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9)
173 #define AFI_PLLE_CONTROL_BYPASS_PCIE2PLLE_CONTROL (1 << 8)
174 #define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1)
175 #define AFI_PLLE_CONTROL_PCIE2PLLE_CONTROL_EN (1 << 0)
176
177 #define AFI_PEXBIAS_CTRL 0x168
178
179 /* Configuration space */
180 #define RP_VEND_XP 0x0F00
181 #define RP_VEND_XP_DL_UP (1 << 30)
182
183 #define RP_VEND_CTL2 0x0fa8
184 #define RP_VEND_CTL2_PCA_ENABLE (1 << 7)
185
186 #define RP_PRIV_MISC 0x0FE0
187 #define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xE << 0)
188 #define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xF << 0)
189
190 #define RP_LINK_CONTROL_STATUS 0x0090
191 #define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000
192 #define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000
193
194 /* PADS space */
195 #define PADS_REFCLK_CFG0 0x000c8
196 #define PADS_REFCLK_CFG1 0x000cc
197
198
199 /* Wait 50 ms (per port) for link. */
200 #define TEGRA_PCIE_LINKUP_TIMEOUT 50000
201
202 /* FPCI Address space */
203 #define FPCI_MAP_IO 0xFDFC000000ULL
204 #define FPCI_MAP_TYPE0_CONFIG 0xFDFC000000ULL
205 #define FPCI_MAP_TYPE1_CONFIG 0xFDFF000000ULL
206 #define FPCI_MAP_EXT_TYPE0_CONFIG 0xFE00000000ULL
207 #define FPCI_MAP_EXT_TYPE1_CONFIG 0xFE10000000ULL
208
209 #define TEGRA_PCIB_MSI_ENABLE
210
211 #define DEBUG
212 #ifdef DEBUG
213 #define debugf(fmt, args...) do { printf(fmt,##args); } while (0)
214 #else
215 #define debugf(fmt, args...)
216 #endif
217
218 /*
219 * Configuration space format:
220 * [27:24] extended register
221 * [23:16] bus
222 * [15:11] slot (device)
223 * [10: 8] function
224 * [ 7: 0] register
225 */
226 #define PCI_CFG_EXT_REG(reg) ((((reg) >> 8) & 0x0f) << 24)
227 #define PCI_CFG_BUS(bus) (((bus) & 0xff) << 16)
228 #define PCI_CFG_DEV(dev) (((dev) & 0x1f) << 11)
229 #define PCI_CFG_FUN(fun) (((fun) & 0x07) << 8)
230 #define PCI_CFG_BASE_REG(reg) ((reg) & 0xff)
231
232 #define PADS_WR4(_sc, _r, _v) bus_write_4((_sc)->pads_mem_res, (_r), (_v))
233 #define PADS_RD4(_sc, _r) bus_read_4((_sc)->pads_mem_res, (_r))
234 #define AFI_WR4(_sc, _r, _v) bus_write_4((_sc)->afi_mem_res, (_r), (_v))
235 #define AFI_RD4(_sc, _r) bus_read_4((_sc)->afi_mem_res, (_r))
236
237 static struct {
238 bus_size_t axi_start;
239 bus_size_t fpci_start;
240 bus_size_t size;
241 } bars[] = {
242 {AFI_AXI_BAR0_START, AFI_FPCI_BAR0, AFI_AXI_BAR0_SZ}, /* BAR 0 */
243 {AFI_AXI_BAR1_START, AFI_FPCI_BAR1, AFI_AXI_BAR1_SZ}, /* BAR 1 */
244 {AFI_AXI_BAR2_START, AFI_FPCI_BAR2, AFI_AXI_BAR2_SZ}, /* BAR 2 */
245 {AFI_AXI_BAR3_START, AFI_FPCI_BAR3, AFI_AXI_BAR3_SZ}, /* BAR 3 */
246 {AFI_AXI_BAR4_START, AFI_FPCI_BAR4, AFI_AXI_BAR4_SZ}, /* BAR 4 */
247 {AFI_AXI_BAR5_START, AFI_FPCI_BAR5, AFI_AXI_BAR5_SZ}, /* BAR 5 */
248 {AFI_AXI_BAR6_START, AFI_FPCI_BAR6, AFI_AXI_BAR6_SZ}, /* BAR 6 */
249 {AFI_AXI_BAR7_START, AFI_FPCI_BAR7, AFI_AXI_BAR7_SZ}, /* BAR 7 */
250 {AFI_AXI_BAR8_START, AFI_FPCI_BAR8, AFI_AXI_BAR8_SZ}, /* BAR 8 */
251 {AFI_MSI_AXI_BAR_ST, AFI_MSI_FPCI_BAR_ST, AFI_MSI_BAR_SZ}, /* MSI 9 */
252 };
253
254
255 struct pcie_soc {
256 char **regulator_names;
257 bool cml_clk;
258 bool pca_enable;
259 uint32_t pads_refclk_cfg0;
260 uint32_t pads_refclk_cfg1;
261 };
262
263 /* Tegra 124 config. */
264 static char *tegra124_reg_names[] = {
265 "avddio-pex-supply",
266 "dvddio-pex-supply",
267 "avdd-pex-pll-supply",
268 "hvdd-pex-supply",
269 "hvdd-pex-pll-e-supply",
270 "vddio-pex-ctl-supply",
271 "avdd-pll-erefe-supply",
272 NULL
273 };
274
275 static struct pcie_soc tegra124_soc = {
276 .regulator_names = tegra124_reg_names,
277 .cml_clk = true,
278 .pca_enable = false,
279 .pads_refclk_cfg0 = 0x44ac44ac,
280 };
281
282 /* Tegra 210 config. */
283 static char *tegra210_reg_names[] = {
284 "avdd-pll-uerefe-supply",
285 "hvddio-pex-supply",
286 "dvddio-pex-supply",
287 "dvdd-pex-pll-supply",
288 "hvdd-pex-pll-e-supply",
289 "vddio-pex-ctl-supply",
290 NULL
291 };
292
293 static struct pcie_soc tegra210_soc = {
294 .regulator_names = tegra210_reg_names,
295 .cml_clk = true,
296 .pca_enable = true,
297 .pads_refclk_cfg0 = 0x90b890b8,
298 };
299
300 /* Compatible devices. */
301 static struct ofw_compat_data compat_data[] = {
302 {"nvidia,tegra124-pcie", (uintptr_t)&tegra124_soc},
303 {"nvidia,tegra210-pcie", (uintptr_t)&tegra210_soc},
304 {NULL, 0},
305 };
306
307 #define TEGRA_FLAG_MSI_USED 0x0001
308 struct tegra_pcib_irqsrc {
309 struct intr_irqsrc isrc;
310 u_int irq;
311 u_int flags;
312 };
313
314 struct tegra_pcib_port {
315 int enabled;
316 int port_idx; /* chip port index */
317 int num_lanes; /* number of lanes */
318 bus_size_t afi_pex_ctrl; /* offset of afi_pex_ctrl */
319 phy_t phy; /* port phy */
320
321 /* Config space properties. */
322 bus_addr_t rp_base_addr; /* PA of config window */
323 bus_size_t rp_size; /* size of config window */
324 bus_space_handle_t cfg_handle; /* handle of config window */
325 };
326
327 #define TEGRA_PCIB_MAX_PORTS 3
328 #define TEGRA_PCIB_MAX_MSI AFI_MSI_INTR_IN_REG * AFI_MSI_REGS
329 struct tegra_pcib_softc {
330 struct ofw_pci_softc ofw_pci;
331 device_t dev;
332 struct pcie_soc *soc;
333 struct mtx mtx;
334 struct resource *pads_mem_res;
335 struct resource *afi_mem_res;
336 struct resource *cfg_mem_res;
337 struct resource *irq_res;
338 struct resource *msi_irq_res;
339 void *intr_cookie;
340 void *msi_intr_cookie;
341
342 struct ofw_pci_range mem_range;
343 struct ofw_pci_range pref_mem_range;
344 struct ofw_pci_range io_range;
345
346 clk_t clk_pex;
347 clk_t clk_afi;
348 clk_t clk_pll_e;
349 clk_t clk_cml;
350 hwreset_t hwreset_pex;
351 hwreset_t hwreset_afi;
352 hwreset_t hwreset_pcie_x;
353 regulator_t regulators[16]; /* Safe maximum */
354
355 vm_offset_t msi_page; /* VA of MSI page */
356 bus_addr_t cfg_base_addr; /* base address of config */
357 bus_size_t cfg_cur_offs; /* currently mapped window */
358 bus_space_handle_t cfg_handle; /* handle of config window */
359 bus_space_tag_t bus_tag; /* tag of config window */
360 int lanes_cfg;
361 int num_ports;
362 struct tegra_pcib_port *ports[TEGRA_PCIB_MAX_PORTS];
363 struct tegra_pcib_irqsrc *isrcs;
364 };
365
366 static int
tegra_pcib_maxslots(device_t dev)367 tegra_pcib_maxslots(device_t dev)
368 {
369 return (16);
370 }
371
372 static int
tegra_pcib_route_interrupt(device_t bus,device_t dev,int pin)373 tegra_pcib_route_interrupt(device_t bus, device_t dev, int pin)
374 {
375 struct tegra_pcib_softc *sc;
376 u_int irq;
377
378 sc = device_get_softc(bus);
379 irq = intr_map_clone_irq(rman_get_start(sc->irq_res));
380 device_printf(bus, "route pin %d for device %d.%d to %u\n",
381 pin, pci_get_slot(dev), pci_get_function(dev),
382 irq);
383
384 return (irq);
385 }
386
387 static int
tegra_pcbib_map_cfg(struct tegra_pcib_softc * sc,u_int bus,u_int slot,u_int func,u_int reg)388 tegra_pcbib_map_cfg(struct tegra_pcib_softc *sc, u_int bus, u_int slot,
389 u_int func, u_int reg)
390 {
391 bus_size_t offs;
392 int flags, rv;
393
394 offs = sc->cfg_base_addr;
395 offs |= PCI_CFG_BUS(bus) | PCI_CFG_DEV(slot) | PCI_CFG_FUN(func) |
396 PCI_CFG_EXT_REG(reg);
397 if ((sc->cfg_handle != 0) && (sc->cfg_cur_offs == offs))
398 return (0);
399 if (sc->cfg_handle != 0)
400 bus_space_unmap(sc->bus_tag, sc->cfg_handle, 0x800);
401
402 #if defined(BUS_SPACE_MAP_NONPOSTED)
403 flags = BUS_SPACE_MAP_NONPOSTED;
404 #else
405 flags = 0;
406 #endif
407 rv = bus_space_map(sc->bus_tag, offs, 0x800, flags, &sc->cfg_handle);
408 if (rv != 0)
409 device_printf(sc->dev, "Cannot map config space\n");
410 else
411 sc->cfg_cur_offs = offs;
412 return (rv);
413 }
414
415 static uint32_t
tegra_pcib_read_config(device_t dev,u_int bus,u_int slot,u_int func,u_int reg,int bytes)416 tegra_pcib_read_config(device_t dev, u_int bus, u_int slot, u_int func,
417 u_int reg, int bytes)
418 {
419 struct tegra_pcib_softc *sc;
420 bus_space_handle_t hndl;
421 uint32_t off;
422 uint32_t val;
423 int rv, i;
424
425 sc = device_get_softc(dev);
426 if (bus == 0) {
427 if (func != 0)
428 return (0xFFFFFFFF);
429 for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) {
430 if ((sc->ports[i] != NULL) &&
431 (sc->ports[i]->port_idx == slot)) {
432 hndl = sc->ports[i]->cfg_handle;
433 off = reg & 0xFFF;
434 break;
435 }
436 }
437 if (i >= TEGRA_PCIB_MAX_PORTS)
438 return (0xFFFFFFFF);
439 } else {
440 rv = tegra_pcbib_map_cfg(sc, bus, slot, func, reg);
441 if (rv != 0)
442 return (0xFFFFFFFF);
443 hndl = sc->cfg_handle;
444 off = PCI_CFG_BASE_REG(reg);
445 }
446
447 val = bus_space_read_4(sc->bus_tag, hndl, off & ~3);
448 switch (bytes) {
449 case 4:
450 break;
451 case 2:
452 if (off & 3)
453 val >>= 16;
454 val &= 0xffff;
455 break;
456 case 1:
457 val >>= ((off & 3) << 3);
458 val &= 0xff;
459 break;
460 }
461 return val;
462 }
463
464 static void
tegra_pcib_write_config(device_t dev,u_int bus,u_int slot,u_int func,u_int reg,uint32_t val,int bytes)465 tegra_pcib_write_config(device_t dev, u_int bus, u_int slot, u_int func,
466 u_int reg, uint32_t val, int bytes)
467 {
468 struct tegra_pcib_softc *sc;
469 bus_space_handle_t hndl;
470 uint32_t off;
471 uint32_t val2;
472 int rv, i;
473
474 sc = device_get_softc(dev);
475 if (bus == 0) {
476 if (func != 0)
477 return;
478 for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) {
479 if ((sc->ports[i] != NULL) &&
480 (sc->ports[i]->port_idx == slot)) {
481 hndl = sc->ports[i]->cfg_handle;
482 off = reg & 0xFFF;
483 break;
484 }
485 }
486 if (i >= TEGRA_PCIB_MAX_PORTS)
487 return;
488 } else {
489 rv = tegra_pcbib_map_cfg(sc, bus, slot, func, reg);
490 if (rv != 0)
491 return;
492 hndl = sc->cfg_handle;
493 off = PCI_CFG_BASE_REG(reg);
494 }
495
496 switch (bytes) {
497 case 4:
498 bus_space_write_4(sc->bus_tag, hndl, off, val);
499 break;
500 case 2:
501 val2 = bus_space_read_4(sc->bus_tag, hndl, off & ~3);
502 val2 &= ~(0xffff << ((off & 3) << 3));
503 val2 |= ((val & 0xffff) << ((off & 3) << 3));
504 bus_space_write_4(sc->bus_tag, hndl, off & ~3, val2);
505 break;
506 case 1:
507 val2 = bus_space_read_4(sc->bus_tag, hndl, off & ~3);
508 val2 &= ~(0xff << ((off & 3) << 3));
509 val2 |= ((val & 0xff) << ((off & 3) << 3));
510 bus_space_write_4(sc->bus_tag, hndl, off & ~3, val2);
511 break;
512 }
513 }
514
tegra_pci_intr(void * arg)515 static int tegra_pci_intr(void *arg)
516 {
517 struct tegra_pcib_softc *sc = arg;
518 uint32_t code, signature;
519
520 code = bus_read_4(sc->afi_mem_res, AFI_INTR_CODE) & AFI_INTR_CODE_MASK;
521 signature = bus_read_4(sc->afi_mem_res, AFI_INTR_SIGNATURE);
522 bus_write_4(sc->afi_mem_res, AFI_INTR_CODE, 0);
523 if (code == AFI_INTR_CODE_INT_CODE_SM_MSG)
524 return(FILTER_STRAY);
525
526 printf("tegra_pci_intr: code %x sig %x\n", code, signature);
527 return (FILTER_HANDLED);
528 }
529
530 /* -----------------------------------------------------------------------
531 *
532 * PCI MSI interface
533 */
534 static int
tegra_pcib_alloc_msi(device_t pci,device_t child,int count,int maxcount,int * irqs)535 tegra_pcib_alloc_msi(device_t pci, device_t child, int count, int maxcount,
536 int *irqs)
537 {
538 phandle_t msi_parent;
539
540 /* XXXX ofw_bus_msimap() don't works for Tegra DT.
541 ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child), &msi_parent,
542 NULL);
543 */
544 msi_parent = OF_xref_from_node(ofw_bus_get_node(pci));
545 return (intr_alloc_msi(pci, child, msi_parent, count, maxcount,
546 irqs));
547 }
548
549 static int
tegra_pcib_release_msi(device_t pci,device_t child,int count,int * irqs)550 tegra_pcib_release_msi(device_t pci, device_t child, int count, int *irqs)
551 {
552 phandle_t msi_parent;
553
554 /* XXXX ofw_bus_msimap() don't works for Tegra DT.
555 ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child), &msi_parent,
556 NULL);
557 */
558 msi_parent = OF_xref_from_node(ofw_bus_get_node(pci));
559 return (intr_release_msi(pci, child, msi_parent, count, irqs));
560 }
561
562 static int
tegra_pcib_map_msi(device_t pci,device_t child,int irq,uint64_t * addr,uint32_t * data)563 tegra_pcib_map_msi(device_t pci, device_t child, int irq, uint64_t *addr,
564 uint32_t *data)
565 {
566 phandle_t msi_parent;
567
568 /* XXXX ofw_bus_msimap() don't works for Tegra DT.
569 ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child), &msi_parent,
570 NULL);
571 */
572 msi_parent = OF_xref_from_node(ofw_bus_get_node(pci));
573 return (intr_map_msi(pci, child, msi_parent, irq, addr, data));
574 }
575
576 #ifdef TEGRA_PCIB_MSI_ENABLE
577
578 /* --------------------------------------------------------------------------
579 *
580 * Interrupts
581 *
582 */
583
584 static inline void
tegra_pcib_isrc_mask(struct tegra_pcib_softc * sc,struct tegra_pcib_irqsrc * tgi,uint32_t val)585 tegra_pcib_isrc_mask(struct tegra_pcib_softc *sc,
586 struct tegra_pcib_irqsrc *tgi, uint32_t val)
587 {
588 uint32_t reg;
589 int offs, bit;
590
591 offs = tgi->irq / AFI_MSI_INTR_IN_REG;
592 bit = 1 << (tgi->irq % AFI_MSI_INTR_IN_REG);
593
594 if (val != 0)
595 AFI_WR4(sc, AFI_MSI_VEC(offs), bit);
596 reg = AFI_RD4(sc, AFI_MSI_EN_VEC(offs));
597 if (val != 0)
598 reg |= bit;
599 else
600 reg &= ~bit;
601 AFI_WR4(sc, AFI_MSI_EN_VEC(offs), reg);
602 }
603
604 static int
tegra_pcib_msi_intr(void * arg)605 tegra_pcib_msi_intr(void *arg)
606 {
607 u_int irq, i, bit, reg;
608 struct tegra_pcib_softc *sc;
609 struct trapframe *tf;
610 struct tegra_pcib_irqsrc *tgi;
611
612 sc = (struct tegra_pcib_softc *)arg;
613 tf = curthread->td_intr_frame;
614
615 for (i = 0; i < AFI_MSI_REGS; i++) {
616 reg = AFI_RD4(sc, AFI_MSI_VEC(i));
617 /* Handle one vector. */
618 while (reg != 0) {
619 bit = ffs(reg) - 1;
620 /* Send EOI */
621 AFI_WR4(sc, AFI_MSI_VEC(i), 1 << bit);
622 irq = i * AFI_MSI_INTR_IN_REG + bit;
623 tgi = &sc->isrcs[irq];
624 if (intr_isrc_dispatch(&tgi->isrc, tf) != 0) {
625 /* Disable stray. */
626 tegra_pcib_isrc_mask(sc, tgi, 0);
627 device_printf(sc->dev,
628 "Stray irq %u disabled\n", irq);
629 }
630 reg = AFI_RD4(sc, AFI_MSI_VEC(i));
631 }
632 }
633 return (FILTER_HANDLED);
634 }
635
636 static int
tegra_pcib_msi_attach(struct tegra_pcib_softc * sc)637 tegra_pcib_msi_attach(struct tegra_pcib_softc *sc)
638 {
639 int error;
640 uint32_t irq;
641 const char *name;
642
643 sc->isrcs = malloc(sizeof(*sc->isrcs) * TEGRA_PCIB_MAX_MSI, M_DEVBUF,
644 M_WAITOK | M_ZERO);
645
646 name = device_get_nameunit(sc->dev);
647 for (irq = 0; irq < TEGRA_PCIB_MAX_MSI; irq++) {
648 sc->isrcs[irq].irq = irq;
649 error = intr_isrc_register(&sc->isrcs[irq].isrc,
650 sc->dev, 0, "%s,%u", name, irq);
651 if (error != 0)
652 return (error); /* XXX deregister ISRCs */
653 }
654 if (intr_msi_register(sc->dev,
655 OF_xref_from_node(ofw_bus_get_node(sc->dev))) != 0)
656 return (ENXIO);
657
658 return (0);
659 }
660
661 static int
tegra_pcib_msi_detach(struct tegra_pcib_softc * sc)662 tegra_pcib_msi_detach(struct tegra_pcib_softc *sc)
663 {
664
665 /*
666 * There has not been established any procedure yet
667 * how to detach PIC from living system correctly.
668 */
669 device_printf(sc->dev, "%s: not implemented yet\n", __func__);
670 return (EBUSY);
671 }
672
673 static void
tegra_pcib_msi_disable_intr(device_t dev,struct intr_irqsrc * isrc)674 tegra_pcib_msi_disable_intr(device_t dev, struct intr_irqsrc *isrc)
675 {
676 struct tegra_pcib_softc *sc;
677 struct tegra_pcib_irqsrc *tgi;
678
679 sc = device_get_softc(dev);
680 tgi = (struct tegra_pcib_irqsrc *)isrc;
681 tegra_pcib_isrc_mask(sc, tgi, 0);
682 }
683
684 static void
tegra_pcib_msi_enable_intr(device_t dev,struct intr_irqsrc * isrc)685 tegra_pcib_msi_enable_intr(device_t dev, struct intr_irqsrc *isrc)
686 {
687 struct tegra_pcib_softc *sc;
688 struct tegra_pcib_irqsrc *tgi;
689
690 sc = device_get_softc(dev);
691 tgi = (struct tegra_pcib_irqsrc *)isrc;
692 tegra_pcib_isrc_mask(sc, tgi, 1);
693 }
694
695 /* MSI interrupts are edge trigered -> do nothing */
696 static void
tegra_pcib_msi_post_filter(device_t dev,struct intr_irqsrc * isrc)697 tegra_pcib_msi_post_filter(device_t dev, struct intr_irqsrc *isrc)
698 {
699 }
700
701 static void
tegra_pcib_msi_post_ithread(device_t dev,struct intr_irqsrc * isrc)702 tegra_pcib_msi_post_ithread(device_t dev, struct intr_irqsrc *isrc)
703 {
704 }
705
706 static void
tegra_pcib_msi_pre_ithread(device_t dev,struct intr_irqsrc * isrc)707 tegra_pcib_msi_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
708 {
709 }
710
711 static int
tegra_pcib_msi_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)712 tegra_pcib_msi_setup_intr(device_t dev, struct intr_irqsrc *isrc,
713 struct resource *res, struct intr_map_data *data)
714 {
715 if (data == NULL || data->type != INTR_MAP_DATA_MSI)
716 return (ENOTSUP);
717
718 if (isrc->isrc_handlers == 0)
719 tegra_pcib_msi_enable_intr(dev, isrc);
720
721 return (0);
722 }
723
724 static int
tegra_pcib_msi_teardown_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)725 tegra_pcib_msi_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
726 struct resource *res, struct intr_map_data *data)
727 {
728 struct tegra_pcib_softc *sc;
729 struct tegra_pcib_irqsrc *tgi;
730
731 sc = device_get_softc(dev);
732 tgi = (struct tegra_pcib_irqsrc *)isrc;
733
734 if (isrc->isrc_handlers == 0)
735 tegra_pcib_isrc_mask(sc, tgi, 0);
736 return (0);
737 }
738
739 static int
tegra_pcib_msi_alloc_msi(device_t dev,device_t child,int count,int maxcount,device_t * pic,struct intr_irqsrc ** srcs)740 tegra_pcib_msi_alloc_msi(device_t dev, device_t child, int count, int maxcount,
741 device_t *pic, struct intr_irqsrc **srcs)
742 {
743 struct tegra_pcib_softc *sc;
744 int i, irq, end_irq;
745 bool found;
746
747 KASSERT(powerof2(count), ("%s: bad count", __func__));
748 KASSERT(powerof2(maxcount), ("%s: bad maxcount", __func__));
749
750 sc = device_get_softc(dev);
751 mtx_lock(&sc->mtx);
752
753 found = false;
754 for (irq = 0; (irq + count - 1) < TEGRA_PCIB_MAX_MSI; irq++) {
755 /* Start on an aligned interrupt */
756 if ((irq & (maxcount - 1)) != 0)
757 continue;
758
759 /* Assume we found a valid range until shown otherwise */
760 found = true;
761
762 /* Check this range is valid */
763 for (end_irq = irq; end_irq < irq + count; end_irq++) {
764 /* This is already used */
765 if ((sc->isrcs[end_irq].flags & TEGRA_FLAG_MSI_USED) ==
766 TEGRA_FLAG_MSI_USED) {
767 found = false;
768 break;
769 }
770 }
771
772 if (found)
773 break;
774 }
775
776 /* Not enough interrupts were found */
777 if (!found || irq == (TEGRA_PCIB_MAX_MSI - 1)) {
778 mtx_unlock(&sc->mtx);
779 return (ENXIO);
780 }
781
782 for (i = 0; i < count; i++) {
783 /* Mark the interrupt as used */
784 sc->isrcs[irq + i].flags |= TEGRA_FLAG_MSI_USED;
785 }
786 mtx_unlock(&sc->mtx);
787
788 for (i = 0; i < count; i++)
789 srcs[i] = (struct intr_irqsrc *)&sc->isrcs[irq + i];
790 *pic = device_get_parent(dev);
791 return (0);
792 }
793
794 static int
tegra_pcib_msi_release_msi(device_t dev,device_t child,int count,struct intr_irqsrc ** isrc)795 tegra_pcib_msi_release_msi(device_t dev, device_t child, int count,
796 struct intr_irqsrc **isrc)
797 {
798 struct tegra_pcib_softc *sc;
799 struct tegra_pcib_irqsrc *ti;
800 int i;
801
802 sc = device_get_softc(dev);
803 mtx_lock(&sc->mtx);
804 for (i = 0; i < count; i++) {
805 ti = (struct tegra_pcib_irqsrc *)isrc[i];
806
807 KASSERT((ti->flags & TEGRA_FLAG_MSI_USED) == TEGRA_FLAG_MSI_USED,
808 ("%s: Trying to release an unused MSI-X interrupt",
809 __func__));
810
811 ti->flags &= ~TEGRA_FLAG_MSI_USED;
812 }
813 mtx_unlock(&sc->mtx);
814 return (0);
815 }
816
817 static int
tegra_pcib_msi_map_msi(device_t dev,device_t child,struct intr_irqsrc * isrc,uint64_t * addr,uint32_t * data)818 tegra_pcib_msi_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
819 uint64_t *addr, uint32_t *data)
820 {
821 struct tegra_pcib_softc *sc = device_get_softc(dev);
822 struct tegra_pcib_irqsrc *ti = (struct tegra_pcib_irqsrc *)isrc;
823
824 *addr = vtophys(sc->msi_page);
825 *data = ti->irq;
826 return (0);
827 }
828 #endif
829
830 /* ------------------------------------------------------------------- */
831 static bus_size_t
tegra_pcib_pex_ctrl(struct tegra_pcib_softc * sc,int port)832 tegra_pcib_pex_ctrl(struct tegra_pcib_softc *sc, int port)
833 {
834 switch (port) {
835 case 0:
836 return (AFI_PEX0_CTRL);
837 case 1:
838 return (AFI_PEX1_CTRL);
839 case 2:
840 return (AFI_PEX2_CTRL);
841 default:
842 panic("invalid port number: %d\n", port);
843 }
844 }
845
846 static int
tegra_pcib_enable_fdt_resources(struct tegra_pcib_softc * sc)847 tegra_pcib_enable_fdt_resources(struct tegra_pcib_softc *sc)
848 {
849 int i, rv;
850
851 rv = hwreset_assert(sc->hwreset_pcie_x);
852 if (rv != 0) {
853 device_printf(sc->dev, "Cannot assert 'pcie_x' reset\n");
854 return (rv);
855 }
856 rv = hwreset_assert(sc->hwreset_afi);
857 if (rv != 0) {
858 device_printf(sc->dev, "Cannot assert 'afi' reset\n");
859 return (rv);
860 }
861 rv = hwreset_assert(sc->hwreset_pex);
862 if (rv != 0) {
863 device_printf(sc->dev, "Cannot assert 'pex' reset\n");
864 return (rv);
865 }
866
867 tegra_powergate_power_off(TEGRA_POWERGATE_PCX);
868
869 /* Regulators. */
870 for (i = 0; i < nitems(sc->regulators); i++) {
871 if (sc->regulators[i] == NULL)
872 continue;
873 rv = regulator_enable(sc->regulators[i]);
874 if (rv != 0) {
875 device_printf(sc->dev,
876 "Cannot enable '%s' regulator\n",
877 sc->soc->regulator_names[i]);
878 return (rv);
879 }
880 }
881
882 rv = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_PCX,
883 sc->clk_pex, sc->hwreset_pex);
884 if (rv != 0) {
885 device_printf(sc->dev, "Cannot enable 'PCX' powergate\n");
886 return (rv);
887 }
888
889 rv = hwreset_deassert(sc->hwreset_afi);
890 if (rv != 0) {
891 device_printf(sc->dev, "Cannot unreset 'afi' reset\n");
892 return (rv);
893 }
894
895 rv = clk_enable(sc->clk_afi);
896 if (rv != 0) {
897 device_printf(sc->dev, "Cannot enable 'afi' clock\n");
898 return (rv);
899 }
900 if (sc->soc->cml_clk) {
901 rv = clk_enable(sc->clk_cml);
902 if (rv != 0) {
903 device_printf(sc->dev, "Cannot enable 'cml' clock\n");
904 return (rv);
905 }
906 }
907 rv = clk_enable(sc->clk_pll_e);
908 if (rv != 0) {
909 device_printf(sc->dev, "Cannot enable 'pll_e' clock\n");
910 return (rv);
911 }
912
913 return (0);
914 }
915
916 static struct tegra_pcib_port *
tegra_pcib_parse_port(struct tegra_pcib_softc * sc,phandle_t node)917 tegra_pcib_parse_port(struct tegra_pcib_softc *sc, phandle_t node)
918 {
919 struct tegra_pcib_port *port;
920 uint32_t tmp[5];
921 char tmpstr[6];
922 int rv;
923
924 port = malloc(sizeof(struct tegra_pcib_port), M_DEVBUF, M_WAITOK);
925
926 rv = OF_getprop(node, "status", tmpstr, sizeof(tmpstr));
927 if (rv <= 0 || strcmp(tmpstr, "okay") == 0 ||
928 strcmp(tmpstr, "ok") == 0)
929 port->enabled = 1;
930 else
931 port->enabled = 0;
932
933 rv = OF_getencprop(node, "assigned-addresses", tmp, sizeof(tmp));
934 if (rv != sizeof(tmp)) {
935 device_printf(sc->dev, "Cannot parse assigned-address: %d\n",
936 rv);
937 goto fail;
938 }
939 port->rp_base_addr = tmp[2];
940 port->rp_size = tmp[4];
941 port->port_idx = OFW_PCI_PHYS_HI_DEVICE(tmp[0]) - 1;
942 if (port->port_idx >= TEGRA_PCIB_MAX_PORTS) {
943 device_printf(sc->dev, "Invalid port index: %d\n",
944 port->port_idx);
945 goto fail;
946 }
947 /* XXX - TODO:
948 * Implement proper function for parsing pci "reg" property:
949 * - it have PCI bus format
950 * - its relative to matching "assigned-addresses"
951 */
952 rv = OF_getencprop(node, "reg", tmp, sizeof(tmp));
953 if (rv != sizeof(tmp)) {
954 device_printf(sc->dev, "Cannot parse reg: %d\n", rv);
955 goto fail;
956 }
957 port->rp_base_addr += tmp[2];
958
959 rv = OF_getencprop(node, "nvidia,num-lanes", &port->num_lanes,
960 sizeof(port->num_lanes));
961 if (rv != sizeof(port->num_lanes)) {
962 device_printf(sc->dev, "Cannot parse nvidia,num-lanes: %d\n",
963 rv);
964 goto fail;
965 }
966 if (port->num_lanes > 4) {
967 device_printf(sc->dev, "Invalid nvidia,num-lanes: %d\n",
968 port->num_lanes);
969 goto fail;
970 }
971
972 port->afi_pex_ctrl = tegra_pcib_pex_ctrl(sc, port->port_idx);
973 sc->lanes_cfg |= port->num_lanes << (4 * port->port_idx);
974
975 /* Phy. */
976 rv = phy_get_by_ofw_name(sc->dev, node, "pcie-0", &port->phy);
977 if (rv != 0) {
978 device_printf(sc->dev,
979 "Cannot get 'pcie-0' phy for port %d\n",
980 port->port_idx);
981 goto fail;
982 }
983
984 return (port);
985 fail:
986 free(port, M_DEVBUF);
987 return (NULL);
988 }
989
990 static int
tegra_pcib_parse_fdt_resources(struct tegra_pcib_softc * sc,phandle_t node)991 tegra_pcib_parse_fdt_resources(struct tegra_pcib_softc *sc, phandle_t node)
992 {
993 phandle_t child;
994 struct tegra_pcib_port *port;
995 int i, rv;
996
997 /* Regulators. */
998 for (i = 0; sc->soc->regulator_names[i] != NULL; i++) {
999 if (i >= nitems(sc->regulators)) {
1000 device_printf(sc->dev,
1001 "Too many regulators present in DT.\n");
1002 return (EOVERFLOW);
1003 }
1004 rv = regulator_get_by_ofw_property(sc->dev, 0,
1005 sc->soc->regulator_names[i], sc->regulators + i);
1006 if (rv != 0) {
1007 device_printf(sc->dev,
1008 "Cannot get '%s' regulator\n",
1009 sc->soc->regulator_names[i]);
1010 return (ENXIO);
1011 }
1012 }
1013
1014 /* Resets. */
1015 rv = hwreset_get_by_ofw_name(sc->dev, 0, "pex", &sc->hwreset_pex);
1016 if (rv != 0) {
1017 device_printf(sc->dev, "Cannot get 'pex' reset\n");
1018 return (ENXIO);
1019 }
1020 rv = hwreset_get_by_ofw_name(sc->dev, 0, "afi", &sc->hwreset_afi);
1021 if (rv != 0) {
1022 device_printf(sc->dev, "Cannot get 'afi' reset\n");
1023 return (ENXIO);
1024 }
1025 rv = hwreset_get_by_ofw_name(sc->dev, 0, "pcie_x", &sc->hwreset_pcie_x);
1026 if (rv != 0) {
1027 device_printf(sc->dev, "Cannot get 'pcie_x' reset\n");
1028 return (ENXIO);
1029 }
1030
1031 /* Clocks. */
1032 rv = clk_get_by_ofw_name(sc->dev, 0, "pex", &sc->clk_pex);
1033 if (rv != 0) {
1034 device_printf(sc->dev, "Cannot get 'pex' clock\n");
1035 return (ENXIO);
1036 }
1037 rv = clk_get_by_ofw_name(sc->dev, 0, "afi", &sc->clk_afi);
1038 if (rv != 0) {
1039 device_printf(sc->dev, "Cannot get 'afi' clock\n");
1040 return (ENXIO);
1041 }
1042 rv = clk_get_by_ofw_name(sc->dev, 0, "pll_e", &sc->clk_pll_e);
1043 if (rv != 0) {
1044 device_printf(sc->dev, "Cannot get 'pll_e' clock\n");
1045 return (ENXIO);
1046 }
1047 if (sc->soc->cml_clk) {
1048 rv = clk_get_by_ofw_name(sc->dev, 0, "cml", &sc->clk_cml);
1049 if (rv != 0) {
1050 device_printf(sc->dev, "Cannot get 'cml' clock\n");
1051 return (ENXIO);
1052 }
1053 }
1054
1055 /* Ports */
1056 sc->num_ports = 0;
1057 for (child = OF_child(node); child != 0; child = OF_peer(child)) {
1058 port = tegra_pcib_parse_port(sc, child);
1059 if (port == NULL) {
1060 device_printf(sc->dev, "Cannot parse PCIe port node\n");
1061 return (ENXIO);
1062 }
1063 sc->ports[sc->num_ports++] = port;
1064 }
1065
1066 return (0);
1067 }
1068
1069 static int
tegra_pcib_decode_ranges(struct tegra_pcib_softc * sc,struct ofw_pci_range * ranges,int nranges)1070 tegra_pcib_decode_ranges(struct tegra_pcib_softc *sc,
1071 struct ofw_pci_range *ranges, int nranges)
1072 {
1073 int i;
1074
1075 for (i = 2; i < nranges; i++) {
1076 if ((ranges[i].pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) ==
1077 OFW_PCI_PHYS_HI_SPACE_IO) {
1078 if (sc->io_range.size != 0) {
1079 device_printf(sc->dev,
1080 "Duplicated IO range found in DT\n");
1081 return (ENXIO);
1082 }
1083 sc->io_range = ranges[i];
1084 }
1085 if (((ranges[i].pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) ==
1086 OFW_PCI_PHYS_HI_SPACE_MEM32)) {
1087 if (ranges[i].pci_hi & OFW_PCI_PHYS_HI_PREFETCHABLE) {
1088 if (sc->pref_mem_range.size != 0) {
1089 device_printf(sc->dev,
1090 "Duplicated memory range found "
1091 "in DT\n");
1092 return (ENXIO);
1093 }
1094 sc->pref_mem_range = ranges[i];
1095 } else {
1096 if (sc->mem_range.size != 0) {
1097 device_printf(sc->dev,
1098 "Duplicated memory range found "
1099 "in DT\n");
1100 return (ENXIO);
1101 }
1102 sc->mem_range = ranges[i];
1103 }
1104 }
1105 }
1106 if ((sc->io_range.size == 0) || (sc->mem_range.size == 0)
1107 || (sc->pref_mem_range.size == 0)) {
1108 device_printf(sc->dev,
1109 " Not all required ranges are found in DT\n");
1110 return (ENXIO);
1111 }
1112 return (0);
1113 }
1114
1115 /*
1116 * Hardware config.
1117 */
1118 static int
tegra_pcib_wait_for_link(struct tegra_pcib_softc * sc,struct tegra_pcib_port * port)1119 tegra_pcib_wait_for_link(struct tegra_pcib_softc *sc,
1120 struct tegra_pcib_port *port)
1121 {
1122 uint32_t reg;
1123 int i;
1124
1125 /* Setup link detection. */
1126 reg = tegra_pcib_read_config(sc->dev, 0, port->port_idx, 0,
1127 RP_PRIV_MISC, 4);
1128 reg &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT;
1129 reg |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT;
1130 tegra_pcib_write_config(sc->dev, 0, port->port_idx, 0,
1131 RP_PRIV_MISC, reg, 4);
1132
1133 for (i = TEGRA_PCIE_LINKUP_TIMEOUT; i > 0; i--) {
1134 reg = tegra_pcib_read_config(sc->dev, 0, port->port_idx, 0,
1135 RP_VEND_XP, 4);
1136 if (reg & RP_VEND_XP_DL_UP)
1137 break;
1138 DELAY(1);
1139 }
1140 if (i <= 0)
1141 return (ETIMEDOUT);
1142
1143 for (i = TEGRA_PCIE_LINKUP_TIMEOUT; i > 0; i--) {
1144 reg = tegra_pcib_read_config(sc->dev, 0, port->port_idx, 0,
1145 RP_LINK_CONTROL_STATUS, 4);
1146 if (reg & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
1147 break;
1148
1149 DELAY(1);
1150 }
1151 if (i <= 0)
1152 return (ETIMEDOUT);
1153 return (0);
1154 }
1155
1156 static void
tegra_pcib_port_enable(struct tegra_pcib_softc * sc,int port_num)1157 tegra_pcib_port_enable(struct tegra_pcib_softc *sc, int port_num)
1158 {
1159 struct tegra_pcib_port *port;
1160 uint32_t reg;
1161 int rv;
1162
1163 port = sc->ports[port_num];
1164
1165 /* Put port to reset. */
1166 reg = AFI_RD4(sc, port->afi_pex_ctrl);
1167 reg &= ~AFI_PEX_CTRL_RST_L;
1168 AFI_WR4(sc, port->afi_pex_ctrl, reg);
1169 AFI_RD4(sc, port->afi_pex_ctrl);
1170 DELAY(10);
1171
1172 /* Enable clocks. */
1173 reg |= AFI_PEX_CTRL_REFCLK_EN;
1174 reg |= AFI_PEX_CTRL_CLKREQ_EN;
1175 reg |= AFI_PEX_CTRL_OVERRIDE_EN;
1176 AFI_WR4(sc, port->afi_pex_ctrl, reg);
1177 AFI_RD4(sc, port->afi_pex_ctrl);
1178 DELAY(100);
1179
1180 /* Release reset. */
1181 reg |= AFI_PEX_CTRL_RST_L;
1182 AFI_WR4(sc, port->afi_pex_ctrl, reg);
1183
1184 if (sc->soc->pca_enable) {
1185 reg = tegra_pcib_read_config(sc->dev, 0, port->port_idx, 0,
1186 RP_VEND_CTL2, 4);
1187 reg |= RP_VEND_CTL2_PCA_ENABLE;
1188 tegra_pcib_write_config(sc->dev, 0, port->port_idx, 0,
1189 RP_VEND_CTL2, reg, 4);
1190 }
1191
1192 rv = tegra_pcib_wait_for_link(sc, port);
1193 if (bootverbose)
1194 device_printf(sc->dev, " port %d (%d lane%s): Link is %s\n",
1195 port->port_idx, port->num_lanes,
1196 port->num_lanes > 1 ? "s": "",
1197 rv == 0 ? "up": "down");
1198 }
1199
1200 static void
tegra_pcib_port_disable(struct tegra_pcib_softc * sc,uint32_t port_num)1201 tegra_pcib_port_disable(struct tegra_pcib_softc *sc, uint32_t port_num)
1202 {
1203 struct tegra_pcib_port *port;
1204 uint32_t reg;
1205
1206 port = sc->ports[port_num];
1207
1208 /* Put port to reset. */
1209 reg = AFI_RD4(sc, port->afi_pex_ctrl);
1210 reg &= ~AFI_PEX_CTRL_RST_L;
1211 AFI_WR4(sc, port->afi_pex_ctrl, reg);
1212 AFI_RD4(sc, port->afi_pex_ctrl);
1213 DELAY(10);
1214
1215 /* Disable clocks. */
1216 reg &= ~AFI_PEX_CTRL_CLKREQ_EN;
1217 reg &= ~AFI_PEX_CTRL_REFCLK_EN;
1218 AFI_WR4(sc, port->afi_pex_ctrl, reg);
1219
1220 if (bootverbose)
1221 device_printf(sc->dev, " port %d (%d lane%s): Disabled\n",
1222 port->port_idx, port->num_lanes,
1223 port->num_lanes > 1 ? "s": "");
1224 }
1225
1226 static void
tegra_pcib_set_bar(struct tegra_pcib_softc * sc,int bar,uint32_t axi,uint64_t fpci,uint32_t size,int is_memory)1227 tegra_pcib_set_bar(struct tegra_pcib_softc *sc, int bar, uint32_t axi,
1228 uint64_t fpci, uint32_t size, int is_memory)
1229 {
1230 uint32_t fpci_reg;
1231 uint32_t axi_reg;
1232 uint32_t size_reg;
1233
1234 axi_reg = axi & ~0xFFF;
1235 size_reg = size >> 12;
1236 fpci_reg = (uint32_t)(fpci >> 8) & ~0xF;
1237 fpci_reg |= is_memory ? 0x1 : 0x0;
1238 AFI_WR4(sc, bars[bar].axi_start, axi_reg);
1239 AFI_WR4(sc, bars[bar].size, size_reg);
1240 AFI_WR4(sc, bars[bar].fpci_start, fpci_reg);
1241 }
1242
1243 static int
tegra_pcib_enable(struct tegra_pcib_softc * sc)1244 tegra_pcib_enable(struct tegra_pcib_softc *sc)
1245 {
1246 int rv;
1247 int i;
1248 uint32_t reg;
1249
1250 rv = tegra_pcib_enable_fdt_resources(sc);
1251 if (rv != 0) {
1252 device_printf(sc->dev, "Cannot enable FDT resources\n");
1253 return (rv);
1254 }
1255
1256 /* Enable PLLE control. */
1257 reg = AFI_RD4(sc, AFI_PLLE_CONTROL);
1258 reg &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
1259 reg |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
1260 AFI_WR4(sc, AFI_PLLE_CONTROL, reg);
1261
1262 /* Set bias pad. */
1263 AFI_WR4(sc, AFI_PEXBIAS_CTRL, 0);
1264
1265 /* Configure mode and ports. */
1266 reg = AFI_RD4(sc, AFI_PCIE_CONFIG);
1267 reg &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK;
1268 if (sc->lanes_cfg == 0x14) {
1269 if (bootverbose)
1270 device_printf(sc->dev,
1271 "Using x1,x4 configuration\n");
1272 reg |= AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_XBAR4_1;
1273 } else if (sc->lanes_cfg == 0x12) {
1274 if (bootverbose)
1275 device_printf(sc->dev,
1276 "Using x1,x2 configuration\n");
1277 reg |= AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_XBAR2_1;
1278 } else {
1279 device_printf(sc->dev,
1280 "Unsupported lanes configuration: 0x%X\n", sc->lanes_cfg);
1281 }
1282 reg |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL;
1283 for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) {
1284 if ((sc->ports[i] != NULL))
1285 reg &=
1286 ~AFI_PCIE_CONFIG_PCIE_DISABLE(sc->ports[i]->port_idx);
1287 }
1288 AFI_WR4(sc, AFI_PCIE_CONFIG, reg);
1289
1290 /* Enable Gen2 support. */
1291 reg = AFI_RD4(sc, AFI_FUSE);
1292 reg &= ~AFI_FUSE_PCIE_T0_GEN2_DIS;
1293 AFI_WR4(sc, AFI_FUSE, reg);
1294
1295 for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) {
1296 if (sc->ports[i] != NULL) {
1297 rv = phy_enable(sc->ports[i]->phy);
1298 if (rv != 0) {
1299 device_printf(sc->dev,
1300 "Cannot enable phy for port %d\n",
1301 sc->ports[i]->port_idx);
1302 return (rv);
1303 }
1304 }
1305 }
1306
1307 /* Configure PCIe reference clock */
1308 PADS_WR4(sc, PADS_REFCLK_CFG0, sc->soc->pads_refclk_cfg0);
1309 if (sc->num_ports > 2)
1310 PADS_WR4(sc, PADS_REFCLK_CFG1, sc->soc->pads_refclk_cfg1);
1311
1312 rv = hwreset_deassert(sc->hwreset_pcie_x);
1313 if (rv != 0) {
1314 device_printf(sc->dev, "Cannot unreset 'pci_x' reset\n");
1315 return (rv);
1316 }
1317
1318 /* Enable config space. */
1319 reg = AFI_RD4(sc, AFI_CONFIGURATION);
1320 reg |= AFI_CONFIGURATION_EN_FPCI;
1321 AFI_WR4(sc, AFI_CONFIGURATION, reg);
1322
1323 /* Enable AFI errors. */
1324 reg = 0;
1325 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_INI_SLVERR);
1326 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_INI_DECERR);
1327 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_TGT_SLVERR);
1328 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_TGT_DECERR);
1329 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_TGT_WRERR);
1330 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_SM_MSG);
1331 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_DFPCI_DECERR);
1332 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_AXI_DECERR);
1333 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_FPCI_TIMEOUT);
1334 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_PE_PRSNT_SENSE);
1335 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_PE_CLKREQ_SENSE);
1336 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_CLKCLAMP_SENSE);
1337 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_RDY4PD_SENSE);
1338 reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_P2P_ERROR);
1339 AFI_WR4(sc, AFI_AFI_INTR_ENABLE, reg);
1340 AFI_WR4(sc, AFI_SM_INTR_ENABLE, 0xffffffff);
1341
1342 /* Enable INT, disable MSI. */
1343 AFI_WR4(sc, AFI_INTR_MASK, AFI_INTR_MASK_INT_MASK);
1344
1345 /* Mask all FPCI errors. */
1346 AFI_WR4(sc, AFI_FPCI_ERROR_MASKS, 0);
1347
1348 /* Setup AFI translation windows. */
1349 /* BAR 0 - type 1 extended configuration. */
1350 tegra_pcib_set_bar(sc, 0, rman_get_start(sc->cfg_mem_res),
1351 FPCI_MAP_EXT_TYPE1_CONFIG, rman_get_size(sc->cfg_mem_res), 0);
1352
1353 /* BAR 1 - downstream I/O. */
1354 tegra_pcib_set_bar(sc, 1, sc->io_range.host, FPCI_MAP_IO,
1355 sc->io_range.size, 0);
1356
1357 /* BAR 2 - downstream prefetchable memory 1:1. */
1358 tegra_pcib_set_bar(sc, 2, sc->pref_mem_range.host,
1359 sc->pref_mem_range.host, sc->pref_mem_range.size, 1);
1360
1361 /* BAR 3 - downstream not prefetchable memory 1:1 .*/
1362 tegra_pcib_set_bar(sc, 3, sc->mem_range.host,
1363 sc->mem_range.host, sc->mem_range.size, 1);
1364
1365 /* BAR 3-8 clear. */
1366 tegra_pcib_set_bar(sc, 4, 0, 0, 0, 0);
1367 tegra_pcib_set_bar(sc, 5, 0, 0, 0, 0);
1368 tegra_pcib_set_bar(sc, 6, 0, 0, 0, 0);
1369 tegra_pcib_set_bar(sc, 7, 0, 0, 0, 0);
1370 tegra_pcib_set_bar(sc, 8, 0, 0, 0, 0);
1371
1372 /* MSI BAR - clear. */
1373 tegra_pcib_set_bar(sc, 9, 0, 0, 0, 0);
1374 return(0);
1375 }
1376
1377 #ifdef TEGRA_PCIB_MSI_ENABLE
1378 static int
tegra_pcib_attach_msi(device_t dev)1379 tegra_pcib_attach_msi(device_t dev)
1380 {
1381 struct tegra_pcib_softc *sc;
1382 uint32_t reg;
1383 int i, rv;
1384
1385 sc = device_get_softc(dev);
1386
1387 sc->msi_page = (uintptr_t)kmem_alloc_contig(PAGE_SIZE, M_WAITOK, 0,
1388 BUS_SPACE_MAXADDR, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
1389
1390 /* MSI BAR */
1391 tegra_pcib_set_bar(sc, 9, vtophys(sc->msi_page), vtophys(sc->msi_page),
1392 PAGE_SIZE, 0);
1393
1394 /* Disable and clear all interrupts. */
1395 for (i = 0; i < AFI_MSI_REGS; i++) {
1396 AFI_WR4(sc, AFI_MSI_EN_VEC(i), 0);
1397 AFI_WR4(sc, AFI_MSI_VEC(i), 0xFFFFFFFF);
1398 }
1399 rv = bus_setup_intr(dev, sc->msi_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
1400 tegra_pcib_msi_intr, NULL, sc, &sc->msi_intr_cookie);
1401 if (rv != 0) {
1402 device_printf(dev, "cannot setup MSI interrupt handler\n");
1403 rv = ENXIO;
1404 goto out;
1405 }
1406
1407 if (tegra_pcib_msi_attach(sc) != 0) {
1408 device_printf(dev, "WARNING: unable to attach PIC\n");
1409 tegra_pcib_msi_detach(sc);
1410 goto out;
1411 }
1412
1413 /* Unmask MSI interrupt. */
1414 reg = AFI_RD4(sc, AFI_INTR_MASK);
1415 reg |= AFI_INTR_MASK_MSI_MASK;
1416 AFI_WR4(sc, AFI_INTR_MASK, reg);
1417
1418 out:
1419 return (rv);
1420 }
1421 #endif
1422
1423 static int
tegra_pcib_probe(device_t dev)1424 tegra_pcib_probe(device_t dev)
1425 {
1426 if (!ofw_bus_status_okay(dev))
1427 return (ENXIO);
1428
1429 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data != 0) {
1430 device_set_desc(dev, "Nvidia Integrated PCI/PCI-E Controller");
1431 return (BUS_PROBE_DEFAULT);
1432 }
1433 return (ENXIO);
1434 }
1435
1436 static int
tegra_pcib_attach(device_t dev)1437 tegra_pcib_attach(device_t dev)
1438 {
1439 struct tegra_pcib_softc *sc;
1440 phandle_t node;
1441 int rv;
1442 int rid;
1443 struct tegra_pcib_port *port;
1444 int i;
1445
1446 sc = device_get_softc(dev);
1447 sc->dev = dev;
1448 mtx_init(&sc->mtx, "msi_mtx", NULL, MTX_DEF);
1449
1450 node = ofw_bus_get_node(dev);
1451 sc->soc = (struct pcie_soc *)ofw_bus_search_compatible(dev,
1452 compat_data)->ocd_data;
1453
1454 rv = tegra_pcib_parse_fdt_resources(sc, node);
1455 if (rv != 0) {
1456 device_printf(dev, "Cannot get FDT resources\n");
1457 return (rv);
1458 }
1459
1460 /* Allocate bus_space resources. */
1461 rid = 0;
1462 sc->pads_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1463 RF_ACTIVE);
1464 if (sc->pads_mem_res == NULL) {
1465 device_printf(dev, "Cannot allocate PADS register\n");
1466 rv = ENXIO;
1467 goto out;
1468 }
1469 /*
1470 * XXX - FIXME
1471 * tag for config space is not filled when RF_ALLOCATED flag is used.
1472 */
1473 sc->bus_tag = rman_get_bustag(sc->pads_mem_res);
1474
1475 rid = 1;
1476 sc->afi_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1477 RF_ACTIVE);
1478 if (sc->afi_mem_res == NULL) {
1479 device_printf(dev, "Cannot allocate AFI register\n");
1480 rv = ENXIO;
1481 goto out;
1482 }
1483
1484 rid = 2;
1485 sc->cfg_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1486 RF_ALLOCATED);
1487 if (sc->cfg_mem_res == NULL) {
1488 device_printf(dev, "Cannot allocate config space memory\n");
1489 rv = ENXIO;
1490 goto out;
1491 }
1492 sc->cfg_base_addr = rman_get_start(sc->cfg_mem_res);
1493
1494 /* Map RP slots */
1495 for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) {
1496 if (sc->ports[i] == NULL)
1497 continue;
1498 port = sc->ports[i];
1499 rv = bus_space_map(sc->bus_tag, port->rp_base_addr,
1500 port->rp_size, 0, &port->cfg_handle);
1501 if (rv != 0) {
1502 device_printf(sc->dev, "Cannot allocate memory for "
1503 "port: %d\n", i);
1504 rv = ENXIO;
1505 goto out;
1506 }
1507 }
1508
1509 /*
1510 * Get PCI interrupt
1511 */
1512 rid = 0;
1513 sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1514 RF_ACTIVE | RF_SHAREABLE);
1515 if (sc->irq_res == NULL) {
1516 device_printf(dev, "Cannot allocate IRQ resources\n");
1517 rv = ENXIO;
1518 goto out;
1519 }
1520
1521 rid = 1;
1522 sc->msi_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1523 RF_ACTIVE);
1524 if (sc->irq_res == NULL) {
1525 device_printf(dev, "Cannot allocate MSI IRQ resources\n");
1526 rv = ENXIO;
1527 goto out;
1528 }
1529
1530 sc->ofw_pci.sc_range_mask = 0x3;
1531 rv = ofw_pcib_init(dev);
1532 if (rv != 0)
1533 goto out;
1534
1535 rv = tegra_pcib_decode_ranges(sc, sc->ofw_pci.sc_range,
1536 sc->ofw_pci.sc_nrange);
1537 if (rv != 0)
1538 goto out;
1539
1540 if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
1541 tegra_pci_intr, NULL, sc, &sc->intr_cookie)) {
1542 device_printf(dev, "cannot setup interrupt handler\n");
1543 rv = ENXIO;
1544 goto out;
1545 }
1546
1547 /*
1548 * Enable PCIE device.
1549 */
1550 rv = tegra_pcib_enable(sc);
1551 if (rv != 0)
1552 goto out;
1553 for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) {
1554 if (sc->ports[i] == NULL)
1555 continue;
1556 if (sc->ports[i]->enabled)
1557 tegra_pcib_port_enable(sc, i);
1558 else
1559 tegra_pcib_port_disable(sc, i);
1560 }
1561
1562 #ifdef TEGRA_PCIB_MSI_ENABLE
1563 rv = tegra_pcib_attach_msi(dev);
1564 if (rv != 0)
1565 goto out;
1566 #endif
1567 device_add_child(dev, "pci", DEVICE_UNIT_ANY);
1568 bus_attach_children(dev);
1569
1570 return (0);
1571
1572 out:
1573
1574 return (rv);
1575 }
1576
1577 static device_method_t tegra_pcib_methods[] = {
1578 /* Device interface */
1579 DEVMETHOD(device_probe, tegra_pcib_probe),
1580 DEVMETHOD(device_attach, tegra_pcib_attach),
1581
1582 /* Bus interface */
1583 DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
1584 DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
1585
1586 /* pcib interface */
1587 DEVMETHOD(pcib_maxslots, tegra_pcib_maxslots),
1588 DEVMETHOD(pcib_read_config, tegra_pcib_read_config),
1589 DEVMETHOD(pcib_write_config, tegra_pcib_write_config),
1590 DEVMETHOD(pcib_route_interrupt, tegra_pcib_route_interrupt),
1591 DEVMETHOD(pcib_alloc_msi, tegra_pcib_alloc_msi),
1592 DEVMETHOD(pcib_release_msi, tegra_pcib_release_msi),
1593 DEVMETHOD(pcib_map_msi, tegra_pcib_map_msi),
1594 DEVMETHOD(pcib_request_feature, pcib_request_feature_allow),
1595
1596 #ifdef TEGRA_PCIB_MSI_ENABLE
1597 /* MSI/MSI-X */
1598 DEVMETHOD(msi_alloc_msi, tegra_pcib_msi_alloc_msi),
1599 DEVMETHOD(msi_release_msi, tegra_pcib_msi_release_msi),
1600 DEVMETHOD(msi_map_msi, tegra_pcib_msi_map_msi),
1601
1602 /* Interrupt controller interface */
1603 DEVMETHOD(pic_disable_intr, tegra_pcib_msi_disable_intr),
1604 DEVMETHOD(pic_enable_intr, tegra_pcib_msi_enable_intr),
1605 DEVMETHOD(pic_setup_intr, tegra_pcib_msi_setup_intr),
1606 DEVMETHOD(pic_teardown_intr, tegra_pcib_msi_teardown_intr),
1607 DEVMETHOD(pic_post_filter, tegra_pcib_msi_post_filter),
1608 DEVMETHOD(pic_post_ithread, tegra_pcib_msi_post_ithread),
1609 DEVMETHOD(pic_pre_ithread, tegra_pcib_msi_pre_ithread),
1610 #endif
1611
1612 /* OFW bus interface */
1613 DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat),
1614 DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model),
1615 DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name),
1616 DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node),
1617 DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type),
1618
1619 DEVMETHOD_END
1620 };
1621
1622 DEFINE_CLASS_1(pcib, tegra_pcib_driver, tegra_pcib_methods,
1623 sizeof(struct tegra_pcib_softc), ofw_pcib_driver);
1624 DRIVER_MODULE(tegra_pcib, simplebus, tegra_pcib_driver, NULL, NULL);
1625