1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2022 Scott Long
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "opt_thunderbolt.h"
30
31 /* PCIe interface for Thunderbolt Native Host Interface */
32 #include <sys/types.h>
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/bus.h>
38 #include <sys/conf.h>
39 #include <sys/malloc.h>
40 #include <sys/sysctl.h>
41 #include <sys/lock.h>
42 #include <sys/param.h>
43 #include <sys/endian.h>
44 #include <sys/taskqueue.h>
45
46 #include <machine/bus.h>
47 #include <machine/resource.h>
48 #include <machine/stdarg.h>
49 #include <sys/rman.h>
50
51 #include <dev/pci/pcireg.h>
52 #include <dev/pci/pcivar.h>
53 #include <dev/pci/pci_private.h>
54
55 #include <dev/thunderbolt/tb_reg.h>
56 #include <dev/thunderbolt/nhi_reg.h>
57 #include <dev/thunderbolt/nhi_var.h>
58 #include <dev/thunderbolt/tbcfg_reg.h>
59 #include <dev/thunderbolt/router_var.h>
60 #include <dev/thunderbolt/tb_debug.h>
61 #include "tb_if.h"
62
63 static int nhi_pci_probe(device_t);
64 static int nhi_pci_attach(device_t);
65 static int nhi_pci_detach(device_t);
66 static int nhi_pci_suspend(device_t);
67 static int nhi_pci_resume(device_t);
68 static void nhi_pci_free(struct nhi_softc *);
69 static int nhi_pci_allocate_interrupts(struct nhi_softc *);
70 static void nhi_pci_free_resources(struct nhi_softc *);
71
72 static device_method_t nhi_methods[] = {
73 DEVMETHOD(device_probe, nhi_pci_probe),
74 DEVMETHOD(device_attach, nhi_pci_attach),
75 DEVMETHOD(device_detach, nhi_pci_detach),
76 DEVMETHOD(device_suspend, nhi_pci_suspend),
77 DEVMETHOD(device_resume, nhi_pci_resume),
78
79 DEVMETHOD(tb_find_ufp, tb_generic_find_ufp),
80 DEVMETHOD(tb_get_debug, tb_generic_get_debug),
81
82 DEVMETHOD_END
83 };
84
85 static driver_t nhi_pci_driver = {
86 "nhi",
87 nhi_methods,
88 sizeof(struct nhi_softc)
89 };
90
91 DRIVER_MODULE_ORDERED(nhi, pci, nhi_pci_driver, NULL, NULL,
92 SI_ORDER_ANY);
93
94 static int
nhi_pci_probe(device_t dev)95 nhi_pci_probe(device_t dev)
96 {
97 if (resource_disabled("tb", 0))
98 return (ENXIO);
99 if ((pci_get_class(dev) == PCIC_SERIALBUS)
100 && (pci_get_subclass(dev) == PCIS_SERIALBUS_USB)
101 && (pci_get_progif(dev) == PCIP_SERIALBUS_USB_USB4)) {
102 device_set_desc(dev, "Generic USB4 NHI");
103 return (BUS_PROBE_DEFAULT);
104 }
105 return (ENXIO);
106 }
107
108 static int
nhi_pci_attach(device_t dev)109 nhi_pci_attach(device_t dev)
110 {
111 devclass_t dc;
112 bus_dma_template_t t;
113 struct nhi_softc *sc;
114 int error = 0;
115
116 sc = device_get_softc(dev);
117 bzero(sc, sizeof(*sc));
118 sc->dev = dev;
119 sc->hwflags = NHI_TYPE_USB4;
120 nhi_get_tunables(sc);
121
122 tb_debug(sc, DBG_INIT|DBG_FULL, "busmaster status was %s\n",
123 (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_BUSMASTEREN)
124 ? "enabled" : "disabled");
125 pci_enable_busmaster(dev);
126
127 sc->ufp = NULL;
128 if ((TB_FIND_UFP(dev, &sc->ufp) != 0) || (sc->ufp == NULL)) {
129 dc = devclass_find("tbolt");
130 if (dc != NULL)
131 sc->ufp = devclass_get_device(dc, device_get_unit(dev));
132 }
133 if (sc->ufp == NULL)
134 tb_printf(sc, "Cannot find Upstream Facing Port\n");
135 else
136 tb_printf(sc, "Upstream Facing Port is %s\n",
137 device_get_nameunit(sc->ufp));
138
139 /* Allocate BAR0 DMA registers */
140 sc->regs_rid = PCIR_BAR(0);
141 if ((sc->regs_resource = bus_alloc_resource_any(dev,
142 SYS_RES_MEMORY, &sc->regs_rid, RF_ACTIVE)) == NULL) {
143 tb_printf(sc, "Cannot allocate PCI registers\n");
144 return (ENXIO);
145 }
146 sc->regs_btag = rman_get_bustag(sc->regs_resource);
147 sc->regs_bhandle = rman_get_bushandle(sc->regs_resource);
148
149 /* Allocate parent DMA tag */
150 bus_dma_template_init(&t, bus_get_dma_tag(dev));
151 if (bus_dma_template_tag(&t, &sc->parent_dmat) != 0) {
152 tb_printf(sc, "Cannot allocate parent DMA tag\n");
153 nhi_pci_free(sc);
154 return (ENOMEM);
155 }
156
157 error = nhi_pci_allocate_interrupts(sc);
158 if (error == 0)
159 error = nhi_attach(sc);
160 if (error != 0)
161 nhi_pci_detach(sc->dev);
162 return (error);
163 }
164
165 static int
nhi_pci_detach(device_t dev)166 nhi_pci_detach(device_t dev)
167 {
168 struct nhi_softc *sc;
169
170 sc = device_get_softc(dev);
171
172 nhi_detach(sc);
173 nhi_pci_free(sc);
174
175 return (0);
176 }
177
178 static int
nhi_pci_suspend(device_t dev)179 nhi_pci_suspend(device_t dev)
180 {
181
182 return (0);
183 }
184
185 static int
nhi_pci_resume(device_t dev)186 nhi_pci_resume(device_t dev)
187 {
188
189 return (0);
190 }
191
192 static void
nhi_pci_free(struct nhi_softc * sc)193 nhi_pci_free(struct nhi_softc *sc)
194 {
195
196 nhi_pci_free_resources(sc);
197
198 if (sc->parent_dmat != NULL) {
199 bus_dma_tag_destroy(sc->parent_dmat);
200 sc->parent_dmat = NULL;
201 }
202
203 if (sc->regs_resource != NULL) {
204 bus_release_resource(sc->dev, SYS_RES_MEMORY,
205 sc->regs_rid, sc->regs_resource);
206 sc->regs_resource = NULL;
207 }
208
209 return;
210 }
211
212 static int
nhi_pci_allocate_interrupts(struct nhi_softc * sc)213 nhi_pci_allocate_interrupts(struct nhi_softc *sc)
214 {
215 int msgs, error = 0;
216
217 /* Map the Pending Bit Array and Vector Table BARs for MSI-X */
218 sc->irq_pba_rid = pci_msix_pba_bar(sc->dev);
219 sc->irq_table_rid = pci_msix_table_bar(sc->dev);
220
221 if (sc->irq_pba_rid != -1)
222 sc->irq_pba = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
223 &sc->irq_pba_rid, RF_ACTIVE);
224 if (sc->irq_table_rid != -1)
225 sc->irq_table = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
226 &sc->irq_table_rid, RF_ACTIVE);
227
228 msgs = pci_msix_count(sc->dev);
229 tb_debug(sc, DBG_INIT|DBG_INTR|DBG_FULL,
230 "Counted %d MSI-X messages\n", msgs);
231 msgs = min(msgs, NHI_MSIX_MAX);
232 msgs = max(msgs, 1);
233 if (msgs != 0) {
234 tb_debug(sc, DBG_INIT|DBG_INTR, "Attempting to allocate %d "
235 "MSI-X interrupts\n", msgs);
236 error = pci_alloc_msix(sc->dev, &msgs);
237 tb_debug(sc, DBG_INIT|DBG_INTR|DBG_FULL,
238 "pci_alloc_msix return msgs= %d, error= %d\n", msgs, error);
239 }
240
241 if ((error != 0) || (msgs <= 0)) {
242 tb_printf(sc, "Failed to allocate any interrupts\n");
243 msgs = 0;
244 }
245
246 sc->msix_count = msgs;
247 return (error);
248 }
249
250 void
nhi_pci_free_interrupts(struct nhi_softc * sc)251 nhi_pci_free_interrupts(struct nhi_softc *sc)
252 {
253 int i;
254
255 for (i = 0; i < sc->msix_count; i++) {
256 bus_teardown_intr(sc->dev, sc->irqs[i], sc->intrhand[i]);
257 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid[i],
258 sc->irqs[i]);
259 }
260
261 pci_release_msi(sc->dev);
262 }
263
264 static void
nhi_pci_free_resources(struct nhi_softc * sc)265 nhi_pci_free_resources(struct nhi_softc *sc)
266 {
267 if (sc->irq_table != NULL) {
268 bus_release_resource(sc->dev, SYS_RES_MEMORY,
269 sc->irq_table_rid, sc->irq_table);
270 sc->irq_table = NULL;
271 }
272
273 if (sc->irq_pba != NULL) {
274 bus_release_resource(sc->dev, SYS_RES_MEMORY,
275 sc->irq_pba_rid, sc->irq_pba);
276 sc->irq_pba = NULL;
277 }
278
279 if (sc->intr_trackers != NULL)
280 free(sc->intr_trackers, M_NHI);
281 return;
282 }
283
284 int
nhi_pci_configure_interrupts(struct nhi_softc * sc)285 nhi_pci_configure_interrupts(struct nhi_softc *sc)
286 {
287 struct nhi_intr_tracker *trkr;
288 int rid, i, error;
289
290 nhi_pci_disable_interrupts(sc);
291
292 sc->intr_trackers = malloc(sizeof(struct nhi_intr_tracker) *
293 sc->msix_count, M_NHI, M_ZERO | M_NOWAIT);
294 if (sc->intr_trackers == NULL) {
295 tb_debug(sc, DBG_INIT, "Cannot allocate intr trackers\n");
296 return (ENOMEM);
297 }
298
299 for (i = 0; i < sc->msix_count; i++) {
300 rid = i + 1;
301 trkr = &sc->intr_trackers[i];
302 trkr->sc = sc;
303 trkr->ring = NULL;
304 trkr->vector = i;
305
306 sc->irq_rid[i] = rid;
307 sc->irqs[i] = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
308 &sc->irq_rid[i], RF_ACTIVE);
309 if (sc->irqs[i] == NULL) {
310 tb_debug(sc, DBG_INIT,
311 "Cannot allocate interrupt RID %d\n",
312 sc->irq_rid[i]);
313 break;
314 }
315 error = bus_setup_intr(sc->dev, sc->irqs[i], INTR_TYPE_BIO |
316 INTR_MPSAFE, NULL, nhi_intr, trkr, &sc->intrhand[i]);
317 if (error) {
318 tb_debug(sc, DBG_INIT,
319 "cannot setup interrupt RID %d\n", sc->irq_rid[i]);
320 break;
321 }
322 }
323
324 tb_debug(sc, DBG_INIT, "Set up %d interrupts\n", sc->msix_count);
325
326 /* Set the interrupt throttle rate to 128us */
327 for (i = 0; i < 16; i ++)
328 nhi_write_reg(sc, NHI_ITR0 + i * 4, 0x1f4);
329
330 return (error);
331 }
332
333 #define NHI_SET_INTERRUPT(offset, mask, val) \
334 do { \
335 reg = offset / 32; \
336 offset %= 32; \
337 ivr[reg] &= ~(mask << offset); \
338 ivr[reg] |= (val << offset); \
339 } while (0)
340
341 void
nhi_pci_enable_interrupt(struct nhi_ring_pair * r)342 nhi_pci_enable_interrupt(struct nhi_ring_pair *r)
343 {
344 struct nhi_softc *sc = r->sc;
345 uint32_t ivr[5];
346 u_int offset, reg;
347
348 tb_debug(sc, DBG_INIT|DBG_INTR, "Enabling interrupts for ring %d\n",
349 r->ring_num);
350 /*
351 * Compute the routing between event type and MSI-X vector.
352 * 4 bits per descriptor.
353 */
354 ivr[0] = nhi_read_reg(sc, NHI_IVR0);
355 ivr[1] = nhi_read_reg(sc, NHI_IVR1);
356 ivr[2] = nhi_read_reg(sc, NHI_IVR2);
357 ivr[3] = nhi_read_reg(sc, NHI_IVR3);
358 ivr[4] = nhi_read_reg(sc, NHI_IVR4);
359
360 /* Program TX */
361 offset = (r->ring_num + IVR_TX_OFFSET) * 4;
362 NHI_SET_INTERRUPT(offset, 0x0f, r->ring_num);
363
364 /* Now program RX */
365 offset = (r->ring_num + IVR_RX_OFFSET) * 4;
366 NHI_SET_INTERRUPT(offset, 0x0f, r->ring_num);
367
368 /* Last, program Nearly Empty. This one always going to vector 15 */
369 offset = (r->ring_num + IVR_NE_OFFSET) * 4;
370 NHI_SET_INTERRUPT(offset, 0x0f, 0x0f);
371
372 nhi_write_reg(sc, NHI_IVR0, ivr[0]);
373 nhi_write_reg(sc, NHI_IVR1, ivr[1]);
374 nhi_write_reg(sc, NHI_IVR2, ivr[2]);
375 nhi_write_reg(sc, NHI_IVR3, ivr[3]);
376 nhi_write_reg(sc, NHI_IVR4, ivr[4]);
377
378 tb_debug(sc, DBG_INIT|DBG_INTR|DBG_FULL,
379 "Wrote IVR 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
380 ivr[0], ivr[1], ivr[2], ivr[3], ivr[4]);
381
382 /* Now do the Interrupt Mask Register, 1 bit per descriptor */
383 ivr[0] = nhi_read_reg(sc, NHI_IMR0);
384 ivr[1] = nhi_read_reg(sc, NHI_IMR1);
385
386 /* Tx */
387 offset = r->ring_num + IMR_TX_OFFSET;
388 NHI_SET_INTERRUPT(offset, 0x01, 1);
389
390 /* Rx */
391 offset = r->ring_num + IMR_RX_OFFSET;
392 NHI_SET_INTERRUPT(offset, 0x01, 1);
393
394 /* NE */
395 offset = r->ring_num + IMR_NE_OFFSET;
396 NHI_SET_INTERRUPT(offset, 0x01, 1);
397
398 nhi_write_reg(sc, NHI_IMR0, ivr[0]);
399 nhi_write_reg(sc, NHI_IMR1, ivr[1]);
400 tb_debug(sc, DBG_INIT|DBG_FULL,
401 "Wrote IMR 0x%08x 0x%08x\n", ivr[0], ivr[1]);
402 }
403
404 void
nhi_pci_disable_interrupts(struct nhi_softc * sc)405 nhi_pci_disable_interrupts(struct nhi_softc *sc)
406 {
407
408 tb_debug(sc, DBG_INIT, "Disabling interrupts\n");
409 nhi_write_reg(sc, NHI_IMR0, 0);
410 nhi_write_reg(sc, NHI_IMR1, 0);
411 nhi_write_reg(sc, NHI_IVR0, 0);
412 nhi_write_reg(sc, NHI_IVR1, 0);
413 nhi_write_reg(sc, NHI_IVR2, 0);
414 nhi_write_reg(sc, NHI_IVR3, 0);
415 nhi_write_reg(sc, NHI_IVR4, 0);
416
417 /* Dummy reads to clear pending bits */
418 nhi_read_reg(sc, NHI_ISR0);
419 nhi_read_reg(sc, NHI_ISR1);
420 }
421