1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2022 Scott Long 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_thunderbolt.h" 30 31 /* PCIe interface for Thunderbolt Native Host Interface */ 32 #include <sys/types.h> 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/bus.h> 38 #include <sys/conf.h> 39 #include <sys/malloc.h> 40 #include <sys/sysctl.h> 41 #include <sys/lock.h> 42 #include <sys/param.h> 43 #include <sys/endian.h> 44 #include <sys/taskqueue.h> 45 46 #include <machine/bus.h> 47 #include <machine/resource.h> 48 #include <machine/stdarg.h> 49 #include <sys/rman.h> 50 51 #include <dev/pci/pcireg.h> 52 #include <dev/pci/pcivar.h> 53 #include <dev/pci/pci_private.h> 54 55 #include <dev/thunderbolt/tb_reg.h> 56 #include <dev/thunderbolt/nhi_reg.h> 57 #include <dev/thunderbolt/nhi_var.h> 58 #include <dev/thunderbolt/tbcfg_reg.h> 59 #include <dev/thunderbolt/router_var.h> 60 #include <dev/thunderbolt/tb_debug.h> 61 #include "tb_if.h" 62 63 static int nhi_pci_probe(device_t); 64 static int nhi_pci_attach(device_t); 65 static int nhi_pci_detach(device_t); 66 static int nhi_pci_suspend(device_t); 67 static int nhi_pci_resume(device_t); 68 static void nhi_pci_free(struct nhi_softc *); 69 static int nhi_pci_allocate_interrupts(struct nhi_softc *); 70 static void nhi_pci_free_resources(struct nhi_softc *); 71 static int nhi_pci_icl_poweron(struct nhi_softc *); 72 73 static device_method_t nhi_methods[] = { 74 DEVMETHOD(device_probe, nhi_pci_probe), 75 DEVMETHOD(device_attach, nhi_pci_attach), 76 DEVMETHOD(device_detach, nhi_pci_detach), 77 DEVMETHOD(device_suspend, nhi_pci_suspend), 78 DEVMETHOD(device_resume, nhi_pci_resume), 79 80 DEVMETHOD(tb_find_ufp, tb_generic_find_ufp), 81 DEVMETHOD(tb_get_debug, tb_generic_get_debug), 82 83 DEVMETHOD_END 84 }; 85 86 static driver_t nhi_pci_driver = { 87 "nhi", 88 nhi_methods, 89 sizeof(struct nhi_softc) 90 }; 91 92 struct nhi_ident { 93 uint16_t vendor; 94 uint16_t device; 95 uint16_t subvendor; 96 uint16_t subdevice; 97 uint32_t flags; 98 const char *desc; 99 } nhi_identifiers[] = { 100 { VENDOR_INTEL, DEVICE_AR_2C_NHI, 0xffff, 0xffff, NHI_TYPE_AR, 101 "Thunderbolt 3 NHI (Alpine Ridge 2C)" }, 102 { VENDOR_INTEL, DEVICE_AR_DP_B_NHI, 0xffff, 0xffff, NHI_TYPE_AR, 103 "Thunderbolt 3 NHI (Alpine Ridge 4C Rev B)" }, 104 { VENDOR_INTEL, DEVICE_AR_DP_C_NHI, 0xffff, 0xffff, NHI_TYPE_AR, 105 "Thunderbolt 3 NHI (Alpine Ridge 4C Rev C)" }, 106 { VENDOR_INTEL, DEVICE_AR_LP_NHI, 0xffff, 0xffff, NHI_TYPE_AR, 107 "Thunderbolt 3 NHI (Alpine Ridge LP 2C)" }, 108 { VENDOR_INTEL, DEVICE_ICL_NHI_0, 0xffff, 0xffff, NHI_TYPE_ICL, 109 "Thunderbolt 3 NHI Port 0 (IceLake)" }, 110 { VENDOR_INTEL, DEVICE_ICL_NHI_1, 0xffff, 0xffff, NHI_TYPE_ICL, 111 "Thunderbolt 3 NHI Port 1 (IceLake)" }, 112 { VENDOR_AMD, DEVICE_PINK_SARDINE_0, 0xffff, 0xffff, NHI_TYPE_USB4, 113 "USB4 NHI Port 0 (Pink Sardine)" }, 114 { VENDOR_AMD, DEVICE_PINK_SARDINE_1, 0xffff, 0xffff, NHI_TYPE_USB4, 115 "USB4 NHI Port 1 (Pink Sardine)" }, 116 { 0, 0, 0, 0, 0, NULL } 117 }; 118 119 DRIVER_MODULE_ORDERED(nhi, pci, nhi_pci_driver, NULL, NULL, 120 SI_ORDER_ANY); 121 122 static struct nhi_ident * 123 nhi_find_ident(device_t dev) 124 { 125 struct nhi_ident *n; 126 127 for (n = nhi_identifiers; n->vendor != 0; n++) { 128 if (n->vendor != pci_get_vendor(dev)) 129 continue; 130 if (n->device != pci_get_device(dev)) 131 continue; 132 if ((n->subvendor != 0xffff) && 133 (n->subvendor != pci_get_subvendor(dev))) 134 continue; 135 if ((n->subdevice != 0xffff) && 136 (n->subdevice != pci_get_subdevice(dev))) 137 continue; 138 return (n); 139 } 140 141 return (NULL); 142 } 143 144 static int 145 nhi_pci_probe(device_t dev) 146 { 147 struct nhi_ident *n; 148 149 if (resource_disabled("tb", 0)) 150 return (ENXIO); 151 if ((n = nhi_find_ident(dev)) != NULL) { 152 device_set_desc(dev, n->desc); 153 return (BUS_PROBE_DEFAULT); 154 } 155 return (ENXIO); 156 } 157 158 static int 159 nhi_pci_attach(device_t dev) 160 { 161 devclass_t dc; 162 bus_dma_template_t t; 163 struct nhi_softc *sc; 164 struct nhi_ident *n; 165 int error = 0; 166 167 sc = device_get_softc(dev); 168 bzero(sc, sizeof(*sc)); 169 sc->dev = dev; 170 n = nhi_find_ident(dev); 171 sc->hwflags = n->flags; 172 nhi_get_tunables(sc); 173 174 tb_debug(sc, DBG_INIT|DBG_FULL, "busmaster status was %s\n", 175 (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_BUSMASTEREN) 176 ? "enabled" : "disabled"); 177 pci_enable_busmaster(dev); 178 179 sc->ufp = NULL; 180 if ((TB_FIND_UFP(dev, &sc->ufp) != 0) || (sc->ufp == NULL)) { 181 dc = devclass_find("tbolt"); 182 if (dc != NULL) 183 sc->ufp = devclass_get_device(dc, device_get_unit(dev)); 184 } 185 if (sc->ufp == NULL) 186 tb_printf(sc, "Cannot find Upstream Facing Port\n"); 187 else 188 tb_printf(sc, "Upstream Facing Port is %s\n", 189 device_get_nameunit(sc->ufp)); 190 191 if (NHI_IS_ICL(sc)) { 192 if ((error = nhi_pci_icl_poweron(sc)) != 0) 193 return (error); 194 } 195 196 197 /* Allocate BAR0 DMA registers */ 198 sc->regs_rid = PCIR_BAR(0); 199 if ((sc->regs_resource = bus_alloc_resource_any(dev, 200 SYS_RES_MEMORY, &sc->regs_rid, RF_ACTIVE)) == NULL) { 201 tb_printf(sc, "Cannot allocate PCI registers\n"); 202 return (ENXIO); 203 } 204 sc->regs_btag = rman_get_bustag(sc->regs_resource); 205 sc->regs_bhandle = rman_get_bushandle(sc->regs_resource); 206 207 /* Allocate parent DMA tag */ 208 bus_dma_template_init(&t, bus_get_dma_tag(dev)); 209 if (bus_dma_template_tag(&t, &sc->parent_dmat) != 0) { 210 tb_printf(sc, "Cannot allocate parent DMA tag\n"); 211 nhi_pci_free(sc); 212 return (ENOMEM); 213 } 214 215 error = nhi_pci_allocate_interrupts(sc); 216 if (error == 0) 217 error = nhi_attach(sc); 218 if (error != 0) 219 nhi_pci_detach(sc->dev); 220 return (error); 221 } 222 223 static int 224 nhi_pci_detach(device_t dev) 225 { 226 struct nhi_softc *sc; 227 228 sc = device_get_softc(dev); 229 230 nhi_detach(sc); 231 nhi_pci_free(sc); 232 233 return (0); 234 } 235 236 static int 237 nhi_pci_suspend(device_t dev) 238 { 239 240 return (0); 241 } 242 243 static int 244 nhi_pci_resume(device_t dev) 245 { 246 247 return (0); 248 } 249 250 static void 251 nhi_pci_free(struct nhi_softc *sc) 252 { 253 254 nhi_pci_free_resources(sc); 255 256 if (sc->parent_dmat != NULL) { 257 bus_dma_tag_destroy(sc->parent_dmat); 258 sc->parent_dmat = NULL; 259 } 260 261 if (sc->regs_resource != NULL) { 262 bus_release_resource(sc->dev, SYS_RES_MEMORY, 263 sc->regs_rid, sc->regs_resource); 264 sc->regs_resource = NULL; 265 } 266 267 return; 268 } 269 270 static int 271 nhi_pci_allocate_interrupts(struct nhi_softc *sc) 272 { 273 int msgs, error = 0; 274 275 /* Map the Pending Bit Array and Vector Table BARs for MSI-X */ 276 sc->irq_pba_rid = pci_msix_pba_bar(sc->dev); 277 sc->irq_table_rid = pci_msix_table_bar(sc->dev); 278 279 if (sc->irq_pba_rid != -1) 280 sc->irq_pba = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 281 &sc->irq_pba_rid, RF_ACTIVE); 282 if (sc->irq_table_rid != -1) 283 sc->irq_table = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 284 &sc->irq_table_rid, RF_ACTIVE); 285 286 msgs = pci_msix_count(sc->dev); 287 tb_debug(sc, DBG_INIT|DBG_INTR|DBG_FULL, 288 "Counted %d MSI-X messages\n", msgs); 289 msgs = min(msgs, NHI_MSIX_MAX); 290 msgs = max(msgs, 1); 291 if (msgs != 0) { 292 tb_debug(sc, DBG_INIT|DBG_INTR, "Attempting to allocate %d " 293 "MSI-X interrupts\n", msgs); 294 error = pci_alloc_msix(sc->dev, &msgs); 295 tb_debug(sc, DBG_INIT|DBG_INTR|DBG_FULL, 296 "pci_alloc_msix return msgs= %d, error= %d\n", msgs, error); 297 } 298 299 if ((error != 0) || (msgs <= 0)) { 300 tb_printf(sc, "Failed to allocate any interrupts\n"); 301 msgs = 0; 302 } 303 304 sc->msix_count = msgs; 305 return (error); 306 } 307 308 void 309 nhi_pci_free_interrupts(struct nhi_softc *sc) 310 { 311 int i; 312 313 for (i = 0; i < sc->msix_count; i++) { 314 bus_teardown_intr(sc->dev, sc->irqs[i], sc->intrhand[i]); 315 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid[i], 316 sc->irqs[i]); 317 } 318 319 pci_release_msi(sc->dev); 320 } 321 322 static void 323 nhi_pci_free_resources(struct nhi_softc *sc) 324 { 325 if (sc->irq_table != NULL) { 326 bus_release_resource(sc->dev, SYS_RES_MEMORY, 327 sc->irq_table_rid, sc->irq_table); 328 sc->irq_table = NULL; 329 } 330 331 if (sc->irq_pba != NULL) { 332 bus_release_resource(sc->dev, SYS_RES_MEMORY, 333 sc->irq_pba_rid, sc->irq_pba); 334 sc->irq_pba = NULL; 335 } 336 337 if (sc->intr_trackers != NULL) 338 free(sc->intr_trackers, M_NHI); 339 return; 340 } 341 342 int 343 nhi_pci_configure_interrupts(struct nhi_softc *sc) 344 { 345 struct nhi_intr_tracker *trkr; 346 int rid, i, error; 347 348 nhi_pci_disable_interrupts(sc); 349 350 sc->intr_trackers = malloc(sizeof(struct nhi_intr_tracker) * 351 sc->msix_count, M_NHI, M_ZERO | M_NOWAIT); 352 if (sc->intr_trackers == NULL) { 353 tb_debug(sc, DBG_INIT, "Cannot allocate intr trackers\n"); 354 return (ENOMEM); 355 } 356 357 for (i = 0; i < sc->msix_count; i++) { 358 rid = i + 1; 359 trkr = &sc->intr_trackers[i]; 360 trkr->sc = sc; 361 trkr->ring = NULL; 362 trkr->vector = i; 363 364 sc->irq_rid[i] = rid; 365 sc->irqs[i] = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 366 &sc->irq_rid[i], RF_ACTIVE); 367 if (sc->irqs[i] == NULL) { 368 tb_debug(sc, DBG_INIT, 369 "Cannot allocate interrupt RID %d\n", 370 sc->irq_rid[i]); 371 break; 372 } 373 error = bus_setup_intr(sc->dev, sc->irqs[i], INTR_TYPE_BIO | 374 INTR_MPSAFE, NULL, nhi_intr, trkr, &sc->intrhand[i]); 375 if (error) { 376 tb_debug(sc, DBG_INIT, 377 "cannot setup interrupt RID %d\n", sc->irq_rid[i]); 378 break; 379 } 380 } 381 382 tb_debug(sc, DBG_INIT, "Set up %d interrupts\n", sc->msix_count); 383 384 /* Set the interrupt throttle rate to 128us */ 385 for (i = 0; i < 16; i ++) 386 nhi_write_reg(sc, NHI_ITR0 + i * 4, 0x1f4); 387 388 return (error); 389 } 390 391 #define NHI_SET_INTERRUPT(offset, mask, val) \ 392 do { \ 393 reg = offset / 32; \ 394 offset %= 32; \ 395 ivr[reg] &= ~(mask << offset); \ 396 ivr[reg] |= (val << offset); \ 397 } while (0) 398 399 void 400 nhi_pci_enable_interrupt(struct nhi_ring_pair *r) 401 { 402 struct nhi_softc *sc = r->sc; 403 uint32_t ivr[5]; 404 u_int offset, reg; 405 406 tb_debug(sc, DBG_INIT|DBG_INTR, "Enabling interrupts for ring %d\n", 407 r->ring_num); 408 /* 409 * Compute the routing between event type and MSI-X vector. 410 * 4 bits per descriptor. 411 */ 412 ivr[0] = nhi_read_reg(sc, NHI_IVR0); 413 ivr[1] = nhi_read_reg(sc, NHI_IVR1); 414 ivr[2] = nhi_read_reg(sc, NHI_IVR2); 415 ivr[3] = nhi_read_reg(sc, NHI_IVR3); 416 ivr[4] = nhi_read_reg(sc, NHI_IVR4); 417 418 /* Program TX */ 419 offset = (r->ring_num + IVR_TX_OFFSET) * 4; 420 NHI_SET_INTERRUPT(offset, 0x0f, r->ring_num); 421 422 /* Now program RX */ 423 offset = (r->ring_num + IVR_RX_OFFSET) * 4; 424 NHI_SET_INTERRUPT(offset, 0x0f, r->ring_num); 425 426 /* Last, program Nearly Empty. This one always going to vector 15 */ 427 offset = (r->ring_num + IVR_NE_OFFSET) * 4; 428 NHI_SET_INTERRUPT(offset, 0x0f, 0x0f); 429 430 nhi_write_reg(sc, NHI_IVR0, ivr[0]); 431 nhi_write_reg(sc, NHI_IVR1, ivr[1]); 432 nhi_write_reg(sc, NHI_IVR2, ivr[2]); 433 nhi_write_reg(sc, NHI_IVR3, ivr[3]); 434 nhi_write_reg(sc, NHI_IVR4, ivr[4]); 435 436 tb_debug(sc, DBG_INIT|DBG_INTR|DBG_FULL, 437 "Wrote IVR 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 438 ivr[0], ivr[1], ivr[2], ivr[3], ivr[4]); 439 440 /* Now do the Interrupt Mask Register, 1 bit per descriptor */ 441 ivr[0] = nhi_read_reg(sc, NHI_IMR0); 442 ivr[1] = nhi_read_reg(sc, NHI_IMR1); 443 444 /* Tx */ 445 offset = r->ring_num + IMR_TX_OFFSET; 446 NHI_SET_INTERRUPT(offset, 0x01, 1); 447 448 /* Rx */ 449 offset = r->ring_num + IMR_RX_OFFSET; 450 NHI_SET_INTERRUPT(offset, 0x01, 1); 451 452 /* NE */ 453 offset = r->ring_num + IMR_NE_OFFSET; 454 NHI_SET_INTERRUPT(offset, 0x01, 1); 455 456 nhi_write_reg(sc, NHI_IMR0, ivr[0]); 457 nhi_write_reg(sc, NHI_IMR1, ivr[1]); 458 tb_debug(sc, DBG_INIT|DBG_FULL, 459 "Wrote IMR 0x%08x 0x%08x\n", ivr[0], ivr[1]); 460 } 461 462 void 463 nhi_pci_disable_interrupts(struct nhi_softc *sc) 464 { 465 466 tb_debug(sc, DBG_INIT, "Disabling interrupts\n"); 467 nhi_write_reg(sc, NHI_IMR0, 0); 468 nhi_write_reg(sc, NHI_IMR1, 0); 469 nhi_write_reg(sc, NHI_IVR0, 0); 470 nhi_write_reg(sc, NHI_IVR1, 0); 471 nhi_write_reg(sc, NHI_IVR2, 0); 472 nhi_write_reg(sc, NHI_IVR3, 0); 473 nhi_write_reg(sc, NHI_IVR4, 0); 474 475 /* Dummy reads to clear pending bits */ 476 nhi_read_reg(sc, NHI_ISR0); 477 nhi_read_reg(sc, NHI_ISR1); 478 } 479 480 /* 481 * Icelake controllers need to be notified of power-on 482 */ 483 static int 484 nhi_pci_icl_poweron(struct nhi_softc *sc) 485 { 486 device_t dev; 487 uint32_t val; 488 int i, error = 0; 489 490 dev = sc->dev; 491 val = pci_read_config(dev, ICL_VSCAP_9, 4); 492 tb_debug(sc, DBG_INIT, "icl_poweron val= 0x%x\n", val); 493 if (val & ICL_VSCAP9_FWREADY) 494 return (0); 495 496 val = pci_read_config(dev, ICL_VSCAP_22, 4); 497 val |= ICL_VSCAP22_FORCEPWR; 498 tb_debug(sc, DBG_INIT|DBG_FULL, "icl_poweron writing 0x%x\n", val); 499 pci_write_config(dev, ICL_VSCAP_22, val, 4); 500 501 error = ETIMEDOUT; 502 for (i = 0; i < 15; i++) { 503 DELAY(1000000); 504 val = pci_read_config(dev, ICL_VSCAP_9, 4); 505 if (val & ICL_VSCAP9_FWREADY) { 506 error = 0; 507 break; 508 } 509 } 510 511 return (error); 512 } 513 514 /* 515 * Icelake and Alderlake controllers store their UUID in PCI config space 516 */ 517 int 518 nhi_pci_get_uuid(struct nhi_softc *sc) 519 { 520 device_t dev; 521 uint32_t val[4]; 522 523 dev = sc->dev; 524 val[0] = pci_read_config(dev, ICL_VSCAP_10, 4); 525 val[1] = pci_read_config(dev, ICL_VSCAP_11, 4); 526 val[2] = 0xffffffff; 527 val[3] = 0xffffffff; 528 529 bcopy(val, &sc->uuid, 16); 530 return (0); 531 } 532