1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2022 Scott Long 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_thunderbolt.h" 30 31 /* Config space access for switches, ports, and devices in TB3 and USB4 */ 32 #include <sys/types.h> 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/bus.h> 38 #include <sys/conf.h> 39 #include <sys/malloc.h> 40 #include <sys/queue.h> 41 #include <sys/sysctl.h> 42 #include <sys/lock.h> 43 #include <sys/mutex.h> 44 #include <sys/taskqueue.h> 45 #include <sys/gsb_crc32.h> 46 #include <sys/endian.h> 47 #include <vm/vm.h> 48 #include <vm/pmap.h> 49 50 #include <machine/bus.h> 51 #include <machine/stdarg.h> 52 53 #include <dev/thunderbolt/nhi_reg.h> 54 #include <dev/thunderbolt/nhi_var.h> 55 #include <dev/thunderbolt/tb_reg.h> 56 #include <dev/thunderbolt/tb_var.h> 57 #include <dev/thunderbolt/tbcfg_reg.h> 58 #include <dev/thunderbolt/router_var.h> 59 #include <dev/thunderbolt/tb_debug.h> 60 61 static int router_alloc_cmd(struct router_softc *, struct router_command **); 62 static void router_free_cmd(struct router_softc *, struct router_command *); 63 static int _tb_router_attach(struct router_softc *); 64 static void router_prepare_cmd(struct router_softc *, struct router_command *, 65 size_t, uint16_t); 66 static void router_prepare_read(struct router_softc *, struct router_command *, 67 size_t); 68 static void router_prepare_write(struct router_softc *, struct router_command *, 69 size_t); 70 static int _tb_config_read(struct router_softc *, u_int, u_int, u_int, u_int, 71 uint32_t *, void *, struct router_command **); 72 static int _tb_config_write(struct router_softc *, u_int, u_int, u_int, u_int, 73 uint32_t *, void *, struct router_command **); 74 static int router_schedule(struct router_softc *, struct router_command *); 75 static int router_schedule_locked(struct router_softc *, 76 struct router_command *); 77 static nhi_ring_cb_t router_complete_intr; 78 static nhi_ring_cb_t router_response_intr; 79 static nhi_ring_cb_t router_notify_intr; 80 81 #define CFG_DEFAULT_RETRIES 3 82 #define CFG_DEFAULT_TIMEOUT 2 83 84 static int 85 router_lookup_device(struct router_softc *sc, tb_route_t route, 86 struct router_softc **dev) 87 { 88 struct router_softc *cursor; 89 uint64_t search_rt, remainder_rt, this_rt; 90 uint8_t hop; 91 92 KASSERT(dev != NULL, ("dev cannot be NULL\n")); 93 94 cursor = tb_config_get_root(sc); 95 remainder_rt = search_rt = route.lo | ((uint64_t)route.hi << 32); 96 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, 97 "%s: Searching for router 0x%016jx\n", __func__, search_rt); 98 99 while (cursor != NULL) { 100 this_rt = TB_ROUTE(cursor); 101 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, 102 "Comparing cursor route 0x%016jx\n", this_rt); 103 if (this_rt == search_rt) 104 break; 105 106 /* Prepare to go to the next hop node in the route */ 107 hop = remainder_rt & 0xff; 108 remainder_rt >>= 8; 109 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, 110 "hop= 0x%02x, remainder= 0x%016jx\n", hop, remainder_rt); 111 112 /* 113 * An adapter index of 0x0 is only for the host interface 114 * adapter on the root route. The only time that 115 * it's valid for searches is when you're looking for the 116 * root route, and that case has already been handled. 117 */ 118 if (hop == 0) { 119 tb_debug(sc, DBG_ROUTER, 120 "End of route chain, route not found\n"); 121 return (ENOENT); 122 } 123 124 if (hop > cursor->max_adap) { 125 tb_debug(sc, DBG_ROUTER, 126 "Route hop out of range for parent\n"); 127 return (EINVAL); 128 } 129 130 if (cursor->adapters == NULL) { 131 tb_debug(sc, DBG_ROUTER, 132 "Error, router not fully initialized\n"); 133 return (EINVAL); 134 } 135 136 cursor = cursor->adapters[hop]; 137 } 138 139 if (cursor == NULL) 140 return (ENOENT); 141 142 *dev = cursor; 143 return (0); 144 } 145 146 static int 147 router_insert(struct router_softc *sc, struct router_softc *parent) 148 { 149 uint64_t this_rt; 150 uint8_t this_hop; 151 152 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "router_insert called\n"); 153 154 if (parent == NULL) { 155 tb_debug(sc, DBG_ROUTER, "Parent cannot be NULL in insert\n"); 156 return (EINVAL); 157 } 158 159 this_rt = TB_ROUTE(sc); 160 if (((this_rt >> (sc->depth * 8)) > 0xffULL) || 161 (parent->depth + 1 != sc->depth)) { 162 tb_debug(sc, DBG_ROUTER, "Added route 0x%08x%08x is not a " 163 "direct child of the parent route 0x%08x%08x\n", 164 sc->route.hi, sc->route.lo, parent->route.hi, 165 parent->route.lo); 166 return (EINVAL); 167 } 168 169 this_hop = (uint8_t)(this_rt >> (sc->depth * 8)); 170 171 tb_debug(sc, DBG_ROUTER, "Inserting route 0x%08x%08x with last hop " 172 "of 0x%02x and depth of %d\n", sc->route.hi, sc->route.lo, 173 this_hop, sc->depth); 174 175 if (this_hop > parent->max_adap) { 176 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, 177 "Inserted route is out of range of the parent\n"); 178 return (EINVAL); 179 } 180 181 if (parent->adapters[this_hop] != NULL) { 182 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, 183 "Inserted route already exists\n"); 184 return (EEXIST); 185 } 186 187 parent->adapters[this_hop] = sc; 188 189 tb_debug(sc, DBG_ROUTER, "Added router 0x%08x%08x to parent " 190 "0x%08x%08x\n", sc->route.hi, sc->route.lo, parent->route.hi, 191 parent->route.lo); 192 return (0); 193 } 194 195 static int 196 router_register_interrupts(struct router_softc *sc) 197 { 198 struct nhi_dispatch tx[] = { { PDF_READ, router_complete_intr, sc }, 199 { PDF_WRITE, router_complete_intr, sc }, 200 { 0, NULL, NULL } }; 201 struct nhi_dispatch rx[] = { { PDF_READ, router_response_intr, sc }, 202 { PDF_WRITE, router_response_intr, sc }, 203 { PDF_NOTIFY, router_notify_intr, sc }, 204 { 0, NULL, NULL } }; 205 206 return (nhi_register_pdf(sc->ring0, tx, rx)); 207 } 208 209 int 210 tb_router_attach(struct router_softc *parent, tb_route_t route) 211 { 212 struct router_softc *sc; 213 214 tb_debug(parent, DBG_ROUTER|DBG_EXTRA, "tb_router_attach called\n"); 215 216 sc = malloc(sizeof(*sc), M_THUNDERBOLT, M_ZERO|M_NOWAIT); 217 if (sc == NULL) { 218 tb_debug(parent, DBG_ROUTER, "Cannot allocate root router\n"); 219 return (ENOMEM); 220 } 221 222 sc->dev = parent->dev; 223 sc->debug = parent->debug; 224 sc->ring0 = parent->ring0; 225 sc->route = route; 226 sc->nsc = parent->nsc; 227 228 mtx_init(&sc->mtx, "tbcfg", "Thunderbolt Router Config", MTX_DEF); 229 TAILQ_INIT(&sc->cmd_queue); 230 231 router_insert(sc, parent); 232 233 return (_tb_router_attach(sc)); 234 } 235 236 int 237 tb_router_attach_root(struct nhi_softc *nsc, tb_route_t route) 238 { 239 struct router_softc *sc; 240 int error; 241 242 tb_debug(nsc, DBG_ROUTER|DBG_EXTRA, "tb_router_attach_root called\n"); 243 244 sc = malloc(sizeof(*sc), M_THUNDERBOLT, M_ZERO|M_NOWAIT); 245 if (sc == NULL) { 246 tb_debug(nsc, DBG_ROUTER, "Cannot allocate root router\n"); 247 return (ENOMEM); 248 } 249 250 sc->dev = nsc->dev; 251 sc->debug = nsc->debug; 252 sc->ring0 = nsc->ring0; 253 sc->route = route; 254 sc->nsc = nsc; 255 256 mtx_init(&sc->mtx, "tbcfg", "Thunderbolt Router Config", MTX_DEF); 257 TAILQ_INIT(&sc->cmd_queue); 258 259 /* 260 * This router is semi-virtual and represents the router that's part 261 * of the NHI DMA engine. Commands can't be issued to the topology 262 * until the NHI is initialized and this router is initialized, so 263 * there's no point in registering router interrupts earlier than this, 264 * even if other routers are found first. 265 */ 266 tb_config_set_root(sc); 267 error = router_register_interrupts(sc); 268 if (error) { 269 tb_router_detach(sc); 270 return (error); 271 } 272 273 error = _tb_router_attach(sc); 274 if (error) 275 return (error); 276 277 bcopy((uint8_t *)sc->uuid, nsc->uuid, 16); 278 return (0); 279 } 280 281 static int 282 _tb_router_attach(struct router_softc *sc) 283 { 284 struct tb_cfg_router *cfg; 285 uint32_t *buf; 286 int error; 287 int up __maybe_unused; 288 289 buf = malloc(9 * 4, M_THUNDERBOLT, M_NOWAIT|M_ZERO); 290 if (buf == NULL) 291 return (ENOMEM); 292 293 error = tb_config_router_read_polled(sc, 0, 9, buf); 294 if (error != 0) { 295 free(buf, M_THUNDERBOLT); 296 return (error); 297 } 298 299 cfg = (struct tb_cfg_router *)buf; 300 up = GET_ROUTER_CS_UPSTREAM_ADAP(cfg); 301 sc->max_adap = GET_ROUTER_CS_MAX_ADAP(cfg); 302 sc->depth = GET_ROUTER_CS_DEPTH(cfg); 303 sc->uuid[0] = cfg->uuid_lo; 304 sc->uuid[1] = cfg->uuid_hi; 305 sc->uuid[2] = 0xffffffff; 306 sc->uuid[3] = 0xffffffff; 307 tb_debug(sc, DBG_ROUTER, 308 "Router upstream_port= %d, max_port= %d, depth= %d\n", 309 up, sc->max_adap, sc->depth); 310 free(buf, M_THUNDERBOLT); 311 312 /* Downstream adapters are indexed in the array allocated here. */ 313 sc->max_adap = MIN(sc->max_adap, ROUTER_CS1_MAX_ADAPTERS); 314 sc->adapters = malloc((1 + sc->max_adap) * sizeof(void *), 315 M_THUNDERBOLT, M_NOWAIT|M_ZERO); 316 if (sc->adapters == NULL) { 317 tb_debug(sc, DBG_ROUTER, 318 "Cannot allocate downstream adapter memory\n"); 319 return (ENOMEM); 320 } 321 322 tb_debug(sc, DBG_ROUTER, "Router created, route 0x%08x%08x\n", 323 sc->route.hi, sc->route.lo); 324 325 return (0); 326 } 327 328 int 329 tb_router_detach(struct router_softc *sc) 330 { 331 332 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "tb_router_deattach called\n"); 333 334 if (TAILQ_FIRST(&sc->cmd_queue) != NULL) 335 return (EBUSY); 336 337 mtx_destroy(&sc->mtx); 338 339 if (sc->adapters != NULL) 340 free(sc->adapters, M_THUNDERBOLT); 341 342 if (sc != NULL) 343 free(sc, M_THUNDERBOLT); 344 345 return (0); 346 } 347 348 static void 349 router_get_config_cb(struct router_softc *sc, struct router_command *cmd, 350 void *arg) 351 { 352 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "router_get_config_cb called\n"); 353 354 /* 355 * Only do the copy if the command didn't have a notify event thrown. 356 * These events serve as asynchronous exception signals, which is 357 * cumbersome. 358 */ 359 if (cmd->ev == 0) 360 bcopy((uint8_t *)cmd->resp_buffer, 361 (uint8_t *)cmd->callback_arg, cmd->dwlen * 4); 362 363 mtx_lock(&sc->mtx); 364 sc->inflight_cmd = NULL; 365 366 if ((cmd->flags & RCMD_POLLED) == 0) 367 wakeup(cmd); 368 else 369 cmd->flags |= RCMD_POLL_COMPLETE; 370 371 router_schedule_locked(sc, NULL); 372 mtx_unlock(&sc->mtx); 373 } 374 375 int 376 tb_config_read(struct router_softc *sc, u_int space, u_int adapter, 377 u_int offset, u_int dwlen, uint32_t *buf) 378 { 379 struct router_command *cmd; 380 int error, retries; 381 382 if ((error = _tb_config_read(sc, space, adapter, offset, dwlen, buf, 383 router_get_config_cb, &cmd)) != 0) 384 return (error); 385 386 retries = cmd->retries; 387 mtx_lock(&sc->mtx); 388 while (retries-- >= 0) { 389 error = router_schedule_locked(sc, cmd); 390 if (error) 391 break; 392 393 error = msleep(cmd, &sc->mtx, 0, "tbtcfg", cmd->timeout * hz); 394 if (error != EWOULDBLOCK) 395 break; 396 sc->inflight_cmd = NULL; 397 tb_debug(sc, DBG_ROUTER, "Config command timed out, retries=%d\n", retries); 398 } 399 400 if (cmd->ev != 0) 401 error = EINVAL; 402 router_free_cmd(sc, cmd); 403 mtx_unlock(&sc->mtx); 404 return (error); 405 } 406 407 int 408 tb_config_read_polled(struct router_softc *sc, u_int space, u_int adapter, 409 u_int offset, u_int dwlen, uint32_t *buf) 410 { 411 struct router_command *cmd; 412 int error, retries, timeout; 413 414 if ((error = _tb_config_read(sc, space, adapter, offset, dwlen, buf, 415 router_get_config_cb, &cmd)) != 0) 416 return (error); 417 418 retries = cmd->retries; 419 cmd->flags |= RCMD_POLLED; 420 timeout = cmd->timeout * 1000000; 421 422 mtx_lock(&sc->mtx); 423 while (retries-- >= 0) { 424 error = router_schedule_locked(sc, cmd); 425 if (error) 426 break; 427 mtx_unlock(&sc->mtx); 428 429 while (timeout > 0) { 430 DELAY(100 * 1000); 431 if ((cmd->flags & RCMD_POLL_COMPLETE) != 0) 432 break; 433 timeout -= 100000; 434 } 435 436 mtx_lock(&sc->mtx); 437 if ((cmd->flags & RCMD_POLL_COMPLETE) == 0) { 438 error = ETIMEDOUT; 439 sc->inflight_cmd = NULL; 440 tb_debug(sc, DBG_ROUTER, "Config command timed out, retries=%d\n", retries); 441 continue; 442 } else 443 break; 444 } 445 446 if (cmd->ev != 0) 447 error = EINVAL; 448 router_free_cmd(sc, cmd); 449 mtx_unlock(&sc->mtx); 450 return (error); 451 } 452 453 int 454 tb_config_read_async(struct router_softc *sc, u_int space, u_int adapter, 455 u_int offset, u_int dwlen, uint32_t *buf, void *cb) 456 { 457 struct router_command *cmd; 458 int error; 459 460 if ((error = _tb_config_read(sc, space, adapter, offset, dwlen, buf, 461 cb, &cmd)) != 0) 462 return (error); 463 464 error = router_schedule(sc, cmd); 465 466 return (error); 467 } 468 469 static int 470 _tb_config_read(struct router_softc *sc, u_int space, u_int adapter, 471 u_int offset, u_int dwlen, uint32_t *buf, void *cb, 472 struct router_command **rcmd) 473 { 474 struct router_command *cmd; 475 struct tb_cfg_read *msg; 476 int error; 477 478 if ((error = router_alloc_cmd(sc, &cmd)) != 0) 479 return (error); 480 481 msg = router_get_frame_data(cmd); 482 bzero(msg, sizeof(*msg)); 483 msg->route.hi = sc->route.hi; 484 msg->route.lo = sc->route.lo; 485 msg->addr_attrs = TB_CONFIG_ADDR(0, space, adapter, dwlen, offset); 486 cmd->callback = cb; 487 cmd->callback_arg = buf; 488 cmd->dwlen = dwlen; 489 router_prepare_read(sc, cmd, sizeof(*msg)); 490 491 if (rcmd != NULL) 492 *rcmd = cmd; 493 494 return (0); 495 } 496 497 int 498 tb_config_write(struct router_softc *sc, u_int space, u_int adapter, 499 u_int offset, u_int dwlen, uint32_t *buf) 500 { 501 struct router_command *cmd; 502 int error, retries; 503 504 if ((error = _tb_config_write(sc, space, adapter, offset, dwlen, buf, 505 router_get_config_cb, &cmd)) != 0) 506 return (error); 507 508 retries = cmd->retries; 509 mtx_lock(&sc->mtx); 510 while (retries-- >= 0) { 511 error = router_schedule_locked(sc, cmd); 512 if (error) 513 break; 514 515 error = msleep(cmd, &sc->mtx, 0, "tbtcfg", cmd->timeout * hz); 516 if (error != EWOULDBLOCK) 517 break; 518 sc->inflight_cmd = NULL; 519 tb_debug(sc, DBG_ROUTER, "Config command timed out, " 520 "retries=%d\n", retries); 521 } 522 523 if (cmd->ev != 0) 524 error = EINVAL; 525 router_free_cmd(sc, cmd); 526 mtx_unlock(&sc->mtx); 527 return (error); 528 } 529 530 static int 531 _tb_config_write(struct router_softc *sc, u_int space, u_int adapter, 532 u_int offset, u_int dwlen, uint32_t *buf, void *cb, 533 struct router_command **rcmd) 534 { 535 struct router_command *cmd; 536 struct tb_cfg_write *msg; 537 /* +4 to account for the CRC at the end. */ 538 size_t msglen = sizeof(*msg) + dwlen * 4 + 4; 539 int error; 540 541 if ((error = router_alloc_cmd(sc, &cmd)) != 0) 542 return (error); 543 544 msg = router_get_frame_data(cmd); 545 bzero(msg, msglen); 546 msg->route.hi = sc->route.hi; 547 msg->route.lo = sc->route.lo; 548 msg->addr_attrs = TB_CONFIG_ADDR(0, space, adapter, dwlen, offset); 549 for (size_t i = 0; i < dwlen; i++) 550 msg->data[i] = buf[i]; 551 cmd->callback = cb; 552 cmd->callback_arg = buf; 553 cmd->dwlen = dwlen; 554 router_prepare_write(sc, cmd, msglen); 555 556 if (rcmd != NULL) 557 *rcmd = cmd; 558 559 return (0); 560 } 561 562 static int 563 router_alloc_cmd(struct router_softc *sc, struct router_command **rcmd) 564 { 565 struct router_command *cmd; 566 567 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "router_alloc_cmd\n"); 568 569 cmd = malloc(sizeof(*cmd), M_THUNDERBOLT, M_ZERO|M_NOWAIT); 570 if (cmd == NULL) { 571 tb_debug(sc, DBG_ROUTER, "Cannot allocate cmd/response\n"); 572 return (ENOMEM); 573 } 574 575 cmd->nhicmd = nhi_alloc_tx_frame(sc->ring0); 576 if (cmd->nhicmd == NULL) { 577 tb_debug(sc, DBG_ROUTER, "Cannot allocate command frame\n"); 578 free(cmd, M_THUNDERBOLT); 579 return (EBUSY); 580 } 581 582 cmd->sc = sc; 583 *rcmd = cmd; 584 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "Allocated command with index %d\n", 585 cmd->nhicmd->idx); 586 587 return (0); 588 } 589 590 static void 591 router_free_cmd(struct router_softc *sc, struct router_command *cmd) 592 { 593 594 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "router_free_cmd\n"); 595 596 if (cmd == NULL) 597 return; 598 599 if (cmd->nhicmd != NULL) { 600 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "Freeing nhi command %d\n", 601 cmd->nhicmd->idx); 602 nhi_free_tx_frame(sc->ring0, cmd->nhicmd); 603 } 604 free(cmd, M_THUNDERBOLT); 605 606 return; 607 } 608 609 static void 610 router_prepare_cmd(struct router_softc *sc, struct router_command *cmd, 611 size_t len, uint16_t pdf) 612 { 613 struct nhi_cmd_frame *nhicmd; 614 uint32_t *msg; 615 int msglen, i; 616 617 KASSERT(cmd != NULL, ("cmd cannot be NULL\n")); 618 KASSERT(len != 0, ("Invalid zero-length command\n")); 619 KASSERT(len % 4 == 0, ("Message must be 32bit padded\n")); 620 621 nhicmd = cmd->nhicmd; 622 msglen = (len - 4) / 4; 623 for (i = 0; i < msglen; i++) 624 nhicmd->data[i] = htobe32(nhicmd->data[i]); 625 626 msg = (uint32_t *)nhicmd->data; 627 /* The last 4 bytes of data is the CRC. Compute CRC for just data. */ 628 msg[msglen] = htobe32(tb_calc_crc(nhicmd->data, len - 4)); 629 630 nhicmd->pdf = pdf; 631 nhicmd->req_len = len; 632 633 nhicmd->timeout = NHI_CMD_TIMEOUT; 634 nhicmd->retries = 0; 635 nhicmd->context = cmd; 636 637 cmd->retries = CFG_DEFAULT_RETRIES; 638 cmd->timeout = CFG_DEFAULT_TIMEOUT; 639 } 640 641 static void 642 router_prepare_read(struct router_softc *sc, struct router_command *cmd, 643 size_t len) 644 { 645 router_prepare_cmd(sc, cmd, len, PDF_READ); 646 647 cmd->nhicmd->resp_buffer = (uint32_t *)cmd->resp_buffer; 648 cmd->nhicmd->resp_len = (cmd->dwlen + 3) * 4; 649 } 650 651 static void 652 router_prepare_write(struct router_softc *sc, struct router_command *cmd, 653 size_t len) 654 { 655 router_prepare_cmd(sc, cmd, len, PDF_WRITE); 656 657 cmd->nhicmd->resp_buffer = (uint32_t *)cmd->resp_buffer; 658 cmd->nhicmd->resp_len = (cmd->dwlen + 3) * 4; 659 } 660 661 static int 662 router_schedule(struct router_softc *sc, struct router_command *cmd) 663 { 664 int error; 665 666 mtx_lock(&sc->mtx); 667 error = router_schedule_locked(sc, cmd); 668 mtx_unlock(&sc->mtx); 669 670 return(error); 671 } 672 673 static int 674 router_schedule_locked(struct router_softc *sc, struct router_command *cmd) 675 { 676 struct nhi_cmd_frame *nhicmd; 677 int error; 678 679 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "router_schedule\n"); 680 681 if (cmd != NULL) 682 TAILQ_INSERT_TAIL(&sc->cmd_queue, cmd, link); 683 684 while ((sc->inflight_cmd == NULL) && 685 ((cmd = TAILQ_FIRST(&sc->cmd_queue)) != NULL)) { 686 687 TAILQ_REMOVE(&sc->cmd_queue, cmd, link); 688 nhicmd = cmd->nhicmd; 689 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, 690 "Scheduling command with index %d\n", nhicmd->idx); 691 sc->inflight_cmd = cmd; 692 if ((error = nhi_tx_schedule(sc->ring0, nhicmd)) != 0) { 693 tb_debug(sc, DBG_ROUTER, "nhi ring error " 694 "%d\n", error); 695 sc->inflight_cmd = NULL; 696 if (error == EBUSY) { 697 TAILQ_INSERT_HEAD(&sc->cmd_queue, cmd, link); 698 error = 0; 699 } 700 break; 701 } 702 } 703 704 return (error); 705 } 706 707 static void 708 router_complete_intr(void *context, union nhi_ring_desc *ring, 709 struct nhi_cmd_frame *nhicmd) 710 { 711 struct router_softc *sc; 712 struct router_command *cmd; 713 714 KASSERT(context != NULL, ("context cannot be NULL\n")); 715 KASSERT(nhicmd != NULL, ("nhicmd cannot be NULL\n")); 716 717 cmd = (struct router_command *)(nhicmd->context); 718 sc = cmd->sc; 719 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "router_complete_intr called\n"); 720 721 if (nhicmd->flags & CMD_RESP_COMPLETE) { 722 cmd->callback(sc, cmd, cmd->callback_arg); 723 } 724 725 return; 726 } 727 728 static void 729 router_response_intr(void *context, union nhi_ring_desc *ring, struct nhi_cmd_frame *nhicmd) 730 { 731 struct router_softc *sc, *dev; 732 struct tb_cfg_read_resp *read; 733 struct tb_cfg_write_resp *write; 734 struct router_command *cmd; 735 tb_route_t route; 736 u_int error, i, eof, len; 737 uint32_t attrs; 738 739 KASSERT(context != NULL, ("context cannot be NULL\n")); 740 741 sc = (struct router_softc *)context; 742 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "router_response_intr called\n"); 743 744 eof = ring->rxpost.eof_len >> RX_BUFFER_DESC_EOF_SHIFT; 745 746 if (eof == PDF_WRITE) { 747 write = (struct tb_cfg_write_resp *)nhicmd->data; 748 route.hi = be32toh(write->route.hi); 749 route.lo = be32toh(write->route.lo); 750 } else { 751 read = (struct tb_cfg_read_resp *)nhicmd->data; 752 route.hi = be32toh(read->route.hi); 753 route.lo = be32toh(read->route.lo); 754 attrs = be32toh(read->addr_attrs); 755 len = (attrs & TB_CFG_SIZE_MASK) >> TB_CFG_SIZE_SHIFT; 756 } 757 758 /* XXX Is this a problem? */ 759 if ((route.hi & 0x80000000) == 0) 760 tb_debug(sc, DBG_ROUTER, "Invalid route\n"); 761 route.hi &= ~0x80000000; 762 763 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "Looking up route 0x%08x%08x\n", 764 route.hi, route.lo); 765 766 error = router_lookup_device(sc, route, &dev); 767 if (error != 0 || dev == NULL) { 768 tb_debug(sc, DBG_ROUTER, "Cannot find device, error= %d\n", 769 error); 770 return; 771 } 772 773 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "Found device %s route 0x%08x%08x, " 774 "inflight_cmd= %p\n", device_get_nameunit(dev->dev), dev->route.hi, 775 dev->route.lo, dev->inflight_cmd); 776 777 cmd = dev->inflight_cmd; 778 if (cmd == NULL) { 779 tb_debug(dev, DBG_ROUTER, "Null inflight cmd\n"); 780 return; 781 } 782 783 if (eof == PDF_READ) { 784 for (i = 0; i < len; i++) 785 cmd->nhicmd->resp_buffer[i] = be32toh(read->data[i]); 786 } 787 788 cmd->nhicmd->flags |= CMD_RESP_COMPLETE; 789 if (cmd->nhicmd->flags & CMD_REQ_COMPLETE) { 790 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "TX_COMPLETE set\n"); 791 cmd->callback(dev, cmd, cmd->callback_arg); 792 } 793 794 return; 795 } 796 797 static void 798 router_notify_intr(void *context, union nhi_ring_desc *ring, struct nhi_cmd_frame *nhicmd) 799 { 800 struct router_softc *sc; 801 struct router_command *cmd; 802 struct tb_cfg_notify event; 803 u_int adap __maybe_unused; 804 u_int ev; 805 806 KASSERT(context != NULL, ("context cannot be NULL\n")); 807 808 sc = (struct router_softc *)context; 809 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "router_notify_intr called\n"); 810 811 event.route.hi = be32toh(nhicmd->data[0]); 812 event.route.lo = be32toh(nhicmd->data[1]); 813 event.event_adap = be32toh(nhicmd->data[2]); 814 815 ev = GET_NOTIFY_EVENT(&event); 816 adap = GET_NOTIFY_ADAPTER(&event); 817 818 tb_debug(sc, DBG_ROUTER, "Event route 0x%08x%08x adap %d code %s\n", 819 event.route.hi, event.route.lo, adap, 820 tb_get_string(ev, tb_notify_event)); 821 822 switch (ev) { 823 case TB_CFG_ERR_CONN: 824 case TB_CFG_ERR_LINK: 825 case TB_CFG_ERR_ADDR: 826 case TB_CFG_ERR_ADP: 827 case TB_CFG_ERR_ENUM: 828 case TB_CFG_ERR_NUA: 829 case TB_CFG_ERR_LEN: 830 case TB_CFG_ERR_HEC: 831 case TB_CFG_ERR_FC: 832 case TB_CFG_ERR_PLUG: 833 case TB_CFG_ERR_LOCK: 834 case TB_CFG_HP_ACK: 835 case TB_CFG_DP_BW: 836 if (sc->inflight_cmd != NULL) { 837 cmd = sc->inflight_cmd; 838 cmd->ev = ev; 839 cmd->callback(sc, cmd, cmd->callback_arg); 840 } 841 break; 842 default: 843 break; 844 } 845 return; 846 } 847 848 int 849 tb_config_next_cap(struct router_softc *sc, struct router_cfg_cap *cap) 850 { 851 union tb_cfg_cap *tbcap; 852 uint32_t *buf; 853 uint16_t current; 854 int error; 855 856 KASSERT(cap != NULL, ("cap cannot be NULL\n")); 857 KASSERT(cap->next_cap != 0, ("next_cap cannot be 0\n")); 858 859 buf = malloc(sizeof(*tbcap), M_THUNDERBOLT, M_NOWAIT|M_ZERO); 860 861 current = cap->next_cap; 862 error = tb_config_read(sc, cap->space, cap->adap, current, 1, buf); 863 if (error) 864 return (error); 865 866 tbcap = (union tb_cfg_cap *)buf; 867 cap->cap_id = tbcap->hdr.cap_id; 868 cap->next_cap = tbcap->hdr.next_cap; 869 cap->current_cap = current; 870 871 if ((cap->space != TB_CFG_CS_ROUTER) && 872 (tbcap->hdr.cap_id != TB_CFG_CAP_VSC)) { 873 free(buf, M_THUNDERBOLT); 874 return (0); 875 } 876 877 tb_config_read(sc, cap->space, cap->adap, current, 2, buf); 878 if (error) { 879 free(buf, M_THUNDERBOLT); 880 return (error); 881 } 882 883 cap->vsc_id = tbcap->vsc.vsc_id; 884 cap->vsc_len = tbcap->vsc.len; 885 if (tbcap->vsc.len == 0) { 886 cap->next_cap = tbcap->vsec.vsec_next_cap; 887 cap->vsec_len = tbcap->vsec.vsec_len; 888 } 889 890 free(buf, M_THUNDERBOLT); 891 return (0); 892 } 893 894 int 895 tb_config_find_cap(struct router_softc *sc, struct router_cfg_cap *cap) 896 { 897 u_int cap_id, vsc_id; 898 int error; 899 900 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, "tb_config_find_cap called\n"); 901 902 cap_id = cap->cap_id; 903 vsc_id = cap->vsc_id; 904 905 cap->cap_id = cap->vsc_id = 0; 906 while ((cap->cap_id != cap_id) || (cap->vsc_id != vsc_id)) { 907 tb_debug(sc, DBG_ROUTER|DBG_EXTRA, 908 "Looking for cap %d at offset %d\n", cap->cap_id, 909 cap->next_cap); 910 if ((cap->next_cap == 0) || 911 (cap->next_cap > TB_CFG_CAP_OFFSET_MAX)) 912 return (EINVAL); 913 error = tb_config_next_cap(sc, cap); 914 if (error) 915 break; 916 } 917 918 return (0); 919 } 920 921 int 922 tb_config_find_router_cap(struct router_softc *sc, u_int cap, u_int vsc, u_int *offset) 923 { 924 struct router_cfg_cap rcap; 925 struct tb_cfg_router *cfg; 926 uint32_t *buf; 927 int error; 928 929 buf = malloc(8 * 4, M_THUNDERBOLT, M_NOWAIT|M_ZERO); 930 if (buf == NULL) 931 return (ENOMEM); 932 933 error = tb_config_router_read(sc, 0, 5, buf); 934 if (error != 0) { 935 free(buf, M_THUNDERBOLT); 936 return (error); 937 } 938 939 cfg = (struct tb_cfg_router *)buf; 940 rcap.space = TB_CFG_CS_ROUTER; 941 rcap.adap = 0; 942 rcap.next_cap = GET_ROUTER_CS_NEXT_CAP(cfg); 943 rcap.cap_id = cap; 944 rcap.vsc_id = vsc; 945 error = tb_config_find_cap(sc, &rcap); 946 if (error == 0) 947 *offset = rcap.current_cap; 948 949 free(buf, M_THUNDERBOLT); 950 return (error); 951 } 952 953 int 954 tb_config_find_router_vsc(struct router_softc *sc, u_int cap, u_int *offset) 955 { 956 957 return (tb_config_find_router_cap(sc, TB_CFG_CAP_VSC, cap, offset)); 958 } 959 960 int 961 tb_config_find_router_vsec(struct router_softc *sc, u_int cap, u_int *offset) 962 { 963 964 return (tb_config_find_router_cap(sc, TB_CFG_CAP_VSEC, cap, offset)); 965 } 966 967 int 968 tb_config_find_adapter_cap(struct router_softc *sc, u_int adap, u_int cap, u_int *offset) 969 { 970 struct router_cfg_cap rcap; 971 struct tb_cfg_adapter *cfg; 972 uint32_t *buf; 973 int error; 974 975 buf = malloc(8 * 4, M_THUNDERBOLT, M_NOWAIT|M_ZERO); 976 if (buf == NULL) 977 return (ENOMEM); 978 979 error = tb_config_adapter_read(sc, adap, 0, 8, buf); 980 if (error != 0) { 981 free(buf, M_THUNDERBOLT); 982 return (error); 983 } 984 985 cfg = (struct tb_cfg_adapter *)buf; 986 rcap.space = TB_CFG_CS_ADAPTER; 987 rcap.adap = adap; 988 rcap.next_cap = GET_ADP_CS_NEXT_CAP(cfg); 989 rcap.cap_id = cap; 990 rcap.vsc_id = 0; 991 error = tb_config_find_cap(sc, &rcap); 992 if (error == 0) 993 *offset = rcap.current_cap; 994 995 free(buf, M_THUNDERBOLT); 996 return (error); 997 } 998 999 int 1000 tb_config_get_lc_uuid(struct router_softc *rsc, uint8_t *uuid) 1001 { 1002 u_int error, offset; 1003 uint32_t buf[8]; 1004 1005 bzero(buf, sizeof(buf)); 1006 1007 error = tb_config_find_router_vsec(rsc, TB_CFG_VSEC_LC, &offset); 1008 if (error != 0) { 1009 tb_debug(rsc, DBG_ROUTER, "Error finding LC registers: %d\n", 1010 error); 1011 return (error); 1012 } 1013 1014 error = tb_config_router_read(rsc, offset + TB_LC_UUID, 4, buf); 1015 if (error != 0) { 1016 tb_debug(rsc, DBG_ROUTER, "Error fetching UUID: %d\n", error); 1017 return (error); 1018 } 1019 1020 bcopy(buf, uuid, 16); 1021 return (0); 1022 } 1023