1 /************************************************************************** 2 3 Copyright (c) 2007, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Chelsio Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ***************************************************************************/ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/kernel.h> 40 #include <sys/bus.h> 41 #include <sys/module.h> 42 #include <sys/pciio.h> 43 #include <sys/conf.h> 44 #include <machine/bus.h> 45 #include <machine/resource.h> 46 #include <sys/bus_dma.h> 47 #include <sys/rman.h> 48 #include <sys/ioccom.h> 49 #include <sys/mbuf.h> 50 #include <sys/linker.h> 51 #include <sys/firmware.h> 52 #include <sys/socket.h> 53 #include <sys/sockio.h> 54 #include <sys/smp.h> 55 #include <sys/sysctl.h> 56 #include <sys/queue.h> 57 #include <sys/taskqueue.h> 58 59 #include <net/bpf.h> 60 #include <net/ethernet.h> 61 #include <net/if.h> 62 #include <net/if_arp.h> 63 #include <net/if_dl.h> 64 #include <net/if_media.h> 65 #include <net/if_types.h> 66 67 #include <netinet/in_systm.h> 68 #include <netinet/in.h> 69 #include <netinet/if_ether.h> 70 #include <netinet/ip.h> 71 #include <netinet/ip.h> 72 #include <netinet/tcp.h> 73 #include <netinet/udp.h> 74 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 #include <dev/pci/pci_private.h> 78 79 #include <dev/cxgb/cxgb_osdep.h> 80 #include <dev/cxgb/common/cxgb_common.h> 81 #include <dev/cxgb/cxgb_ioctl.h> 82 #include <dev/cxgb/common/cxgb_regs.h> 83 #include <dev/cxgb/common/cxgb_t3_cpl.h> 84 #include <dev/cxgb/common/cxgb_firmware_exports.h> 85 86 #include <dev/cxgb/sys/mvec.h> 87 88 89 #ifdef PRIV_SUPPORTED 90 #include <sys/priv.h> 91 #endif 92 93 static int cxgb_setup_msix(adapter_t *, int); 94 static void cxgb_init(void *); 95 static void cxgb_init_locked(struct port_info *); 96 static void cxgb_stop_locked(struct port_info *); 97 static void cxgb_set_rxmode(struct port_info *); 98 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 99 static void cxgb_start(struct ifnet *); 100 static void cxgb_start_proc(void *, int ncount); 101 static int cxgb_media_change(struct ifnet *); 102 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 103 static int setup_sge_qsets(adapter_t *); 104 static void cxgb_async_intr(void *); 105 static void cxgb_ext_intr_handler(void *, int); 106 static void cxgb_tick(void *); 107 static void setup_rss(adapter_t *sc); 108 109 /* Attachment glue for the PCI controller end of the device. Each port of 110 * the device is attached separately, as defined later. 111 */ 112 static int cxgb_controller_probe(device_t); 113 static int cxgb_controller_attach(device_t); 114 static int cxgb_controller_detach(device_t); 115 static void cxgb_free(struct adapter *); 116 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 117 unsigned int end); 118 static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf); 119 static int cxgb_get_regs_len(void); 120 121 static device_method_t cxgb_controller_methods[] = { 122 DEVMETHOD(device_probe, cxgb_controller_probe), 123 DEVMETHOD(device_attach, cxgb_controller_attach), 124 DEVMETHOD(device_detach, cxgb_controller_detach), 125 126 /* bus interface */ 127 DEVMETHOD(bus_print_child, bus_generic_print_child), 128 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 129 130 { 0, 0 } 131 }; 132 133 static driver_t cxgb_controller_driver = { 134 "cxgbc", 135 cxgb_controller_methods, 136 sizeof(struct adapter) 137 }; 138 139 static devclass_t cxgb_controller_devclass; 140 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 141 142 /* 143 * Attachment glue for the ports. Attachment is done directly to the 144 * controller device. 145 */ 146 static int cxgb_port_probe(device_t); 147 static int cxgb_port_attach(device_t); 148 static int cxgb_port_detach(device_t); 149 150 static device_method_t cxgb_port_methods[] = { 151 DEVMETHOD(device_probe, cxgb_port_probe), 152 DEVMETHOD(device_attach, cxgb_port_attach), 153 DEVMETHOD(device_detach, cxgb_port_detach), 154 { 0, 0 } 155 }; 156 157 static driver_t cxgb_port_driver = { 158 "cxgb", 159 cxgb_port_methods, 160 0 161 }; 162 163 static d_ioctl_t cxgb_extension_ioctl; 164 165 static devclass_t cxgb_port_devclass; 166 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 167 168 #define SGE_MSIX_COUNT (SGE_QSETS + 1) 169 170 extern int collapse_mbufs; 171 /* 172 * The driver uses the best interrupt scheme available on a platform in the 173 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 174 * of these schemes the driver may consider as follows: 175 * 176 * msi = 2: choose from among all three options 177 * msi = 1 : only consider MSI and pin interrupts 178 * msi = 0: force pin interrupts 179 */ 180 static int msi_allowed = 2; 181 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 182 183 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 184 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 185 "MSI-X, MSI, INTx selector"); 186 /* 187 * Multiple queues need further tuning 188 */ 189 static int singleq = 1; 190 191 enum { 192 MAX_TXQ_ENTRIES = 16384, 193 MAX_CTRL_TXQ_ENTRIES = 1024, 194 MAX_RSPQ_ENTRIES = 16384, 195 MAX_RX_BUFFERS = 16384, 196 MAX_RX_JUMBO_BUFFERS = 16384, 197 MIN_TXQ_ENTRIES = 4, 198 MIN_CTRL_TXQ_ENTRIES = 4, 199 MIN_RSPQ_ENTRIES = 32, 200 MIN_FL_ENTRIES = 32 201 }; 202 203 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 204 205 /* Table for probing the cards. The desc field isn't actually used */ 206 struct cxgb_ident { 207 uint16_t vendor; 208 uint16_t device; 209 int index; 210 char *desc; 211 } cxgb_identifiers[] = { 212 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 213 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 214 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 215 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 216 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 217 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 218 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 219 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 220 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 221 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 222 {0, 0, 0, NULL} 223 }; 224 225 static struct cxgb_ident * 226 cxgb_get_ident(device_t dev) 227 { 228 struct cxgb_ident *id; 229 230 for (id = cxgb_identifiers; id->desc != NULL; id++) { 231 if ((id->vendor == pci_get_vendor(dev)) && 232 (id->device == pci_get_device(dev))) { 233 return (id); 234 } 235 } 236 return (NULL); 237 } 238 239 static const struct adapter_info * 240 cxgb_get_adapter_info(device_t dev) 241 { 242 struct cxgb_ident *id; 243 const struct adapter_info *ai; 244 245 id = cxgb_get_ident(dev); 246 if (id == NULL) 247 return (NULL); 248 249 ai = t3_get_adapter_info(id->index); 250 251 return (ai); 252 } 253 254 static int 255 cxgb_controller_probe(device_t dev) 256 { 257 const struct adapter_info *ai; 258 char *ports, buf[80]; 259 260 ai = cxgb_get_adapter_info(dev); 261 if (ai == NULL) 262 return (ENXIO); 263 264 if (ai->nports == 1) 265 ports = "port"; 266 else 267 ports = "ports"; 268 269 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, ai->nports, ports); 270 device_set_desc_copy(dev, buf); 271 return (BUS_PROBE_DEFAULT); 272 } 273 274 static int 275 cxgb_fw_download(adapter_t *sc, device_t dev) 276 { 277 char buf[32]; 278 #ifdef FIRMWARE_LATEST 279 const struct firmware *fw; 280 #else 281 struct firmware *fw; 282 #endif 283 int status; 284 285 snprintf(&buf[0], sizeof(buf), "t3fw%d%d", FW_VERSION_MAJOR, 286 FW_VERSION_MINOR); 287 288 fw = firmware_get(buf); 289 290 291 if (fw == NULL) { 292 device_printf(dev, "Could not find firmware image %s\n", buf); 293 return ENOENT; 294 } 295 296 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 297 298 firmware_put(fw, FIRMWARE_UNLOAD); 299 300 return (status); 301 } 302 303 304 static int 305 cxgb_controller_attach(device_t dev) 306 { 307 driver_intr_t *cxgb_intr = NULL; 308 device_t child; 309 const struct adapter_info *ai; 310 struct adapter *sc; 311 int i, reg, msi_needed, msi_count = 0, error = 0; 312 uint32_t vers; 313 int port_qsets = 1; 314 315 sc = device_get_softc(dev); 316 sc->dev = dev; 317 318 /* find the PCIe link width and set max read request to 4KB*/ 319 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 320 uint16_t lnk, pectl; 321 lnk = pci_read_config(dev, reg + 0x12, 2); 322 sc->link_width = (lnk >> 4) & 0x3f; 323 324 pectl = pci_read_config(dev, reg + 0x8, 2); 325 pectl = (pectl & ~0x7000) | (5 << 12); 326 pci_write_config(dev, reg + 0x8, pectl, 2); 327 } 328 if (sc->link_width != 0 && sc->link_width <= 4) { 329 device_printf(sc->dev, 330 "PCIe x%d Link, expect reduced performance\n", 331 sc->link_width); 332 } 333 334 pci_enable_busmaster(dev); 335 336 /* 337 * Allocate the registers and make them available to the driver. 338 * The registers that we care about for NIC mode are in BAR 0 339 */ 340 sc->regs_rid = PCIR_BAR(0); 341 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 342 &sc->regs_rid, RF_ACTIVE)) == NULL) { 343 device_printf(dev, "Cannot allocate BAR\n"); 344 return (ENXIO); 345 } 346 347 mtx_init(&sc->sge.reg_lock, "SGE reg lock", NULL, MTX_DEF); 348 mtx_init(&sc->lock, "cxgb controller lock", NULL, MTX_DEF); 349 mtx_init(&sc->mdio_lock, "cxgb mdio", NULL, MTX_DEF); 350 351 sc->bt = rman_get_bustag(sc->regs_res); 352 sc->bh = rman_get_bushandle(sc->regs_res); 353 sc->mmio_len = rman_get_size(sc->regs_res); 354 355 ai = cxgb_get_adapter_info(dev); 356 if (t3_prep_adapter(sc, ai, 1) < 0) { 357 error = ENODEV; 358 goto out; 359 } 360 361 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 362 * enough messages for the queue sets. If that fails, try falling 363 * back to MSI. If that fails, then try falling back to the legacy 364 * interrupt pin model. 365 */ 366 #ifdef MSI_SUPPORTED 367 368 sc->msix_regs_rid = 0x20; 369 if ((msi_allowed >= 2) && 370 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 371 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 372 373 msi_needed = msi_count = SGE_MSIX_COUNT; 374 375 if ((pci_alloc_msix(dev, &msi_count) != 0) || 376 (msi_count != msi_needed)) { 377 device_printf(dev, "msix allocation failed" 378 " will try msi\n"); 379 msi_count = 0; 380 pci_release_msi(dev); 381 bus_release_resource(dev, SYS_RES_MEMORY, 382 sc->msix_regs_rid, sc->msix_regs_res); 383 sc->msix_regs_res = NULL; 384 } else { 385 sc->flags |= USING_MSIX; 386 cxgb_intr = t3_intr_msix; 387 } 388 } 389 390 if ((msi_allowed >= 1) && (msi_count == 0)) { 391 msi_count = 1; 392 if (pci_alloc_msi(dev, &msi_count)) { 393 device_printf(dev, "alloc msi failed - will try INTx\n"); 394 msi_count = 0; 395 pci_release_msi(dev); 396 } else { 397 sc->flags |= USING_MSI; 398 sc->irq_rid = 1; 399 cxgb_intr = t3_intr_msi; 400 } 401 } 402 #endif 403 if (msi_count == 0) { 404 device_printf(dev, "using line interrupts\n"); 405 sc->irq_rid = 0; 406 cxgb_intr = t3b_intr; 407 } 408 409 410 /* Create a private taskqueue thread for handling driver events */ 411 #ifdef TASKQUEUE_CURRENT 412 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 413 taskqueue_thread_enqueue, &sc->tq); 414 #else 415 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 416 taskqueue_thread_enqueue, &sc->tq); 417 #endif 418 if (sc->tq == NULL) { 419 device_printf(dev, "failed to allocate controller task queue\n"); 420 goto out; 421 } 422 423 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 424 device_get_nameunit(dev)); 425 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 426 427 428 /* Create a periodic callout for checking adapter status */ 429 callout_init_mtx(&sc->cxgb_tick_ch, &sc->lock, CALLOUT_RETURNUNLOCKED); 430 431 if (t3_check_fw_version(sc) != 0) { 432 /* 433 * Warn user that a firmware update will be attempted in init. 434 */ 435 device_printf(dev, "firmware needs to be updated to version %d.%d\n", 436 FW_VERSION_MAJOR, FW_VERSION_MINOR); 437 sc->flags &= ~FW_UPTODATE; 438 } else { 439 sc->flags |= FW_UPTODATE; 440 } 441 442 if (t3_init_hw(sc, 0) != 0) { 443 device_printf(dev, "hw initialization failed\n"); 444 error = ENXIO; 445 goto out; 446 } 447 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 448 449 450 if ((singleq == 0) && (sc->flags & USING_MSIX)) 451 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); 452 453 /* 454 * Create a child device for each MAC. The ethernet attachment 455 * will be done in these children. 456 */ 457 for (i = 0; i < (sc)->params.nports; i++) { 458 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 459 device_printf(dev, "failed to add child port\n"); 460 error = EINVAL; 461 goto out; 462 } 463 sc->portdev[i] = child; 464 sc->port[i].adapter = sc; 465 sc->port[i].nqsets = port_qsets; 466 sc->port[i].first_qset = i*port_qsets; 467 sc->port[i].port = i; 468 device_set_softc(child, &sc->port[i]); 469 } 470 if ((error = bus_generic_attach(dev)) != 0) 471 goto out;; 472 473 if ((error = setup_sge_qsets(sc)) != 0) 474 goto out; 475 476 setup_rss(sc); 477 478 /* If it's MSI or INTx, allocate a single interrupt for everything */ 479 if ((sc->flags & USING_MSIX) == 0) { 480 if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 481 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 482 device_printf(dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid); 483 error = EINVAL; 484 goto out; 485 } 486 device_printf(dev, "allocated irq_res=%p\n", sc->irq_res); 487 488 if (bus_setup_intr(dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 489 #ifdef INTR_FILTERS 490 NULL, 491 #endif 492 cxgb_intr, sc, &sc->intr_tag)) { 493 device_printf(dev, "Cannot set up interrupt\n"); 494 error = EINVAL; 495 goto out; 496 } 497 } else { 498 cxgb_setup_msix(sc, msi_count); 499 } 500 501 sc->params.stats_update_period = 1; 502 503 /* initialize sge private state */ 504 t3_sge_init_sw(sc); 505 506 t3_led_ready(sc); 507 508 error = t3_get_fw_version(sc, &vers); 509 if (error) 510 goto out; 511 512 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d", G_FW_VERSION_MAJOR(vers), 513 G_FW_VERSION_MINOR(vers)); 514 515 t3_add_sysctls(sc); 516 517 out: 518 if (error) 519 cxgb_free(sc); 520 521 return (error); 522 } 523 524 static int 525 cxgb_controller_detach(device_t dev) 526 { 527 struct adapter *sc; 528 529 sc = device_get_softc(dev); 530 531 cxgb_free(sc); 532 533 return (0); 534 } 535 536 static void 537 cxgb_free(struct adapter *sc) 538 { 539 int i; 540 541 /* 542 * XXX need to drain the ifq by hand until 543 * it is taught about mbuf iovecs 544 */ 545 546 callout_drain(&sc->cxgb_tick_ch); 547 548 t3_sge_deinit_sw(sc); 549 550 if (sc->tq != NULL) { 551 taskqueue_drain(sc->tq, &sc->ext_intr_task); 552 taskqueue_free(sc->tq); 553 } 554 555 for (i = 0; i < (sc)->params.nports; ++i) { 556 if (sc->portdev[i] != NULL) 557 device_delete_child(sc->dev, sc->portdev[i]); 558 } 559 560 bus_generic_detach(sc->dev); 561 562 t3_free_sge_resources(sc); 563 t3_sge_free(sc); 564 565 for (i = 0; i < SGE_QSETS; i++) { 566 if (sc->msix_intr_tag[i] != NULL) { 567 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 568 sc->msix_intr_tag[i]); 569 } 570 if (sc->msix_irq_res[i] != NULL) { 571 bus_release_resource(sc->dev, SYS_RES_IRQ, 572 sc->msix_irq_rid[i], sc->msix_irq_res[i]); 573 } 574 } 575 576 if (sc->intr_tag != NULL) { 577 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 578 } 579 580 if (sc->irq_res != NULL) { 581 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", 582 sc->irq_rid, sc->irq_res); 583 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 584 sc->irq_res); 585 } 586 #ifdef MSI_SUPPORTED 587 if (sc->flags & (USING_MSI | USING_MSIX)) { 588 device_printf(sc->dev, "releasing msi message(s)\n"); 589 pci_release_msi(sc->dev); 590 } 591 #endif 592 if (sc->msix_regs_res != NULL) { 593 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 594 sc->msix_regs_res); 595 } 596 597 if (sc->regs_res != NULL) 598 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 599 sc->regs_res); 600 601 mtx_destroy(&sc->mdio_lock); 602 mtx_destroy(&sc->sge.reg_lock); 603 mtx_destroy(&sc->lock); 604 605 return; 606 } 607 608 /** 609 * setup_sge_qsets - configure SGE Tx/Rx/response queues 610 * @sc: the controller softc 611 * 612 * Determines how many sets of SGE queues to use and initializes them. 613 * We support multiple queue sets per port if we have MSI-X, otherwise 614 * just one queue set per port. 615 */ 616 static int 617 setup_sge_qsets(adapter_t *sc) 618 { 619 int i, j, err, irq_idx, qset_idx; 620 u_int ntxq = 3; 621 622 if ((err = t3_sge_alloc(sc)) != 0) { 623 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 624 return (err); 625 } 626 627 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 628 irq_idx = -1; 629 else 630 irq_idx = 0; 631 632 for (qset_idx = 0, i = 0; i < (sc)->params.nports; ++i) { 633 struct port_info *pi = &sc->port[i]; 634 635 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) { 636 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 637 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 638 &sc->params.sge.qset[qset_idx], ntxq, pi); 639 if (err) { 640 t3_free_sge_resources(sc); 641 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err); 642 return (err); 643 } 644 } 645 } 646 647 return (0); 648 } 649 650 static int 651 cxgb_setup_msix(adapter_t *sc, int msix_count) 652 { 653 int i, j, k, nqsets, rid; 654 655 /* The first message indicates link changes and error conditions */ 656 sc->irq_rid = 1; 657 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 658 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 659 device_printf(sc->dev, "Cannot allocate msix interrupt\n"); 660 return (EINVAL); 661 } 662 663 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 664 #ifdef INTR_FILTERS 665 NULL, 666 #endif 667 cxgb_async_intr, sc, &sc->intr_tag)) { 668 device_printf(sc->dev, "Cannot set up interrupt\n"); 669 return (EINVAL); 670 } 671 for (i = 0, k = 0; i < (sc)->params.nports; ++i) { 672 nqsets = sc->port[i].nqsets; 673 for (j = 0; j < nqsets; ++j, k++) { 674 struct sge_qset *qs = &sc->sge.qs[k]; 675 676 rid = k + 2; 677 if (cxgb_debug) 678 printf("rid=%d ", rid); 679 if ((sc->msix_irq_res[k] = bus_alloc_resource_any( 680 sc->dev, SYS_RES_IRQ, &rid, 681 RF_SHAREABLE | RF_ACTIVE)) == NULL) { 682 device_printf(sc->dev, "Cannot allocate " 683 "interrupt for message %d\n", rid); 684 return (EINVAL); 685 } 686 sc->msix_irq_rid[k] = rid; 687 if (bus_setup_intr(sc->dev, sc->msix_irq_res[j], 688 INTR_MPSAFE|INTR_TYPE_NET, 689 #ifdef INTR_FILTERS 690 NULL, 691 #endif 692 t3_intr_msix, qs, &sc->msix_intr_tag[k])) { 693 device_printf(sc->dev, "Cannot set up " 694 "interrupt for message %d\n", rid); 695 return (EINVAL); 696 } 697 } 698 } 699 700 701 return (0); 702 } 703 704 static int 705 cxgb_port_probe(device_t dev) 706 { 707 struct port_info *p; 708 char buf[80]; 709 710 p = device_get_softc(dev); 711 712 snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc); 713 device_set_desc_copy(dev, buf); 714 return (0); 715 } 716 717 718 static int 719 cxgb_makedev(struct port_info *pi) 720 { 721 struct cdevsw *cxgb_cdevsw; 722 723 if ((cxgb_cdevsw = malloc(sizeof(struct cdevsw), M_DEVBUF, M_NOWAIT|M_ZERO)) == NULL) 724 return (ENOMEM); 725 726 cxgb_cdevsw->d_version = D_VERSION; 727 cxgb_cdevsw->d_name = strdup(pi->ifp->if_xname, M_DEVBUF); 728 cxgb_cdevsw->d_ioctl = cxgb_extension_ioctl; 729 730 pi->port_cdev = make_dev(cxgb_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 731 pi->ifp->if_xname); 732 733 if (pi->port_cdev == NULL) 734 return (ENOMEM); 735 736 pi->port_cdev->si_drv1 = (void *)pi; 737 738 return (0); 739 } 740 741 742 #ifdef TSO_SUPPORTED 743 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU) 744 /* Don't enable TSO6 yet */ 745 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU) 746 #else 747 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 748 /* Don't enable TSO6 yet */ 749 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 750 #define IFCAP_TSO4 0x0 751 #define CSUM_TSO 0x0 752 #endif 753 754 755 static int 756 cxgb_port_attach(device_t dev) 757 { 758 struct port_info *p; 759 struct ifnet *ifp; 760 int media_flags; 761 int err; 762 char buf[64]; 763 764 p = device_get_softc(dev); 765 766 snprintf(buf, sizeof(buf), "cxgb port %d", p->port); 767 mtx_init(&p->lock, buf, 0, MTX_DEF); 768 769 /* Allocate an ifnet object and set it up */ 770 ifp = p->ifp = if_alloc(IFT_ETHER); 771 if (ifp == NULL) { 772 device_printf(dev, "Cannot allocate ifnet\n"); 773 return (ENOMEM); 774 } 775 776 /* 777 * Note that there is currently no watchdog timer. 778 */ 779 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 780 ifp->if_init = cxgb_init; 781 ifp->if_softc = p; 782 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 783 ifp->if_ioctl = cxgb_ioctl; 784 ifp->if_start = cxgb_start; 785 ifp->if_timer = 0; /* Disable ifnet watchdog */ 786 ifp->if_watchdog = NULL; 787 788 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE; 789 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 790 IFQ_SET_READY(&ifp->if_snd); 791 792 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 793 ifp->if_capabilities |= CXGB_CAP; 794 ifp->if_capenable |= CXGB_CAP_ENABLE; 795 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 796 ifp->if_baudrate = 100000000; 797 798 ether_ifattach(ifp, p->hw_addr); 799 #ifdef DEFAULT_JUMBO 800 ifp->if_mtu = 9000; 801 #endif 802 if ((err = cxgb_makedev(p)) != 0) { 803 printf("makedev failed %d\n", err); 804 return (err); 805 } 806 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 807 cxgb_media_status); 808 809 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) 810 media_flags = IFM_ETHER | IFM_10G_CX4; 811 else if (!strcmp(p->port_type->desc, "10GBASE-SR")) 812 media_flags = IFM_ETHER | IFM_10G_SR; 813 else if (!strcmp(p->port_type->desc, "10GBASE-XR")) 814 media_flags = IFM_ETHER | IFM_10G_LR; 815 else { 816 printf("unsupported media type %s\n", p->port_type->desc); 817 return (ENXIO); 818 } 819 820 ifmedia_add(&p->media, media_flags, 0, NULL); 821 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 822 ifmedia_set(&p->media, media_flags); 823 824 snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port); 825 #ifdef TASKQUEUE_CURRENT 826 /* Create a port for handling TX without starvation */ 827 p->tq = taskqueue_create(buf, M_NOWAIT, 828 taskqueue_thread_enqueue, &p->tq); 829 #else 830 /* Create a port for handling TX without starvation */ 831 p->tq = taskqueue_create_fast(buf, M_NOWAIT, 832 taskqueue_thread_enqueue, &p->tq); 833 #endif 834 835 836 if (p->tq == NULL) { 837 device_printf(dev, "failed to allocate port task queue\n"); 838 return (ENOMEM); 839 } 840 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq", 841 device_get_nameunit(dev)); 842 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp); 843 844 845 return (0); 846 } 847 848 static int 849 cxgb_port_detach(device_t dev) 850 { 851 struct port_info *p; 852 853 p = device_get_softc(dev); 854 mtx_destroy(&p->lock); 855 if (p->tq != NULL) { 856 taskqueue_drain(p->tq, &p->start_task); 857 taskqueue_free(p->tq); 858 p->tq = NULL; 859 } 860 861 ether_ifdetach(p->ifp); 862 if_free(p->ifp); 863 864 destroy_dev(p->port_cdev); 865 866 867 return (0); 868 } 869 870 void 871 t3_fatal_err(struct adapter *sc) 872 { 873 u_int fw_status[4]; 874 875 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 876 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 877 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 878 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 879 } 880 881 int 882 t3_os_find_pci_capability(adapter_t *sc, int cap) 883 { 884 device_t dev; 885 struct pci_devinfo *dinfo; 886 pcicfgregs *cfg; 887 uint32_t status; 888 uint8_t ptr; 889 890 dev = sc->dev; 891 dinfo = device_get_ivars(dev); 892 cfg = &dinfo->cfg; 893 894 status = pci_read_config(dev, PCIR_STATUS, 2); 895 if (!(status & PCIM_STATUS_CAPPRESENT)) 896 return (0); 897 898 switch (cfg->hdrtype & PCIM_HDRTYPE) { 899 case 0: 900 case 1: 901 ptr = PCIR_CAP_PTR; 902 break; 903 case 2: 904 ptr = PCIR_CAP_PTR_2; 905 break; 906 default: 907 return (0); 908 break; 909 } 910 ptr = pci_read_config(dev, ptr, 1); 911 912 while (ptr != 0) { 913 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 914 return (ptr); 915 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 916 } 917 918 return (0); 919 } 920 921 int 922 t3_os_pci_save_state(struct adapter *sc) 923 { 924 device_t dev; 925 struct pci_devinfo *dinfo; 926 927 dev = sc->dev; 928 dinfo = device_get_ivars(dev); 929 930 pci_cfg_save(dev, dinfo, 0); 931 return (0); 932 } 933 934 int 935 t3_os_pci_restore_state(struct adapter *sc) 936 { 937 device_t dev; 938 struct pci_devinfo *dinfo; 939 940 dev = sc->dev; 941 dinfo = device_get_ivars(dev); 942 943 pci_cfg_restore(dev, dinfo); 944 return (0); 945 } 946 947 /** 948 * t3_os_link_changed - handle link status changes 949 * @adapter: the adapter associated with the link change 950 * @port_id: the port index whose limk status has changed 951 * @link_stat: the new status of the link 952 * @speed: the new speed setting 953 * @duplex: the new duplex setting 954 * @fc: the new flow-control setting 955 * 956 * This is the OS-dependent handler for link status changes. The OS 957 * neutral handler takes care of most of the processing for these events, 958 * then calls this handler for any OS-specific processing. 959 */ 960 void 961 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 962 int duplex, int fc) 963 { 964 struct port_info *pi = &adapter->port[port_id]; 965 966 if ((pi->ifp->if_flags & IFF_UP) == 0) 967 return; 968 969 if (link_status) 970 if_link_state_change(pi->ifp, LINK_STATE_UP); 971 else 972 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 973 974 } 975 976 977 /* 978 * Interrupt-context handler for external (PHY) interrupts. 979 */ 980 void 981 t3_os_ext_intr_handler(adapter_t *sc) 982 { 983 if (cxgb_debug) 984 printf("t3_os_ext_intr_handler\n"); 985 /* 986 * Schedule a task to handle external interrupts as they may be slow 987 * and we use a mutex to protect MDIO registers. We disable PHY 988 * interrupts in the meantime and let the task reenable them when 989 * it's done. 990 */ 991 if (sc->slow_intr_mask) { 992 sc->slow_intr_mask &= ~F_T3DBG; 993 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 994 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 995 } 996 } 997 998 void 999 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1000 { 1001 1002 /* 1003 * The ifnet might not be allocated before this gets called, 1004 * as this is called early on in attach by t3_prep_adapter 1005 * save the address off in the port structure 1006 */ 1007 if (cxgb_debug) 1008 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1009 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1010 } 1011 1012 /** 1013 * link_start - enable a port 1014 * @p: the port to enable 1015 * 1016 * Performs the MAC and PHY actions needed to enable a port. 1017 */ 1018 static void 1019 cxgb_link_start(struct port_info *p) 1020 { 1021 struct ifnet *ifp; 1022 struct t3_rx_mode rm; 1023 struct cmac *mac = &p->mac; 1024 1025 ifp = p->ifp; 1026 1027 t3_init_rx_mode(&rm, p); 1028 t3_mac_reset(mac); 1029 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN); 1030 t3_mac_set_address(mac, 0, p->hw_addr); 1031 t3_mac_set_rx_mode(mac, &rm); 1032 t3_link_start(&p->phy, mac, &p->link_config); 1033 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1034 } 1035 1036 /** 1037 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1038 * @adap: the adapter 1039 * 1040 * Sets up RSS to distribute packets to multiple receive queues. We 1041 * configure the RSS CPU lookup table to distribute to the number of HW 1042 * receive queues, and the response queue lookup table to narrow that 1043 * down to the response queues actually configured for each port. 1044 * We always configure the RSS mapping for two ports since the mapping 1045 * table has plenty of entries. 1046 */ 1047 static void 1048 setup_rss(adapter_t *adap) 1049 { 1050 int i; 1051 u_int nq0 = adap->port[0].nqsets; 1052 u_int nq1 = max((u_int)adap->port[1].nqsets, 1U); 1053 uint8_t cpus[SGE_QSETS + 1]; 1054 uint16_t rspq_map[RSS_TABLE_SIZE]; 1055 1056 for (i = 0; i < SGE_QSETS; ++i) 1057 cpus[i] = i; 1058 cpus[SGE_QSETS] = 0xff; 1059 1060 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1061 rspq_map[i] = i % nq0; 1062 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0; 1063 } 1064 1065 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1066 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | 1067 V_RRCPLCPUSIZE(6), cpus, rspq_map); 1068 } 1069 1070 static void 1071 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1072 int hi, int port) 1073 { 1074 struct mbuf *m; 1075 struct mngt_pktsched_wr *req; 1076 1077 m = m_gethdr(M_NOWAIT, MT_DATA); 1078 if (m) { 1079 req = (struct mngt_pktsched_wr *)m->m_data; 1080 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1081 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1082 req->sched = sched; 1083 req->idx = qidx; 1084 req->min = lo; 1085 req->max = hi; 1086 req->binding = port; 1087 m->m_len = m->m_pkthdr.len = sizeof(*req); 1088 t3_mgmt_tx(adap, m); 1089 } 1090 } 1091 1092 static void 1093 bind_qsets(adapter_t *sc) 1094 { 1095 int i, j; 1096 1097 for (i = 0; i < (sc)->params.nports; ++i) { 1098 const struct port_info *pi = adap2pinfo(sc, i); 1099 1100 for (j = 0; j < pi->nqsets; ++j) 1101 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1102 -1, i); 1103 } 1104 } 1105 1106 static void 1107 cxgb_init(void *arg) 1108 { 1109 struct port_info *p = arg; 1110 1111 PORT_LOCK(p); 1112 cxgb_init_locked(p); 1113 PORT_UNLOCK(p); 1114 } 1115 1116 static void 1117 cxgb_init_locked(struct port_info *p) 1118 { 1119 struct ifnet *ifp; 1120 adapter_t *sc = p->adapter; 1121 int error; 1122 1123 mtx_assert(&p->lock, MA_OWNED); 1124 1125 ifp = p->ifp; 1126 if ((sc->flags & FW_UPTODATE) == 0) { 1127 device_printf(sc->dev, "updating firmware to version %d.%d\n", 1128 FW_VERSION_MAJOR, FW_VERSION_MINOR); 1129 if ((error = cxgb_fw_download(sc, sc->dev)) != 0) { 1130 device_printf(sc->dev, "firmware download failed err: %d" 1131 "interface will be unavailable\n", error); 1132 return; 1133 } 1134 sc->flags |= FW_UPTODATE; 1135 } 1136 1137 cxgb_link_start(p); 1138 ADAPTER_LOCK(p->adapter); 1139 if (p->adapter->open_device_map == 0) 1140 t3_intr_clear(sc); 1141 t3_sge_start(sc); 1142 1143 p->adapter->open_device_map |= (1 << p->port); 1144 ADAPTER_UNLOCK(p->adapter); 1145 t3_intr_enable(sc); 1146 t3_port_intr_enable(sc, p->port); 1147 1148 if ((p->adapter->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX) 1149 bind_qsets(sc); 1150 p->adapter->flags |= QUEUES_BOUND; 1151 1152 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, 1153 cxgb_tick, sc); 1154 1155 1156 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1157 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1158 } 1159 1160 static void 1161 cxgb_set_rxmode(struct port_info *p) 1162 { 1163 struct t3_rx_mode rm; 1164 struct cmac *mac = &p->mac; 1165 1166 mtx_assert(&p->lock, MA_OWNED); 1167 1168 t3_init_rx_mode(&rm, p); 1169 t3_mac_set_rx_mode(mac, &rm); 1170 } 1171 1172 static void 1173 cxgb_stop_locked(struct port_info *p) 1174 { 1175 struct ifnet *ifp; 1176 1177 mtx_assert(&p->lock, MA_OWNED); 1178 mtx_assert(&p->adapter->lock, MA_NOTOWNED); 1179 1180 ifp = p->ifp; 1181 1182 ADAPTER_LOCK(p->adapter); 1183 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1184 p->adapter->open_device_map &= ~(1 << p->port); 1185 if (p->adapter->open_device_map == 0) 1186 t3_intr_disable(p->adapter); 1187 ADAPTER_UNLOCK(p->adapter); 1188 t3_port_intr_disable(p->adapter, p->port); 1189 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1190 1191 } 1192 1193 static int 1194 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1195 { 1196 struct port_info *p = ifp->if_softc; 1197 struct ifaddr *ifa = (struct ifaddr *)data; 1198 struct ifreq *ifr = (struct ifreq *)data; 1199 int flags, error = 0; 1200 uint32_t mask; 1201 1202 /* 1203 * XXX need to check that we aren't in the middle of an unload 1204 */ 1205 switch (command) { 1206 case SIOCSIFMTU: 1207 if ((ifr->ifr_mtu < ETHERMIN) || 1208 (ifr->ifr_mtu > ETHER_MAX_LEN_JUMBO)) 1209 error = EINVAL; 1210 else if (ifp->if_mtu != ifr->ifr_mtu) { 1211 PORT_LOCK(p); 1212 ifp->if_mtu = ifr->ifr_mtu; 1213 t3_mac_set_mtu(&p->mac, ifp->if_mtu + ETHER_HDR_LEN); 1214 PORT_UNLOCK(p); 1215 } 1216 break; 1217 case SIOCSIFADDR: 1218 case SIOCGIFADDR: 1219 if (ifa->ifa_addr->sa_family == AF_INET) { 1220 ifp->if_flags |= IFF_UP; 1221 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1222 cxgb_init(p); 1223 } 1224 arp_ifinit(ifp, ifa); 1225 } else 1226 error = ether_ioctl(ifp, command, data); 1227 break; 1228 case SIOCSIFFLAGS: 1229 if (ifp->if_flags & IFF_UP) { 1230 PORT_LOCK(p); 1231 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1232 flags = p->if_flags; 1233 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1234 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 1235 cxgb_set_rxmode(p); 1236 1237 } else 1238 cxgb_init_locked(p); 1239 p->if_flags = ifp->if_flags; 1240 PORT_UNLOCK(p); 1241 } else { 1242 callout_drain(&p->adapter->cxgb_tick_ch); 1243 PORT_LOCK(p); 1244 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1245 cxgb_stop_locked(p); 1246 } else { 1247 adapter_t *sc = p->adapter; 1248 callout_reset(&sc->cxgb_tick_ch, 1249 sc->params.stats_update_period * hz, 1250 cxgb_tick, sc); 1251 } 1252 PORT_UNLOCK(p); 1253 } 1254 1255 1256 break; 1257 case SIOCSIFMEDIA: 1258 case SIOCGIFMEDIA: 1259 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 1260 break; 1261 case SIOCSIFCAP: 1262 PORT_LOCK(p); 1263 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1264 if (mask & IFCAP_TXCSUM) { 1265 if (IFCAP_TXCSUM & ifp->if_capenable) { 1266 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 1267 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 1268 | CSUM_TSO); 1269 } else { 1270 ifp->if_capenable |= IFCAP_TXCSUM; 1271 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1272 } 1273 } else if (mask & IFCAP_RXCSUM) { 1274 if (IFCAP_RXCSUM & ifp->if_capenable) { 1275 ifp->if_capenable &= ~IFCAP_RXCSUM; 1276 } else { 1277 ifp->if_capenable |= IFCAP_RXCSUM; 1278 } 1279 } 1280 if (mask & IFCAP_TSO4) { 1281 if (IFCAP_TSO4 & ifp->if_capenable) { 1282 ifp->if_capenable &= ~IFCAP_TSO4; 1283 ifp->if_hwassist &= ~CSUM_TSO; 1284 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 1285 ifp->if_capenable |= IFCAP_TSO4; 1286 ifp->if_hwassist |= CSUM_TSO; 1287 } else { 1288 if (cxgb_debug) 1289 printf("cxgb requires tx checksum offload" 1290 " be enabled to use TSO\n"); 1291 error = EINVAL; 1292 } 1293 } 1294 PORT_UNLOCK(p); 1295 break; 1296 default: 1297 error = ether_ioctl(ifp, command, data); 1298 break; 1299 } 1300 1301 return (error); 1302 } 1303 1304 static int 1305 cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) 1306 { 1307 struct sge_qset *qs; 1308 struct sge_txq *txq; 1309 struct port_info *p = ifp->if_softc; 1310 struct mbuf *m0, *m = NULL; 1311 int err, in_use_init; 1312 1313 1314 if (!p->link_config.link_ok) 1315 return (ENXIO); 1316 1317 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1318 return (ENOBUFS); 1319 1320 qs = &p->adapter->sge.qs[p->first_qset]; 1321 txq = &qs->txq[TXQ_ETH]; 1322 err = 0; 1323 1324 mtx_lock(&txq->lock); 1325 in_use_init = txq->in_use; 1326 while ((txq->in_use - in_use_init < txmax) && 1327 (txq->size > txq->in_use + TX_MAX_DESC)) { 1328 IFQ_DRV_DEQUEUE(&ifp->if_snd, m); 1329 if (m == NULL) 1330 break; 1331 /* 1332 * Convert chain to M_IOVEC 1333 */ 1334 KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early")); 1335 m0 = m; 1336 #ifdef INVARIANTS 1337 /* 1338 * Clean up after net stack sloppiness 1339 * before calling m_sanity 1340 */ 1341 m0 = m->m_next; 1342 while (m0) { 1343 m0->m_flags &= ~M_PKTHDR; 1344 m0 = m0->m_next; 1345 } 1346 m_sanity(m0, 0); 1347 m0 = m; 1348 #endif 1349 if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES && 1350 m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) { 1351 if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) { 1352 m = m0; 1353 m_collapse(m, TX_MAX_SEGS, &m0); 1354 } else 1355 break; 1356 } 1357 m = m0; 1358 if ((err = t3_encap(p, &m)) != 0) 1359 break; 1360 BPF_MTAP(ifp, m); 1361 } 1362 mtx_unlock(&txq->lock); 1363 1364 if (__predict_false(err)) { 1365 if (cxgb_debug) 1366 printf("would set OFLAGS\n"); 1367 if (err == ENOMEM) { 1368 IFQ_LOCK(&ifp->if_snd); 1369 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1370 IFQ_UNLOCK(&ifp->if_snd); 1371 } 1372 } 1373 if (err == 0 && m == NULL) 1374 err = ENOBUFS; 1375 1376 return (err); 1377 } 1378 1379 static void 1380 cxgb_start_proc(void *arg, int ncount) 1381 { 1382 struct ifnet *ifp = arg; 1383 struct port_info *pi = ifp->if_softc; 1384 struct sge_qset *qs; 1385 struct sge_txq *txq; 1386 int error = 0; 1387 1388 qs = &pi->adapter->sge.qs[pi->first_qset]; 1389 txq = &qs->txq[TXQ_ETH]; 1390 1391 while (error == 0) { 1392 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC) 1393 taskqueue_enqueue(pi->adapter->tq, 1394 &pi->adapter->timer_reclaim_task); 1395 1396 error = cxgb_start_tx(ifp, TX_START_MAX_DESC); 1397 } 1398 } 1399 1400 static void 1401 cxgb_start(struct ifnet *ifp) 1402 { 1403 struct port_info *pi = ifp->if_softc; 1404 struct sge_qset *qs; 1405 struct sge_txq *txq; 1406 int err; 1407 1408 qs = &pi->adapter->sge.qs[pi->first_qset]; 1409 txq = &qs->txq[TXQ_ETH]; 1410 1411 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC) 1412 taskqueue_enqueue(pi->adapter->tq, 1413 &pi->adapter->timer_reclaim_task); 1414 1415 err = cxgb_start_tx(ifp, TX_START_MAX_DESC); 1416 1417 if (err == 0) 1418 taskqueue_enqueue(pi->tq, &pi->start_task); 1419 } 1420 1421 1422 static int 1423 cxgb_media_change(struct ifnet *ifp) 1424 { 1425 if_printf(ifp, "media change not supported\n"); 1426 return (ENXIO); 1427 } 1428 1429 static void 1430 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1431 { 1432 struct port_info *p = ifp->if_softc; 1433 1434 ifmr->ifm_status = IFM_AVALID; 1435 ifmr->ifm_active = IFM_ETHER; 1436 1437 if (!p->link_config.link_ok) 1438 return; 1439 1440 ifmr->ifm_status |= IFM_ACTIVE; 1441 1442 if (p->link_config.duplex) 1443 ifmr->ifm_active |= IFM_FDX; 1444 else 1445 ifmr->ifm_active |= IFM_HDX; 1446 } 1447 1448 static void 1449 cxgb_async_intr(void *data) 1450 { 1451 adapter_t *sc = data; 1452 1453 if (cxgb_debug) 1454 device_printf(sc->dev, "cxgb_async_intr\n"); 1455 1456 t3_slow_intr_handler(sc); 1457 1458 } 1459 1460 static void 1461 cxgb_ext_intr_handler(void *arg, int count) 1462 { 1463 adapter_t *sc = (adapter_t *)arg; 1464 1465 if (cxgb_debug) 1466 printf("cxgb_ext_intr_handler\n"); 1467 1468 t3_phy_intr_handler(sc); 1469 1470 /* Now reenable external interrupts */ 1471 if (sc->slow_intr_mask) { 1472 sc->slow_intr_mask |= F_T3DBG; 1473 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 1474 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1475 } 1476 } 1477 1478 static void 1479 check_link_status(adapter_t *sc) 1480 { 1481 int i; 1482 1483 for (i = 0; i < (sc)->params.nports; ++i) { 1484 struct port_info *p = &sc->port[i]; 1485 1486 if (!(p->port_type->caps & SUPPORTED_IRQ)) 1487 t3_link_changed(sc, i); 1488 } 1489 } 1490 1491 static void 1492 check_t3b2_mac(struct adapter *adapter) 1493 { 1494 int i; 1495 1496 for_each_port(adapter, i) { 1497 struct port_info *p = &adapter->port[i]; 1498 struct ifnet *ifp = p->ifp; 1499 int status; 1500 1501 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1502 continue; 1503 1504 status = 0; 1505 PORT_LOCK(p); 1506 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 1507 status = t3b2_mac_watchdog_task(&p->mac); 1508 if (status == 1) 1509 p->mac.stats.num_toggled++; 1510 else if (status == 2) { 1511 struct cmac *mac = &p->mac; 1512 1513 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN); 1514 t3_mac_set_address(mac, 0, p->hw_addr); 1515 cxgb_set_rxmode(p); 1516 t3_link_start(&p->phy, mac, &p->link_config); 1517 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1518 t3_port_intr_enable(adapter, p->port); 1519 p->mac.stats.num_resets++; 1520 } 1521 PORT_UNLOCK(p); 1522 } 1523 } 1524 1525 static void 1526 cxgb_tick(void *arg) 1527 { 1528 adapter_t *sc = (adapter_t *)arg; 1529 const struct adapter_params *p = &sc->params; 1530 1531 if (p->linkpoll_period) 1532 check_link_status(sc); 1533 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, 1534 cxgb_tick, sc); 1535 1536 /* 1537 * adapter lock can currently only be acquire after the 1538 * port lock 1539 */ 1540 ADAPTER_UNLOCK(sc); 1541 if (p->rev == T3_REV_B2) 1542 check_t3b2_mac(sc); 1543 1544 } 1545 1546 static int 1547 in_range(int val, int lo, int hi) 1548 { 1549 return val < 0 || (val <= hi && val >= lo); 1550 } 1551 1552 static int 1553 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 1554 int fflag, struct thread *td) 1555 { 1556 int mmd, error = 0; 1557 struct port_info *pi = dev->si_drv1; 1558 adapter_t *sc = pi->adapter; 1559 1560 #ifdef PRIV_SUPPORTED 1561 if (priv_check(td, PRIV_DRIVER)) { 1562 if (cxgb_debug) 1563 printf("user does not have access to privileged ioctls\n"); 1564 return (EPERM); 1565 } 1566 #else 1567 if (suser(td)) { 1568 if (cxgb_debug) 1569 printf("user does not have access to privileged ioctls\n"); 1570 return (EPERM); 1571 } 1572 #endif 1573 1574 switch (cmd) { 1575 case SIOCGMIIREG: { 1576 uint32_t val; 1577 struct cphy *phy = &pi->phy; 1578 struct mii_data *mid = (struct mii_data *)data; 1579 1580 if (!phy->mdio_read) 1581 return (EOPNOTSUPP); 1582 if (is_10G(sc)) { 1583 mmd = mid->phy_id >> 8; 1584 if (!mmd) 1585 mmd = MDIO_DEV_PCS; 1586 else if (mmd > MDIO_DEV_XGXS) 1587 return -EINVAL; 1588 1589 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 1590 mid->reg_num, &val); 1591 } else 1592 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 1593 mid->reg_num & 0x1f, &val); 1594 if (error == 0) 1595 mid->val_out = val; 1596 break; 1597 } 1598 case SIOCSMIIREG: { 1599 struct cphy *phy = &pi->phy; 1600 struct mii_data *mid = (struct mii_data *)data; 1601 1602 if (!phy->mdio_write) 1603 return (EOPNOTSUPP); 1604 if (is_10G(sc)) { 1605 mmd = mid->phy_id >> 8; 1606 if (!mmd) 1607 mmd = MDIO_DEV_PCS; 1608 else if (mmd > MDIO_DEV_XGXS) 1609 return (EINVAL); 1610 1611 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 1612 mmd, mid->reg_num, mid->val_in); 1613 } else 1614 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 1615 mid->reg_num & 0x1f, 1616 mid->val_in); 1617 break; 1618 } 1619 case CHELSIO_SETREG: { 1620 struct ch_reg *edata = (struct ch_reg *)data; 1621 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 1622 return (EFAULT); 1623 t3_write_reg(sc, edata->addr, edata->val); 1624 break; 1625 } 1626 case CHELSIO_GETREG: { 1627 struct ch_reg *edata = (struct ch_reg *)data; 1628 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 1629 return (EFAULT); 1630 edata->val = t3_read_reg(sc, edata->addr); 1631 break; 1632 } 1633 case CHELSIO_GET_SGE_CONTEXT: { 1634 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 1635 mtx_lock(&sc->sge.reg_lock); 1636 switch (ecntxt->cntxt_type) { 1637 case CNTXT_TYPE_EGRESS: 1638 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 1639 ecntxt->data); 1640 break; 1641 case CNTXT_TYPE_FL: 1642 error = t3_sge_read_fl(sc, ecntxt->cntxt_id, 1643 ecntxt->data); 1644 break; 1645 case CNTXT_TYPE_RSP: 1646 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id, 1647 ecntxt->data); 1648 break; 1649 case CNTXT_TYPE_CQ: 1650 error = t3_sge_read_cq(sc, ecntxt->cntxt_id, 1651 ecntxt->data); 1652 break; 1653 default: 1654 error = EINVAL; 1655 break; 1656 } 1657 mtx_unlock(&sc->sge.reg_lock); 1658 break; 1659 } 1660 case CHELSIO_GET_SGE_DESC: { 1661 struct ch_desc *edesc = (struct ch_desc *)data; 1662 int ret; 1663 if (edesc->queue_num >= SGE_QSETS * 6) 1664 return (EINVAL); 1665 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 1666 edesc->queue_num % 6, edesc->idx, edesc->data); 1667 if (ret < 0) 1668 return (EINVAL); 1669 edesc->size = ret; 1670 break; 1671 } 1672 case CHELSIO_SET_QSET_PARAMS: { 1673 struct qset_params *q; 1674 struct ch_qset_params *t = (struct ch_qset_params *)data; 1675 1676 if (t->qset_idx >= SGE_QSETS) 1677 return -EINVAL; 1678 if (!in_range(t->intr_lat, 0, M_NEWTIMER) || 1679 !in_range(t->cong_thres, 0, 255) || 1680 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES, 1681 MAX_TXQ_ENTRIES) || 1682 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES, 1683 MAX_TXQ_ENTRIES) || 1684 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES, 1685 MAX_CTRL_TXQ_ENTRIES) || 1686 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) || 1687 !in_range(t->fl_size[1], MIN_FL_ENTRIES, 1688 MAX_RX_JUMBO_BUFFERS) || 1689 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES)) 1690 return -EINVAL; 1691 if ((sc->flags & FULL_INIT_DONE) && 1692 (t->rspq_size >= 0 || t->fl_size[0] >= 0 || 1693 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 || 1694 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 || 1695 t->polling >= 0 || t->cong_thres >= 0)) 1696 return -EBUSY; 1697 1698 q = &sc->params.sge.qset[t->qset_idx]; 1699 1700 if (t->rspq_size >= 0) 1701 q->rspq_size = t->rspq_size; 1702 if (t->fl_size[0] >= 0) 1703 q->fl_size = t->fl_size[0]; 1704 if (t->fl_size[1] >= 0) 1705 q->jumbo_size = t->fl_size[1]; 1706 if (t->txq_size[0] >= 0) 1707 q->txq_size[0] = t->txq_size[0]; 1708 if (t->txq_size[1] >= 0) 1709 q->txq_size[1] = t->txq_size[1]; 1710 if (t->txq_size[2] >= 0) 1711 q->txq_size[2] = t->txq_size[2]; 1712 if (t->cong_thres >= 0) 1713 q->cong_thres = t->cong_thres; 1714 if (t->intr_lat >= 0) { 1715 struct sge_qset *qs = &sc->sge.qs[t->qset_idx]; 1716 1717 q->coalesce_nsecs = t->intr_lat*1000; 1718 t3_update_qset_coalesce(qs, q); 1719 } 1720 break; 1721 } 1722 case CHELSIO_GET_QSET_PARAMS: { 1723 struct qset_params *q; 1724 struct ch_qset_params *t = (struct ch_qset_params *)data; 1725 1726 if (t->qset_idx >= SGE_QSETS) 1727 return (EINVAL); 1728 1729 q = &(sc)->params.sge.qset[t->qset_idx]; 1730 t->rspq_size = q->rspq_size; 1731 t->txq_size[0] = q->txq_size[0]; 1732 t->txq_size[1] = q->txq_size[1]; 1733 t->txq_size[2] = q->txq_size[2]; 1734 t->fl_size[0] = q->fl_size; 1735 t->fl_size[1] = q->jumbo_size; 1736 t->polling = q->polling; 1737 t->intr_lat = q->coalesce_nsecs / 1000; 1738 t->cong_thres = q->cong_thres; 1739 break; 1740 } 1741 case CHELSIO_SET_QSET_NUM: { 1742 struct ch_reg *edata = (struct ch_reg *)data; 1743 unsigned int port_idx = pi->port; 1744 1745 if (sc->flags & FULL_INIT_DONE) 1746 return (EBUSY); 1747 if (edata->val < 1 || 1748 (edata->val > 1 && !(sc->flags & USING_MSIX))) 1749 return (EINVAL); 1750 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS) 1751 return (EINVAL); 1752 sc->port[port_idx].nqsets = edata->val; 1753 /* 1754 * XXX we're hardcoding ourselves to 2 ports 1755 * just like the LEENUX 1756 */ 1757 sc->port[1].first_qset = sc->port[0].nqsets; 1758 break; 1759 } 1760 case CHELSIO_GET_QSET_NUM: { 1761 struct ch_reg *edata = (struct ch_reg *)data; 1762 edata->val = pi->nqsets; 1763 break; 1764 } 1765 #ifdef notyet 1766 /* 1767 * XXX FreeBSD driver does not currently support any 1768 * offload functionality 1769 */ 1770 case CHELSIO_LOAD_FW: 1771 case CHELSIO_DEVUP: 1772 case CHELSIO_SETMTUTAB: 1773 case CHELSIO_GET_PM: 1774 case CHELSIO_SET_PM: 1775 case CHELSIO_READ_TCAM_WORD: 1776 return (EOPNOTSUPP); 1777 break; 1778 #endif 1779 case CHELSIO_GET_MEM: { 1780 struct ch_mem_range *t = (struct ch_mem_range *)data; 1781 struct mc7 *mem; 1782 uint8_t *useraddr; 1783 u64 buf[32]; 1784 1785 if (!is_offload(sc)) 1786 return (EOPNOTSUPP); 1787 if (!(sc->flags & FULL_INIT_DONE)) 1788 return (EIO); /* need the memory controllers */ 1789 if ((t->addr & 0x7) || (t->len & 0x7)) 1790 return (EINVAL); 1791 if (t->mem_id == MEM_CM) 1792 mem = &sc->cm; 1793 else if (t->mem_id == MEM_PMRX) 1794 mem = &sc->pmrx; 1795 else if (t->mem_id == MEM_PMTX) 1796 mem = &sc->pmtx; 1797 else 1798 return (EINVAL); 1799 1800 /* 1801 * Version scheme: 1802 * bits 0..9: chip version 1803 * bits 10..15: chip revision 1804 */ 1805 t->version = 3 | (sc->params.rev << 10); 1806 1807 /* 1808 * Read 256 bytes at a time as len can be large and we don't 1809 * want to use huge intermediate buffers. 1810 */ 1811 useraddr = (uint8_t *)(t + 1); /* advance to start of buffer */ 1812 while (t->len) { 1813 unsigned int chunk = min(t->len, sizeof(buf)); 1814 1815 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf); 1816 if (error) 1817 return (-error); 1818 if (copyout(buf, useraddr, chunk)) 1819 return (EFAULT); 1820 useraddr += chunk; 1821 t->addr += chunk; 1822 t->len -= chunk; 1823 } 1824 break; 1825 } 1826 case CHELSIO_SET_TRACE_FILTER: { 1827 struct ch_trace *t = (struct ch_trace *)data; 1828 const struct trace_params *tp; 1829 1830 tp = (const struct trace_params *)&t->sip; 1831 if (t->config_tx) 1832 t3_config_trace_filter(sc, tp, 0, t->invert_match, 1833 t->trace_tx); 1834 if (t->config_rx) 1835 t3_config_trace_filter(sc, tp, 1, t->invert_match, 1836 t->trace_rx); 1837 break; 1838 } 1839 case CHELSIO_SET_PKTSCHED: { 1840 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 1841 if (sc->open_device_map == 0) 1842 return (EAGAIN); 1843 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 1844 p->binding); 1845 break; 1846 } 1847 case CHELSIO_IFCONF_GETREGS: { 1848 struct ifconf_regs *regs = (struct ifconf_regs *)data; 1849 int reglen = cxgb_get_regs_len(); 1850 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT); 1851 if (buf == NULL) { 1852 return (ENOMEM); 1853 } if (regs->len > reglen) 1854 regs->len = reglen; 1855 else if (regs->len < reglen) { 1856 error = E2BIG; 1857 goto done; 1858 } 1859 cxgb_get_regs(sc, regs, buf); 1860 error = copyout(buf, regs->data, reglen); 1861 1862 done: 1863 free(buf, M_DEVBUF); 1864 1865 break; 1866 } 1867 default: 1868 return (EOPNOTSUPP); 1869 break; 1870 } 1871 1872 return (error); 1873 } 1874 1875 static __inline void 1876 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 1877 unsigned int end) 1878 { 1879 uint32_t *p = (uint32_t *)buf + start; 1880 1881 for ( ; start <= end; start += sizeof(uint32_t)) 1882 *p++ = t3_read_reg(ap, start); 1883 } 1884 1885 #define T3_REGMAP_SIZE (3 * 1024) 1886 static int 1887 cxgb_get_regs_len(void) 1888 { 1889 return T3_REGMAP_SIZE; 1890 } 1891 #undef T3_REGMAP_SIZE 1892 1893 static void 1894 cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf) 1895 { 1896 1897 /* 1898 * Version scheme: 1899 * bits 0..9: chip version 1900 * bits 10..15: chip revision 1901 * bit 31: set for PCIe cards 1902 */ 1903 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 1904 1905 /* 1906 * We skip the MAC statistics registers because they are clear-on-read. 1907 * Also reading multi-register stats would need to synchronize with the 1908 * periodic mac stats accumulation. Hard to justify the complexity. 1909 */ 1910 memset(buf, 0, REGDUMP_SIZE); 1911 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 1912 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 1913 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 1914 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 1915 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 1916 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 1917 XGM_REG(A_XGM_SERDES_STAT3, 1)); 1918 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 1919 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 1920 } 1921