1 /************************************************************************** 2 3 Copyright (c) 2007, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Chelsio Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ***************************************************************************/ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/kernel.h> 40 #include <sys/bus.h> 41 #include <sys/module.h> 42 #include <sys/pciio.h> 43 #include <sys/conf.h> 44 #include <machine/bus.h> 45 #include <machine/resource.h> 46 #include <sys/bus_dma.h> 47 #include <sys/rman.h> 48 #include <sys/ioccom.h> 49 #include <sys/mbuf.h> 50 #include <sys/linker.h> 51 #include <sys/firmware.h> 52 #include <sys/socket.h> 53 #include <sys/sockio.h> 54 #include <sys/smp.h> 55 #include <sys/sysctl.h> 56 #include <sys/queue.h> 57 #include <sys/taskqueue.h> 58 59 #include <net/bpf.h> 60 #include <net/ethernet.h> 61 #include <net/if.h> 62 #include <net/if_arp.h> 63 #include <net/if_dl.h> 64 #include <net/if_media.h> 65 #include <net/if_types.h> 66 67 #include <netinet/in_systm.h> 68 #include <netinet/in.h> 69 #include <netinet/if_ether.h> 70 #include <netinet/ip.h> 71 #include <netinet/ip.h> 72 #include <netinet/tcp.h> 73 #include <netinet/udp.h> 74 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 #include <dev/pci/pci_private.h> 78 79 #include <dev/cxgb/cxgb_osdep.h> 80 #include <dev/cxgb/common/cxgb_common.h> 81 #include <dev/cxgb/cxgb_ioctl.h> 82 #include <dev/cxgb/common/cxgb_regs.h> 83 #include <dev/cxgb/common/cxgb_t3_cpl.h> 84 #include <dev/cxgb/common/cxgb_firmware_exports.h> 85 86 #include <dev/cxgb/sys/mvec.h> 87 88 89 #ifdef PRIV_SUPPORTED 90 #include <sys/priv.h> 91 #endif 92 93 static int cxgb_setup_msix(adapter_t *, int); 94 static void cxgb_init(void *); 95 static void cxgb_init_locked(struct port_info *); 96 static void cxgb_stop_locked(struct port_info *); 97 static void cxgb_set_rxmode(struct port_info *); 98 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 99 static void cxgb_start(struct ifnet *); 100 static void cxgb_start_proc(void *, int ncount); 101 static int cxgb_media_change(struct ifnet *); 102 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 103 static int setup_sge_qsets(adapter_t *); 104 static void cxgb_async_intr(void *); 105 static void cxgb_ext_intr_handler(void *, int); 106 static void cxgb_tick(void *); 107 static void setup_rss(adapter_t *sc); 108 109 /* Attachment glue for the PCI controller end of the device. Each port of 110 * the device is attached separately, as defined later. 111 */ 112 static int cxgb_controller_probe(device_t); 113 static int cxgb_controller_attach(device_t); 114 static int cxgb_controller_detach(device_t); 115 static void cxgb_free(struct adapter *); 116 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 117 unsigned int end); 118 static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf); 119 static int cxgb_get_regs_len(void); 120 121 static device_method_t cxgb_controller_methods[] = { 122 DEVMETHOD(device_probe, cxgb_controller_probe), 123 DEVMETHOD(device_attach, cxgb_controller_attach), 124 DEVMETHOD(device_detach, cxgb_controller_detach), 125 126 /* bus interface */ 127 DEVMETHOD(bus_print_child, bus_generic_print_child), 128 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 129 130 { 0, 0 } 131 }; 132 133 static driver_t cxgb_controller_driver = { 134 "cxgbc", 135 cxgb_controller_methods, 136 sizeof(struct adapter) 137 }; 138 139 static devclass_t cxgb_controller_devclass; 140 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 141 142 /* 143 * Attachment glue for the ports. Attachment is done directly to the 144 * controller device. 145 */ 146 static int cxgb_port_probe(device_t); 147 static int cxgb_port_attach(device_t); 148 static int cxgb_port_detach(device_t); 149 150 static device_method_t cxgb_port_methods[] = { 151 DEVMETHOD(device_probe, cxgb_port_probe), 152 DEVMETHOD(device_attach, cxgb_port_attach), 153 DEVMETHOD(device_detach, cxgb_port_detach), 154 { 0, 0 } 155 }; 156 157 static driver_t cxgb_port_driver = { 158 "cxgb", 159 cxgb_port_methods, 160 0 161 }; 162 163 static d_ioctl_t cxgb_extension_ioctl; 164 165 static devclass_t cxgb_port_devclass; 166 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 167 168 #define SGE_MSIX_COUNT (SGE_QSETS + 1) 169 170 extern int collapse_mbufs; 171 /* 172 * The driver uses the best interrupt scheme available on a platform in the 173 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 174 * of these schemes the driver may consider as follows: 175 * 176 * msi = 2: choose from among all three options 177 * msi = 1 : only consider MSI and pin interrupts 178 * msi = 0: force pin interrupts 179 */ 180 static int msi_allowed = 2; 181 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 182 183 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 184 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 185 "MSI-X, MSI, INTx selector"); 186 187 enum { 188 MAX_TXQ_ENTRIES = 16384, 189 MAX_CTRL_TXQ_ENTRIES = 1024, 190 MAX_RSPQ_ENTRIES = 16384, 191 MAX_RX_BUFFERS = 16384, 192 MAX_RX_JUMBO_BUFFERS = 16384, 193 MIN_TXQ_ENTRIES = 4, 194 MIN_CTRL_TXQ_ENTRIES = 4, 195 MIN_RSPQ_ENTRIES = 32, 196 MIN_FL_ENTRIES = 32 197 }; 198 199 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 200 201 /* Table for probing the cards. The desc field isn't actually used */ 202 struct cxgb_ident { 203 uint16_t vendor; 204 uint16_t device; 205 int index; 206 char *desc; 207 } cxgb_identifiers[] = { 208 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 209 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 210 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 211 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 212 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 213 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 214 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 215 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 216 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 217 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 218 {0, 0, 0, NULL} 219 }; 220 221 static struct cxgb_ident * 222 cxgb_get_ident(device_t dev) 223 { 224 struct cxgb_ident *id; 225 226 for (id = cxgb_identifiers; id->desc != NULL; id++) { 227 if ((id->vendor == pci_get_vendor(dev)) && 228 (id->device == pci_get_device(dev))) { 229 return (id); 230 } 231 } 232 return (NULL); 233 } 234 235 static const struct adapter_info * 236 cxgb_get_adapter_info(device_t dev) 237 { 238 struct cxgb_ident *id; 239 const struct adapter_info *ai; 240 241 id = cxgb_get_ident(dev); 242 if (id == NULL) 243 return (NULL); 244 245 ai = t3_get_adapter_info(id->index); 246 247 return (ai); 248 } 249 250 static int 251 cxgb_controller_probe(device_t dev) 252 { 253 const struct adapter_info *ai; 254 char *ports, buf[80]; 255 256 ai = cxgb_get_adapter_info(dev); 257 if (ai == NULL) 258 return (ENXIO); 259 260 if (ai->nports == 1) 261 ports = "port"; 262 else 263 ports = "ports"; 264 265 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, ai->nports, ports); 266 device_set_desc_copy(dev, buf); 267 return (BUS_PROBE_DEFAULT); 268 } 269 270 static int 271 cxgb_fw_download(adapter_t *sc, device_t dev) 272 { 273 char buf[32]; 274 #ifdef FIRMWARE_LATEST 275 const struct firmware *fw; 276 #else 277 struct firmware *fw; 278 #endif 279 int status; 280 281 snprintf(&buf[0], sizeof(buf), "t3fw%d%d", FW_VERSION_MAJOR, 282 FW_VERSION_MINOR); 283 284 fw = firmware_get(buf); 285 286 287 if (fw == NULL) { 288 device_printf(dev, "Could not find firmware image %s\n", buf); 289 return ENOENT; 290 } 291 292 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 293 294 firmware_put(fw, FIRMWARE_UNLOAD); 295 296 return (status); 297 } 298 299 300 static int 301 cxgb_controller_attach(device_t dev) 302 { 303 driver_intr_t *cxgb_intr = NULL; 304 device_t child; 305 const struct adapter_info *ai; 306 struct adapter *sc; 307 int i, reg, msi_needed, msi_count = 0, error = 0; 308 uint32_t vers; 309 int port_qsets = 1; 310 311 sc = device_get_softc(dev); 312 sc->dev = dev; 313 314 /* find the PCIe link width and set max read request to 4KB*/ 315 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 316 uint16_t lnk, pectl; 317 lnk = pci_read_config(dev, reg + 0x12, 2); 318 sc->link_width = (lnk >> 4) & 0x3f; 319 320 pectl = pci_read_config(dev, reg + 0x8, 2); 321 pectl = (pectl & ~0x7000) | (5 << 12); 322 pci_write_config(dev, reg + 0x8, pectl, 2); 323 } 324 if (sc->link_width != 0 && sc->link_width <= 4) { 325 device_printf(sc->dev, 326 "PCIe x%d Link, expect reduced performance\n", 327 sc->link_width); 328 } 329 330 pci_enable_busmaster(dev); 331 332 /* 333 * Allocate the registers and make them available to the driver. 334 * The registers that we care about for NIC mode are in BAR 0 335 */ 336 sc->regs_rid = PCIR_BAR(0); 337 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 338 &sc->regs_rid, RF_ACTIVE)) == NULL) { 339 device_printf(dev, "Cannot allocate BAR\n"); 340 return (ENXIO); 341 } 342 343 mtx_init(&sc->sge.reg_lock, "SGE reg lock", NULL, MTX_DEF); 344 mtx_init(&sc->lock, "cxgb controller lock", NULL, MTX_DEF); 345 mtx_init(&sc->mdio_lock, "cxgb mdio", NULL, MTX_DEF); 346 347 sc->bt = rman_get_bustag(sc->regs_res); 348 sc->bh = rman_get_bushandle(sc->regs_res); 349 sc->mmio_len = rman_get_size(sc->regs_res); 350 351 ai = cxgb_get_adapter_info(dev); 352 if (t3_prep_adapter(sc, ai, 1) < 0) { 353 error = ENODEV; 354 goto out; 355 } 356 357 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 358 * enough messages for the queue sets. If that fails, try falling 359 * back to MSI. If that fails, then try falling back to the legacy 360 * interrupt pin model. 361 */ 362 #ifdef MSI_SUPPORTED 363 364 sc->msix_regs_rid = 0x20; 365 if ((msi_allowed >= 2) && 366 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 367 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 368 369 msi_needed = msi_count = SGE_MSIX_COUNT; 370 371 if ((pci_alloc_msix(dev, &msi_count) != 0) || 372 (msi_count != msi_needed)) { 373 device_printf(dev, "msix allocation failed" 374 " will try msi\n"); 375 msi_count = 0; 376 pci_release_msi(dev); 377 bus_release_resource(dev, SYS_RES_MEMORY, 378 sc->msix_regs_rid, sc->msix_regs_res); 379 sc->msix_regs_res = NULL; 380 } else { 381 sc->flags |= USING_MSIX; 382 cxgb_intr = t3_intr_msix; 383 } 384 } 385 386 if ((msi_allowed >= 1) && (msi_count == 0)) { 387 msi_count = 1; 388 if (pci_alloc_msi(dev, &msi_count)) { 389 device_printf(dev, "alloc msi failed - will try INTx\n"); 390 msi_count = 0; 391 pci_release_msi(dev); 392 } else { 393 sc->flags |= USING_MSI; 394 sc->irq_rid = 1; 395 cxgb_intr = t3_intr_msi; 396 } 397 } 398 #endif 399 if (msi_count == 0) { 400 device_printf(dev, "using line interrupts\n"); 401 sc->irq_rid = 0; 402 cxgb_intr = t3b_intr; 403 } 404 405 406 /* Create a private taskqueue thread for handling driver events */ 407 #ifdef TASKQUEUE_CURRENT 408 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 409 taskqueue_thread_enqueue, &sc->tq); 410 #else 411 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 412 taskqueue_thread_enqueue, &sc->tq); 413 #endif 414 if (sc->tq == NULL) { 415 device_printf(dev, "failed to allocate controller task queue\n"); 416 goto out; 417 } 418 419 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 420 device_get_nameunit(dev)); 421 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 422 423 424 /* Create a periodic callout for checking adapter status */ 425 callout_init_mtx(&sc->cxgb_tick_ch, &sc->lock, CALLOUT_RETURNUNLOCKED); 426 427 if (t3_check_fw_version(sc) != 0) { 428 /* 429 * Warn user that a firmware update will be attempted in init. 430 */ 431 device_printf(dev, "firmware needs to be updated to version %d.%d\n", 432 FW_VERSION_MAJOR, FW_VERSION_MINOR); 433 sc->flags &= ~FW_UPTODATE; 434 } else { 435 sc->flags |= FW_UPTODATE; 436 } 437 438 if (t3_init_hw(sc, 0) != 0) { 439 device_printf(dev, "hw initialization failed\n"); 440 error = ENXIO; 441 goto out; 442 } 443 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 444 445 if (sc->flags & USING_MSIX) 446 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); 447 448 /* 449 * Create a child device for each MAC. The ethernet attachment 450 * will be done in these children. 451 */ 452 for (i = 0; i < (sc)->params.nports; i++) { 453 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 454 device_printf(dev, "failed to add child port\n"); 455 error = EINVAL; 456 goto out; 457 } 458 sc->portdev[i] = child; 459 sc->port[i].adapter = sc; 460 sc->port[i].nqsets = port_qsets; 461 sc->port[i].first_qset = i*port_qsets; 462 sc->port[i].port = i; 463 device_set_softc(child, &sc->port[i]); 464 } 465 if ((error = bus_generic_attach(dev)) != 0) 466 goto out;; 467 468 if ((error = setup_sge_qsets(sc)) != 0) 469 goto out; 470 471 setup_rss(sc); 472 473 /* If it's MSI or INTx, allocate a single interrupt for everything */ 474 if ((sc->flags & USING_MSIX) == 0) { 475 if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 476 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 477 device_printf(dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid); 478 error = EINVAL; 479 goto out; 480 } 481 device_printf(dev, "allocated irq_res=%p\n", sc->irq_res); 482 483 if (bus_setup_intr(dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 484 #ifdef INTR_FILTERS 485 NULL, 486 #endif 487 cxgb_intr, sc, &sc->intr_tag)) { 488 device_printf(dev, "Cannot set up interrupt\n"); 489 error = EINVAL; 490 goto out; 491 } 492 } else { 493 cxgb_setup_msix(sc, msi_count); 494 } 495 496 sc->params.stats_update_period = 1; 497 498 /* initialize sge private state */ 499 t3_sge_init_sw(sc); 500 501 t3_led_ready(sc); 502 503 error = t3_get_fw_version(sc, &vers); 504 if (error) 505 goto out; 506 507 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d", G_FW_VERSION_MAJOR(vers), 508 G_FW_VERSION_MINOR(vers)); 509 510 t3_add_sysctls(sc); 511 512 out: 513 if (error) 514 cxgb_free(sc); 515 516 return (error); 517 } 518 519 static int 520 cxgb_controller_detach(device_t dev) 521 { 522 struct adapter *sc; 523 524 sc = device_get_softc(dev); 525 526 cxgb_free(sc); 527 528 return (0); 529 } 530 531 static void 532 cxgb_free(struct adapter *sc) 533 { 534 int i; 535 536 /* 537 * XXX need to drain the ifq by hand until 538 * it is taught about mbuf iovecs 539 */ 540 541 callout_drain(&sc->cxgb_tick_ch); 542 543 t3_sge_deinit_sw(sc); 544 545 if (sc->tq != NULL) { 546 taskqueue_drain(sc->tq, &sc->ext_intr_task); 547 taskqueue_free(sc->tq); 548 } 549 550 for (i = 0; i < (sc)->params.nports; ++i) { 551 if (sc->portdev[i] != NULL) 552 device_delete_child(sc->dev, sc->portdev[i]); 553 } 554 555 bus_generic_detach(sc->dev); 556 557 t3_free_sge_resources(sc); 558 t3_sge_free(sc); 559 560 for (i = 0; i < SGE_QSETS; i++) { 561 if (sc->msix_intr_tag[i] != NULL) { 562 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 563 sc->msix_intr_tag[i]); 564 } 565 if (sc->msix_irq_res[i] != NULL) { 566 bus_release_resource(sc->dev, SYS_RES_IRQ, 567 sc->msix_irq_rid[i], sc->msix_irq_res[i]); 568 } 569 } 570 571 if (sc->intr_tag != NULL) { 572 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 573 } 574 575 if (sc->irq_res != NULL) { 576 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", 577 sc->irq_rid, sc->irq_res); 578 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 579 sc->irq_res); 580 } 581 #ifdef MSI_SUPPORTED 582 if (sc->flags & (USING_MSI | USING_MSIX)) { 583 device_printf(sc->dev, "releasing msi message(s)\n"); 584 pci_release_msi(sc->dev); 585 } 586 #endif 587 if (sc->msix_regs_res != NULL) { 588 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 589 sc->msix_regs_res); 590 } 591 592 if (sc->regs_res != NULL) 593 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 594 sc->regs_res); 595 596 mtx_destroy(&sc->mdio_lock); 597 mtx_destroy(&sc->sge.reg_lock); 598 mtx_destroy(&sc->lock); 599 600 return; 601 } 602 603 /** 604 * setup_sge_qsets - configure SGE Tx/Rx/response queues 605 * @sc: the controller softc 606 * 607 * Determines how many sets of SGE queues to use and initializes them. 608 * We support multiple queue sets per port if we have MSI-X, otherwise 609 * just one queue set per port. 610 */ 611 static int 612 setup_sge_qsets(adapter_t *sc) 613 { 614 int i, j, err, irq_idx, qset_idx; 615 u_int ntxq = 3; 616 617 if ((err = t3_sge_alloc(sc)) != 0) { 618 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 619 return (err); 620 } 621 622 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 623 irq_idx = -1; 624 else 625 irq_idx = 0; 626 627 for (qset_idx = 0, i = 0; i < (sc)->params.nports; ++i) { 628 struct port_info *pi = &sc->port[i]; 629 630 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) { 631 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 632 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 633 &sc->params.sge.qset[qset_idx], ntxq, pi); 634 if (err) { 635 t3_free_sge_resources(sc); 636 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err); 637 return (err); 638 } 639 } 640 } 641 642 return (0); 643 } 644 645 static int 646 cxgb_setup_msix(adapter_t *sc, int msix_count) 647 { 648 int i, j, k, nqsets, rid; 649 650 /* The first message indicates link changes and error conditions */ 651 sc->irq_rid = 1; 652 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 653 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 654 device_printf(sc->dev, "Cannot allocate msix interrupt\n"); 655 return (EINVAL); 656 } 657 658 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 659 #ifdef INTR_FILTERS 660 NULL, 661 #endif 662 cxgb_async_intr, sc, &sc->intr_tag)) { 663 device_printf(sc->dev, "Cannot set up interrupt\n"); 664 return (EINVAL); 665 } 666 for (i = 0, k = 0; i < (sc)->params.nports; ++i) { 667 nqsets = sc->port[i].nqsets; 668 for (j = 0; j < nqsets; ++j, k++) { 669 struct sge_qset *qs = &sc->sge.qs[k]; 670 671 rid = k + 2; 672 if (cxgb_debug) 673 printf("rid=%d ", rid); 674 if ((sc->msix_irq_res[k] = bus_alloc_resource_any( 675 sc->dev, SYS_RES_IRQ, &rid, 676 RF_SHAREABLE | RF_ACTIVE)) == NULL) { 677 device_printf(sc->dev, "Cannot allocate " 678 "interrupt for message %d\n", rid); 679 return (EINVAL); 680 } 681 sc->msix_irq_rid[k] = rid; 682 if (bus_setup_intr(sc->dev, sc->msix_irq_res[j], 683 INTR_MPSAFE|INTR_TYPE_NET, 684 #ifdef INTR_FILTERS 685 NULL, 686 #endif 687 t3_intr_msix, qs, &sc->msix_intr_tag[k])) { 688 device_printf(sc->dev, "Cannot set up " 689 "interrupt for message %d\n", rid); 690 return (EINVAL); 691 } 692 } 693 } 694 695 696 return (0); 697 } 698 699 static int 700 cxgb_port_probe(device_t dev) 701 { 702 struct port_info *p; 703 char buf[80]; 704 705 p = device_get_softc(dev); 706 707 snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc); 708 device_set_desc_copy(dev, buf); 709 return (0); 710 } 711 712 713 static int 714 cxgb_makedev(struct port_info *pi) 715 { 716 struct cdevsw *cxgb_cdevsw; 717 718 if ((cxgb_cdevsw = malloc(sizeof(struct cdevsw), M_DEVBUF, M_NOWAIT|M_ZERO)) == NULL) 719 return (ENOMEM); 720 721 cxgb_cdevsw->d_version = D_VERSION; 722 cxgb_cdevsw->d_name = strdup(pi->ifp->if_xname, M_DEVBUF); 723 cxgb_cdevsw->d_ioctl = cxgb_extension_ioctl; 724 725 pi->port_cdev = make_dev(cxgb_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 726 pi->ifp->if_xname); 727 728 if (pi->port_cdev == NULL) 729 return (ENOMEM); 730 731 pi->port_cdev->si_drv1 = (void *)pi; 732 733 return (0); 734 } 735 736 737 #ifdef TSO_SUPPORTED 738 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU) 739 /* Don't enable TSO6 yet */ 740 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU) 741 #else 742 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 743 /* Don't enable TSO6 yet */ 744 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 745 #define IFCAP_TSO4 0x0 746 #define CSUM_TSO 0x0 747 #endif 748 749 750 static int 751 cxgb_port_attach(device_t dev) 752 { 753 struct port_info *p; 754 struct ifnet *ifp; 755 int media_flags; 756 int err; 757 char buf[64]; 758 759 p = device_get_softc(dev); 760 761 snprintf(buf, sizeof(buf), "cxgb port %d", p->port); 762 mtx_init(&p->lock, buf, 0, MTX_DEF); 763 764 /* Allocate an ifnet object and set it up */ 765 ifp = p->ifp = if_alloc(IFT_ETHER); 766 if (ifp == NULL) { 767 device_printf(dev, "Cannot allocate ifnet\n"); 768 return (ENOMEM); 769 } 770 771 /* 772 * Note that there is currently no watchdog timer. 773 */ 774 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 775 ifp->if_init = cxgb_init; 776 ifp->if_softc = p; 777 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 778 ifp->if_ioctl = cxgb_ioctl; 779 ifp->if_start = cxgb_start; 780 ifp->if_timer = 0; /* Disable ifnet watchdog */ 781 ifp->if_watchdog = NULL; 782 783 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE; 784 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 785 IFQ_SET_READY(&ifp->if_snd); 786 787 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 788 ifp->if_capabilities |= CXGB_CAP; 789 ifp->if_capenable |= CXGB_CAP_ENABLE; 790 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 791 ifp->if_baudrate = 100000000; 792 793 ether_ifattach(ifp, p->hw_addr); 794 #ifdef DEFAULT_JUMBO 795 ifp->if_mtu = 9000; 796 #endif 797 if ((err = cxgb_makedev(p)) != 0) { 798 printf("makedev failed %d\n", err); 799 return (err); 800 } 801 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 802 cxgb_media_status); 803 804 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) 805 media_flags = IFM_ETHER | IFM_10G_CX4; 806 else if (!strcmp(p->port_type->desc, "10GBASE-SR")) 807 media_flags = IFM_ETHER | IFM_10G_SR; 808 else if (!strcmp(p->port_type->desc, "10GBASE-XR")) 809 media_flags = IFM_ETHER | IFM_10G_LR; 810 else { 811 printf("unsupported media type %s\n", p->port_type->desc); 812 return (ENXIO); 813 } 814 815 ifmedia_add(&p->media, media_flags, 0, NULL); 816 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 817 ifmedia_set(&p->media, media_flags); 818 819 snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port); 820 #ifdef TASKQUEUE_CURRENT 821 /* Create a port for handling TX without starvation */ 822 p->tq = taskqueue_create(buf, M_NOWAIT, 823 taskqueue_thread_enqueue, &p->tq); 824 #else 825 /* Create a port for handling TX without starvation */ 826 p->tq = taskqueue_create_fast(buf, M_NOWAIT, 827 taskqueue_thread_enqueue, &p->tq); 828 #endif 829 830 831 if (p->tq == NULL) { 832 device_printf(dev, "failed to allocate port task queue\n"); 833 return (ENOMEM); 834 } 835 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq", 836 device_get_nameunit(dev)); 837 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp); 838 839 840 return (0); 841 } 842 843 static int 844 cxgb_port_detach(device_t dev) 845 { 846 struct port_info *p; 847 848 p = device_get_softc(dev); 849 mtx_destroy(&p->lock); 850 if (p->tq != NULL) { 851 taskqueue_drain(p->tq, &p->start_task); 852 taskqueue_free(p->tq); 853 p->tq = NULL; 854 } 855 856 ether_ifdetach(p->ifp); 857 if_free(p->ifp); 858 859 destroy_dev(p->port_cdev); 860 861 862 return (0); 863 } 864 865 void 866 t3_fatal_err(struct adapter *sc) 867 { 868 u_int fw_status[4]; 869 870 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 871 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 872 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 873 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 874 } 875 876 int 877 t3_os_find_pci_capability(adapter_t *sc, int cap) 878 { 879 device_t dev; 880 struct pci_devinfo *dinfo; 881 pcicfgregs *cfg; 882 uint32_t status; 883 uint8_t ptr; 884 885 dev = sc->dev; 886 dinfo = device_get_ivars(dev); 887 cfg = &dinfo->cfg; 888 889 status = pci_read_config(dev, PCIR_STATUS, 2); 890 if (!(status & PCIM_STATUS_CAPPRESENT)) 891 return (0); 892 893 switch (cfg->hdrtype & PCIM_HDRTYPE) { 894 case 0: 895 case 1: 896 ptr = PCIR_CAP_PTR; 897 break; 898 case 2: 899 ptr = PCIR_CAP_PTR_2; 900 break; 901 default: 902 return (0); 903 break; 904 } 905 ptr = pci_read_config(dev, ptr, 1); 906 907 while (ptr != 0) { 908 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 909 return (ptr); 910 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 911 } 912 913 return (0); 914 } 915 916 int 917 t3_os_pci_save_state(struct adapter *sc) 918 { 919 device_t dev; 920 struct pci_devinfo *dinfo; 921 922 dev = sc->dev; 923 dinfo = device_get_ivars(dev); 924 925 pci_cfg_save(dev, dinfo, 0); 926 return (0); 927 } 928 929 int 930 t3_os_pci_restore_state(struct adapter *sc) 931 { 932 device_t dev; 933 struct pci_devinfo *dinfo; 934 935 dev = sc->dev; 936 dinfo = device_get_ivars(dev); 937 938 pci_cfg_restore(dev, dinfo); 939 return (0); 940 } 941 942 /** 943 * t3_os_link_changed - handle link status changes 944 * @adapter: the adapter associated with the link change 945 * @port_id: the port index whose limk status has changed 946 * @link_stat: the new status of the link 947 * @speed: the new speed setting 948 * @duplex: the new duplex setting 949 * @fc: the new flow-control setting 950 * 951 * This is the OS-dependent handler for link status changes. The OS 952 * neutral handler takes care of most of the processing for these events, 953 * then calls this handler for any OS-specific processing. 954 */ 955 void 956 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 957 int duplex, int fc) 958 { 959 struct port_info *pi = &adapter->port[port_id]; 960 961 if ((pi->ifp->if_flags & IFF_UP) == 0) 962 return; 963 964 if (link_status) 965 if_link_state_change(pi->ifp, LINK_STATE_UP); 966 else 967 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 968 969 } 970 971 972 /* 973 * Interrupt-context handler for external (PHY) interrupts. 974 */ 975 void 976 t3_os_ext_intr_handler(adapter_t *sc) 977 { 978 if (cxgb_debug) 979 printf("t3_os_ext_intr_handler\n"); 980 /* 981 * Schedule a task to handle external interrupts as they may be slow 982 * and we use a mutex to protect MDIO registers. We disable PHY 983 * interrupts in the meantime and let the task reenable them when 984 * it's done. 985 */ 986 if (sc->slow_intr_mask) { 987 sc->slow_intr_mask &= ~F_T3DBG; 988 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 989 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 990 } 991 } 992 993 void 994 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 995 { 996 997 /* 998 * The ifnet might not be allocated before this gets called, 999 * as this is called early on in attach by t3_prep_adapter 1000 * save the address off in the port structure 1001 */ 1002 if (cxgb_debug) 1003 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1004 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1005 } 1006 1007 /** 1008 * link_start - enable a port 1009 * @p: the port to enable 1010 * 1011 * Performs the MAC and PHY actions needed to enable a port. 1012 */ 1013 static void 1014 cxgb_link_start(struct port_info *p) 1015 { 1016 struct ifnet *ifp; 1017 struct t3_rx_mode rm; 1018 struct cmac *mac = &p->mac; 1019 1020 ifp = p->ifp; 1021 1022 t3_init_rx_mode(&rm, p); 1023 t3_mac_reset(mac); 1024 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN); 1025 t3_mac_set_address(mac, 0, p->hw_addr); 1026 t3_mac_set_rx_mode(mac, &rm); 1027 t3_link_start(&p->phy, mac, &p->link_config); 1028 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1029 } 1030 1031 /** 1032 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1033 * @adap: the adapter 1034 * 1035 * Sets up RSS to distribute packets to multiple receive queues. We 1036 * configure the RSS CPU lookup table to distribute to the number of HW 1037 * receive queues, and the response queue lookup table to narrow that 1038 * down to the response queues actually configured for each port. 1039 * We always configure the RSS mapping for two ports since the mapping 1040 * table has plenty of entries. 1041 */ 1042 static void 1043 setup_rss(adapter_t *adap) 1044 { 1045 int i; 1046 u_int nq0 = adap->port[0].nqsets; 1047 u_int nq1 = max((u_int)adap->port[1].nqsets, 1U); 1048 uint8_t cpus[SGE_QSETS + 1]; 1049 uint16_t rspq_map[RSS_TABLE_SIZE]; 1050 1051 for (i = 0; i < SGE_QSETS; ++i) 1052 cpus[i] = i; 1053 cpus[SGE_QSETS] = 0xff; 1054 1055 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1056 rspq_map[i] = i % nq0; 1057 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0; 1058 } 1059 1060 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1061 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | 1062 V_RRCPLCPUSIZE(6), cpus, rspq_map); 1063 } 1064 1065 static void 1066 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1067 int hi, int port) 1068 { 1069 struct mbuf *m; 1070 struct mngt_pktsched_wr *req; 1071 1072 m = m_gethdr(M_NOWAIT, MT_DATA); 1073 if (m) { 1074 req = (struct mngt_pktsched_wr *)m->m_data; 1075 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1076 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1077 req->sched = sched; 1078 req->idx = qidx; 1079 req->min = lo; 1080 req->max = hi; 1081 req->binding = port; 1082 m->m_len = m->m_pkthdr.len = sizeof(*req); 1083 t3_mgmt_tx(adap, m); 1084 } 1085 } 1086 1087 static void 1088 bind_qsets(adapter_t *sc) 1089 { 1090 int i, j; 1091 1092 for (i = 0; i < (sc)->params.nports; ++i) { 1093 const struct port_info *pi = adap2pinfo(sc, i); 1094 1095 for (j = 0; j < pi->nqsets; ++j) 1096 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1097 -1, i); 1098 } 1099 } 1100 1101 static void 1102 cxgb_init(void *arg) 1103 { 1104 struct port_info *p = arg; 1105 1106 PORT_LOCK(p); 1107 cxgb_init_locked(p); 1108 PORT_UNLOCK(p); 1109 } 1110 1111 static void 1112 cxgb_init_locked(struct port_info *p) 1113 { 1114 struct ifnet *ifp; 1115 adapter_t *sc = p->adapter; 1116 int error; 1117 1118 mtx_assert(&p->lock, MA_OWNED); 1119 1120 ifp = p->ifp; 1121 if ((sc->flags & FW_UPTODATE) == 0) { 1122 device_printf(sc->dev, "updating firmware to version %d.%d\n", 1123 FW_VERSION_MAJOR, FW_VERSION_MINOR); 1124 if ((error = cxgb_fw_download(sc, sc->dev)) != 0) { 1125 device_printf(sc->dev, "firmware download failed err: %d" 1126 "interface will be unavailable\n", error); 1127 return; 1128 } 1129 sc->flags |= FW_UPTODATE; 1130 } 1131 1132 cxgb_link_start(p); 1133 ADAPTER_LOCK(p->adapter); 1134 if (p->adapter->open_device_map == 0) 1135 t3_intr_clear(sc); 1136 t3_sge_start(sc); 1137 1138 p->adapter->open_device_map |= (1 << p->port); 1139 ADAPTER_UNLOCK(p->adapter); 1140 t3_intr_enable(sc); 1141 t3_port_intr_enable(sc, p->port); 1142 1143 if ((p->adapter->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX) 1144 bind_qsets(sc); 1145 p->adapter->flags |= QUEUES_BOUND; 1146 1147 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, 1148 cxgb_tick, sc); 1149 1150 1151 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1152 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1153 } 1154 1155 static void 1156 cxgb_set_rxmode(struct port_info *p) 1157 { 1158 struct t3_rx_mode rm; 1159 struct cmac *mac = &p->mac; 1160 1161 mtx_assert(&p->lock, MA_OWNED); 1162 1163 t3_init_rx_mode(&rm, p); 1164 t3_mac_set_rx_mode(mac, &rm); 1165 } 1166 1167 static void 1168 cxgb_stop_locked(struct port_info *p) 1169 { 1170 struct ifnet *ifp; 1171 1172 mtx_assert(&p->lock, MA_OWNED); 1173 mtx_assert(&p->adapter->lock, MA_NOTOWNED); 1174 1175 ifp = p->ifp; 1176 1177 ADAPTER_LOCK(p->adapter); 1178 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1179 p->adapter->open_device_map &= ~(1 << p->port); 1180 if (p->adapter->open_device_map == 0) 1181 t3_intr_disable(p->adapter); 1182 ADAPTER_UNLOCK(p->adapter); 1183 t3_port_intr_disable(p->adapter, p->port); 1184 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1185 1186 } 1187 1188 static int 1189 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1190 { 1191 struct port_info *p = ifp->if_softc; 1192 struct ifaddr *ifa = (struct ifaddr *)data; 1193 struct ifreq *ifr = (struct ifreq *)data; 1194 int flags, error = 0; 1195 uint32_t mask; 1196 1197 /* 1198 * XXX need to check that we aren't in the middle of an unload 1199 */ 1200 switch (command) { 1201 case SIOCSIFMTU: 1202 if ((ifr->ifr_mtu < ETHERMIN) || 1203 (ifr->ifr_mtu > ETHER_MAX_LEN_JUMBO)) 1204 error = EINVAL; 1205 else if (ifp->if_mtu != ifr->ifr_mtu) { 1206 PORT_LOCK(p); 1207 ifp->if_mtu = ifr->ifr_mtu; 1208 t3_mac_set_mtu(&p->mac, ifp->if_mtu + ETHER_HDR_LEN); 1209 PORT_UNLOCK(p); 1210 } 1211 break; 1212 case SIOCSIFADDR: 1213 case SIOCGIFADDR: 1214 if (ifa->ifa_addr->sa_family == AF_INET) { 1215 ifp->if_flags |= IFF_UP; 1216 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1217 cxgb_init(p); 1218 } 1219 arp_ifinit(ifp, ifa); 1220 } else 1221 error = ether_ioctl(ifp, command, data); 1222 break; 1223 case SIOCSIFFLAGS: 1224 if (ifp->if_flags & IFF_UP) { 1225 PORT_LOCK(p); 1226 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1227 flags = p->if_flags; 1228 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1229 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 1230 cxgb_set_rxmode(p); 1231 1232 } else 1233 cxgb_init_locked(p); 1234 p->if_flags = ifp->if_flags; 1235 PORT_UNLOCK(p); 1236 } else { 1237 callout_drain(&p->adapter->cxgb_tick_ch); 1238 PORT_LOCK(p); 1239 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1240 cxgb_stop_locked(p); 1241 } else { 1242 adapter_t *sc = p->adapter; 1243 callout_reset(&sc->cxgb_tick_ch, 1244 sc->params.stats_update_period * hz, 1245 cxgb_tick, sc); 1246 } 1247 PORT_UNLOCK(p); 1248 } 1249 1250 1251 break; 1252 case SIOCSIFMEDIA: 1253 case SIOCGIFMEDIA: 1254 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 1255 break; 1256 case SIOCSIFCAP: 1257 PORT_LOCK(p); 1258 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1259 if (mask & IFCAP_TXCSUM) { 1260 if (IFCAP_TXCSUM & ifp->if_capenable) { 1261 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 1262 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 1263 | CSUM_TSO); 1264 } else { 1265 ifp->if_capenable |= IFCAP_TXCSUM; 1266 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1267 } 1268 } else if (mask & IFCAP_RXCSUM) { 1269 if (IFCAP_RXCSUM & ifp->if_capenable) { 1270 ifp->if_capenable &= ~IFCAP_RXCSUM; 1271 } else { 1272 ifp->if_capenable |= IFCAP_RXCSUM; 1273 } 1274 } 1275 if (mask & IFCAP_TSO4) { 1276 if (IFCAP_TSO4 & ifp->if_capenable) { 1277 ifp->if_capenable &= ~IFCAP_TSO4; 1278 ifp->if_hwassist &= ~CSUM_TSO; 1279 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 1280 ifp->if_capenable |= IFCAP_TSO4; 1281 ifp->if_hwassist |= CSUM_TSO; 1282 } else { 1283 if (cxgb_debug) 1284 printf("cxgb requires tx checksum offload" 1285 " be enabled to use TSO\n"); 1286 error = EINVAL; 1287 } 1288 } 1289 PORT_UNLOCK(p); 1290 break; 1291 default: 1292 error = ether_ioctl(ifp, command, data); 1293 break; 1294 } 1295 1296 return (error); 1297 } 1298 1299 static int 1300 cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) 1301 { 1302 struct sge_qset *qs; 1303 struct sge_txq *txq; 1304 struct port_info *p = ifp->if_softc; 1305 struct mbuf *m0, *m = NULL; 1306 int err, in_use_init; 1307 1308 1309 if (!p->link_config.link_ok) 1310 return (ENXIO); 1311 1312 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1313 return (ENOBUFS); 1314 1315 qs = &p->adapter->sge.qs[p->first_qset]; 1316 txq = &qs->txq[TXQ_ETH]; 1317 err = 0; 1318 1319 mtx_lock(&txq->lock); 1320 in_use_init = txq->in_use; 1321 while ((txq->in_use - in_use_init < txmax) && 1322 (txq->size > txq->in_use + TX_MAX_DESC)) { 1323 IFQ_DRV_DEQUEUE(&ifp->if_snd, m); 1324 if (m == NULL) 1325 break; 1326 /* 1327 * Convert chain to M_IOVEC 1328 */ 1329 KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early")); 1330 m0 = m; 1331 #ifdef INVARIANTS 1332 /* 1333 * Clean up after net stack sloppiness 1334 * before calling m_sanity 1335 */ 1336 m0 = m->m_next; 1337 while (m0) { 1338 m0->m_flags &= ~M_PKTHDR; 1339 m0 = m0->m_next; 1340 } 1341 m_sanity(m0, 0); 1342 m0 = m; 1343 #endif 1344 if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES && 1345 m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) { 1346 if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) { 1347 m = m0; 1348 m_collapse(m, TX_MAX_SEGS, &m0); 1349 } else 1350 break; 1351 } 1352 m = m0; 1353 if ((err = t3_encap(p, &m)) != 0) 1354 break; 1355 BPF_MTAP(ifp, m); 1356 } 1357 mtx_unlock(&txq->lock); 1358 1359 if (__predict_false(err)) { 1360 if (cxgb_debug) 1361 printf("would set OFLAGS\n"); 1362 if (err == ENOMEM) { 1363 IFQ_LOCK(&ifp->if_snd); 1364 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1365 IFQ_UNLOCK(&ifp->if_snd); 1366 } 1367 } 1368 if (err == 0 && m == NULL) 1369 err = ENOBUFS; 1370 1371 return (err); 1372 } 1373 1374 static void 1375 cxgb_start_proc(void *arg, int ncount) 1376 { 1377 struct ifnet *ifp = arg; 1378 struct port_info *pi = ifp->if_softc; 1379 struct sge_qset *qs; 1380 struct sge_txq *txq; 1381 int error = 0; 1382 1383 qs = &pi->adapter->sge.qs[pi->first_qset]; 1384 txq = &qs->txq[TXQ_ETH]; 1385 1386 while (error == 0) { 1387 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC) 1388 taskqueue_enqueue(pi->adapter->tq, 1389 &pi->adapter->timer_reclaim_task); 1390 1391 error = cxgb_start_tx(ifp, TX_START_MAX_DESC); 1392 } 1393 } 1394 1395 static void 1396 cxgb_start(struct ifnet *ifp) 1397 { 1398 struct port_info *pi = ifp->if_softc; 1399 struct sge_qset *qs; 1400 struct sge_txq *txq; 1401 int err; 1402 1403 qs = &pi->adapter->sge.qs[pi->first_qset]; 1404 txq = &qs->txq[TXQ_ETH]; 1405 1406 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC) 1407 taskqueue_enqueue(pi->adapter->tq, 1408 &pi->adapter->timer_reclaim_task); 1409 1410 err = cxgb_start_tx(ifp, TX_START_MAX_DESC); 1411 1412 if (err == 0) 1413 taskqueue_enqueue(pi->tq, &pi->start_task); 1414 } 1415 1416 1417 static int 1418 cxgb_media_change(struct ifnet *ifp) 1419 { 1420 if_printf(ifp, "media change not supported\n"); 1421 return (ENXIO); 1422 } 1423 1424 static void 1425 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1426 { 1427 struct port_info *p = ifp->if_softc; 1428 1429 ifmr->ifm_status = IFM_AVALID; 1430 ifmr->ifm_active = IFM_ETHER; 1431 1432 if (!p->link_config.link_ok) 1433 return; 1434 1435 ifmr->ifm_status |= IFM_ACTIVE; 1436 1437 if (p->link_config.duplex) 1438 ifmr->ifm_active |= IFM_FDX; 1439 else 1440 ifmr->ifm_active |= IFM_HDX; 1441 } 1442 1443 static void 1444 cxgb_async_intr(void *data) 1445 { 1446 adapter_t *sc = data; 1447 1448 if (cxgb_debug) 1449 device_printf(sc->dev, "cxgb_async_intr\n"); 1450 1451 t3_slow_intr_handler(sc); 1452 1453 } 1454 1455 static void 1456 cxgb_ext_intr_handler(void *arg, int count) 1457 { 1458 adapter_t *sc = (adapter_t *)arg; 1459 1460 if (cxgb_debug) 1461 printf("cxgb_ext_intr_handler\n"); 1462 1463 t3_phy_intr_handler(sc); 1464 1465 /* Now reenable external interrupts */ 1466 if (sc->slow_intr_mask) { 1467 sc->slow_intr_mask |= F_T3DBG; 1468 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 1469 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1470 } 1471 } 1472 1473 static void 1474 check_link_status(adapter_t *sc) 1475 { 1476 int i; 1477 1478 for (i = 0; i < (sc)->params.nports; ++i) { 1479 struct port_info *p = &sc->port[i]; 1480 1481 if (!(p->port_type->caps & SUPPORTED_IRQ)) 1482 t3_link_changed(sc, i); 1483 } 1484 } 1485 1486 static void 1487 check_t3b2_mac(struct adapter *adapter) 1488 { 1489 int i; 1490 1491 for_each_port(adapter, i) { 1492 struct port_info *p = &adapter->port[i]; 1493 struct ifnet *ifp = p->ifp; 1494 int status; 1495 1496 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1497 continue; 1498 1499 status = 0; 1500 PORT_LOCK(p); 1501 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 1502 status = t3b2_mac_watchdog_task(&p->mac); 1503 if (status == 1) 1504 p->mac.stats.num_toggled++; 1505 else if (status == 2) { 1506 struct cmac *mac = &p->mac; 1507 1508 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN); 1509 t3_mac_set_address(mac, 0, p->hw_addr); 1510 cxgb_set_rxmode(p); 1511 t3_link_start(&p->phy, mac, &p->link_config); 1512 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1513 t3_port_intr_enable(adapter, p->port); 1514 p->mac.stats.num_resets++; 1515 } 1516 PORT_UNLOCK(p); 1517 } 1518 } 1519 1520 static void 1521 cxgb_tick(void *arg) 1522 { 1523 adapter_t *sc = (adapter_t *)arg; 1524 const struct adapter_params *p = &sc->params; 1525 1526 if (p->linkpoll_period) 1527 check_link_status(sc); 1528 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, 1529 cxgb_tick, sc); 1530 1531 /* 1532 * adapter lock can currently only be acquire after the 1533 * port lock 1534 */ 1535 ADAPTER_UNLOCK(sc); 1536 if (p->rev == T3_REV_B2) 1537 check_t3b2_mac(sc); 1538 1539 } 1540 1541 static int 1542 in_range(int val, int lo, int hi) 1543 { 1544 return val < 0 || (val <= hi && val >= lo); 1545 } 1546 1547 static int 1548 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 1549 int fflag, struct thread *td) 1550 { 1551 int mmd, error = 0; 1552 struct port_info *pi = dev->si_drv1; 1553 adapter_t *sc = pi->adapter; 1554 1555 #ifdef PRIV_SUPPORTED 1556 if (priv_check(td, PRIV_DRIVER)) { 1557 if (cxgb_debug) 1558 printf("user does not have access to privileged ioctls\n"); 1559 return (EPERM); 1560 } 1561 #else 1562 if (suser(td)) { 1563 if (cxgb_debug) 1564 printf("user does not have access to privileged ioctls\n"); 1565 return (EPERM); 1566 } 1567 #endif 1568 1569 switch (cmd) { 1570 case SIOCGMIIREG: { 1571 uint32_t val; 1572 struct cphy *phy = &pi->phy; 1573 struct mii_data *mid = (struct mii_data *)data; 1574 1575 if (!phy->mdio_read) 1576 return (EOPNOTSUPP); 1577 if (is_10G(sc)) { 1578 mmd = mid->phy_id >> 8; 1579 if (!mmd) 1580 mmd = MDIO_DEV_PCS; 1581 else if (mmd > MDIO_DEV_XGXS) 1582 return -EINVAL; 1583 1584 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 1585 mid->reg_num, &val); 1586 } else 1587 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 1588 mid->reg_num & 0x1f, &val); 1589 if (error == 0) 1590 mid->val_out = val; 1591 break; 1592 } 1593 case SIOCSMIIREG: { 1594 struct cphy *phy = &pi->phy; 1595 struct mii_data *mid = (struct mii_data *)data; 1596 1597 if (!phy->mdio_write) 1598 return (EOPNOTSUPP); 1599 if (is_10G(sc)) { 1600 mmd = mid->phy_id >> 8; 1601 if (!mmd) 1602 mmd = MDIO_DEV_PCS; 1603 else if (mmd > MDIO_DEV_XGXS) 1604 return (EINVAL); 1605 1606 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 1607 mmd, mid->reg_num, mid->val_in); 1608 } else 1609 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 1610 mid->reg_num & 0x1f, 1611 mid->val_in); 1612 break; 1613 } 1614 case CHELSIO_SETREG: { 1615 struct ch_reg *edata = (struct ch_reg *)data; 1616 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 1617 return (EFAULT); 1618 t3_write_reg(sc, edata->addr, edata->val); 1619 break; 1620 } 1621 case CHELSIO_GETREG: { 1622 struct ch_reg *edata = (struct ch_reg *)data; 1623 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 1624 return (EFAULT); 1625 edata->val = t3_read_reg(sc, edata->addr); 1626 break; 1627 } 1628 case CHELSIO_GET_SGE_CONTEXT: { 1629 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 1630 mtx_lock(&sc->sge.reg_lock); 1631 switch (ecntxt->cntxt_type) { 1632 case CNTXT_TYPE_EGRESS: 1633 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 1634 ecntxt->data); 1635 break; 1636 case CNTXT_TYPE_FL: 1637 error = t3_sge_read_fl(sc, ecntxt->cntxt_id, 1638 ecntxt->data); 1639 break; 1640 case CNTXT_TYPE_RSP: 1641 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id, 1642 ecntxt->data); 1643 break; 1644 case CNTXT_TYPE_CQ: 1645 error = t3_sge_read_cq(sc, ecntxt->cntxt_id, 1646 ecntxt->data); 1647 break; 1648 default: 1649 error = EINVAL; 1650 break; 1651 } 1652 mtx_unlock(&sc->sge.reg_lock); 1653 break; 1654 } 1655 case CHELSIO_GET_SGE_DESC: { 1656 struct ch_desc *edesc = (struct ch_desc *)data; 1657 int ret; 1658 if (edesc->queue_num >= SGE_QSETS * 6) 1659 return (EINVAL); 1660 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 1661 edesc->queue_num % 6, edesc->idx, edesc->data); 1662 if (ret < 0) 1663 return (EINVAL); 1664 edesc->size = ret; 1665 break; 1666 } 1667 case CHELSIO_SET_QSET_PARAMS: { 1668 struct qset_params *q; 1669 struct ch_qset_params *t = (struct ch_qset_params *)data; 1670 1671 if (t->qset_idx >= SGE_QSETS) 1672 return -EINVAL; 1673 if (!in_range(t->intr_lat, 0, M_NEWTIMER) || 1674 !in_range(t->cong_thres, 0, 255) || 1675 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES, 1676 MAX_TXQ_ENTRIES) || 1677 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES, 1678 MAX_TXQ_ENTRIES) || 1679 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES, 1680 MAX_CTRL_TXQ_ENTRIES) || 1681 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) || 1682 !in_range(t->fl_size[1], MIN_FL_ENTRIES, 1683 MAX_RX_JUMBO_BUFFERS) || 1684 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES)) 1685 return -EINVAL; 1686 if ((sc->flags & FULL_INIT_DONE) && 1687 (t->rspq_size >= 0 || t->fl_size[0] >= 0 || 1688 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 || 1689 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 || 1690 t->polling >= 0 || t->cong_thres >= 0)) 1691 return -EBUSY; 1692 1693 q = &sc->params.sge.qset[t->qset_idx]; 1694 1695 if (t->rspq_size >= 0) 1696 q->rspq_size = t->rspq_size; 1697 if (t->fl_size[0] >= 0) 1698 q->fl_size = t->fl_size[0]; 1699 if (t->fl_size[1] >= 0) 1700 q->jumbo_size = t->fl_size[1]; 1701 if (t->txq_size[0] >= 0) 1702 q->txq_size[0] = t->txq_size[0]; 1703 if (t->txq_size[1] >= 0) 1704 q->txq_size[1] = t->txq_size[1]; 1705 if (t->txq_size[2] >= 0) 1706 q->txq_size[2] = t->txq_size[2]; 1707 if (t->cong_thres >= 0) 1708 q->cong_thres = t->cong_thres; 1709 if (t->intr_lat >= 0) { 1710 struct sge_qset *qs = &sc->sge.qs[t->qset_idx]; 1711 1712 q->coalesce_nsecs = t->intr_lat*1000; 1713 t3_update_qset_coalesce(qs, q); 1714 } 1715 break; 1716 } 1717 case CHELSIO_GET_QSET_PARAMS: { 1718 struct qset_params *q; 1719 struct ch_qset_params *t = (struct ch_qset_params *)data; 1720 1721 if (t->qset_idx >= SGE_QSETS) 1722 return (EINVAL); 1723 1724 q = &(sc)->params.sge.qset[t->qset_idx]; 1725 t->rspq_size = q->rspq_size; 1726 t->txq_size[0] = q->txq_size[0]; 1727 t->txq_size[1] = q->txq_size[1]; 1728 t->txq_size[2] = q->txq_size[2]; 1729 t->fl_size[0] = q->fl_size; 1730 t->fl_size[1] = q->jumbo_size; 1731 t->polling = q->polling; 1732 t->intr_lat = q->coalesce_nsecs / 1000; 1733 t->cong_thres = q->cong_thres; 1734 break; 1735 } 1736 case CHELSIO_SET_QSET_NUM: { 1737 struct ch_reg *edata = (struct ch_reg *)data; 1738 unsigned int port_idx = pi->port; 1739 1740 if (sc->flags & FULL_INIT_DONE) 1741 return (EBUSY); 1742 if (edata->val < 1 || 1743 (edata->val > 1 && !(sc->flags & USING_MSIX))) 1744 return (EINVAL); 1745 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS) 1746 return (EINVAL); 1747 sc->port[port_idx].nqsets = edata->val; 1748 /* 1749 * XXX we're hardcoding ourselves to 2 ports 1750 * just like the LEENUX 1751 */ 1752 sc->port[1].first_qset = sc->port[0].nqsets; 1753 break; 1754 } 1755 case CHELSIO_GET_QSET_NUM: { 1756 struct ch_reg *edata = (struct ch_reg *)data; 1757 edata->val = pi->nqsets; 1758 break; 1759 } 1760 #ifdef notyet 1761 /* 1762 * XXX FreeBSD driver does not currently support any 1763 * offload functionality 1764 */ 1765 case CHELSIO_LOAD_FW: 1766 case CHELSIO_DEVUP: 1767 case CHELSIO_SETMTUTAB: 1768 case CHELSIO_GET_PM: 1769 case CHELSIO_SET_PM: 1770 case CHELSIO_READ_TCAM_WORD: 1771 return (EOPNOTSUPP); 1772 break; 1773 #endif 1774 case CHELSIO_GET_MEM: { 1775 struct ch_mem_range *t = (struct ch_mem_range *)data; 1776 struct mc7 *mem; 1777 uint8_t *useraddr; 1778 u64 buf[32]; 1779 1780 if (!is_offload(sc)) 1781 return (EOPNOTSUPP); 1782 if (!(sc->flags & FULL_INIT_DONE)) 1783 return (EIO); /* need the memory controllers */ 1784 if ((t->addr & 0x7) || (t->len & 0x7)) 1785 return (EINVAL); 1786 if (t->mem_id == MEM_CM) 1787 mem = &sc->cm; 1788 else if (t->mem_id == MEM_PMRX) 1789 mem = &sc->pmrx; 1790 else if (t->mem_id == MEM_PMTX) 1791 mem = &sc->pmtx; 1792 else 1793 return (EINVAL); 1794 1795 /* 1796 * Version scheme: 1797 * bits 0..9: chip version 1798 * bits 10..15: chip revision 1799 */ 1800 t->version = 3 | (sc->params.rev << 10); 1801 1802 /* 1803 * Read 256 bytes at a time as len can be large and we don't 1804 * want to use huge intermediate buffers. 1805 */ 1806 useraddr = (uint8_t *)(t + 1); /* advance to start of buffer */ 1807 while (t->len) { 1808 unsigned int chunk = min(t->len, sizeof(buf)); 1809 1810 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf); 1811 if (error) 1812 return (-error); 1813 if (copyout(buf, useraddr, chunk)) 1814 return (EFAULT); 1815 useraddr += chunk; 1816 t->addr += chunk; 1817 t->len -= chunk; 1818 } 1819 break; 1820 } 1821 case CHELSIO_SET_TRACE_FILTER: { 1822 struct ch_trace *t = (struct ch_trace *)data; 1823 const struct trace_params *tp; 1824 1825 tp = (const struct trace_params *)&t->sip; 1826 if (t->config_tx) 1827 t3_config_trace_filter(sc, tp, 0, t->invert_match, 1828 t->trace_tx); 1829 if (t->config_rx) 1830 t3_config_trace_filter(sc, tp, 1, t->invert_match, 1831 t->trace_rx); 1832 break; 1833 } 1834 case CHELSIO_SET_PKTSCHED: { 1835 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 1836 if (sc->open_device_map == 0) 1837 return (EAGAIN); 1838 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 1839 p->binding); 1840 break; 1841 } 1842 case CHELSIO_IFCONF_GETREGS: { 1843 struct ifconf_regs *regs = (struct ifconf_regs *)data; 1844 int reglen = cxgb_get_regs_len(); 1845 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT); 1846 if (buf == NULL) { 1847 return (ENOMEM); 1848 } if (regs->len > reglen) 1849 regs->len = reglen; 1850 else if (regs->len < reglen) { 1851 error = E2BIG; 1852 goto done; 1853 } 1854 cxgb_get_regs(sc, regs, buf); 1855 error = copyout(buf, regs->data, reglen); 1856 1857 done: 1858 free(buf, M_DEVBUF); 1859 1860 break; 1861 } 1862 default: 1863 return (EOPNOTSUPP); 1864 break; 1865 } 1866 1867 return (error); 1868 } 1869 1870 static __inline void 1871 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 1872 unsigned int end) 1873 { 1874 uint32_t *p = (uint32_t *)buf + start; 1875 1876 for ( ; start <= end; start += sizeof(uint32_t)) 1877 *p++ = t3_read_reg(ap, start); 1878 } 1879 1880 #define T3_REGMAP_SIZE (3 * 1024) 1881 static int 1882 cxgb_get_regs_len(void) 1883 { 1884 return T3_REGMAP_SIZE; 1885 } 1886 #undef T3_REGMAP_SIZE 1887 1888 static void 1889 cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf) 1890 { 1891 1892 /* 1893 * Version scheme: 1894 * bits 0..9: chip version 1895 * bits 10..15: chip revision 1896 * bit 31: set for PCIe cards 1897 */ 1898 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 1899 1900 /* 1901 * We skip the MAC statistics registers because they are clear-on-read. 1902 * Also reading multi-register stats would need to synchronize with the 1903 * periodic mac stats accumulation. Hard to justify the complexity. 1904 */ 1905 memset(buf, 0, REGDUMP_SIZE); 1906 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 1907 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 1908 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 1909 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 1910 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 1911 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 1912 XGM_REG(A_XGM_SERDES_STAT3, 1)); 1913 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 1914 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 1915 } 1916