1 /************************************************************************** 2 3 Copyright (c) 2007, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Chelsio Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ***************************************************************************/ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/kernel.h> 40 #include <sys/bus.h> 41 #include <sys/module.h> 42 #include <sys/pciio.h> 43 #include <sys/conf.h> 44 #include <machine/bus.h> 45 #include <machine/resource.h> 46 #include <sys/bus_dma.h> 47 #include <sys/rman.h> 48 #include <sys/ioccom.h> 49 #include <sys/mbuf.h> 50 #include <sys/linker.h> 51 #include <sys/firmware.h> 52 #include <sys/socket.h> 53 #include <sys/sockio.h> 54 #include <sys/smp.h> 55 #include <sys/sysctl.h> 56 #include <sys/queue.h> 57 #include <sys/taskqueue.h> 58 59 60 61 #include <net/bpf.h> 62 #include <net/ethernet.h> 63 #include <net/if.h> 64 #include <net/if_arp.h> 65 #include <net/if_dl.h> 66 #include <net/if_media.h> 67 #include <net/if_types.h> 68 69 #include <netinet/in_systm.h> 70 #include <netinet/in.h> 71 #include <netinet/if_ether.h> 72 #include <netinet/ip.h> 73 #include <netinet/ip.h> 74 #include <netinet/tcp.h> 75 #include <netinet/udp.h> 76 77 #include <dev/pci/pcireg.h> 78 #include <dev/pci/pcivar.h> 79 #include <dev/pci/pci_private.h> 80 81 #include <dev/cxgb/cxgb_osdep.h> 82 #include <dev/cxgb/common/cxgb_common.h> 83 #include <dev/cxgb/cxgb_ioctl.h> 84 #include <dev/cxgb/common/cxgb_regs.h> 85 #include <dev/cxgb/common/cxgb_t3_cpl.h> 86 #include <dev/cxgb/common/cxgb_firmware_exports.h> 87 88 89 #ifdef PRIV_SUPPORTED 90 #include <sys/priv.h> 91 #endif 92 93 static int cxgb_setup_msix(adapter_t *, int); 94 static void cxgb_init(void *); 95 static void cxgb_init_locked(struct port_info *); 96 static void cxgb_stop_locked(struct port_info *); 97 static void cxgb_set_rxmode(struct port_info *); 98 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 99 static void cxgb_start(struct ifnet *); 100 static void cxgb_start_proc(void *, int ncount); 101 static int cxgb_media_change(struct ifnet *); 102 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 103 static int setup_sge_qsets(adapter_t *); 104 static void cxgb_async_intr(void *); 105 static void cxgb_ext_intr_handler(void *, int); 106 static void cxgb_tick(void *); 107 static void setup_rss(adapter_t *sc); 108 109 /* Attachment glue for the PCI controller end of the device. Each port of 110 * the device is attached separately, as defined later. 111 */ 112 static int cxgb_controller_probe(device_t); 113 static int cxgb_controller_attach(device_t); 114 static int cxgb_controller_detach(device_t); 115 static void cxgb_free(struct adapter *); 116 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 117 unsigned int end); 118 static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf); 119 static int cxgb_get_regs_len(void); 120 121 static device_method_t cxgb_controller_methods[] = { 122 DEVMETHOD(device_probe, cxgb_controller_probe), 123 DEVMETHOD(device_attach, cxgb_controller_attach), 124 DEVMETHOD(device_detach, cxgb_controller_detach), 125 126 /* bus interface */ 127 DEVMETHOD(bus_print_child, bus_generic_print_child), 128 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 129 130 { 0, 0 } 131 }; 132 133 static driver_t cxgb_controller_driver = { 134 "cxgbc", 135 cxgb_controller_methods, 136 sizeof(struct adapter) 137 }; 138 139 static devclass_t cxgb_controller_devclass; 140 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 141 142 /* 143 * Attachment glue for the ports. Attachment is done directly to the 144 * controller device. 145 */ 146 static int cxgb_port_probe(device_t); 147 static int cxgb_port_attach(device_t); 148 static int cxgb_port_detach(device_t); 149 150 static device_method_t cxgb_port_methods[] = { 151 DEVMETHOD(device_probe, cxgb_port_probe), 152 DEVMETHOD(device_attach, cxgb_port_attach), 153 DEVMETHOD(device_detach, cxgb_port_detach), 154 { 0, 0 } 155 }; 156 157 static driver_t cxgb_port_driver = { 158 "cxgb", 159 cxgb_port_methods, 160 0 161 }; 162 163 static d_ioctl_t cxgb_extension_ioctl; 164 165 static devclass_t cxgb_port_devclass; 166 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 167 168 #define SGE_MSIX_COUNT (SGE_QSETS + 1) 169 170 /* 171 * The driver uses the best interrupt scheme available on a platform in the 172 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 173 * of these schemes the driver may consider as follows: 174 * 175 * msi = 2: choose from among all three options 176 * msi = 1 : only consider MSI and pin interrupts 177 * msi = 0: force pin interrupts 178 */ 179 static int msi_allowed = 2; 180 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 181 182 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 183 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 184 "MSI-X, MSI, INTx selector"); 185 186 enum { 187 MAX_TXQ_ENTRIES = 16384, 188 MAX_CTRL_TXQ_ENTRIES = 1024, 189 MAX_RSPQ_ENTRIES = 16384, 190 MAX_RX_BUFFERS = 16384, 191 MAX_RX_JUMBO_BUFFERS = 16384, 192 MIN_TXQ_ENTRIES = 4, 193 MIN_CTRL_TXQ_ENTRIES = 4, 194 MIN_RSPQ_ENTRIES = 32, 195 MIN_FL_ENTRIES = 32 196 }; 197 198 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 199 200 /* Table for probing the cards. The desc field isn't actually used */ 201 struct cxgb_ident { 202 uint16_t vendor; 203 uint16_t device; 204 int index; 205 char *desc; 206 } cxgb_identifiers[] = { 207 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 208 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 209 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 210 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 211 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 212 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 213 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 214 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 215 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 216 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 217 {0, 0, 0, NULL} 218 }; 219 220 static struct cxgb_ident * 221 cxgb_get_ident(device_t dev) 222 { 223 struct cxgb_ident *id; 224 225 for (id = cxgb_identifiers; id->desc != NULL; id++) { 226 if ((id->vendor == pci_get_vendor(dev)) && 227 (id->device == pci_get_device(dev))) { 228 return (id); 229 } 230 } 231 return (NULL); 232 } 233 234 static const struct adapter_info * 235 cxgb_get_adapter_info(device_t dev) 236 { 237 struct cxgb_ident *id; 238 const struct adapter_info *ai; 239 240 id = cxgb_get_ident(dev); 241 if (id == NULL) 242 return (NULL); 243 244 ai = t3_get_adapter_info(id->index); 245 246 return (ai); 247 } 248 249 static int 250 cxgb_controller_probe(device_t dev) 251 { 252 const struct adapter_info *ai; 253 char *ports, buf[80]; 254 255 ai = cxgb_get_adapter_info(dev); 256 if (ai == NULL) 257 return (ENXIO); 258 259 if (ai->nports == 1) 260 ports = "port"; 261 else 262 ports = "ports"; 263 264 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, ai->nports, ports); 265 device_set_desc_copy(dev, buf); 266 return (BUS_PROBE_DEFAULT); 267 } 268 269 static int 270 cxgb_fw_download(adapter_t *sc, device_t dev) 271 { 272 char buf[32]; 273 #ifdef FIRMWARE_LATEST 274 const struct firmware *fw; 275 #else 276 struct firmware *fw; 277 #endif 278 int status; 279 280 snprintf(&buf[0], sizeof(buf), "t3fw%d%d", FW_VERSION_MAJOR, 281 FW_VERSION_MINOR); 282 283 fw = firmware_get(buf); 284 285 286 if (fw == NULL) { 287 device_printf(dev, "Could not find firmware image %s\n", buf); 288 return ENOENT; 289 } 290 291 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 292 293 firmware_put(fw, FIRMWARE_UNLOAD); 294 295 return (status); 296 } 297 298 299 static int 300 cxgb_controller_attach(device_t dev) 301 { 302 driver_intr_t *cxgb_intr = NULL; 303 device_t child; 304 const struct adapter_info *ai; 305 struct adapter *sc; 306 int i, reg, msi_needed, msi_count = 0, error = 0; 307 uint32_t vers; 308 int port_qsets = 1; 309 310 sc = device_get_softc(dev); 311 sc->dev = dev; 312 313 /* find the PCIe link width and set max read request to 4KB*/ 314 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 315 uint16_t lnk, pectl; 316 lnk = pci_read_config(dev, reg + 0x12, 2); 317 sc->link_width = (lnk >> 4) & 0x3f; 318 319 pectl = pci_read_config(dev, reg + 0x8, 2); 320 pectl = (pectl & ~0x7000) | (5 << 12); 321 pci_write_config(dev, reg + 0x8, pectl, 2); 322 } 323 if (sc->link_width != 0 && sc->link_width <= 4) { 324 device_printf(sc->dev, 325 "PCIe x%d Link, expect reduced performance\n", 326 sc->link_width); 327 } 328 329 pci_enable_busmaster(dev); 330 331 /* 332 * Allocate the registers and make them available to the driver. 333 * The registers that we care about for NIC mode are in BAR 0 334 */ 335 sc->regs_rid = PCIR_BAR(0); 336 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 337 &sc->regs_rid, RF_ACTIVE)) == NULL) { 338 device_printf(dev, "Cannot allocate BAR\n"); 339 return (ENXIO); 340 } 341 342 mtx_init(&sc->sge.reg_lock, "SGE reg lock", NULL, MTX_DEF); 343 mtx_init(&sc->lock, "cxgb controller lock", NULL, MTX_DEF); 344 mtx_init(&sc->mdio_lock, "cxgb mdio", NULL, MTX_DEF); 345 346 sc->bt = rman_get_bustag(sc->regs_res); 347 sc->bh = rman_get_bushandle(sc->regs_res); 348 sc->mmio_len = rman_get_size(sc->regs_res); 349 350 ai = cxgb_get_adapter_info(dev); 351 if (t3_prep_adapter(sc, ai, 1) < 0) { 352 error = ENODEV; 353 goto out; 354 } 355 356 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 357 * enough messages for the queue sets. If that fails, try falling 358 * back to MSI. If that fails, then try falling back to the legacy 359 * interrupt pin model. 360 */ 361 #ifdef MSI_SUPPORTED 362 363 sc->msix_regs_rid = 0x20; 364 if ((msi_allowed >= 2) && 365 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 366 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 367 368 msi_needed = msi_count = SGE_MSIX_COUNT; 369 370 if ((pci_alloc_msix(dev, &msi_count) != 0) || 371 (msi_count != msi_needed)) { 372 device_printf(dev, "msix allocation failed" 373 " will try msi\n"); 374 msi_count = 0; 375 pci_release_msi(dev); 376 bus_release_resource(dev, SYS_RES_MEMORY, 377 sc->msix_regs_rid, sc->msix_regs_res); 378 sc->msix_regs_res = NULL; 379 } else { 380 sc->flags |= USING_MSIX; 381 cxgb_intr = t3_intr_msix; 382 } 383 } 384 385 if ((msi_allowed >= 1) && (msi_count == 0)) { 386 msi_count = 1; 387 if (pci_alloc_msi(dev, &msi_count)) { 388 device_printf(dev, "alloc msi failed - will try INTx\n"); 389 msi_count = 0; 390 pci_release_msi(dev); 391 } else { 392 sc->flags |= USING_MSI; 393 sc->irq_rid = 1; 394 cxgb_intr = t3_intr_msi; 395 } 396 } 397 #endif 398 if (msi_count == 0) { 399 device_printf(dev, "using line interrupts\n"); 400 sc->irq_rid = 0; 401 cxgb_intr = t3b_intr; 402 } 403 404 405 /* Create a private taskqueue thread for handling driver events */ 406 #ifdef TASKQUEUE_CURRENT 407 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 408 taskqueue_thread_enqueue, &sc->tq); 409 #else 410 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 411 taskqueue_thread_enqueue, &sc->tq); 412 #endif 413 if (sc->tq == NULL) { 414 device_printf(dev, "failed to allocate controller task queue\n"); 415 goto out; 416 } 417 418 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 419 device_get_nameunit(dev)); 420 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 421 422 423 /* Create a periodic callout for checking adapter status */ 424 callout_init_mtx(&sc->cxgb_tick_ch, &sc->lock, CALLOUT_RETURNUNLOCKED); 425 426 if (t3_check_fw_version(sc) != 0) { 427 /* 428 * Warn user that a firmware update will be attempted in init. 429 */ 430 device_printf(dev, "firmware needs to be updated to version %d.%d\n", 431 FW_VERSION_MAJOR, FW_VERSION_MINOR); 432 sc->flags &= ~FW_UPTODATE; 433 } else { 434 sc->flags |= FW_UPTODATE; 435 } 436 437 if (t3_init_hw(sc, 0) != 0) { 438 device_printf(dev, "hw initialization failed\n"); 439 error = ENXIO; 440 goto out; 441 } 442 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 443 444 if (sc->flags & USING_MSIX) 445 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); 446 447 /* 448 * Create a child device for each MAC. The ethernet attachment 449 * will be done in these children. 450 */ 451 for (i = 0; i < (sc)->params.nports; i++) { 452 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 453 device_printf(dev, "failed to add child port\n"); 454 error = EINVAL; 455 goto out; 456 } 457 sc->portdev[i] = child; 458 sc->port[i].adapter = sc; 459 sc->port[i].nqsets = port_qsets; 460 sc->port[i].first_qset = i*port_qsets; 461 sc->port[i].port = i; 462 device_set_softc(child, &sc->port[i]); 463 } 464 if ((error = bus_generic_attach(dev)) != 0) 465 goto out;; 466 467 if ((error = setup_sge_qsets(sc)) != 0) 468 goto out; 469 470 setup_rss(sc); 471 472 /* If it's MSI or INTx, allocate a single interrupt for everything */ 473 if ((sc->flags & USING_MSIX) == 0) { 474 if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 475 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 476 device_printf(dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid); 477 error = EINVAL; 478 goto out; 479 } 480 device_printf(dev, "allocated irq_res=%p\n", sc->irq_res); 481 482 if (bus_setup_intr(dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 483 #ifdef INTR_FILTERS 484 NULL, 485 #endif 486 cxgb_intr, sc, &sc->intr_tag)) { 487 device_printf(dev, "Cannot set up interrupt\n"); 488 error = EINVAL; 489 goto out; 490 } 491 } else { 492 cxgb_setup_msix(sc, msi_count); 493 } 494 495 sc->params.stats_update_period = 1; 496 497 /* initialize sge private state */ 498 t3_sge_init_sw(sc); 499 500 t3_led_ready(sc); 501 502 error = t3_get_fw_version(sc, &vers); 503 if (error) 504 goto out; 505 506 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d", G_FW_VERSION_MAJOR(vers), 507 G_FW_VERSION_MINOR(vers)); 508 509 t3_add_sysctls(sc); 510 511 out: 512 if (error) 513 cxgb_free(sc); 514 515 return (error); 516 } 517 518 static int 519 cxgb_controller_detach(device_t dev) 520 { 521 struct adapter *sc; 522 523 sc = device_get_softc(dev); 524 525 cxgb_free(sc); 526 527 return (0); 528 } 529 530 static void 531 cxgb_free(struct adapter *sc) 532 { 533 int i; 534 535 callout_drain(&sc->cxgb_tick_ch); 536 537 t3_sge_deinit_sw(sc); 538 539 if (sc->tq != NULL) { 540 taskqueue_drain(sc->tq, &sc->ext_intr_task); 541 taskqueue_free(sc->tq); 542 } 543 544 for (i = 0; i < (sc)->params.nports; ++i) { 545 if (sc->portdev[i] != NULL) 546 device_delete_child(sc->dev, sc->portdev[i]); 547 } 548 549 bus_generic_detach(sc->dev); 550 551 t3_free_sge_resources(sc); 552 t3_sge_free(sc); 553 554 for (i = 0; i < SGE_QSETS; i++) { 555 if (sc->msix_intr_tag[i] != NULL) { 556 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 557 sc->msix_intr_tag[i]); 558 } 559 if (sc->msix_irq_res[i] != NULL) { 560 bus_release_resource(sc->dev, SYS_RES_IRQ, 561 sc->msix_irq_rid[i], sc->msix_irq_res[i]); 562 } 563 } 564 565 if (sc->intr_tag != NULL) { 566 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 567 } 568 569 if (sc->irq_res != NULL) { 570 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", 571 sc->irq_rid, sc->irq_res); 572 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 573 sc->irq_res); 574 } 575 #ifdef MSI_SUPPORTED 576 if (sc->flags & (USING_MSI | USING_MSIX)) { 577 device_printf(sc->dev, "releasing msi message(s)\n"); 578 pci_release_msi(sc->dev); 579 } 580 #endif 581 if (sc->msix_regs_res != NULL) { 582 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 583 sc->msix_regs_res); 584 } 585 586 if (sc->regs_res != NULL) 587 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 588 sc->regs_res); 589 590 mtx_destroy(&sc->mdio_lock); 591 mtx_destroy(&sc->sge.reg_lock); 592 mtx_destroy(&sc->lock); 593 594 return; 595 } 596 597 /** 598 * setup_sge_qsets - configure SGE Tx/Rx/response queues 599 * @sc: the controller softc 600 * 601 * Determines how many sets of SGE queues to use and initializes them. 602 * We support multiple queue sets per port if we have MSI-X, otherwise 603 * just one queue set per port. 604 */ 605 static int 606 setup_sge_qsets(adapter_t *sc) 607 { 608 int i, j, err, irq_idx, qset_idx; 609 u_int ntxq = 3; 610 611 if ((err = t3_sge_alloc(sc)) != 0) { 612 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 613 return (err); 614 } 615 616 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 617 irq_idx = -1; 618 else 619 irq_idx = 0; 620 621 for (qset_idx = 0, i = 0; i < (sc)->params.nports; ++i) { 622 struct port_info *pi = &sc->port[i]; 623 624 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) { 625 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 626 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 627 &sc->params.sge.qset[qset_idx], ntxq, pi); 628 if (err) { 629 t3_free_sge_resources(sc); 630 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err); 631 return (err); 632 } 633 } 634 } 635 636 return (0); 637 } 638 639 static int 640 cxgb_setup_msix(adapter_t *sc, int msix_count) 641 { 642 int i, j, k, nqsets, rid; 643 644 /* The first message indicates link changes and error conditions */ 645 sc->irq_rid = 1; 646 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 647 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 648 device_printf(sc->dev, "Cannot allocate msix interrupt\n"); 649 return (EINVAL); 650 } 651 652 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 653 #ifdef INTR_FILTERS 654 NULL, 655 #endif 656 cxgb_async_intr, sc, &sc->intr_tag)) { 657 device_printf(sc->dev, "Cannot set up interrupt\n"); 658 return (EINVAL); 659 } 660 for (i = 0, k = 0; i < (sc)->params.nports; ++i) { 661 nqsets = sc->port[i].nqsets; 662 for (j = 0; j < nqsets; ++j, k++) { 663 struct sge_qset *qs = &sc->sge.qs[k]; 664 665 rid = k + 2; 666 if (cxgb_debug) 667 printf("rid=%d ", rid); 668 if ((sc->msix_irq_res[k] = bus_alloc_resource_any( 669 sc->dev, SYS_RES_IRQ, &rid, 670 RF_SHAREABLE | RF_ACTIVE)) == NULL) { 671 device_printf(sc->dev, "Cannot allocate " 672 "interrupt for message %d\n", rid); 673 return (EINVAL); 674 } 675 sc->msix_irq_rid[k] = rid; 676 if (bus_setup_intr(sc->dev, sc->msix_irq_res[j], 677 INTR_MPSAFE|INTR_TYPE_NET, 678 #ifdef INTR_FILTERS 679 NULL, 680 #endif 681 t3_intr_msix, qs, &sc->msix_intr_tag[k])) { 682 device_printf(sc->dev, "Cannot set up " 683 "interrupt for message %d\n", rid); 684 return (EINVAL); 685 } 686 } 687 } 688 689 690 return (0); 691 } 692 693 static int 694 cxgb_port_probe(device_t dev) 695 { 696 struct port_info *p; 697 char buf[80]; 698 699 p = device_get_softc(dev); 700 701 snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc); 702 device_set_desc_copy(dev, buf); 703 return (0); 704 } 705 706 707 static int 708 cxgb_makedev(struct port_info *pi) 709 { 710 struct cdevsw *cxgb_cdevsw; 711 712 if ((cxgb_cdevsw = malloc(sizeof(struct cdevsw), M_DEVBUF, M_NOWAIT|M_ZERO)) == NULL) 713 return (ENOMEM); 714 715 cxgb_cdevsw->d_version = D_VERSION; 716 cxgb_cdevsw->d_name = strdup(pi->ifp->if_xname, M_DEVBUF); 717 cxgb_cdevsw->d_ioctl = cxgb_extension_ioctl; 718 719 pi->port_cdev = make_dev(cxgb_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 720 pi->ifp->if_xname); 721 722 if (pi->port_cdev == NULL) 723 return (ENOMEM); 724 725 pi->port_cdev->si_drv1 = (void *)pi; 726 727 return (0); 728 } 729 730 731 #ifdef TSO_SUPPORTED 732 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU) 733 /* Don't enable TSO6 yet */ 734 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU) 735 #else 736 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 737 /* Don't enable TSO6 yet */ 738 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 739 #define IFCAP_TSO4 0x0 740 #define CSUM_TSO 0x0 741 #endif 742 743 744 static int 745 cxgb_port_attach(device_t dev) 746 { 747 struct port_info *p; 748 struct ifnet *ifp; 749 int media_flags; 750 int err; 751 char buf[64]; 752 753 p = device_get_softc(dev); 754 755 snprintf(buf, sizeof(buf), "cxgb port %d", p->port); 756 mtx_init(&p->lock, buf, 0, MTX_DEF); 757 758 /* Allocate an ifnet object and set it up */ 759 ifp = p->ifp = if_alloc(IFT_ETHER); 760 if (ifp == NULL) { 761 device_printf(dev, "Cannot allocate ifnet\n"); 762 return (ENOMEM); 763 } 764 765 /* 766 * Note that there is currently no watchdog timer. 767 */ 768 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 769 ifp->if_init = cxgb_init; 770 ifp->if_softc = p; 771 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 772 ifp->if_ioctl = cxgb_ioctl; 773 ifp->if_start = cxgb_start; 774 ifp->if_timer = 0; /* Disable ifnet watchdog */ 775 ifp->if_watchdog = NULL; 776 777 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE; 778 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 779 IFQ_SET_READY(&ifp->if_snd); 780 781 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 782 ifp->if_capabilities |= CXGB_CAP; 783 ifp->if_capenable |= CXGB_CAP_ENABLE; 784 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 785 ifp->if_baudrate = 100000000; 786 787 ether_ifattach(ifp, p->hw_addr); 788 #ifdef DEFAULT_JUMBO 789 ifp->if_mtu = 9000; 790 #endif 791 if ((err = cxgb_makedev(p)) != 0) { 792 printf("makedev failed %d\n", err); 793 return (err); 794 } 795 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 796 cxgb_media_status); 797 798 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) 799 media_flags = IFM_ETHER | IFM_10G_CX4; 800 else if (!strcmp(p->port_type->desc, "10GBASE-SR")) 801 media_flags = IFM_ETHER | IFM_10G_SR; 802 else if (!strcmp(p->port_type->desc, "10GBASE-XR")) 803 media_flags = IFM_ETHER | IFM_10G_LR; 804 else { 805 printf("unsupported media type %s\n", p->port_type->desc); 806 return (ENXIO); 807 } 808 809 ifmedia_add(&p->media, media_flags, 0, NULL); 810 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 811 ifmedia_set(&p->media, media_flags); 812 813 snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port); 814 #ifdef TASKQUEUE_CURRENT 815 /* Create a port for handling TX without starvation */ 816 p->tq = taskqueue_create(buf, M_NOWAIT, 817 taskqueue_thread_enqueue, &p->tq); 818 #else 819 /* Create a port for handling TX without starvation */ 820 p->tq = taskqueue_create_fast(buf, M_NOWAIT, 821 taskqueue_thread_enqueue, &p->tq); 822 #endif 823 824 825 if (p->tq == NULL) { 826 device_printf(dev, "failed to allocate port task queue\n"); 827 return (ENOMEM); 828 } 829 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq", 830 device_get_nameunit(dev)); 831 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp); 832 833 834 return (0); 835 } 836 837 static int 838 cxgb_port_detach(device_t dev) 839 { 840 struct port_info *p; 841 842 p = device_get_softc(dev); 843 mtx_destroy(&p->lock); 844 if (p->tq != NULL) { 845 taskqueue_drain(p->tq, &p->start_task); 846 taskqueue_free(p->tq); 847 p->tq = NULL; 848 } 849 850 ether_ifdetach(p->ifp); 851 if_free(p->ifp); 852 853 destroy_dev(p->port_cdev); 854 855 856 return (0); 857 } 858 859 void 860 t3_fatal_err(struct adapter *sc) 861 { 862 u_int fw_status[4]; 863 864 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 865 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 866 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 867 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 868 } 869 870 int 871 t3_os_find_pci_capability(adapter_t *sc, int cap) 872 { 873 device_t dev; 874 struct pci_devinfo *dinfo; 875 pcicfgregs *cfg; 876 uint32_t status; 877 uint8_t ptr; 878 879 dev = sc->dev; 880 dinfo = device_get_ivars(dev); 881 cfg = &dinfo->cfg; 882 883 status = pci_read_config(dev, PCIR_STATUS, 2); 884 if (!(status & PCIM_STATUS_CAPPRESENT)) 885 return (0); 886 887 switch (cfg->hdrtype & PCIM_HDRTYPE) { 888 case 0: 889 case 1: 890 ptr = PCIR_CAP_PTR; 891 break; 892 case 2: 893 ptr = PCIR_CAP_PTR_2; 894 break; 895 default: 896 return (0); 897 break; 898 } 899 ptr = pci_read_config(dev, ptr, 1); 900 901 while (ptr != 0) { 902 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 903 return (ptr); 904 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 905 } 906 907 return (0); 908 } 909 910 int 911 t3_os_pci_save_state(struct adapter *sc) 912 { 913 device_t dev; 914 struct pci_devinfo *dinfo; 915 916 dev = sc->dev; 917 dinfo = device_get_ivars(dev); 918 919 pci_cfg_save(dev, dinfo, 0); 920 return (0); 921 } 922 923 int 924 t3_os_pci_restore_state(struct adapter *sc) 925 { 926 device_t dev; 927 struct pci_devinfo *dinfo; 928 929 dev = sc->dev; 930 dinfo = device_get_ivars(dev); 931 932 pci_cfg_restore(dev, dinfo); 933 return (0); 934 } 935 936 /** 937 * t3_os_link_changed - handle link status changes 938 * @adapter: the adapter associated with the link change 939 * @port_id: the port index whose limk status has changed 940 * @link_stat: the new status of the link 941 * @speed: the new speed setting 942 * @duplex: the new duplex setting 943 * @fc: the new flow-control setting 944 * 945 * This is the OS-dependent handler for link status changes. The OS 946 * neutral handler takes care of most of the processing for these events, 947 * then calls this handler for any OS-specific processing. 948 */ 949 void 950 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 951 int duplex, int fc) 952 { 953 struct port_info *pi = &adapter->port[port_id]; 954 955 if ((pi->ifp->if_flags & IFF_UP) == 0) 956 return; 957 958 if (link_status) 959 if_link_state_change(pi->ifp, LINK_STATE_UP); 960 else 961 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 962 963 } 964 965 966 /* 967 * Interrupt-context handler for external (PHY) interrupts. 968 */ 969 void 970 t3_os_ext_intr_handler(adapter_t *sc) 971 { 972 if (cxgb_debug) 973 printf("t3_os_ext_intr_handler\n"); 974 /* 975 * Schedule a task to handle external interrupts as they may be slow 976 * and we use a mutex to protect MDIO registers. We disable PHY 977 * interrupts in the meantime and let the task reenable them when 978 * it's done. 979 */ 980 if (sc->slow_intr_mask) { 981 sc->slow_intr_mask &= ~F_T3DBG; 982 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 983 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 984 } 985 } 986 987 void 988 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 989 { 990 991 /* 992 * The ifnet might not be allocated before this gets called, 993 * as this is called early on in attach by t3_prep_adapter 994 * save the address off in the port structure 995 */ 996 if (cxgb_debug) 997 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 998 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 999 } 1000 1001 /** 1002 * link_start - enable a port 1003 * @p: the port to enable 1004 * 1005 * Performs the MAC and PHY actions needed to enable a port. 1006 */ 1007 static void 1008 cxgb_link_start(struct port_info *p) 1009 { 1010 struct ifnet *ifp; 1011 struct t3_rx_mode rm; 1012 struct cmac *mac = &p->mac; 1013 1014 ifp = p->ifp; 1015 1016 t3_init_rx_mode(&rm, p); 1017 t3_mac_reset(mac); 1018 t3_mac_set_mtu(mac, ifp->if_mtu); 1019 t3_mac_set_address(mac, 0, p->hw_addr); 1020 t3_mac_set_rx_mode(mac, &rm); 1021 t3_link_start(&p->phy, mac, &p->link_config); 1022 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1023 } 1024 1025 /** 1026 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1027 * @adap: the adapter 1028 * 1029 * Sets up RSS to distribute packets to multiple receive queues. We 1030 * configure the RSS CPU lookup table to distribute to the number of HW 1031 * receive queues, and the response queue lookup table to narrow that 1032 * down to the response queues actually configured for each port. 1033 * We always configure the RSS mapping for two ports since the mapping 1034 * table has plenty of entries. 1035 */ 1036 static void 1037 setup_rss(adapter_t *adap) 1038 { 1039 int i; 1040 u_int nq0 = adap->port[0].nqsets; 1041 u_int nq1 = max((u_int)adap->port[1].nqsets, 1U); 1042 uint8_t cpus[SGE_QSETS + 1]; 1043 uint16_t rspq_map[RSS_TABLE_SIZE]; 1044 1045 for (i = 0; i < SGE_QSETS; ++i) 1046 cpus[i] = i; 1047 cpus[SGE_QSETS] = 0xff; 1048 1049 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1050 rspq_map[i] = i % nq0; 1051 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0; 1052 } 1053 1054 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1055 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | 1056 V_RRCPLCPUSIZE(6), cpus, rspq_map); 1057 } 1058 1059 static void 1060 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1061 int hi, int port) 1062 { 1063 struct mbuf *m; 1064 struct mngt_pktsched_wr *req; 1065 1066 m = m_gethdr(M_NOWAIT, MT_DATA); 1067 if (m) { 1068 req = (struct mngt_pktsched_wr *)m->m_data; 1069 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1070 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1071 req->sched = sched; 1072 req->idx = qidx; 1073 req->min = lo; 1074 req->max = hi; 1075 req->binding = port; 1076 m->m_len = m->m_pkthdr.len = sizeof(*req); 1077 t3_mgmt_tx(adap, m); 1078 } 1079 } 1080 1081 static void 1082 bind_qsets(adapter_t *sc) 1083 { 1084 int i, j; 1085 1086 for (i = 0; i < (sc)->params.nports; ++i) { 1087 const struct port_info *pi = adap2pinfo(sc, i); 1088 1089 for (j = 0; j < pi->nqsets; ++j) 1090 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1091 -1, i); 1092 } 1093 } 1094 1095 static void 1096 cxgb_init(void *arg) 1097 { 1098 struct port_info *p = arg; 1099 1100 PORT_LOCK(p); 1101 cxgb_init_locked(p); 1102 PORT_UNLOCK(p); 1103 } 1104 1105 static void 1106 cxgb_init_locked(struct port_info *p) 1107 { 1108 struct ifnet *ifp; 1109 adapter_t *sc = p->adapter; 1110 int error; 1111 1112 mtx_assert(&p->lock, MA_OWNED); 1113 1114 ifp = p->ifp; 1115 if ((sc->flags & FW_UPTODATE) == 0) { 1116 device_printf(sc->dev, "updating firmware to version %d.%d\n", 1117 FW_VERSION_MAJOR, FW_VERSION_MINOR); 1118 if ((error = cxgb_fw_download(sc, sc->dev)) != 0) { 1119 device_printf(sc->dev, "firmware download failed err: %d" 1120 "interface will be unavailable\n", error); 1121 return; 1122 } 1123 sc->flags |= FW_UPTODATE; 1124 } 1125 1126 cxgb_link_start(p); 1127 ADAPTER_LOCK(p->adapter); 1128 if (p->adapter->open_device_map == 0) 1129 t3_intr_clear(sc); 1130 t3_sge_start(sc); 1131 1132 p->adapter->open_device_map |= (1 << p->port); 1133 ADAPTER_UNLOCK(p->adapter); 1134 t3_intr_enable(sc); 1135 t3_port_intr_enable(sc, p->port); 1136 1137 if ((p->adapter->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX) 1138 bind_qsets(sc); 1139 p->adapter->flags |= QUEUES_BOUND; 1140 1141 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, 1142 cxgb_tick, sc); 1143 1144 1145 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1146 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1147 } 1148 1149 static void 1150 cxgb_set_rxmode(struct port_info *p) 1151 { 1152 struct t3_rx_mode rm; 1153 struct cmac *mac = &p->mac; 1154 1155 mtx_assert(&p->lock, MA_OWNED); 1156 1157 t3_init_rx_mode(&rm, p); 1158 t3_mac_set_rx_mode(mac, &rm); 1159 } 1160 1161 static void 1162 cxgb_stop_locked(struct port_info *p) 1163 { 1164 struct ifnet *ifp; 1165 1166 mtx_assert(&p->lock, MA_OWNED); 1167 mtx_assert(&p->adapter->lock, MA_NOTOWNED); 1168 1169 ifp = p->ifp; 1170 1171 ADAPTER_LOCK(p->adapter); 1172 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1173 p->adapter->open_device_map &= ~(1 << p->port); 1174 if (p->adapter->open_device_map == 0) 1175 t3_intr_disable(p->adapter); 1176 ADAPTER_UNLOCK(p->adapter); 1177 t3_port_intr_disable(p->adapter, p->port); 1178 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1179 1180 } 1181 1182 static int 1183 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1184 { 1185 struct port_info *p = ifp->if_softc; 1186 struct ifaddr *ifa = (struct ifaddr *)data; 1187 struct ifreq *ifr = (struct ifreq *)data; 1188 int flags, error = 0; 1189 uint32_t mask; 1190 1191 switch (command) { 1192 case SIOCSIFMTU: 1193 if ((ifr->ifr_mtu < ETHERMIN) || 1194 (ifr->ifr_mtu > ETHER_MAX_LEN_JUMBO)) 1195 error = EINVAL; 1196 else if (ifp->if_mtu != ifr->ifr_mtu) { 1197 PORT_LOCK(p); 1198 ifp->if_mtu = ifr->ifr_mtu; 1199 t3_mac_set_mtu(&p->mac, ifp->if_mtu); 1200 PORT_UNLOCK(p); 1201 } 1202 break; 1203 case SIOCSIFADDR: 1204 case SIOCGIFADDR: 1205 if (ifa->ifa_addr->sa_family == AF_INET) { 1206 ifp->if_flags |= IFF_UP; 1207 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1208 cxgb_init(p); 1209 } 1210 arp_ifinit(ifp, ifa); 1211 } else 1212 error = ether_ioctl(ifp, command, data); 1213 break; 1214 case SIOCSIFFLAGS: 1215 1216 if (ifp->if_flags & IFF_UP) { 1217 PORT_LOCK(p); 1218 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1219 flags = p->if_flags; 1220 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1221 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 1222 cxgb_set_rxmode(p); 1223 1224 } else 1225 cxgb_init_locked(p); 1226 p->if_flags = ifp->if_flags; 1227 PORT_UNLOCK(p); 1228 } else { 1229 callout_drain(&p->adapter->cxgb_tick_ch); 1230 PORT_LOCK(p); 1231 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1232 cxgb_stop_locked(p); 1233 } else { 1234 adapter_t *sc = p->adapter; 1235 callout_reset(&sc->cxgb_tick_ch, 1236 sc->params.stats_update_period * hz, 1237 cxgb_tick, sc); 1238 } 1239 PORT_UNLOCK(p); 1240 } 1241 1242 1243 break; 1244 case SIOCSIFMEDIA: 1245 case SIOCGIFMEDIA: 1246 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 1247 break; 1248 case SIOCSIFCAP: 1249 PORT_LOCK(p); 1250 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1251 if (mask & IFCAP_TXCSUM) { 1252 if (IFCAP_TXCSUM & ifp->if_capenable) { 1253 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 1254 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 1255 | CSUM_TSO); 1256 } else { 1257 ifp->if_capenable |= IFCAP_TXCSUM; 1258 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1259 } 1260 } else if (mask & IFCAP_RXCSUM) { 1261 if (IFCAP_RXCSUM & ifp->if_capenable) { 1262 ifp->if_capenable &= ~IFCAP_RXCSUM; 1263 } else { 1264 ifp->if_capenable |= IFCAP_RXCSUM; 1265 } 1266 } 1267 if (mask & IFCAP_TSO4) { 1268 if (IFCAP_TSO4 & ifp->if_capenable) { 1269 ifp->if_capenable &= ~IFCAP_TSO4; 1270 ifp->if_hwassist &= ~CSUM_TSO; 1271 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 1272 ifp->if_capenable |= IFCAP_TSO4; 1273 ifp->if_hwassist |= CSUM_TSO; 1274 } else { 1275 if (cxgb_debug) 1276 printf("cxgb requires tx checksum offload" 1277 " be enabled to use TSO\n"); 1278 error = EINVAL; 1279 } 1280 } 1281 PORT_UNLOCK(p); 1282 break; 1283 default: 1284 error = ether_ioctl(ifp, command, data); 1285 break; 1286 } 1287 1288 return (error); 1289 } 1290 1291 static int 1292 cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) 1293 { 1294 struct sge_qset *qs; 1295 struct sge_txq *txq; 1296 struct port_info *p = ifp->if_softc; 1297 struct mbuf *m = NULL; 1298 int err, in_use_init; 1299 1300 1301 if (!p->link_config.link_ok) 1302 return (ENXIO); 1303 1304 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1305 return (ENOBUFS); 1306 1307 qs = &p->adapter->sge.qs[p->first_qset]; 1308 txq = &qs->txq[TXQ_ETH]; 1309 err = 0; 1310 1311 mtx_lock(&txq->lock); 1312 in_use_init = txq->in_use; 1313 while ((txq->in_use - in_use_init < txmax) && 1314 (txq->size > txq->in_use + TX_MAX_DESC)) { 1315 IFQ_DRV_DEQUEUE(&ifp->if_snd, m); 1316 if (m == NULL) 1317 break; 1318 if ((err = t3_encap(p, &m)) != 0) 1319 break; 1320 BPF_MTAP(ifp, m); 1321 } 1322 mtx_unlock(&txq->lock); 1323 1324 if (__predict_false(err)) { 1325 if (cxgb_debug) 1326 printf("would set OFLAGS\n"); 1327 if (err == ENOMEM) { 1328 IFQ_LOCK(&ifp->if_snd); 1329 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1330 IFQ_UNLOCK(&ifp->if_snd); 1331 } 1332 } 1333 if (err == 0 && m == NULL) 1334 err = ENOBUFS; 1335 1336 return (err); 1337 } 1338 1339 static void 1340 cxgb_start_proc(void *arg, int ncount) 1341 { 1342 struct ifnet *ifp = arg; 1343 struct port_info *pi = ifp->if_softc; 1344 struct sge_qset *qs; 1345 struct sge_txq *txq; 1346 int error = 0; 1347 1348 qs = &pi->adapter->sge.qs[pi->first_qset]; 1349 txq = &qs->txq[TXQ_ETH]; 1350 1351 while (error == 0) { 1352 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC) 1353 taskqueue_enqueue(pi->adapter->tq, 1354 &pi->adapter->timer_reclaim_task); 1355 1356 error = cxgb_start_tx(ifp, TX_START_MAX_DESC); 1357 } 1358 } 1359 1360 static void 1361 cxgb_start(struct ifnet *ifp) 1362 { 1363 struct port_info *pi = ifp->if_softc; 1364 struct sge_qset *qs; 1365 struct sge_txq *txq; 1366 int err; 1367 1368 qs = &pi->adapter->sge.qs[pi->first_qset]; 1369 txq = &qs->txq[TXQ_ETH]; 1370 1371 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC) 1372 taskqueue_enqueue(pi->adapter->tq, 1373 &pi->adapter->timer_reclaim_task); 1374 1375 err = cxgb_start_tx(ifp, TX_START_MAX_DESC); 1376 1377 if (err == 0) 1378 taskqueue_enqueue(pi->tq, &pi->start_task); 1379 } 1380 1381 1382 static int 1383 cxgb_media_change(struct ifnet *ifp) 1384 { 1385 if_printf(ifp, "media change not supported\n"); 1386 return (ENXIO); 1387 } 1388 1389 static void 1390 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1391 { 1392 struct port_info *p = ifp->if_softc; 1393 1394 ifmr->ifm_status = IFM_AVALID; 1395 ifmr->ifm_active = IFM_ETHER; 1396 1397 if (!p->link_config.link_ok) 1398 return; 1399 1400 ifmr->ifm_status |= IFM_ACTIVE; 1401 1402 if (p->link_config.duplex) 1403 ifmr->ifm_active |= IFM_FDX; 1404 else 1405 ifmr->ifm_active |= IFM_HDX; 1406 } 1407 1408 static void 1409 cxgb_async_intr(void *data) 1410 { 1411 adapter_t *sc = data; 1412 1413 if (cxgb_debug) 1414 device_printf(sc->dev, "cxgb_async_intr\n"); 1415 1416 t3_slow_intr_handler(sc); 1417 1418 } 1419 1420 static void 1421 cxgb_ext_intr_handler(void *arg, int count) 1422 { 1423 adapter_t *sc = (adapter_t *)arg; 1424 1425 if (cxgb_debug) 1426 printf("cxgb_ext_intr_handler\n"); 1427 1428 t3_phy_intr_handler(sc); 1429 1430 /* Now reenable external interrupts */ 1431 if (sc->slow_intr_mask) { 1432 sc->slow_intr_mask |= F_T3DBG; 1433 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 1434 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1435 } 1436 } 1437 1438 static void 1439 check_link_status(adapter_t *sc) 1440 { 1441 int i; 1442 1443 for (i = 0; i < (sc)->params.nports; ++i) { 1444 struct port_info *p = &sc->port[i]; 1445 1446 if (!(p->port_type->caps & SUPPORTED_IRQ)) 1447 t3_link_changed(sc, i); 1448 } 1449 } 1450 1451 static void 1452 check_t3b2_mac(struct adapter *adapter) 1453 { 1454 int i; 1455 1456 for_each_port(adapter, i) { 1457 struct port_info *p = &adapter->port[i]; 1458 struct ifnet *ifp = p->ifp; 1459 int status; 1460 1461 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1462 continue; 1463 1464 status = 0; 1465 PORT_LOCK(p); 1466 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 1467 status = t3b2_mac_watchdog_task(&p->mac); 1468 if (status == 1) 1469 p->mac.stats.num_toggled++; 1470 else if (status == 2) { 1471 struct cmac *mac = &p->mac; 1472 1473 t3_mac_set_mtu(mac, ifp->if_mtu); 1474 t3_mac_set_address(mac, 0, p->hw_addr); 1475 cxgb_set_rxmode(p); 1476 t3_link_start(&p->phy, mac, &p->link_config); 1477 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1478 t3_port_intr_enable(adapter, p->port); 1479 p->mac.stats.num_resets++; 1480 } 1481 PORT_UNLOCK(p); 1482 } 1483 } 1484 1485 static void 1486 cxgb_tick(void *arg) 1487 { 1488 adapter_t *sc = (adapter_t *)arg; 1489 const struct adapter_params *p = &sc->params; 1490 1491 if (p->linkpoll_period) 1492 check_link_status(sc); 1493 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, 1494 cxgb_tick, sc); 1495 1496 /* 1497 * adapter lock can currently only be acquire after the 1498 * port lock 1499 */ 1500 ADAPTER_UNLOCK(sc); 1501 if (p->rev == T3_REV_B2) 1502 check_t3b2_mac(sc); 1503 1504 } 1505 1506 static int 1507 in_range(int val, int lo, int hi) 1508 { 1509 return val < 0 || (val <= hi && val >= lo); 1510 } 1511 1512 static int 1513 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 1514 int fflag, struct thread *td) 1515 { 1516 int mmd, error = 0; 1517 struct port_info *pi = dev->si_drv1; 1518 adapter_t *sc = pi->adapter; 1519 1520 #ifdef PRIV_SUPPORTED 1521 if (priv_check(td, PRIV_DRIVER)) { 1522 if (cxgb_debug) 1523 printf("user does not have access to privileged ioctls\n"); 1524 return (EPERM); 1525 } 1526 #else 1527 if (suser(td)) { 1528 if (cxgb_debug) 1529 printf("user does not have access to privileged ioctls\n"); 1530 return (EPERM); 1531 } 1532 #endif 1533 1534 switch (cmd) { 1535 case SIOCGMIIREG: { 1536 uint32_t val; 1537 struct cphy *phy = &pi->phy; 1538 struct mii_data *mid = (struct mii_data *)data; 1539 1540 if (!phy->mdio_read) 1541 return (EOPNOTSUPP); 1542 if (is_10G(sc)) { 1543 mmd = mid->phy_id >> 8; 1544 if (!mmd) 1545 mmd = MDIO_DEV_PCS; 1546 else if (mmd > MDIO_DEV_XGXS) 1547 return -EINVAL; 1548 1549 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 1550 mid->reg_num, &val); 1551 } else 1552 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 1553 mid->reg_num & 0x1f, &val); 1554 if (error == 0) 1555 mid->val_out = val; 1556 break; 1557 } 1558 case SIOCSMIIREG: { 1559 struct cphy *phy = &pi->phy; 1560 struct mii_data *mid = (struct mii_data *)data; 1561 1562 if (!phy->mdio_write) 1563 return (EOPNOTSUPP); 1564 if (is_10G(sc)) { 1565 mmd = mid->phy_id >> 8; 1566 if (!mmd) 1567 mmd = MDIO_DEV_PCS; 1568 else if (mmd > MDIO_DEV_XGXS) 1569 return (EINVAL); 1570 1571 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 1572 mmd, mid->reg_num, mid->val_in); 1573 } else 1574 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 1575 mid->reg_num & 0x1f, 1576 mid->val_in); 1577 break; 1578 } 1579 case CHELSIO_SETREG: { 1580 struct ch_reg *edata = (struct ch_reg *)data; 1581 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 1582 return (EFAULT); 1583 t3_write_reg(sc, edata->addr, edata->val); 1584 break; 1585 } 1586 case CHELSIO_GETREG: { 1587 struct ch_reg *edata = (struct ch_reg *)data; 1588 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 1589 return (EFAULT); 1590 edata->val = t3_read_reg(sc, edata->addr); 1591 break; 1592 } 1593 case CHELSIO_GET_SGE_CONTEXT: { 1594 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 1595 mtx_lock(&sc->sge.reg_lock); 1596 switch (ecntxt->cntxt_type) { 1597 case CNTXT_TYPE_EGRESS: 1598 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 1599 ecntxt->data); 1600 break; 1601 case CNTXT_TYPE_FL: 1602 error = t3_sge_read_fl(sc, ecntxt->cntxt_id, 1603 ecntxt->data); 1604 break; 1605 case CNTXT_TYPE_RSP: 1606 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id, 1607 ecntxt->data); 1608 break; 1609 case CNTXT_TYPE_CQ: 1610 error = t3_sge_read_cq(sc, ecntxt->cntxt_id, 1611 ecntxt->data); 1612 break; 1613 default: 1614 error = EINVAL; 1615 break; 1616 } 1617 mtx_unlock(&sc->sge.reg_lock); 1618 break; 1619 } 1620 case CHELSIO_GET_SGE_DESC: { 1621 struct ch_desc *edesc = (struct ch_desc *)data; 1622 int ret; 1623 if (edesc->queue_num >= SGE_QSETS * 6) 1624 return (EINVAL); 1625 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 1626 edesc->queue_num % 6, edesc->idx, edesc->data); 1627 if (ret < 0) 1628 return (EINVAL); 1629 edesc->size = ret; 1630 break; 1631 } 1632 case CHELSIO_SET_QSET_PARAMS: { 1633 struct qset_params *q; 1634 struct ch_qset_params *t = (struct ch_qset_params *)data; 1635 1636 if (t->qset_idx >= SGE_QSETS) 1637 return -EINVAL; 1638 if (!in_range(t->intr_lat, 0, M_NEWTIMER) || 1639 !in_range(t->cong_thres, 0, 255) || 1640 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES, 1641 MAX_TXQ_ENTRIES) || 1642 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES, 1643 MAX_TXQ_ENTRIES) || 1644 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES, 1645 MAX_CTRL_TXQ_ENTRIES) || 1646 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) || 1647 !in_range(t->fl_size[1], MIN_FL_ENTRIES, 1648 MAX_RX_JUMBO_BUFFERS) || 1649 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES)) 1650 return -EINVAL; 1651 if ((sc->flags & FULL_INIT_DONE) && 1652 (t->rspq_size >= 0 || t->fl_size[0] >= 0 || 1653 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 || 1654 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 || 1655 t->polling >= 0 || t->cong_thres >= 0)) 1656 return -EBUSY; 1657 1658 q = &sc->params.sge.qset[t->qset_idx]; 1659 1660 if (t->rspq_size >= 0) 1661 q->rspq_size = t->rspq_size; 1662 if (t->fl_size[0] >= 0) 1663 q->fl_size = t->fl_size[0]; 1664 if (t->fl_size[1] >= 0) 1665 q->jumbo_size = t->fl_size[1]; 1666 if (t->txq_size[0] >= 0) 1667 q->txq_size[0] = t->txq_size[0]; 1668 if (t->txq_size[1] >= 0) 1669 q->txq_size[1] = t->txq_size[1]; 1670 if (t->txq_size[2] >= 0) 1671 q->txq_size[2] = t->txq_size[2]; 1672 if (t->cong_thres >= 0) 1673 q->cong_thres = t->cong_thres; 1674 if (t->intr_lat >= 0) { 1675 struct sge_qset *qs = &sc->sge.qs[t->qset_idx]; 1676 1677 q->coalesce_nsecs = t->intr_lat*1000; 1678 t3_update_qset_coalesce(qs, q); 1679 } 1680 break; 1681 } 1682 case CHELSIO_GET_QSET_PARAMS: { 1683 struct qset_params *q; 1684 struct ch_qset_params *t = (struct ch_qset_params *)data; 1685 1686 if (t->qset_idx >= SGE_QSETS) 1687 return (EINVAL); 1688 1689 q = &(sc)->params.sge.qset[t->qset_idx]; 1690 t->rspq_size = q->rspq_size; 1691 t->txq_size[0] = q->txq_size[0]; 1692 t->txq_size[1] = q->txq_size[1]; 1693 t->txq_size[2] = q->txq_size[2]; 1694 t->fl_size[0] = q->fl_size; 1695 t->fl_size[1] = q->jumbo_size; 1696 t->polling = q->polling; 1697 t->intr_lat = q->coalesce_nsecs / 1000; 1698 t->cong_thres = q->cong_thres; 1699 break; 1700 } 1701 case CHELSIO_SET_QSET_NUM: { 1702 struct ch_reg *edata = (struct ch_reg *)data; 1703 unsigned int port_idx = pi->port; 1704 1705 if (sc->flags & FULL_INIT_DONE) 1706 return (EBUSY); 1707 if (edata->val < 1 || 1708 (edata->val > 1 && !(sc->flags & USING_MSIX))) 1709 return (EINVAL); 1710 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS) 1711 return (EINVAL); 1712 sc->port[port_idx].nqsets = edata->val; 1713 /* 1714 * XXX we're hardcoding ourselves to 2 ports 1715 * just like the LEENUX 1716 */ 1717 sc->port[1].first_qset = sc->port[0].nqsets; 1718 break; 1719 } 1720 case CHELSIO_GET_QSET_NUM: { 1721 struct ch_reg *edata = (struct ch_reg *)data; 1722 edata->val = pi->nqsets; 1723 break; 1724 } 1725 #ifdef notyet 1726 /* 1727 * XXX FreeBSD driver does not currently support any 1728 * offload functionality 1729 */ 1730 case CHELSIO_LOAD_FW: 1731 case CHELSIO_DEVUP: 1732 case CHELSIO_SETMTUTAB: 1733 case CHELSIO_GET_PM: 1734 case CHELSIO_SET_PM: 1735 case CHELSIO_READ_TCAM_WORD: 1736 return (EOPNOTSUPP); 1737 break; 1738 #endif 1739 case CHELSIO_GET_MEM: { 1740 struct ch_mem_range *t = (struct ch_mem_range *)data; 1741 struct mc7 *mem; 1742 uint8_t *useraddr; 1743 u64 buf[32]; 1744 1745 if (!is_offload(sc)) 1746 return (EOPNOTSUPP); 1747 if (!(sc->flags & FULL_INIT_DONE)) 1748 return (EIO); /* need the memory controllers */ 1749 if ((t->addr & 0x7) || (t->len & 0x7)) 1750 return (EINVAL); 1751 if (t->mem_id == MEM_CM) 1752 mem = &sc->cm; 1753 else if (t->mem_id == MEM_PMRX) 1754 mem = &sc->pmrx; 1755 else if (t->mem_id == MEM_PMTX) 1756 mem = &sc->pmtx; 1757 else 1758 return (EINVAL); 1759 1760 /* 1761 * Version scheme: 1762 * bits 0..9: chip version 1763 * bits 10..15: chip revision 1764 */ 1765 t->version = 3 | (sc->params.rev << 10); 1766 1767 /* 1768 * Read 256 bytes at a time as len can be large and we don't 1769 * want to use huge intermediate buffers. 1770 */ 1771 useraddr = (uint8_t *)(t + 1); /* advance to start of buffer */ 1772 while (t->len) { 1773 unsigned int chunk = min(t->len, sizeof(buf)); 1774 1775 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf); 1776 if (error) 1777 return (-error); 1778 if (copyout(buf, useraddr, chunk)) 1779 return (EFAULT); 1780 useraddr += chunk; 1781 t->addr += chunk; 1782 t->len -= chunk; 1783 } 1784 break; 1785 } 1786 case CHELSIO_SET_TRACE_FILTER: { 1787 struct ch_trace *t = (struct ch_trace *)data; 1788 const struct trace_params *tp; 1789 1790 tp = (const struct trace_params *)&t->sip; 1791 if (t->config_tx) 1792 t3_config_trace_filter(sc, tp, 0, t->invert_match, 1793 t->trace_tx); 1794 if (t->config_rx) 1795 t3_config_trace_filter(sc, tp, 1, t->invert_match, 1796 t->trace_rx); 1797 break; 1798 } 1799 case CHELSIO_SET_PKTSCHED: { 1800 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 1801 if (sc->open_device_map == 0) 1802 return (EAGAIN); 1803 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 1804 p->binding); 1805 break; 1806 } 1807 case CHELSIO_IFCONF_GETREGS: { 1808 struct ifconf_regs *regs = (struct ifconf_regs *)data; 1809 int reglen = cxgb_get_regs_len(); 1810 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT); 1811 if (buf == NULL) { 1812 return (ENOMEM); 1813 } if (regs->len > reglen) 1814 regs->len = reglen; 1815 else if (regs->len < reglen) { 1816 error = E2BIG; 1817 goto done; 1818 } 1819 cxgb_get_regs(sc, regs, buf); 1820 error = copyout(buf, regs->data, reglen); 1821 1822 done: 1823 free(buf, M_DEVBUF); 1824 1825 break; 1826 } 1827 default: 1828 return (EOPNOTSUPP); 1829 break; 1830 } 1831 1832 return (error); 1833 } 1834 1835 static __inline void 1836 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 1837 unsigned int end) 1838 { 1839 uint32_t *p = (uint32_t *)buf + start; 1840 1841 for ( ; start <= end; start += sizeof(uint32_t)) 1842 *p++ = t3_read_reg(ap, start); 1843 } 1844 1845 #define T3_REGMAP_SIZE (3 * 1024) 1846 static int 1847 cxgb_get_regs_len(void) 1848 { 1849 return T3_REGMAP_SIZE; 1850 } 1851 #undef T3_REGMAP_SIZE 1852 1853 static void 1854 cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf) 1855 { 1856 1857 /* 1858 * Version scheme: 1859 * bits 0..9: chip version 1860 * bits 10..15: chip revision 1861 * bit 31: set for PCIe cards 1862 */ 1863 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 1864 1865 /* 1866 * We skip the MAC statistics registers because they are clear-on-read. 1867 * Also reading multi-register stats would need to synchronize with the 1868 * periodic mac stats accumulation. Hard to justify the complexity. 1869 */ 1870 memset(buf, 0, REGDUMP_SIZE); 1871 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 1872 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 1873 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 1874 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 1875 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 1876 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 1877 XGM_REG(A_XGM_SERDES_STAT3, 1)); 1878 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 1879 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 1880 } 1881