1 /************************************************************************** 2 3 Copyright (c) 2007, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 POSSIBILITY OF SUCH DAMAGE. 27 28 ***************************************************************************/ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/bus.h> 37 #include <sys/module.h> 38 #include <sys/pciio.h> 39 #include <sys/conf.h> 40 #include <machine/bus.h> 41 #include <machine/resource.h> 42 #include <sys/bus_dma.h> 43 #include <sys/rman.h> 44 #include <sys/ioccom.h> 45 #include <sys/mbuf.h> 46 #include <sys/linker.h> 47 #include <sys/firmware.h> 48 #include <sys/socket.h> 49 #include <sys/sockio.h> 50 #include <sys/smp.h> 51 #include <sys/sysctl.h> 52 #include <sys/syslog.h> 53 #include <sys/queue.h> 54 #include <sys/taskqueue.h> 55 #include <sys/proc.h> 56 57 #include <net/bpf.h> 58 #include <net/ethernet.h> 59 #include <net/if.h> 60 #include <net/if_arp.h> 61 #include <net/if_dl.h> 62 #include <net/if_media.h> 63 #include <net/if_types.h> 64 65 #include <netinet/in_systm.h> 66 #include <netinet/in.h> 67 #include <netinet/if_ether.h> 68 #include <netinet/ip.h> 69 #include <netinet/ip.h> 70 #include <netinet/tcp.h> 71 #include <netinet/udp.h> 72 73 #include <dev/pci/pcireg.h> 74 #include <dev/pci/pcivar.h> 75 #include <dev/pci/pci_private.h> 76 77 #ifdef CONFIG_DEFINED 78 #include <cxgb_include.h> 79 #else 80 #include <dev/cxgb/cxgb_include.h> 81 #endif 82 83 #ifdef PRIV_SUPPORTED 84 #include <sys/priv.h> 85 #endif 86 87 #ifdef IFNET_MULTIQUEUE 88 #include <machine/intr_machdep.h> 89 #endif 90 91 static int cxgb_setup_msix(adapter_t *, int); 92 static void cxgb_teardown_msix(adapter_t *); 93 static void cxgb_init(void *); 94 static void cxgb_init_locked(struct port_info *); 95 static void cxgb_stop_locked(struct port_info *); 96 static void cxgb_set_rxmode(struct port_info *); 97 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 98 static int cxgb_media_change(struct ifnet *); 99 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 100 static int setup_sge_qsets(adapter_t *); 101 static void cxgb_async_intr(void *); 102 static void cxgb_ext_intr_handler(void *, int); 103 static void cxgb_tick_handler(void *, int); 104 static void cxgb_down_locked(struct adapter *sc); 105 static void cxgb_tick(void *); 106 static void setup_rss(adapter_t *sc); 107 108 #ifndef IFNET_MULTIQUEUE 109 static void cxgb_start_proc(void *, int ncount); 110 #endif 111 112 /* Attachment glue for the PCI controller end of the device. Each port of 113 * the device is attached separately, as defined later. 114 */ 115 static int cxgb_controller_probe(device_t); 116 static int cxgb_controller_attach(device_t); 117 static int cxgb_controller_detach(device_t); 118 static void cxgb_free(struct adapter *); 119 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 120 unsigned int end); 121 static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf); 122 static int cxgb_get_regs_len(void); 123 static int offload_open(struct port_info *pi); 124 static void touch_bars(device_t dev); 125 static int offload_close(struct t3cdev *tdev); 126 127 static device_method_t cxgb_controller_methods[] = { 128 DEVMETHOD(device_probe, cxgb_controller_probe), 129 DEVMETHOD(device_attach, cxgb_controller_attach), 130 DEVMETHOD(device_detach, cxgb_controller_detach), 131 132 /* bus interface */ 133 DEVMETHOD(bus_print_child, bus_generic_print_child), 134 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 135 136 { 0, 0 } 137 }; 138 139 static driver_t cxgb_controller_driver = { 140 "cxgbc", 141 cxgb_controller_methods, 142 sizeof(struct adapter) 143 }; 144 145 static devclass_t cxgb_controller_devclass; 146 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 147 148 /* 149 * Attachment glue for the ports. Attachment is done directly to the 150 * controller device. 151 */ 152 static int cxgb_port_probe(device_t); 153 static int cxgb_port_attach(device_t); 154 static int cxgb_port_detach(device_t); 155 156 static device_method_t cxgb_port_methods[] = { 157 DEVMETHOD(device_probe, cxgb_port_probe), 158 DEVMETHOD(device_attach, cxgb_port_attach), 159 DEVMETHOD(device_detach, cxgb_port_detach), 160 { 0, 0 } 161 }; 162 163 static driver_t cxgb_port_driver = { 164 "cxgb", 165 cxgb_port_methods, 166 0 167 }; 168 169 static d_ioctl_t cxgb_extension_ioctl; 170 static d_open_t cxgb_extension_open; 171 static d_close_t cxgb_extension_close; 172 173 static struct cdevsw cxgb_cdevsw = { 174 .d_version = D_VERSION, 175 .d_flags = 0, 176 .d_open = cxgb_extension_open, 177 .d_close = cxgb_extension_close, 178 .d_ioctl = cxgb_extension_ioctl, 179 .d_name = "cxgb", 180 }; 181 182 static devclass_t cxgb_port_devclass; 183 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 184 185 #define SGE_MSIX_COUNT (SGE_QSETS + 1) 186 187 /* 188 * The driver uses the best interrupt scheme available on a platform in the 189 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 190 * of these schemes the driver may consider as follows: 191 * 192 * msi = 2: choose from among all three options 193 * msi = 1 : only consider MSI and pin interrupts 194 * msi = 0: force pin interrupts 195 */ 196 static int msi_allowed = 2; 197 198 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 199 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 200 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 201 "MSI-X, MSI, INTx selector"); 202 203 /* 204 * The driver enables offload as a default. 205 * To disable it, use ofld_disable = 1. 206 */ 207 static int ofld_disable = 0; 208 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable); 209 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0, 210 "disable ULP offload"); 211 212 /* 213 * The driver uses an auto-queue algorithm by default. 214 * To disable it and force a single queue-set per port, use singleq = 1. 215 */ 216 static int singleq = 0; 217 TUNABLE_INT("hw.cxgb.singleq", &singleq); 218 SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0, 219 "use a single queue-set per port"); 220 221 #ifndef IFNET_MULTIQUEUE 222 int cxgb_txq_buf_ring_size = 0; 223 #endif 224 225 enum { 226 MAX_TXQ_ENTRIES = 16384, 227 MAX_CTRL_TXQ_ENTRIES = 1024, 228 MAX_RSPQ_ENTRIES = 16384, 229 MAX_RX_BUFFERS = 16384, 230 MAX_RX_JUMBO_BUFFERS = 16384, 231 MIN_TXQ_ENTRIES = 4, 232 MIN_CTRL_TXQ_ENTRIES = 4, 233 MIN_RSPQ_ENTRIES = 32, 234 MIN_FL_ENTRIES = 32, 235 MIN_FL_JUMBO_ENTRIES = 32 236 }; 237 238 struct filter_info { 239 u32 sip; 240 u32 sip_mask; 241 u32 dip; 242 u16 sport; 243 u16 dport; 244 u32 vlan:12; 245 u32 vlan_prio:3; 246 u32 mac_hit:1; 247 u32 mac_idx:4; 248 u32 mac_vld:1; 249 u32 pkt_type:2; 250 u32 report_filter_id:1; 251 u32 pass:1; 252 u32 rss:1; 253 u32 qset:3; 254 u32 locked:1; 255 u32 valid:1; 256 }; 257 258 enum { FILTER_NO_VLAN_PRI = 7 }; 259 260 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 261 262 /* Table for probing the cards. The desc field isn't actually used */ 263 struct cxgb_ident { 264 uint16_t vendor; 265 uint16_t device; 266 int index; 267 char *desc; 268 } cxgb_identifiers[] = { 269 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 270 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 271 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 272 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 273 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 274 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 275 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 276 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 277 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 278 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 279 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 280 {0, 0, 0, NULL} 281 }; 282 283 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 284 285 static __inline void 286 check_pkt_coalesce(struct sge_qset *qs) 287 { 288 struct adapter *sc; 289 struct sge_txq *txq; 290 291 txq = &qs->txq[TXQ_ETH]; 292 sc = qs->port->adapter; 293 294 if (sc->tunq_fill[qs->idx] && (txq->in_use < (txq->size - (txq->size>>2)))) 295 sc->tunq_fill[qs->idx] = 0; 296 else if (!sc->tunq_fill[qs->idx] && (txq->in_use > (txq->size - (txq->size>>2)))) 297 sc->tunq_fill[qs->idx] = 1; 298 } 299 300 static __inline char 301 t3rev2char(struct adapter *adapter) 302 { 303 char rev = 'z'; 304 305 switch(adapter->params.rev) { 306 case T3_REV_A: 307 rev = 'a'; 308 break; 309 case T3_REV_B: 310 case T3_REV_B2: 311 rev = 'b'; 312 break; 313 case T3_REV_C: 314 rev = 'c'; 315 break; 316 } 317 return rev; 318 } 319 320 static struct cxgb_ident * 321 cxgb_get_ident(device_t dev) 322 { 323 struct cxgb_ident *id; 324 325 for (id = cxgb_identifiers; id->desc != NULL; id++) { 326 if ((id->vendor == pci_get_vendor(dev)) && 327 (id->device == pci_get_device(dev))) { 328 return (id); 329 } 330 } 331 return (NULL); 332 } 333 334 static const struct adapter_info * 335 cxgb_get_adapter_info(device_t dev) 336 { 337 struct cxgb_ident *id; 338 const struct adapter_info *ai; 339 340 id = cxgb_get_ident(dev); 341 if (id == NULL) 342 return (NULL); 343 344 ai = t3_get_adapter_info(id->index); 345 346 return (ai); 347 } 348 349 static int 350 cxgb_controller_probe(device_t dev) 351 { 352 const struct adapter_info *ai; 353 char *ports, buf[80]; 354 int nports; 355 356 ai = cxgb_get_adapter_info(dev); 357 if (ai == NULL) 358 return (ENXIO); 359 360 nports = ai->nports0 + ai->nports1; 361 if (nports == 1) 362 ports = "port"; 363 else 364 ports = "ports"; 365 366 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, nports, ports); 367 device_set_desc_copy(dev, buf); 368 return (BUS_PROBE_DEFAULT); 369 } 370 371 #define FW_FNAME "t3fw%d%d%d" 372 #define TPEEPROM_NAME "t3%ctpe%d%d%d" 373 #define TPSRAM_NAME "t3%cps%d%d%d" 374 375 static int 376 upgrade_fw(adapter_t *sc) 377 { 378 char buf[32]; 379 #ifdef FIRMWARE_LATEST 380 const struct firmware *fw; 381 #else 382 struct firmware *fw; 383 #endif 384 int status; 385 386 snprintf(&buf[0], sizeof(buf), FW_FNAME, FW_VERSION_MAJOR, 387 FW_VERSION_MINOR, FW_VERSION_MICRO); 388 389 fw = firmware_get(buf); 390 391 if (fw == NULL) { 392 device_printf(sc->dev, "Could not find firmware image %s\n", buf); 393 return (ENOENT); 394 } else 395 device_printf(sc->dev, "updating firmware on card with %s\n", buf); 396 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 397 398 device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status); 399 400 firmware_put(fw, FIRMWARE_UNLOAD); 401 402 return (status); 403 } 404 405 static int 406 cxgb_controller_attach(device_t dev) 407 { 408 device_t child; 409 const struct adapter_info *ai; 410 struct adapter *sc; 411 int i, error = 0; 412 uint32_t vers; 413 int port_qsets = 1; 414 #ifdef MSI_SUPPORTED 415 int msi_needed, reg; 416 #endif 417 sc = device_get_softc(dev); 418 sc->dev = dev; 419 sc->msi_count = 0; 420 ai = cxgb_get_adapter_info(dev); 421 422 /* 423 * XXX not really related but a recent addition 424 */ 425 #ifdef MSI_SUPPORTED 426 /* find the PCIe link width and set max read request to 4KB*/ 427 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 428 uint16_t lnk, pectl; 429 lnk = pci_read_config(dev, reg + 0x12, 2); 430 sc->link_width = (lnk >> 4) & 0x3f; 431 432 pectl = pci_read_config(dev, reg + 0x8, 2); 433 pectl = (pectl & ~0x7000) | (5 << 12); 434 pci_write_config(dev, reg + 0x8, pectl, 2); 435 } 436 437 if (sc->link_width != 0 && sc->link_width <= 4 && 438 (ai->nports0 + ai->nports1) <= 2) { 439 device_printf(sc->dev, 440 "PCIe x%d Link, expect reduced performance\n", 441 sc->link_width); 442 } 443 #endif 444 touch_bars(dev); 445 pci_enable_busmaster(dev); 446 /* 447 * Allocate the registers and make them available to the driver. 448 * The registers that we care about for NIC mode are in BAR 0 449 */ 450 sc->regs_rid = PCIR_BAR(0); 451 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 452 &sc->regs_rid, RF_ACTIVE)) == NULL) { 453 device_printf(dev, "Cannot allocate BAR\n"); 454 return (ENXIO); 455 } 456 457 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 458 device_get_unit(dev)); 459 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 460 461 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 462 device_get_unit(dev)); 463 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 464 device_get_unit(dev)); 465 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 466 device_get_unit(dev)); 467 468 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_DEF); 469 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 470 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 471 472 sc->bt = rman_get_bustag(sc->regs_res); 473 sc->bh = rman_get_bushandle(sc->regs_res); 474 sc->mmio_len = rman_get_size(sc->regs_res); 475 476 if (t3_prep_adapter(sc, ai, 1) < 0) { 477 printf("prep adapter failed\n"); 478 error = ENODEV; 479 goto out; 480 } 481 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 482 * enough messages for the queue sets. If that fails, try falling 483 * back to MSI. If that fails, then try falling back to the legacy 484 * interrupt pin model. 485 */ 486 #ifdef MSI_SUPPORTED 487 488 sc->msix_regs_rid = 0x20; 489 if ((msi_allowed >= 2) && 490 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 491 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 492 493 msi_needed = sc->msi_count = SGE_MSIX_COUNT; 494 495 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) || 496 (sc->msi_count != msi_needed)) { 497 device_printf(dev, "msix allocation failed - msi_count = %d" 498 " msi_needed=%d will try msi err=%d\n", sc->msi_count, 499 msi_needed, error); 500 sc->msi_count = 0; 501 pci_release_msi(dev); 502 bus_release_resource(dev, SYS_RES_MEMORY, 503 sc->msix_regs_rid, sc->msix_regs_res); 504 sc->msix_regs_res = NULL; 505 } else { 506 sc->flags |= USING_MSIX; 507 sc->cxgb_intr = t3_intr_msix; 508 } 509 } 510 511 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 512 sc->msi_count = 1; 513 if (pci_alloc_msi(dev, &sc->msi_count)) { 514 device_printf(dev, "alloc msi failed - will try INTx\n"); 515 sc->msi_count = 0; 516 pci_release_msi(dev); 517 } else { 518 sc->flags |= USING_MSI; 519 sc->irq_rid = 1; 520 sc->cxgb_intr = t3_intr_msi; 521 } 522 } 523 #endif 524 if (sc->msi_count == 0) { 525 device_printf(dev, "using line interrupts\n"); 526 sc->irq_rid = 0; 527 sc->cxgb_intr = t3b_intr; 528 } 529 530 531 /* Create a private taskqueue thread for handling driver events */ 532 #ifdef TASKQUEUE_CURRENT 533 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 534 taskqueue_thread_enqueue, &sc->tq); 535 #else 536 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 537 taskqueue_thread_enqueue, &sc->tq); 538 #endif 539 if (sc->tq == NULL) { 540 device_printf(dev, "failed to allocate controller task queue\n"); 541 goto out; 542 } 543 544 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 545 device_get_nameunit(dev)); 546 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 547 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 548 549 550 /* Create a periodic callout for checking adapter status */ 551 callout_init(&sc->cxgb_tick_ch, TRUE); 552 553 if (t3_check_fw_version(sc) != 0) { 554 /* 555 * Warn user that a firmware update will be attempted in init. 556 */ 557 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 558 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 559 sc->flags &= ~FW_UPTODATE; 560 } else { 561 sc->flags |= FW_UPTODATE; 562 } 563 564 if (t3_check_tpsram_version(sc) != 0) { 565 /* 566 * Warn user that a firmware update will be attempted in init. 567 */ 568 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 569 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 570 sc->flags &= ~TPS_UPTODATE; 571 } else { 572 sc->flags |= TPS_UPTODATE; 573 } 574 575 if ((sc->flags & USING_MSIX) && !singleq) 576 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); 577 578 /* 579 * Create a child device for each MAC. The ethernet attachment 580 * will be done in these children. 581 */ 582 for (i = 0; i < (sc)->params.nports; i++) { 583 struct port_info *pi; 584 585 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 586 device_printf(dev, "failed to add child port\n"); 587 error = EINVAL; 588 goto out; 589 } 590 pi = &sc->port[i]; 591 pi->adapter = sc; 592 pi->nqsets = port_qsets; 593 pi->first_qset = i*port_qsets; 594 pi->port_id = i; 595 pi->tx_chan = i >= ai->nports0; 596 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 597 sc->rxpkt_map[pi->txpkt_intf] = i; 598 sc->port[i].tx_chan = i >= ai->nports0; 599 sc->portdev[i] = child; 600 device_set_softc(child, pi); 601 } 602 if ((error = bus_generic_attach(dev)) != 0) 603 goto out; 604 605 /* 606 * XXX need to poll for link status 607 */ 608 sc->params.stats_update_period = 1; 609 610 /* initialize sge private state */ 611 t3_sge_init_adapter(sc); 612 613 t3_led_ready(sc); 614 615 cxgb_offload_init(); 616 if (is_offload(sc)) { 617 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); 618 cxgb_adapter_ofld(sc); 619 } 620 error = t3_get_fw_version(sc, &vers); 621 if (error) 622 goto out; 623 624 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 625 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 626 G_FW_VERSION_MICRO(vers)); 627 628 t3_add_attach_sysctls(sc); 629 out: 630 if (error) 631 cxgb_free(sc); 632 633 return (error); 634 } 635 636 static int 637 cxgb_controller_detach(device_t dev) 638 { 639 struct adapter *sc; 640 641 sc = device_get_softc(dev); 642 643 cxgb_free(sc); 644 645 return (0); 646 } 647 648 static void 649 cxgb_free(struct adapter *sc) 650 { 651 int i; 652 653 654 #ifdef IFNET_MULTIQUEUE 655 cxgb_pcpu_shutdown_threads(sc); 656 #endif 657 ADAPTER_LOCK(sc); 658 /* 659 * drops the lock 660 */ 661 cxgb_down_locked(sc); 662 663 #ifdef MSI_SUPPORTED 664 if (sc->flags & (USING_MSI | USING_MSIX)) { 665 device_printf(sc->dev, "releasing msi message(s)\n"); 666 pci_release_msi(sc->dev); 667 } else { 668 device_printf(sc->dev, "no msi message to release\n"); 669 } 670 #endif 671 if (sc->msix_regs_res != NULL) { 672 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 673 sc->msix_regs_res); 674 } 675 676 if (sc->tq != NULL) { 677 taskqueue_drain(sc->tq, &sc->ext_intr_task); 678 taskqueue_drain(sc->tq, &sc->tick_task); 679 } 680 t3_sge_deinit_sw(sc); 681 /* 682 * Wait for last callout 683 */ 684 685 DELAY(hz*100); 686 687 for (i = 0; i < (sc)->params.nports; ++i) { 688 if (sc->portdev[i] != NULL) 689 device_delete_child(sc->dev, sc->portdev[i]); 690 } 691 692 bus_generic_detach(sc->dev); 693 if (sc->tq != NULL) 694 taskqueue_free(sc->tq); 695 if (is_offload(sc)) { 696 cxgb_adapter_unofld(sc); 697 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT)) 698 offload_close(&sc->tdev); 699 else 700 printf("cxgb_free: DEVMAP_BIT not set\n"); 701 } else 702 printf("not offloading set\n"); 703 #ifndef IFNET_MULTIQUEUE 704 t3_free_sge_resources(sc); 705 #endif 706 free(sc->filters, M_DEVBUF); 707 t3_sge_free(sc); 708 709 cxgb_offload_exit(); 710 711 if (sc->regs_res != NULL) 712 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 713 sc->regs_res); 714 715 MTX_DESTROY(&sc->mdio_lock); 716 MTX_DESTROY(&sc->sge.reg_lock); 717 MTX_DESTROY(&sc->elmer_lock); 718 ADAPTER_LOCK_DEINIT(sc); 719 } 720 721 /** 722 * setup_sge_qsets - configure SGE Tx/Rx/response queues 723 * @sc: the controller softc 724 * 725 * Determines how many sets of SGE queues to use and initializes them. 726 * We support multiple queue sets per port if we have MSI-X, otherwise 727 * just one queue set per port. 728 */ 729 static int 730 setup_sge_qsets(adapter_t *sc) 731 { 732 int i, j, err, irq_idx = 0, qset_idx = 0; 733 u_int ntxq = SGE_TXQ_PER_SET; 734 735 if ((err = t3_sge_alloc(sc)) != 0) { 736 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 737 return (err); 738 } 739 740 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 741 irq_idx = -1; 742 743 for (i = 0; i < (sc)->params.nports; i++) { 744 struct port_info *pi = &sc->port[i]; 745 746 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 747 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 748 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 749 &sc->params.sge.qset[qset_idx], ntxq, pi); 750 if (err) { 751 t3_free_sge_resources(sc); 752 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", 753 err); 754 return (err); 755 } 756 } 757 } 758 759 return (0); 760 } 761 762 static void 763 cxgb_teardown_msix(adapter_t *sc) 764 { 765 int i, nqsets; 766 767 for (nqsets = i = 0; i < (sc)->params.nports; i++) 768 nqsets += sc->port[i].nqsets; 769 770 for (i = 0; i < nqsets; i++) { 771 if (sc->msix_intr_tag[i] != NULL) { 772 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 773 sc->msix_intr_tag[i]); 774 sc->msix_intr_tag[i] = NULL; 775 } 776 if (sc->msix_irq_res[i] != NULL) { 777 bus_release_resource(sc->dev, SYS_RES_IRQ, 778 sc->msix_irq_rid[i], sc->msix_irq_res[i]); 779 sc->msix_irq_res[i] = NULL; 780 } 781 } 782 } 783 784 static int 785 cxgb_setup_msix(adapter_t *sc, int msix_count) 786 { 787 int i, j, k, nqsets, rid; 788 789 /* The first message indicates link changes and error conditions */ 790 sc->irq_rid = 1; 791 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 792 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 793 device_printf(sc->dev, "Cannot allocate msix interrupt\n"); 794 return (EINVAL); 795 } 796 797 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 798 #ifdef INTR_FILTERS 799 NULL, 800 #endif 801 cxgb_async_intr, sc, &sc->intr_tag)) { 802 device_printf(sc->dev, "Cannot set up interrupt\n"); 803 return (EINVAL); 804 } 805 for (i = k = 0; i < (sc)->params.nports; i++) { 806 nqsets = sc->port[i].nqsets; 807 for (j = 0; j < nqsets; j++, k++) { 808 struct sge_qset *qs = &sc->sge.qs[k]; 809 810 rid = k + 2; 811 if (cxgb_debug) 812 printf("rid=%d ", rid); 813 if ((sc->msix_irq_res[k] = bus_alloc_resource_any( 814 sc->dev, SYS_RES_IRQ, &rid, 815 RF_SHAREABLE | RF_ACTIVE)) == NULL) { 816 device_printf(sc->dev, "Cannot allocate " 817 "interrupt for message %d\n", rid); 818 return (EINVAL); 819 } 820 sc->msix_irq_rid[k] = rid; 821 printf("setting up interrupt for port=%d\n", 822 qs->port->port_id); 823 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k], 824 INTR_MPSAFE|INTR_TYPE_NET, 825 #ifdef INTR_FILTERS 826 NULL, 827 #endif 828 t3_intr_msix, qs, &sc->msix_intr_tag[k])) { 829 device_printf(sc->dev, "Cannot set up " 830 "interrupt for message %d\n", rid); 831 return (EINVAL); 832 } 833 #ifdef IFNET_MULTIQUEUE 834 if (singleq == 0) { 835 int vector = rman_get_start(sc->msix_irq_res[k]); 836 if (bootverbose) 837 device_printf(sc->dev, "binding vector=%d to cpu=%d\n", vector, k % mp_ncpus); 838 intr_bind(vector, k % mp_ncpus); 839 } 840 #endif 841 } 842 } 843 844 return (0); 845 } 846 847 static int 848 cxgb_port_probe(device_t dev) 849 { 850 struct port_info *p; 851 char buf[80]; 852 853 p = device_get_softc(dev); 854 855 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, p->port_type->desc); 856 device_set_desc_copy(dev, buf); 857 return (0); 858 } 859 860 861 static int 862 cxgb_makedev(struct port_info *pi) 863 { 864 865 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 866 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp)); 867 868 if (pi->port_cdev == NULL) 869 return (ENOMEM); 870 871 pi->port_cdev->si_drv1 = (void *)pi; 872 873 return (0); 874 } 875 876 877 #ifdef TSO_SUPPORTED 878 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU) 879 /* Don't enable TSO6 yet */ 880 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU) 881 #else 882 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 883 /* Don't enable TSO6 yet */ 884 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 885 #define IFCAP_TSO4 0x0 886 #define IFCAP_TSO6 0x0 887 #define CSUM_TSO 0x0 888 #endif 889 890 891 static int 892 cxgb_port_attach(device_t dev) 893 { 894 struct port_info *p; 895 struct ifnet *ifp; 896 int err, media_flags; 897 898 p = device_get_softc(dev); 899 900 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 901 device_get_unit(device_get_parent(dev)), p->port_id); 902 PORT_LOCK_INIT(p, p->lockbuf); 903 904 /* Allocate an ifnet object and set it up */ 905 ifp = p->ifp = if_alloc(IFT_ETHER); 906 if (ifp == NULL) { 907 device_printf(dev, "Cannot allocate ifnet\n"); 908 return (ENOMEM); 909 } 910 911 /* 912 * Note that there is currently no watchdog timer. 913 */ 914 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 915 ifp->if_init = cxgb_init; 916 ifp->if_softc = p; 917 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 918 ifp->if_ioctl = cxgb_ioctl; 919 ifp->if_start = cxgb_start; 920 921 #ifdef IFNET_MULTIQUEUE 922 ifp->if_flags |= IFF_MULTIQ; 923 ifp->if_mq_start = cxgb_pcpu_start; 924 #endif 925 926 ifp->if_timer = 0; /* Disable ifnet watchdog */ 927 ifp->if_watchdog = NULL; 928 929 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE; 930 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 931 IFQ_SET_READY(&ifp->if_snd); 932 933 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 934 ifp->if_capabilities |= CXGB_CAP; 935 ifp->if_capenable |= CXGB_CAP_ENABLE; 936 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 937 /* 938 * disable TSO on 4-port - it isn't supported by the firmware yet 939 */ 940 if (p->adapter->params.nports > 2) { 941 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6); 942 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6); 943 ifp->if_hwassist &= ~CSUM_TSO; 944 } 945 946 ether_ifattach(ifp, p->hw_addr); 947 /* 948 * Only default to jumbo frames on 10GigE 949 */ 950 if (p->adapter->params.nports <= 2) 951 ifp->if_mtu = 9000; 952 if ((err = cxgb_makedev(p)) != 0) { 953 printf("makedev failed %d\n", err); 954 return (err); 955 } 956 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 957 cxgb_media_status); 958 959 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) { 960 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX; 961 } else if (!strcmp(p->port_type->desc, "10GBASE-SR")) { 962 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX; 963 } else if (!strcmp(p->port_type->desc, "10GBASE-XR")) { 964 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX; 965 } else if (!strcmp(p->port_type->desc, "10/100/1000BASE-T")) { 966 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL); 967 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX, 968 0, NULL); 969 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX, 970 0, NULL); 971 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 972 0, NULL); 973 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 974 0, NULL); 975 media_flags = 0; 976 } else { 977 printf("unsupported media type %s\n", p->port_type->desc); 978 return (ENXIO); 979 } 980 if (media_flags) { 981 ifmedia_add(&p->media, media_flags, 0, NULL); 982 ifmedia_set(&p->media, media_flags); 983 } else { 984 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 985 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO); 986 } 987 988 989 snprintf(p->taskqbuf, TASKQ_NAME_LEN, "cxgb_port_taskq%d", p->port_id); 990 #ifdef TASKQUEUE_CURRENT 991 /* Create a port for handling TX without starvation */ 992 p->tq = taskqueue_create(p->taskqbuf, M_NOWAIT, 993 taskqueue_thread_enqueue, &p->tq); 994 #else 995 /* Create a port for handling TX without starvation */ 996 p->tq = taskqueue_create_fast(p->taskqbuf, M_NOWAIT, 997 taskqueue_thread_enqueue, &p->tq); 998 #endif 999 #ifndef IFNET_MULTIQUEUE 1000 if (p->tq == NULL) { 1001 device_printf(dev, "failed to allocate port task queue\n"); 1002 return (ENOMEM); 1003 } 1004 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq", 1005 device_get_nameunit(dev)); 1006 1007 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp); 1008 #endif 1009 t3_sge_init_port(p); 1010 1011 return (0); 1012 } 1013 1014 static int 1015 cxgb_port_detach(device_t dev) 1016 { 1017 struct port_info *p; 1018 1019 p = device_get_softc(dev); 1020 1021 PORT_LOCK(p); 1022 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING) 1023 cxgb_stop_locked(p); 1024 PORT_UNLOCK(p); 1025 1026 if (p->tq != NULL) { 1027 taskqueue_drain(p->tq, &p->start_task); 1028 taskqueue_free(p->tq); 1029 p->tq = NULL; 1030 } 1031 1032 ether_ifdetach(p->ifp); 1033 printf("waiting for callout to stop ..."); 1034 DELAY(1000000); 1035 printf("done\n"); 1036 /* 1037 * the lock may be acquired in ifdetach 1038 */ 1039 PORT_LOCK_DEINIT(p); 1040 if_free(p->ifp); 1041 1042 if (p->port_cdev != NULL) 1043 destroy_dev(p->port_cdev); 1044 1045 return (0); 1046 } 1047 1048 void 1049 t3_fatal_err(struct adapter *sc) 1050 { 1051 u_int fw_status[4]; 1052 1053 if (sc->flags & FULL_INIT_DONE) { 1054 t3_sge_stop(sc); 1055 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1056 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1057 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1058 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1059 t3_intr_disable(sc); 1060 } 1061 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1062 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1063 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1064 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1065 } 1066 1067 int 1068 t3_os_find_pci_capability(adapter_t *sc, int cap) 1069 { 1070 device_t dev; 1071 struct pci_devinfo *dinfo; 1072 pcicfgregs *cfg; 1073 uint32_t status; 1074 uint8_t ptr; 1075 1076 dev = sc->dev; 1077 dinfo = device_get_ivars(dev); 1078 cfg = &dinfo->cfg; 1079 1080 status = pci_read_config(dev, PCIR_STATUS, 2); 1081 if (!(status & PCIM_STATUS_CAPPRESENT)) 1082 return (0); 1083 1084 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1085 case 0: 1086 case 1: 1087 ptr = PCIR_CAP_PTR; 1088 break; 1089 case 2: 1090 ptr = PCIR_CAP_PTR_2; 1091 break; 1092 default: 1093 return (0); 1094 break; 1095 } 1096 ptr = pci_read_config(dev, ptr, 1); 1097 1098 while (ptr != 0) { 1099 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1100 return (ptr); 1101 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1102 } 1103 1104 return (0); 1105 } 1106 1107 int 1108 t3_os_pci_save_state(struct adapter *sc) 1109 { 1110 device_t dev; 1111 struct pci_devinfo *dinfo; 1112 1113 dev = sc->dev; 1114 dinfo = device_get_ivars(dev); 1115 1116 pci_cfg_save(dev, dinfo, 0); 1117 return (0); 1118 } 1119 1120 int 1121 t3_os_pci_restore_state(struct adapter *sc) 1122 { 1123 device_t dev; 1124 struct pci_devinfo *dinfo; 1125 1126 dev = sc->dev; 1127 dinfo = device_get_ivars(dev); 1128 1129 pci_cfg_restore(dev, dinfo); 1130 return (0); 1131 } 1132 1133 /** 1134 * t3_os_link_changed - handle link status changes 1135 * @adapter: the adapter associated with the link change 1136 * @port_id: the port index whose limk status has changed 1137 * @link_stat: the new status of the link 1138 * @speed: the new speed setting 1139 * @duplex: the new duplex setting 1140 * @fc: the new flow-control setting 1141 * 1142 * This is the OS-dependent handler for link status changes. The OS 1143 * neutral handler takes care of most of the processing for these events, 1144 * then calls this handler for any OS-specific processing. 1145 */ 1146 void 1147 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1148 int duplex, int fc) 1149 { 1150 struct port_info *pi = &adapter->port[port_id]; 1151 struct cmac *mac = &adapter->port[port_id].mac; 1152 1153 if ((pi->ifp->if_flags & IFF_UP) == 0) 1154 return; 1155 1156 if (link_status) { 1157 t3_mac_enable(mac, MAC_DIRECTION_RX); 1158 if_link_state_change(pi->ifp, LINK_STATE_UP); 1159 } else { 1160 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 1161 pi->phy.ops->power_down(&pi->phy, 1); 1162 t3_mac_disable(mac, MAC_DIRECTION_RX); 1163 t3_link_start(&pi->phy, mac, &pi->link_config); 1164 } 1165 } 1166 1167 /* 1168 * Interrupt-context handler for external (PHY) interrupts. 1169 */ 1170 void 1171 t3_os_ext_intr_handler(adapter_t *sc) 1172 { 1173 if (cxgb_debug) 1174 printf("t3_os_ext_intr_handler\n"); 1175 /* 1176 * Schedule a task to handle external interrupts as they may be slow 1177 * and we use a mutex to protect MDIO registers. We disable PHY 1178 * interrupts in the meantime and let the task reenable them when 1179 * it's done. 1180 */ 1181 ADAPTER_LOCK(sc); 1182 if (sc->slow_intr_mask) { 1183 sc->slow_intr_mask &= ~F_T3DBG; 1184 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1185 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 1186 } 1187 ADAPTER_UNLOCK(sc); 1188 } 1189 1190 void 1191 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1192 { 1193 1194 /* 1195 * The ifnet might not be allocated before this gets called, 1196 * as this is called early on in attach by t3_prep_adapter 1197 * save the address off in the port structure 1198 */ 1199 if (cxgb_debug) 1200 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1201 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1202 } 1203 1204 /** 1205 * link_start - enable a port 1206 * @p: the port to enable 1207 * 1208 * Performs the MAC and PHY actions needed to enable a port. 1209 */ 1210 static void 1211 cxgb_link_start(struct port_info *p) 1212 { 1213 struct ifnet *ifp; 1214 struct t3_rx_mode rm; 1215 struct cmac *mac = &p->mac; 1216 1217 ifp = p->ifp; 1218 1219 t3_init_rx_mode(&rm, p); 1220 if (!mac->multiport) 1221 t3_mac_reset(mac); 1222 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1223 t3_mac_set_address(mac, 0, p->hw_addr); 1224 t3_mac_set_rx_mode(mac, &rm); 1225 t3_link_start(&p->phy, mac, &p->link_config); 1226 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1227 } 1228 1229 /** 1230 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1231 * @adap: the adapter 1232 * 1233 * Sets up RSS to distribute packets to multiple receive queues. We 1234 * configure the RSS CPU lookup table to distribute to the number of HW 1235 * receive queues, and the response queue lookup table to narrow that 1236 * down to the response queues actually configured for each port. 1237 * We always configure the RSS mapping for two ports since the mapping 1238 * table has plenty of entries. 1239 */ 1240 static void 1241 setup_rss(adapter_t *adap) 1242 { 1243 int i; 1244 u_int nq[2]; 1245 uint8_t cpus[SGE_QSETS + 1]; 1246 uint16_t rspq_map[RSS_TABLE_SIZE]; 1247 1248 for (i = 0; i < SGE_QSETS; ++i) 1249 cpus[i] = i; 1250 cpus[SGE_QSETS] = 0xff; 1251 1252 nq[0] = nq[1] = 0; 1253 for_each_port(adap, i) { 1254 const struct port_info *pi = adap2pinfo(adap, i); 1255 1256 nq[pi->tx_chan] += pi->nqsets; 1257 } 1258 nq[0] = max(nq[0], 1U); 1259 nq[1] = max(nq[1], 1U); 1260 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1261 rspq_map[i] = i % nq[0]; 1262 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq[1]) + nq[0]; 1263 } 1264 /* Calculate the reverse RSS map table */ 1265 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1266 if (adap->rrss_map[rspq_map[i]] == 0xff) 1267 adap->rrss_map[rspq_map[i]] = i; 1268 1269 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1270 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1271 V_RRCPLCPUSIZE(6), cpus, rspq_map); 1272 1273 } 1274 1275 /* 1276 * Sends an mbuf to an offload queue driver 1277 * after dealing with any active network taps. 1278 */ 1279 static inline int 1280 offload_tx(struct t3cdev *tdev, struct mbuf *m) 1281 { 1282 int ret; 1283 1284 ret = t3_offload_tx(tdev, m); 1285 return (ret); 1286 } 1287 1288 static int 1289 write_smt_entry(struct adapter *adapter, int idx) 1290 { 1291 struct port_info *pi = &adapter->port[idx]; 1292 struct cpl_smt_write_req *req; 1293 struct mbuf *m; 1294 1295 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1296 return (ENOMEM); 1297 1298 req = mtod(m, struct cpl_smt_write_req *); 1299 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req); 1300 1301 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1302 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 1303 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 1304 req->iff = idx; 1305 memset(req->src_mac1, 0, sizeof(req->src_mac1)); 1306 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 1307 1308 m_set_priority(m, 1); 1309 1310 offload_tx(&adapter->tdev, m); 1311 1312 return (0); 1313 } 1314 1315 static int 1316 init_smt(struct adapter *adapter) 1317 { 1318 int i; 1319 1320 for_each_port(adapter, i) 1321 write_smt_entry(adapter, i); 1322 return 0; 1323 } 1324 1325 static void 1326 init_port_mtus(adapter_t *adapter) 1327 { 1328 unsigned int mtus = adapter->port[0].ifp->if_mtu; 1329 1330 if (adapter->port[1].ifp) 1331 mtus |= adapter->port[1].ifp->if_mtu << 16; 1332 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); 1333 } 1334 1335 static void 1336 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1337 int hi, int port) 1338 { 1339 struct mbuf *m; 1340 struct mngt_pktsched_wr *req; 1341 1342 m = m_gethdr(M_DONTWAIT, MT_DATA); 1343 if (m) { 1344 req = mtod(m, struct mngt_pktsched_wr *); 1345 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1346 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1347 req->sched = sched; 1348 req->idx = qidx; 1349 req->min = lo; 1350 req->max = hi; 1351 req->binding = port; 1352 m->m_len = m->m_pkthdr.len = sizeof(*req); 1353 t3_mgmt_tx(adap, m); 1354 } 1355 } 1356 1357 static void 1358 bind_qsets(adapter_t *sc) 1359 { 1360 int i, j; 1361 1362 #ifdef IFNET_MULTIQUEUE 1363 cxgb_pcpu_startup_threads(sc); 1364 #endif 1365 1366 for (i = 0; i < (sc)->params.nports; ++i) { 1367 const struct port_info *pi = adap2pinfo(sc, i); 1368 1369 for (j = 0; j < pi->nqsets; ++j) { 1370 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1371 -1, pi->tx_chan); 1372 1373 } 1374 } 1375 } 1376 1377 static void 1378 update_tpeeprom(struct adapter *adap) 1379 { 1380 #ifdef FIRMWARE_LATEST 1381 const struct firmware *tpeeprom; 1382 #else 1383 struct firmware *tpeeprom; 1384 #endif 1385 1386 char buf[64]; 1387 uint32_t version; 1388 unsigned int major, minor; 1389 int ret, len; 1390 char rev; 1391 1392 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1393 1394 major = G_TP_VERSION_MAJOR(version); 1395 minor = G_TP_VERSION_MINOR(version); 1396 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1397 return; 1398 1399 rev = t3rev2char(adap); 1400 1401 snprintf(buf, sizeof(buf), TPEEPROM_NAME, rev, 1402 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1403 1404 tpeeprom = firmware_get(buf); 1405 if (tpeeprom == NULL) { 1406 device_printf(adap->dev, "could not load TP EEPROM: unable to load %s\n", 1407 buf); 1408 return; 1409 } 1410 1411 len = tpeeprom->datasize - 4; 1412 1413 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1414 if (ret) 1415 goto release_tpeeprom; 1416 1417 if (len != TP_SRAM_LEN) { 1418 device_printf(adap->dev, "%s length is wrong len=%d expected=%d\n", buf, len, TP_SRAM_LEN); 1419 return; 1420 } 1421 1422 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1423 TP_SRAM_OFFSET); 1424 1425 if (!ret) { 1426 device_printf(adap->dev, 1427 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1428 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1429 } else 1430 device_printf(adap->dev, "Protocol SRAM image update in EEPROM failed\n"); 1431 1432 release_tpeeprom: 1433 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1434 1435 return; 1436 } 1437 1438 static int 1439 update_tpsram(struct adapter *adap) 1440 { 1441 #ifdef FIRMWARE_LATEST 1442 const struct firmware *tpsram; 1443 #else 1444 struct firmware *tpsram; 1445 #endif 1446 char buf[64]; 1447 int ret; 1448 char rev; 1449 1450 rev = t3rev2char(adap); 1451 if (!rev) 1452 return 0; 1453 1454 update_tpeeprom(adap); 1455 1456 snprintf(buf, sizeof(buf), TPSRAM_NAME, rev, 1457 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1458 1459 tpsram = firmware_get(buf); 1460 if (tpsram == NULL){ 1461 device_printf(adap->dev, "could not load TP SRAM: unable to load %s\n", 1462 buf); 1463 return (EINVAL); 1464 } else 1465 device_printf(adap->dev, "updating TP SRAM with %s\n", buf); 1466 1467 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1468 if (ret) 1469 goto release_tpsram; 1470 1471 ret = t3_set_proto_sram(adap, tpsram->data); 1472 if (ret) 1473 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1474 1475 release_tpsram: 1476 firmware_put(tpsram, FIRMWARE_UNLOAD); 1477 1478 return ret; 1479 } 1480 1481 /** 1482 * cxgb_up - enable the adapter 1483 * @adap: adapter being enabled 1484 * 1485 * Called when the first port is enabled, this function performs the 1486 * actions necessary to make an adapter operational, such as completing 1487 * the initialization of HW modules, and enabling interrupts. 1488 * 1489 */ 1490 static int 1491 cxgb_up(struct adapter *sc) 1492 { 1493 int err = 0; 1494 1495 if ((sc->flags & FULL_INIT_DONE) == 0) { 1496 1497 if ((sc->flags & FW_UPTODATE) == 0) 1498 if ((err = upgrade_fw(sc))) 1499 goto out; 1500 if ((sc->flags & TPS_UPTODATE) == 0) 1501 if ((err = update_tpsram(sc))) 1502 goto out; 1503 err = t3_init_hw(sc, 0); 1504 if (err) 1505 goto out; 1506 1507 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1508 1509 err = setup_sge_qsets(sc); 1510 if (err) 1511 goto out; 1512 1513 setup_rss(sc); 1514 t3_add_configured_sysctls(sc); 1515 sc->flags |= FULL_INIT_DONE; 1516 } 1517 1518 t3_intr_clear(sc); 1519 1520 /* If it's MSI or INTx, allocate a single interrupt for everything */ 1521 if ((sc->flags & USING_MSIX) == 0) { 1522 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 1523 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 1524 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", 1525 sc->irq_rid); 1526 err = EINVAL; 1527 goto out; 1528 } 1529 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res); 1530 1531 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 1532 #ifdef INTR_FILTERS 1533 NULL, 1534 #endif 1535 sc->cxgb_intr, sc, &sc->intr_tag)) { 1536 device_printf(sc->dev, "Cannot set up interrupt\n"); 1537 err = EINVAL; 1538 goto irq_err; 1539 } 1540 } else { 1541 cxgb_setup_msix(sc, sc->msi_count); 1542 } 1543 1544 t3_sge_start(sc); 1545 t3_intr_enable(sc); 1546 1547 if (!(sc->flags & QUEUES_BOUND)) { 1548 printf("bind qsets\n"); 1549 bind_qsets(sc); 1550 sc->flags |= QUEUES_BOUND; 1551 } 1552 out: 1553 return (err); 1554 irq_err: 1555 CH_ERR(sc, "request_irq failed, err %d\n", err); 1556 goto out; 1557 } 1558 1559 1560 /* 1561 * Release resources when all the ports and offloading have been stopped. 1562 */ 1563 static void 1564 cxgb_down_locked(struct adapter *sc) 1565 { 1566 int i; 1567 1568 t3_sge_stop(sc); 1569 t3_intr_disable(sc); 1570 1571 if (sc->intr_tag != NULL) { 1572 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 1573 sc->intr_tag = NULL; 1574 } 1575 if (sc->irq_res != NULL) { 1576 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", 1577 sc->irq_rid, sc->irq_res); 1578 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 1579 sc->irq_res); 1580 sc->irq_res = NULL; 1581 } 1582 1583 if (sc->flags & USING_MSIX) 1584 cxgb_teardown_msix(sc); 1585 ADAPTER_UNLOCK(sc); 1586 1587 callout_stop(&sc->cxgb_tick_ch); 1588 callout_stop(&sc->sge_timer_ch); 1589 callout_drain(&sc->cxgb_tick_ch); 1590 callout_drain(&sc->sge_timer_ch); 1591 1592 if (sc->tq != NULL) { 1593 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1594 for (i = 0; i < sc->params.nports; i++) 1595 taskqueue_drain(sc->tq, &sc->port[i].timer_reclaim_task); 1596 } 1597 } 1598 1599 static int 1600 offload_open(struct port_info *pi) 1601 { 1602 struct adapter *adapter = pi->adapter; 1603 struct t3cdev *tdev = &adapter->tdev; 1604 #ifdef notyet 1605 T3CDEV(pi->ifp); 1606 #endif 1607 int adap_up = adapter->open_device_map & PORT_MASK; 1608 int err = 0; 1609 1610 printf("device_map=0x%x\n", adapter->open_device_map); 1611 if (atomic_cmpset_int(&adapter->open_device_map, 1612 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)), 1613 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0) 1614 return (0); 1615 1616 1617 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1618 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", adapter->open_device_map); 1619 ADAPTER_LOCK(pi->adapter); 1620 if (!adap_up) 1621 err = cxgb_up(adapter); 1622 ADAPTER_UNLOCK(pi->adapter); 1623 if (err) 1624 return (err); 1625 1626 t3_tp_set_offload_mode(adapter, 1); 1627 tdev->lldev = pi->ifp; 1628 err = cxgb_offload_activate(adapter); 1629 if (err) 1630 goto out; 1631 1632 init_port_mtus(adapter); 1633 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, 1634 adapter->params.b_wnd, 1635 adapter->params.rev == 0 ? 1636 adapter->port[0].ifp->if_mtu : 0xffff); 1637 init_smt(adapter); 1638 1639 /* Call back all registered clients */ 1640 cxgb_add_clients(tdev); 1641 1642 out: 1643 /* restore them in case the offload module has changed them */ 1644 if (err) { 1645 t3_tp_set_offload_mode(adapter, 0); 1646 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1647 cxgb_set_dummy_ops(tdev); 1648 } 1649 return (err); 1650 } 1651 1652 static int 1653 offload_close(struct t3cdev *tdev) 1654 { 1655 struct adapter *adapter = tdev2adap(tdev); 1656 1657 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) { 1658 printf("offload_close: DEVMAP_BIT not set\n"); 1659 1660 return (0); 1661 } 1662 1663 /* Call back all registered clients */ 1664 cxgb_remove_clients(tdev); 1665 tdev->lldev = NULL; 1666 cxgb_set_dummy_ops(tdev); 1667 t3_tp_set_offload_mode(adapter, 0); 1668 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1669 1670 ADAPTER_LOCK(adapter); 1671 if (!adapter->open_device_map) 1672 cxgb_down_locked(adapter); 1673 else 1674 ADAPTER_UNLOCK(adapter); 1675 cxgb_offload_deactivate(adapter); 1676 return (0); 1677 } 1678 1679 1680 static void 1681 cxgb_init(void *arg) 1682 { 1683 struct port_info *p = arg; 1684 1685 PORT_LOCK(p); 1686 cxgb_init_locked(p); 1687 PORT_UNLOCK(p); 1688 } 1689 1690 static void 1691 cxgb_init_locked(struct port_info *p) 1692 { 1693 struct ifnet *ifp; 1694 adapter_t *sc = p->adapter; 1695 int err; 1696 1697 PORT_LOCK_ASSERT_OWNED(p); 1698 ifp = p->ifp; 1699 1700 ADAPTER_LOCK(p->adapter); 1701 if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) { 1702 ADAPTER_UNLOCK(p->adapter); 1703 cxgb_stop_locked(p); 1704 return; 1705 } 1706 if (p->adapter->open_device_map == 0) { 1707 t3_intr_clear(sc); 1708 t3_sge_init_adapter(sc); 1709 } 1710 setbit(&p->adapter->open_device_map, p->port_id); 1711 ADAPTER_UNLOCK(p->adapter); 1712 1713 if (is_offload(sc) && !ofld_disable) { 1714 err = offload_open(p); 1715 if (err) 1716 log(LOG_WARNING, 1717 "Could not initialize offload capabilities\n"); 1718 else 1719 printf("offload opened\n"); 1720 } 1721 cxgb_link_start(p); 1722 t3_link_changed(sc, p->port_id); 1723 ifp->if_baudrate = p->link_config.speed * 1000000; 1724 1725 device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id); 1726 t3_port_intr_enable(sc, p->port_id); 1727 1728 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 1729 1730 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1731 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1732 } 1733 1734 static void 1735 cxgb_set_rxmode(struct port_info *p) 1736 { 1737 struct t3_rx_mode rm; 1738 struct cmac *mac = &p->mac; 1739 1740 PORT_LOCK_ASSERT_OWNED(p); 1741 1742 t3_init_rx_mode(&rm, p); 1743 t3_mac_set_rx_mode(mac, &rm); 1744 } 1745 1746 static void 1747 cxgb_stop_locked(struct port_info *p) 1748 { 1749 struct ifnet *ifp; 1750 1751 PORT_LOCK_ASSERT_OWNED(p); 1752 ADAPTER_LOCK_ASSERT_NOTOWNED(p->adapter); 1753 1754 ifp = p->ifp; 1755 t3_port_intr_disable(p->adapter, p->port_id); 1756 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1757 p->phy.ops->power_down(&p->phy, 1); 1758 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1759 1760 ADAPTER_LOCK(p->adapter); 1761 clrbit(&p->adapter->open_device_map, p->port_id); 1762 1763 if (p->adapter->open_device_map == 0) { 1764 cxgb_down_locked(p->adapter); 1765 } else 1766 ADAPTER_UNLOCK(p->adapter); 1767 1768 } 1769 1770 static int 1771 cxgb_set_mtu(struct port_info *p, int mtu) 1772 { 1773 struct ifnet *ifp = p->ifp; 1774 int error = 0; 1775 1776 if ((mtu < ETHERMIN) || (mtu > ETHER_MAX_LEN_JUMBO)) 1777 error = EINVAL; 1778 else if (ifp->if_mtu != mtu) { 1779 PORT_LOCK(p); 1780 ifp->if_mtu = mtu; 1781 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1782 callout_stop(&p->adapter->cxgb_tick_ch); 1783 cxgb_stop_locked(p); 1784 cxgb_init_locked(p); 1785 } 1786 PORT_UNLOCK(p); 1787 } 1788 return (error); 1789 } 1790 1791 static int 1792 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1793 { 1794 struct port_info *p = ifp->if_softc; 1795 struct ifaddr *ifa = (struct ifaddr *)data; 1796 struct ifreq *ifr = (struct ifreq *)data; 1797 int flags, error = 0; 1798 uint32_t mask; 1799 1800 /* 1801 * XXX need to check that we aren't in the middle of an unload 1802 */ 1803 switch (command) { 1804 case SIOCSIFMTU: 1805 error = cxgb_set_mtu(p, ifr->ifr_mtu); 1806 break; 1807 case SIOCSIFADDR: 1808 case SIOCGIFADDR: 1809 PORT_LOCK(p); 1810 if (ifa->ifa_addr->sa_family == AF_INET) { 1811 ifp->if_flags |= IFF_UP; 1812 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1813 cxgb_init_locked(p); 1814 arp_ifinit(ifp, ifa); 1815 } else 1816 error = ether_ioctl(ifp, command, data); 1817 PORT_UNLOCK(p); 1818 break; 1819 case SIOCSIFFLAGS: 1820 callout_drain(&p->adapter->cxgb_tick_ch); 1821 PORT_LOCK(p); 1822 if (ifp->if_flags & IFF_UP) { 1823 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1824 flags = p->if_flags; 1825 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1826 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 1827 cxgb_set_rxmode(p); 1828 } else 1829 cxgb_init_locked(p); 1830 p->if_flags = ifp->if_flags; 1831 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1832 cxgb_stop_locked(p); 1833 1834 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1835 adapter_t *sc = p->adapter; 1836 callout_reset(&sc->cxgb_tick_ch, hz, 1837 cxgb_tick, sc); 1838 } 1839 PORT_UNLOCK(p); 1840 break; 1841 case SIOCSIFMEDIA: 1842 case SIOCGIFMEDIA: 1843 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 1844 break; 1845 case SIOCSIFCAP: 1846 PORT_LOCK(p); 1847 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1848 if (mask & IFCAP_TXCSUM) { 1849 if (IFCAP_TXCSUM & ifp->if_capenable) { 1850 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 1851 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 1852 | CSUM_TSO); 1853 } else { 1854 ifp->if_capenable |= IFCAP_TXCSUM; 1855 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1856 } 1857 } else if (mask & IFCAP_RXCSUM) { 1858 if (IFCAP_RXCSUM & ifp->if_capenable) { 1859 ifp->if_capenable &= ~IFCAP_RXCSUM; 1860 } else { 1861 ifp->if_capenable |= IFCAP_RXCSUM; 1862 } 1863 } 1864 if (mask & IFCAP_TSO4) { 1865 if (IFCAP_TSO4 & ifp->if_capenable) { 1866 ifp->if_capenable &= ~IFCAP_TSO4; 1867 ifp->if_hwassist &= ~CSUM_TSO; 1868 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 1869 ifp->if_capenable |= IFCAP_TSO4; 1870 ifp->if_hwassist |= CSUM_TSO; 1871 } else { 1872 if (cxgb_debug) 1873 printf("cxgb requires tx checksum offload" 1874 " be enabled to use TSO\n"); 1875 error = EINVAL; 1876 } 1877 } 1878 PORT_UNLOCK(p); 1879 break; 1880 default: 1881 error = ether_ioctl(ifp, command, data); 1882 break; 1883 } 1884 return (error); 1885 } 1886 1887 int 1888 cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax) 1889 { 1890 struct sge_txq *txq; 1891 int err, in_use_init, count; 1892 struct mbuf **m_vec; 1893 1894 txq = &qs->txq[TXQ_ETH]; 1895 m_vec = txq->txq_m_vec; 1896 in_use_init = txq->in_use; 1897 err = 0; 1898 while ((txq->in_use - in_use_init < txmax) && 1899 (txq->size > txq->in_use + TX_MAX_DESC)) { 1900 check_pkt_coalesce(qs); 1901 count = cxgb_dequeue_packet(ifp, txq, m_vec); 1902 if (count == 0) 1903 break; 1904 ETHER_BPF_MTAP(ifp, m_vec[0]); 1905 1906 if ((err = t3_encap(qs, m_vec, count)) != 0) 1907 break; 1908 txq->txq_enqueued += count; 1909 } 1910 #ifndef IFNET_MULTIQUEUE 1911 if (__predict_false(err)) { 1912 if (err == ENOMEM) { 1913 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1914 IFQ_LOCK(&ifp->if_snd); 1915 IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]); 1916 IFQ_UNLOCK(&ifp->if_snd); 1917 } 1918 } 1919 if (err == 0 && m_vec[0] == NULL) { 1920 err = ENOBUFS; 1921 } 1922 else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) && 1923 (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { 1924 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1925 err = ENOSPC; 1926 } 1927 #else 1928 if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC)) { 1929 err = ENOSPC; 1930 setbit(&qs->txq_stopped, TXQ_ETH); 1931 } 1932 if (err == ENOMEM) { 1933 int i; 1934 /* 1935 * Sub-optimal :-/ 1936 */ 1937 for (i = 0; i < count; i++) 1938 m_freem(m_vec[i]); 1939 } 1940 #endif 1941 return (err); 1942 } 1943 1944 #ifndef IFNET_MULTIQUEUE 1945 static int 1946 cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) 1947 { 1948 struct sge_qset *qs; 1949 struct sge_txq *txq; 1950 struct port_info *p = ifp->if_softc; 1951 int err; 1952 1953 if (!p->link_config.link_ok) 1954 return (ENXIO); 1955 1956 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 1957 return (ENOBUFS); 1958 } 1959 1960 qs = &p->adapter->sge.qs[p->first_qset]; 1961 txq = &qs->txq[TXQ_ETH]; 1962 err = 0; 1963 1964 if (txq->flags & TXQ_TRANSMITTING) 1965 return (EINPROGRESS); 1966 1967 mtx_lock(&txq->lock); 1968 txq->flags |= TXQ_TRANSMITTING; 1969 cxgb_tx_common(ifp, qs, txmax); 1970 txq->flags &= ~TXQ_TRANSMITTING; 1971 mtx_unlock(&txq->lock); 1972 1973 return (err); 1974 } 1975 1976 static void 1977 cxgb_start_proc(void *arg, int ncount) 1978 { 1979 struct ifnet *ifp = arg; 1980 struct port_info *pi = ifp->if_softc; 1981 struct sge_qset *qs; 1982 struct sge_txq *txq; 1983 int error; 1984 1985 qs = &pi->adapter->sge.qs[pi->first_qset]; 1986 txq = &qs->txq[TXQ_ETH]; 1987 1988 do { 1989 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC >> 2) 1990 taskqueue_enqueue(pi->tq, &txq->qreclaim_task); 1991 1992 error = cxgb_start_tx(ifp, TX_START_MAX_DESC); 1993 } while (error == 0); 1994 } 1995 1996 int 1997 cxgb_dequeue_packet(struct ifnet *ifp, struct sge_txq *unused, struct mbuf **m_vec) 1998 { 1999 2000 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_vec[0]); 2001 return (m_vec[0] ? 1 : 0); 2002 } 2003 2004 void 2005 cxgb_start(struct ifnet *ifp) 2006 { 2007 struct port_info *pi = ifp->if_softc; 2008 struct sge_qset *qs; 2009 struct sge_txq *txq; 2010 int err; 2011 2012 qs = &pi->adapter->sge.qs[pi->first_qset]; 2013 txq = &qs->txq[TXQ_ETH]; 2014 2015 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC >> 2) 2016 taskqueue_enqueue(pi->tq, 2017 &txq->qreclaim_task); 2018 2019 err = cxgb_start_tx(ifp, TX_START_MAX_DESC); 2020 2021 if (err == 0) 2022 taskqueue_enqueue(pi->tq, &pi->start_task); 2023 } 2024 #endif 2025 2026 static int 2027 cxgb_media_change(struct ifnet *ifp) 2028 { 2029 if_printf(ifp, "media change not supported\n"); 2030 return (ENXIO); 2031 } 2032 2033 static void 2034 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2035 { 2036 struct port_info *p = ifp->if_softc; 2037 2038 ifmr->ifm_status = IFM_AVALID; 2039 ifmr->ifm_active = IFM_ETHER; 2040 2041 if (!p->link_config.link_ok) 2042 return; 2043 2044 ifmr->ifm_status |= IFM_ACTIVE; 2045 2046 switch (p->link_config.speed) { 2047 case 10: 2048 ifmr->ifm_active |= IFM_10_T; 2049 break; 2050 case 100: 2051 ifmr->ifm_active |= IFM_100_TX; 2052 break; 2053 case 1000: 2054 ifmr->ifm_active |= IFM_1000_T; 2055 break; 2056 } 2057 2058 if (p->link_config.duplex) 2059 ifmr->ifm_active |= IFM_FDX; 2060 else 2061 ifmr->ifm_active |= IFM_HDX; 2062 } 2063 2064 static void 2065 cxgb_async_intr(void *data) 2066 { 2067 adapter_t *sc = data; 2068 2069 if (cxgb_debug) 2070 device_printf(sc->dev, "cxgb_async_intr\n"); 2071 /* 2072 * May need to sleep - defer to taskqueue 2073 */ 2074 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2075 } 2076 2077 static void 2078 cxgb_ext_intr_handler(void *arg, int count) 2079 { 2080 adapter_t *sc = (adapter_t *)arg; 2081 2082 if (cxgb_debug) 2083 printf("cxgb_ext_intr_handler\n"); 2084 2085 t3_phy_intr_handler(sc); 2086 2087 /* Now reenable external interrupts */ 2088 ADAPTER_LOCK(sc); 2089 if (sc->slow_intr_mask) { 2090 sc->slow_intr_mask |= F_T3DBG; 2091 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 2092 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 2093 } 2094 ADAPTER_UNLOCK(sc); 2095 } 2096 2097 static void 2098 check_link_status(adapter_t *sc) 2099 { 2100 int i; 2101 2102 for (i = 0; i < (sc)->params.nports; ++i) { 2103 struct port_info *p = &sc->port[i]; 2104 2105 if (!(p->port_type->caps & SUPPORTED_IRQ)) 2106 t3_link_changed(sc, i); 2107 p->ifp->if_baudrate = p->link_config.speed * 1000000; 2108 } 2109 } 2110 2111 static void 2112 check_t3b2_mac(struct adapter *adapter) 2113 { 2114 int i; 2115 2116 for_each_port(adapter, i) { 2117 struct port_info *p = &adapter->port[i]; 2118 struct ifnet *ifp = p->ifp; 2119 int status; 2120 2121 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2122 continue; 2123 2124 status = 0; 2125 PORT_LOCK(p); 2126 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 2127 status = t3b2_mac_watchdog_task(&p->mac); 2128 if (status == 1) 2129 p->mac.stats.num_toggled++; 2130 else if (status == 2) { 2131 struct cmac *mac = &p->mac; 2132 2133 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN 2134 + ETHER_VLAN_ENCAP_LEN); 2135 t3_mac_set_address(mac, 0, p->hw_addr); 2136 cxgb_set_rxmode(p); 2137 t3_link_start(&p->phy, mac, &p->link_config); 2138 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2139 t3_port_intr_enable(adapter, p->port_id); 2140 p->mac.stats.num_resets++; 2141 } 2142 PORT_UNLOCK(p); 2143 } 2144 } 2145 2146 static void 2147 cxgb_tick(void *arg) 2148 { 2149 adapter_t *sc = (adapter_t *)arg; 2150 int i, running = 0; 2151 2152 for_each_port(sc, i) { 2153 2154 struct port_info *p = &sc->port[i]; 2155 struct ifnet *ifp = p->ifp; 2156 PORT_LOCK(p); 2157 2158 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 2159 running = 1; 2160 PORT_UNLOCK(p); 2161 } 2162 2163 if (running == 0) 2164 return; 2165 2166 taskqueue_enqueue(sc->tq, &sc->tick_task); 2167 2168 if (sc->open_device_map != 0) 2169 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 2170 } 2171 2172 static void 2173 cxgb_tick_handler(void *arg, int count) 2174 { 2175 adapter_t *sc = (adapter_t *)arg; 2176 const struct adapter_params *p = &sc->params; 2177 2178 ADAPTER_LOCK(sc); 2179 if (p->linkpoll_period) 2180 check_link_status(sc); 2181 2182 /* 2183 * adapter lock can currently only be acquire after the 2184 * port lock 2185 */ 2186 ADAPTER_UNLOCK(sc); 2187 2188 if (p->rev == T3_REV_B2 && p->nports < 4) 2189 check_t3b2_mac(sc); 2190 } 2191 2192 static void 2193 touch_bars(device_t dev) 2194 { 2195 /* 2196 * Don't enable yet 2197 */ 2198 #if !defined(__LP64__) && 0 2199 u32 v; 2200 2201 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2202 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2203 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2204 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2205 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2206 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2207 #endif 2208 } 2209 2210 static int 2211 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2212 { 2213 uint8_t *buf; 2214 int err = 0; 2215 u32 aligned_offset, aligned_len, *p; 2216 struct adapter *adapter = pi->adapter; 2217 2218 2219 aligned_offset = offset & ~3; 2220 aligned_len = (len + (offset & 3) + 3) & ~3; 2221 2222 if (aligned_offset != offset || aligned_len != len) { 2223 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2224 if (!buf) 2225 return (ENOMEM); 2226 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2227 if (!err && aligned_len > 4) 2228 err = t3_seeprom_read(adapter, 2229 aligned_offset + aligned_len - 4, 2230 (u32 *)&buf[aligned_len - 4]); 2231 if (err) 2232 goto out; 2233 memcpy(buf + (offset & 3), data, len); 2234 } else 2235 buf = (uint8_t *)(uintptr_t)data; 2236 2237 err = t3_seeprom_wp(adapter, 0); 2238 if (err) 2239 goto out; 2240 2241 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2242 err = t3_seeprom_write(adapter, aligned_offset, *p); 2243 aligned_offset += 4; 2244 } 2245 2246 if (!err) 2247 err = t3_seeprom_wp(adapter, 1); 2248 out: 2249 if (buf != data) 2250 free(buf, M_DEVBUF); 2251 return err; 2252 } 2253 2254 2255 static int 2256 in_range(int val, int lo, int hi) 2257 { 2258 return val < 0 || (val <= hi && val >= lo); 2259 } 2260 2261 static int 2262 cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td) 2263 { 2264 return (0); 2265 } 2266 2267 static int 2268 cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td) 2269 { 2270 return (0); 2271 } 2272 2273 static int 2274 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2275 int fflag, struct thread *td) 2276 { 2277 int mmd, error = 0; 2278 struct port_info *pi = dev->si_drv1; 2279 adapter_t *sc = pi->adapter; 2280 2281 #ifdef PRIV_SUPPORTED 2282 if (priv_check(td, PRIV_DRIVER)) { 2283 if (cxgb_debug) 2284 printf("user does not have access to privileged ioctls\n"); 2285 return (EPERM); 2286 } 2287 #else 2288 if (suser(td)) { 2289 if (cxgb_debug) 2290 printf("user does not have access to privileged ioctls\n"); 2291 return (EPERM); 2292 } 2293 #endif 2294 2295 switch (cmd) { 2296 case SIOCGMIIREG: { 2297 uint32_t val; 2298 struct cphy *phy = &pi->phy; 2299 struct mii_data *mid = (struct mii_data *)data; 2300 2301 if (!phy->mdio_read) 2302 return (EOPNOTSUPP); 2303 if (is_10G(sc)) { 2304 mmd = mid->phy_id >> 8; 2305 if (!mmd) 2306 mmd = MDIO_DEV_PCS; 2307 else if (mmd > MDIO_DEV_XGXS) 2308 return (EINVAL); 2309 2310 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2311 mid->reg_num, &val); 2312 } else 2313 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2314 mid->reg_num & 0x1f, &val); 2315 if (error == 0) 2316 mid->val_out = val; 2317 break; 2318 } 2319 case SIOCSMIIREG: { 2320 struct cphy *phy = &pi->phy; 2321 struct mii_data *mid = (struct mii_data *)data; 2322 2323 if (!phy->mdio_write) 2324 return (EOPNOTSUPP); 2325 if (is_10G(sc)) { 2326 mmd = mid->phy_id >> 8; 2327 if (!mmd) 2328 mmd = MDIO_DEV_PCS; 2329 else if (mmd > MDIO_DEV_XGXS) 2330 return (EINVAL); 2331 2332 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2333 mmd, mid->reg_num, mid->val_in); 2334 } else 2335 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2336 mid->reg_num & 0x1f, 2337 mid->val_in); 2338 break; 2339 } 2340 case CHELSIO_SETREG: { 2341 struct ch_reg *edata = (struct ch_reg *)data; 2342 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2343 return (EFAULT); 2344 t3_write_reg(sc, edata->addr, edata->val); 2345 break; 2346 } 2347 case CHELSIO_GETREG: { 2348 struct ch_reg *edata = (struct ch_reg *)data; 2349 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2350 return (EFAULT); 2351 edata->val = t3_read_reg(sc, edata->addr); 2352 break; 2353 } 2354 case CHELSIO_GET_SGE_CONTEXT: { 2355 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2356 mtx_lock(&sc->sge.reg_lock); 2357 switch (ecntxt->cntxt_type) { 2358 case CNTXT_TYPE_EGRESS: 2359 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2360 ecntxt->data); 2361 break; 2362 case CNTXT_TYPE_FL: 2363 error = t3_sge_read_fl(sc, ecntxt->cntxt_id, 2364 ecntxt->data); 2365 break; 2366 case CNTXT_TYPE_RSP: 2367 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2368 ecntxt->data); 2369 break; 2370 case CNTXT_TYPE_CQ: 2371 error = t3_sge_read_cq(sc, ecntxt->cntxt_id, 2372 ecntxt->data); 2373 break; 2374 default: 2375 error = EINVAL; 2376 break; 2377 } 2378 mtx_unlock(&sc->sge.reg_lock); 2379 break; 2380 } 2381 case CHELSIO_GET_SGE_DESC: { 2382 struct ch_desc *edesc = (struct ch_desc *)data; 2383 int ret; 2384 if (edesc->queue_num >= SGE_QSETS * 6) 2385 return (EINVAL); 2386 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2387 edesc->queue_num % 6, edesc->idx, edesc->data); 2388 if (ret < 0) 2389 return (EINVAL); 2390 edesc->size = ret; 2391 break; 2392 } 2393 case CHELSIO_SET_QSET_PARAMS: { 2394 struct qset_params *q; 2395 struct ch_qset_params *t = (struct ch_qset_params *)data; 2396 2397 if (t->qset_idx >= SGE_QSETS) 2398 return (EINVAL); 2399 if (!in_range(t->intr_lat, 0, M_NEWTIMER) || 2400 !in_range(t->cong_thres, 0, 255) || 2401 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES, 2402 MAX_TXQ_ENTRIES) || 2403 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES, 2404 MAX_TXQ_ENTRIES) || 2405 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES, 2406 MAX_CTRL_TXQ_ENTRIES) || 2407 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) || 2408 !in_range(t->fl_size[1], MIN_FL_ENTRIES, 2409 MAX_RX_JUMBO_BUFFERS) || 2410 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES)) 2411 return (EINVAL); 2412 if ((sc->flags & FULL_INIT_DONE) && 2413 (t->rspq_size >= 0 || t->fl_size[0] >= 0 || 2414 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 || 2415 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 || 2416 t->polling >= 0 || t->cong_thres >= 0)) 2417 return (EBUSY); 2418 2419 q = &sc->params.sge.qset[t->qset_idx]; 2420 2421 if (t->rspq_size >= 0) 2422 q->rspq_size = t->rspq_size; 2423 if (t->fl_size[0] >= 0) 2424 q->fl_size = t->fl_size[0]; 2425 if (t->fl_size[1] >= 0) 2426 q->jumbo_size = t->fl_size[1]; 2427 if (t->txq_size[0] >= 0) 2428 q->txq_size[0] = t->txq_size[0]; 2429 if (t->txq_size[1] >= 0) 2430 q->txq_size[1] = t->txq_size[1]; 2431 if (t->txq_size[2] >= 0) 2432 q->txq_size[2] = t->txq_size[2]; 2433 if (t->cong_thres >= 0) 2434 q->cong_thres = t->cong_thres; 2435 if (t->intr_lat >= 0) { 2436 struct sge_qset *qs = &sc->sge.qs[t->qset_idx]; 2437 2438 q->coalesce_nsecs = t->intr_lat*1000; 2439 t3_update_qset_coalesce(qs, q); 2440 } 2441 break; 2442 } 2443 case CHELSIO_GET_QSET_PARAMS: { 2444 struct qset_params *q; 2445 struct ch_qset_params *t = (struct ch_qset_params *)data; 2446 2447 if (t->qset_idx >= SGE_QSETS) 2448 return (EINVAL); 2449 2450 q = &(sc)->params.sge.qset[t->qset_idx]; 2451 t->rspq_size = q->rspq_size; 2452 t->txq_size[0] = q->txq_size[0]; 2453 t->txq_size[1] = q->txq_size[1]; 2454 t->txq_size[2] = q->txq_size[2]; 2455 t->fl_size[0] = q->fl_size; 2456 t->fl_size[1] = q->jumbo_size; 2457 t->polling = q->polling; 2458 t->intr_lat = q->coalesce_nsecs / 1000; 2459 t->cong_thres = q->cong_thres; 2460 break; 2461 } 2462 case CHELSIO_SET_QSET_NUM: { 2463 struct ch_reg *edata = (struct ch_reg *)data; 2464 unsigned int port_idx = pi->port_id; 2465 2466 if (sc->flags & FULL_INIT_DONE) 2467 return (EBUSY); 2468 if (edata->val < 1 || 2469 (edata->val > 1 && !(sc->flags & USING_MSIX))) 2470 return (EINVAL); 2471 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS) 2472 return (EINVAL); 2473 sc->port[port_idx].nqsets = edata->val; 2474 sc->port[0].first_qset = 0; 2475 /* 2476 * XXX hardcode ourselves to 2 ports just like LEEENUX 2477 */ 2478 sc->port[1].first_qset = sc->port[0].nqsets; 2479 break; 2480 } 2481 case CHELSIO_GET_QSET_NUM: { 2482 struct ch_reg *edata = (struct ch_reg *)data; 2483 edata->val = pi->nqsets; 2484 break; 2485 } 2486 #ifdef notyet 2487 case CHELSIO_LOAD_FW: 2488 case CHELSIO_GET_PM: 2489 case CHELSIO_SET_PM: 2490 return (EOPNOTSUPP); 2491 break; 2492 #endif 2493 case CHELSIO_SETMTUTAB: { 2494 struct ch_mtus *m = (struct ch_mtus *)data; 2495 int i; 2496 2497 if (!is_offload(sc)) 2498 return (EOPNOTSUPP); 2499 if (offload_running(sc)) 2500 return (EBUSY); 2501 if (m->nmtus != NMTUS) 2502 return (EINVAL); 2503 if (m->mtus[0] < 81) /* accommodate SACK */ 2504 return (EINVAL); 2505 2506 /* 2507 * MTUs must be in ascending order 2508 */ 2509 for (i = 1; i < NMTUS; ++i) 2510 if (m->mtus[i] < m->mtus[i - 1]) 2511 return (EINVAL); 2512 2513 memcpy(sc->params.mtus, m->mtus, 2514 sizeof(sc->params.mtus)); 2515 break; 2516 } 2517 case CHELSIO_GETMTUTAB: { 2518 struct ch_mtus *m = (struct ch_mtus *)data; 2519 2520 if (!is_offload(sc)) 2521 return (EOPNOTSUPP); 2522 2523 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2524 m->nmtus = NMTUS; 2525 break; 2526 } 2527 case CHELSIO_DEVUP: 2528 if (!is_offload(sc)) 2529 return (EOPNOTSUPP); 2530 return offload_open(pi); 2531 break; 2532 case CHELSIO_GET_MEM: { 2533 struct ch_mem_range *t = (struct ch_mem_range *)data; 2534 struct mc7 *mem; 2535 uint8_t *useraddr; 2536 u64 buf[32]; 2537 2538 if (!is_offload(sc)) 2539 return (EOPNOTSUPP); 2540 if (!(sc->flags & FULL_INIT_DONE)) 2541 return (EIO); /* need the memory controllers */ 2542 if ((t->addr & 0x7) || (t->len & 0x7)) 2543 return (EINVAL); 2544 if (t->mem_id == MEM_CM) 2545 mem = &sc->cm; 2546 else if (t->mem_id == MEM_PMRX) 2547 mem = &sc->pmrx; 2548 else if (t->mem_id == MEM_PMTX) 2549 mem = &sc->pmtx; 2550 else 2551 return (EINVAL); 2552 2553 /* 2554 * Version scheme: 2555 * bits 0..9: chip version 2556 * bits 10..15: chip revision 2557 */ 2558 t->version = 3 | (sc->params.rev << 10); 2559 2560 /* 2561 * Read 256 bytes at a time as len can be large and we don't 2562 * want to use huge intermediate buffers. 2563 */ 2564 useraddr = (uint8_t *)t->buf; 2565 while (t->len) { 2566 unsigned int chunk = min(t->len, sizeof(buf)); 2567 2568 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf); 2569 if (error) 2570 return (-error); 2571 if (copyout(buf, useraddr, chunk)) 2572 return (EFAULT); 2573 useraddr += chunk; 2574 t->addr += chunk; 2575 t->len -= chunk; 2576 } 2577 break; 2578 } 2579 case CHELSIO_READ_TCAM_WORD: { 2580 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2581 2582 if (!is_offload(sc)) 2583 return (EOPNOTSUPP); 2584 if (!(sc->flags & FULL_INIT_DONE)) 2585 return (EIO); /* need MC5 */ 2586 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2587 break; 2588 } 2589 case CHELSIO_SET_TRACE_FILTER: { 2590 struct ch_trace *t = (struct ch_trace *)data; 2591 const struct trace_params *tp; 2592 2593 tp = (const struct trace_params *)&t->sip; 2594 if (t->config_tx) 2595 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2596 t->trace_tx); 2597 if (t->config_rx) 2598 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2599 t->trace_rx); 2600 break; 2601 } 2602 case CHELSIO_SET_PKTSCHED: { 2603 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2604 if (sc->open_device_map == 0) 2605 return (EAGAIN); 2606 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2607 p->binding); 2608 break; 2609 } 2610 case CHELSIO_IFCONF_GETREGS: { 2611 struct ifconf_regs *regs = (struct ifconf_regs *)data; 2612 int reglen = cxgb_get_regs_len(); 2613 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT); 2614 if (buf == NULL) { 2615 return (ENOMEM); 2616 } if (regs->len > reglen) 2617 regs->len = reglen; 2618 else if (regs->len < reglen) { 2619 error = E2BIG; 2620 goto done; 2621 } 2622 cxgb_get_regs(sc, regs, buf); 2623 error = copyout(buf, regs->data, reglen); 2624 2625 done: 2626 free(buf, M_DEVBUF); 2627 2628 break; 2629 } 2630 case CHELSIO_SET_HW_SCHED: { 2631 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2632 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2633 2634 if ((sc->flags & FULL_INIT_DONE) == 0) 2635 return (EAGAIN); /* need TP to be initialized */ 2636 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2637 !in_range(t->channel, 0, 1) || 2638 !in_range(t->kbps, 0, 10000000) || 2639 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2640 !in_range(t->flow_ipg, 0, 2641 dack_ticks_to_usec(sc, 0x7ff))) 2642 return (EINVAL); 2643 2644 if (t->kbps >= 0) { 2645 error = t3_config_sched(sc, t->kbps, t->sched); 2646 if (error < 0) 2647 return (-error); 2648 } 2649 if (t->class_ipg >= 0) 2650 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2651 if (t->flow_ipg >= 0) { 2652 t->flow_ipg *= 1000; /* us -> ns */ 2653 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2654 } 2655 if (t->mode >= 0) { 2656 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2657 2658 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2659 bit, t->mode ? bit : 0); 2660 } 2661 if (t->channel >= 0) 2662 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2663 1 << t->sched, t->channel << t->sched); 2664 break; 2665 } 2666 default: 2667 return (EOPNOTSUPP); 2668 break; 2669 } 2670 2671 return (error); 2672 } 2673 2674 static __inline void 2675 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 2676 unsigned int end) 2677 { 2678 uint32_t *p = (uint32_t *)buf + start; 2679 2680 for ( ; start <= end; start += sizeof(uint32_t)) 2681 *p++ = t3_read_reg(ap, start); 2682 } 2683 2684 #define T3_REGMAP_SIZE (3 * 1024) 2685 static int 2686 cxgb_get_regs_len(void) 2687 { 2688 return T3_REGMAP_SIZE; 2689 } 2690 #undef T3_REGMAP_SIZE 2691 2692 static void 2693 cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf) 2694 { 2695 2696 /* 2697 * Version scheme: 2698 * bits 0..9: chip version 2699 * bits 10..15: chip revision 2700 * bit 31: set for PCIe cards 2701 */ 2702 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 2703 2704 /* 2705 * We skip the MAC statistics registers because they are clear-on-read. 2706 * Also reading multi-register stats would need to synchronize with the 2707 * periodic mac stats accumulation. Hard to justify the complexity. 2708 */ 2709 memset(buf, 0, REGDUMP_SIZE); 2710 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 2711 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 2712 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 2713 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 2714 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 2715 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 2716 XGM_REG(A_XGM_SERDES_STAT3, 1)); 2717 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 2718 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 2719 } 2720