1 /************************************************************************** 2 3 Copyright (c) 2007, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 POSSIBILITY OF SUCH DAMAGE. 27 28 ***************************************************************************/ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/bus.h> 37 #include <sys/module.h> 38 #include <sys/pciio.h> 39 #include <sys/conf.h> 40 #include <machine/bus.h> 41 #include <machine/resource.h> 42 #include <sys/bus_dma.h> 43 #include <sys/rman.h> 44 #include <sys/ioccom.h> 45 #include <sys/mbuf.h> 46 #include <sys/linker.h> 47 #include <sys/firmware.h> 48 #include <sys/socket.h> 49 #include <sys/sockio.h> 50 #include <sys/smp.h> 51 #include <sys/sysctl.h> 52 #include <sys/syslog.h> 53 #include <sys/queue.h> 54 #include <sys/taskqueue.h> 55 #include <sys/proc.h> 56 57 #include <net/bpf.h> 58 #include <net/ethernet.h> 59 #include <net/if.h> 60 #include <net/if_arp.h> 61 #include <net/if_dl.h> 62 #include <net/if_media.h> 63 #include <net/if_types.h> 64 65 #include <netinet/in_systm.h> 66 #include <netinet/in.h> 67 #include <netinet/if_ether.h> 68 #include <netinet/ip.h> 69 #include <netinet/ip.h> 70 #include <netinet/tcp.h> 71 #include <netinet/udp.h> 72 73 #include <dev/pci/pcireg.h> 74 #include <dev/pci/pcivar.h> 75 #include <dev/pci/pci_private.h> 76 77 #ifdef CONFIG_DEFINED 78 #include <cxgb_include.h> 79 #else 80 #include <dev/cxgb/cxgb_include.h> 81 #endif 82 83 #ifdef PRIV_SUPPORTED 84 #include <sys/priv.h> 85 #endif 86 87 #include <machine/intr_machdep.h> 88 89 static int cxgb_setup_msix(adapter_t *, int); 90 static void cxgb_teardown_msix(adapter_t *); 91 static void cxgb_init(void *); 92 static void cxgb_init_locked(struct port_info *); 93 static void cxgb_stop_locked(struct port_info *); 94 static void cxgb_set_rxmode(struct port_info *); 95 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 96 static int cxgb_media_change(struct ifnet *); 97 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 98 static int setup_sge_qsets(adapter_t *); 99 static void cxgb_async_intr(void *); 100 static void cxgb_ext_intr_handler(void *, int); 101 static void cxgb_tick_handler(void *, int); 102 static void cxgb_down_locked(struct adapter *sc); 103 static void cxgb_tick(void *); 104 static void setup_rss(adapter_t *sc); 105 106 #ifndef IFNET_MULTIQUEUE 107 static void cxgb_start_proc(void *, int ncount); 108 #endif 109 110 /* Attachment glue for the PCI controller end of the device. Each port of 111 * the device is attached separately, as defined later. 112 */ 113 static int cxgb_controller_probe(device_t); 114 static int cxgb_controller_attach(device_t); 115 static int cxgb_controller_detach(device_t); 116 static void cxgb_free(struct adapter *); 117 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 118 unsigned int end); 119 static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf); 120 static int cxgb_get_regs_len(void); 121 static int offload_open(struct port_info *pi); 122 static void touch_bars(device_t dev); 123 static int offload_close(struct t3cdev *tdev); 124 125 static device_method_t cxgb_controller_methods[] = { 126 DEVMETHOD(device_probe, cxgb_controller_probe), 127 DEVMETHOD(device_attach, cxgb_controller_attach), 128 DEVMETHOD(device_detach, cxgb_controller_detach), 129 130 /* bus interface */ 131 DEVMETHOD(bus_print_child, bus_generic_print_child), 132 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 133 134 { 0, 0 } 135 }; 136 137 static driver_t cxgb_controller_driver = { 138 "cxgbc", 139 cxgb_controller_methods, 140 sizeof(struct adapter) 141 }; 142 143 static devclass_t cxgb_controller_devclass; 144 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 145 146 /* 147 * Attachment glue for the ports. Attachment is done directly to the 148 * controller device. 149 */ 150 static int cxgb_port_probe(device_t); 151 static int cxgb_port_attach(device_t); 152 static int cxgb_port_detach(device_t); 153 154 static device_method_t cxgb_port_methods[] = { 155 DEVMETHOD(device_probe, cxgb_port_probe), 156 DEVMETHOD(device_attach, cxgb_port_attach), 157 DEVMETHOD(device_detach, cxgb_port_detach), 158 { 0, 0 } 159 }; 160 161 static driver_t cxgb_port_driver = { 162 "cxgb", 163 cxgb_port_methods, 164 0 165 }; 166 167 static d_ioctl_t cxgb_extension_ioctl; 168 static d_open_t cxgb_extension_open; 169 static d_close_t cxgb_extension_close; 170 171 static struct cdevsw cxgb_cdevsw = { 172 .d_version = D_VERSION, 173 .d_flags = 0, 174 .d_open = cxgb_extension_open, 175 .d_close = cxgb_extension_close, 176 .d_ioctl = cxgb_extension_ioctl, 177 .d_name = "cxgb", 178 }; 179 180 static devclass_t cxgb_port_devclass; 181 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 182 183 #define SGE_MSIX_COUNT (SGE_QSETS + 1) 184 185 /* 186 * The driver uses the best interrupt scheme available on a platform in the 187 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 188 * of these schemes the driver may consider as follows: 189 * 190 * msi = 2: choose from among all three options 191 * msi = 1 : only consider MSI and pin interrupts 192 * msi = 0: force pin interrupts 193 */ 194 static int msi_allowed = 2; 195 196 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 197 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 198 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 199 "MSI-X, MSI, INTx selector"); 200 201 /* 202 * The driver enables offload as a default. 203 * To disable it, use ofld_disable = 1. 204 */ 205 static int ofld_disable = 0; 206 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable); 207 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0, 208 "disable ULP offload"); 209 210 /* 211 * The driver uses an auto-queue algorithm by default. 212 * To disable it and force a single queue-set per port, use singleq = 1. 213 */ 214 static int singleq = 0; 215 TUNABLE_INT("hw.cxgb.singleq", &singleq); 216 SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0, 217 "use a single queue-set per port"); 218 219 #ifndef IFNET_MULTIQUEUE 220 int cxgb_txq_buf_ring_size = 0; 221 #endif 222 223 enum { 224 MAX_TXQ_ENTRIES = 16384, 225 MAX_CTRL_TXQ_ENTRIES = 1024, 226 MAX_RSPQ_ENTRIES = 16384, 227 MAX_RX_BUFFERS = 16384, 228 MAX_RX_JUMBO_BUFFERS = 16384, 229 MIN_TXQ_ENTRIES = 4, 230 MIN_CTRL_TXQ_ENTRIES = 4, 231 MIN_RSPQ_ENTRIES = 32, 232 MIN_FL_ENTRIES = 32, 233 MIN_FL_JUMBO_ENTRIES = 32 234 }; 235 236 struct filter_info { 237 u32 sip; 238 u32 sip_mask; 239 u32 dip; 240 u16 sport; 241 u16 dport; 242 u32 vlan:12; 243 u32 vlan_prio:3; 244 u32 mac_hit:1; 245 u32 mac_idx:4; 246 u32 mac_vld:1; 247 u32 pkt_type:2; 248 u32 report_filter_id:1; 249 u32 pass:1; 250 u32 rss:1; 251 u32 qset:3; 252 u32 locked:1; 253 u32 valid:1; 254 }; 255 256 enum { FILTER_NO_VLAN_PRI = 7 }; 257 258 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 259 260 /* Table for probing the cards. The desc field isn't actually used */ 261 struct cxgb_ident { 262 uint16_t vendor; 263 uint16_t device; 264 int index; 265 char *desc; 266 } cxgb_identifiers[] = { 267 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 268 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 269 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 270 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 271 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 272 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 273 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 274 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 275 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 276 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 277 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 278 {0, 0, 0, NULL} 279 }; 280 281 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 282 283 static __inline void 284 check_pkt_coalesce(struct sge_qset *qs) 285 { 286 struct adapter *sc; 287 struct sge_txq *txq; 288 289 txq = &qs->txq[TXQ_ETH]; 290 sc = qs->port->adapter; 291 292 if (sc->tunq_fill[qs->idx] && (txq->in_use < (txq->size - (txq->size>>2)))) 293 sc->tunq_fill[qs->idx] = 0; 294 else if (!sc->tunq_fill[qs->idx] && (txq->in_use > (txq->size - (txq->size>>2)))) 295 sc->tunq_fill[qs->idx] = 1; 296 } 297 298 static __inline char 299 t3rev2char(struct adapter *adapter) 300 { 301 char rev = 'z'; 302 303 switch(adapter->params.rev) { 304 case T3_REV_A: 305 rev = 'a'; 306 break; 307 case T3_REV_B: 308 case T3_REV_B2: 309 rev = 'b'; 310 break; 311 case T3_REV_C: 312 rev = 'c'; 313 break; 314 } 315 return rev; 316 } 317 318 static struct cxgb_ident * 319 cxgb_get_ident(device_t dev) 320 { 321 struct cxgb_ident *id; 322 323 for (id = cxgb_identifiers; id->desc != NULL; id++) { 324 if ((id->vendor == pci_get_vendor(dev)) && 325 (id->device == pci_get_device(dev))) { 326 return (id); 327 } 328 } 329 return (NULL); 330 } 331 332 static const struct adapter_info * 333 cxgb_get_adapter_info(device_t dev) 334 { 335 struct cxgb_ident *id; 336 const struct adapter_info *ai; 337 338 id = cxgb_get_ident(dev); 339 if (id == NULL) 340 return (NULL); 341 342 ai = t3_get_adapter_info(id->index); 343 344 return (ai); 345 } 346 347 static int 348 cxgb_controller_probe(device_t dev) 349 { 350 const struct adapter_info *ai; 351 char *ports, buf[80]; 352 int nports; 353 354 ai = cxgb_get_adapter_info(dev); 355 if (ai == NULL) 356 return (ENXIO); 357 358 nports = ai->nports0 + ai->nports1; 359 if (nports == 1) 360 ports = "port"; 361 else 362 ports = "ports"; 363 364 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, nports, ports); 365 device_set_desc_copy(dev, buf); 366 return (BUS_PROBE_DEFAULT); 367 } 368 369 #define FW_FNAME "t3fw%d%d%d" 370 #define TPEEPROM_NAME "t3%ctpe%d%d%d" 371 #define TPSRAM_NAME "t3%cps%d%d%d" 372 373 static int 374 upgrade_fw(adapter_t *sc) 375 { 376 char buf[32]; 377 #ifdef FIRMWARE_LATEST 378 const struct firmware *fw; 379 #else 380 struct firmware *fw; 381 #endif 382 int status; 383 384 snprintf(&buf[0], sizeof(buf), FW_FNAME, FW_VERSION_MAJOR, 385 FW_VERSION_MINOR, FW_VERSION_MICRO); 386 387 fw = firmware_get(buf); 388 389 if (fw == NULL) { 390 device_printf(sc->dev, "Could not find firmware image %s\n", buf); 391 return (ENOENT); 392 } else 393 device_printf(sc->dev, "updating firmware on card with %s\n", buf); 394 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 395 396 device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status); 397 398 firmware_put(fw, FIRMWARE_UNLOAD); 399 400 return (status); 401 } 402 403 static int 404 cxgb_controller_attach(device_t dev) 405 { 406 device_t child; 407 const struct adapter_info *ai; 408 struct adapter *sc; 409 int i, error = 0; 410 uint32_t vers; 411 int port_qsets = 1; 412 #ifdef MSI_SUPPORTED 413 int msi_needed, reg; 414 #endif 415 sc = device_get_softc(dev); 416 sc->dev = dev; 417 sc->msi_count = 0; 418 ai = cxgb_get_adapter_info(dev); 419 420 /* 421 * XXX not really related but a recent addition 422 */ 423 #ifdef MSI_SUPPORTED 424 /* find the PCIe link width and set max read request to 4KB*/ 425 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 426 uint16_t lnk, pectl; 427 lnk = pci_read_config(dev, reg + 0x12, 2); 428 sc->link_width = (lnk >> 4) & 0x3f; 429 430 pectl = pci_read_config(dev, reg + 0x8, 2); 431 pectl = (pectl & ~0x7000) | (5 << 12); 432 pci_write_config(dev, reg + 0x8, pectl, 2); 433 } 434 435 if (sc->link_width != 0 && sc->link_width <= 4 && 436 (ai->nports0 + ai->nports1) <= 2) { 437 device_printf(sc->dev, 438 "PCIe x%d Link, expect reduced performance\n", 439 sc->link_width); 440 } 441 #endif 442 touch_bars(dev); 443 pci_enable_busmaster(dev); 444 /* 445 * Allocate the registers and make them available to the driver. 446 * The registers that we care about for NIC mode are in BAR 0 447 */ 448 sc->regs_rid = PCIR_BAR(0); 449 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 450 &sc->regs_rid, RF_ACTIVE)) == NULL) { 451 device_printf(dev, "Cannot allocate BAR\n"); 452 return (ENXIO); 453 } 454 455 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 456 device_get_unit(dev)); 457 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 458 459 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 460 device_get_unit(dev)); 461 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 462 device_get_unit(dev)); 463 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 464 device_get_unit(dev)); 465 466 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_DEF); 467 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 468 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 469 470 sc->bt = rman_get_bustag(sc->regs_res); 471 sc->bh = rman_get_bushandle(sc->regs_res); 472 sc->mmio_len = rman_get_size(sc->regs_res); 473 474 if (t3_prep_adapter(sc, ai, 1) < 0) { 475 printf("prep adapter failed\n"); 476 error = ENODEV; 477 goto out; 478 } 479 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 480 * enough messages for the queue sets. If that fails, try falling 481 * back to MSI. If that fails, then try falling back to the legacy 482 * interrupt pin model. 483 */ 484 #ifdef MSI_SUPPORTED 485 486 sc->msix_regs_rid = 0x20; 487 if ((msi_allowed >= 2) && 488 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 489 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 490 491 msi_needed = sc->msi_count = SGE_MSIX_COUNT; 492 493 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) || 494 (sc->msi_count != msi_needed)) { 495 device_printf(dev, "msix allocation failed - msi_count = %d" 496 " msi_needed=%d will try msi err=%d\n", sc->msi_count, 497 msi_needed, error); 498 sc->msi_count = 0; 499 pci_release_msi(dev); 500 bus_release_resource(dev, SYS_RES_MEMORY, 501 sc->msix_regs_rid, sc->msix_regs_res); 502 sc->msix_regs_res = NULL; 503 } else { 504 sc->flags |= USING_MSIX; 505 sc->cxgb_intr = t3_intr_msix; 506 } 507 } 508 509 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 510 sc->msi_count = 1; 511 if (pci_alloc_msi(dev, &sc->msi_count)) { 512 device_printf(dev, "alloc msi failed - will try INTx\n"); 513 sc->msi_count = 0; 514 pci_release_msi(dev); 515 } else { 516 sc->flags |= USING_MSI; 517 sc->irq_rid = 1; 518 sc->cxgb_intr = t3_intr_msi; 519 } 520 } 521 #endif 522 if (sc->msi_count == 0) { 523 device_printf(dev, "using line interrupts\n"); 524 sc->irq_rid = 0; 525 sc->cxgb_intr = t3b_intr; 526 } 527 528 529 /* Create a private taskqueue thread for handling driver events */ 530 #ifdef TASKQUEUE_CURRENT 531 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 532 taskqueue_thread_enqueue, &sc->tq); 533 #else 534 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 535 taskqueue_thread_enqueue, &sc->tq); 536 #endif 537 if (sc->tq == NULL) { 538 device_printf(dev, "failed to allocate controller task queue\n"); 539 goto out; 540 } 541 542 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 543 device_get_nameunit(dev)); 544 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 545 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 546 547 548 /* Create a periodic callout for checking adapter status */ 549 callout_init(&sc->cxgb_tick_ch, TRUE); 550 551 if (t3_check_fw_version(sc) != 0) { 552 /* 553 * Warn user that a firmware update will be attempted in init. 554 */ 555 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 556 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 557 sc->flags &= ~FW_UPTODATE; 558 } else { 559 sc->flags |= FW_UPTODATE; 560 } 561 562 if (t3_check_tpsram_version(sc) != 0) { 563 /* 564 * Warn user that a firmware update will be attempted in init. 565 */ 566 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 567 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 568 sc->flags &= ~TPS_UPTODATE; 569 } else { 570 sc->flags |= TPS_UPTODATE; 571 } 572 573 if ((sc->flags & USING_MSIX) && !singleq) 574 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); 575 576 /* 577 * Create a child device for each MAC. The ethernet attachment 578 * will be done in these children. 579 */ 580 for (i = 0; i < (sc)->params.nports; i++) { 581 struct port_info *pi; 582 583 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 584 device_printf(dev, "failed to add child port\n"); 585 error = EINVAL; 586 goto out; 587 } 588 pi = &sc->port[i]; 589 pi->adapter = sc; 590 pi->nqsets = port_qsets; 591 pi->first_qset = i*port_qsets; 592 pi->port_id = i; 593 pi->tx_chan = i >= ai->nports0; 594 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 595 sc->rxpkt_map[pi->txpkt_intf] = i; 596 sc->port[i].tx_chan = i >= ai->nports0; 597 sc->portdev[i] = child; 598 device_set_softc(child, pi); 599 } 600 if ((error = bus_generic_attach(dev)) != 0) 601 goto out; 602 603 /* 604 * XXX need to poll for link status 605 */ 606 sc->params.stats_update_period = 1; 607 608 /* initialize sge private state */ 609 t3_sge_init_adapter(sc); 610 611 t3_led_ready(sc); 612 613 cxgb_offload_init(); 614 if (is_offload(sc)) { 615 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); 616 cxgb_adapter_ofld(sc); 617 } 618 error = t3_get_fw_version(sc, &vers); 619 if (error) 620 goto out; 621 622 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 623 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 624 G_FW_VERSION_MICRO(vers)); 625 626 t3_add_attach_sysctls(sc); 627 out: 628 if (error) 629 cxgb_free(sc); 630 631 return (error); 632 } 633 634 static int 635 cxgb_controller_detach(device_t dev) 636 { 637 struct adapter *sc; 638 639 sc = device_get_softc(dev); 640 641 cxgb_free(sc); 642 643 return (0); 644 } 645 646 static void 647 cxgb_free(struct adapter *sc) 648 { 649 int i; 650 651 652 #ifdef IFNET_MULTIQUEUE 653 cxgb_pcpu_shutdown_threads(sc); 654 #endif 655 ADAPTER_LOCK(sc); 656 /* 657 * drops the lock 658 */ 659 cxgb_down_locked(sc); 660 661 #ifdef MSI_SUPPORTED 662 if (sc->flags & (USING_MSI | USING_MSIX)) { 663 device_printf(sc->dev, "releasing msi message(s)\n"); 664 pci_release_msi(sc->dev); 665 } else { 666 device_printf(sc->dev, "no msi message to release\n"); 667 } 668 #endif 669 if (sc->msix_regs_res != NULL) { 670 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 671 sc->msix_regs_res); 672 } 673 674 if (sc->tq != NULL) { 675 taskqueue_drain(sc->tq, &sc->ext_intr_task); 676 taskqueue_drain(sc->tq, &sc->tick_task); 677 } 678 t3_sge_deinit_sw(sc); 679 /* 680 * Wait for last callout 681 */ 682 683 DELAY(hz*100); 684 685 for (i = 0; i < (sc)->params.nports; ++i) { 686 if (sc->portdev[i] != NULL) 687 device_delete_child(sc->dev, sc->portdev[i]); 688 } 689 690 bus_generic_detach(sc->dev); 691 if (sc->tq != NULL) 692 taskqueue_free(sc->tq); 693 if (is_offload(sc)) { 694 cxgb_adapter_unofld(sc); 695 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT)) 696 offload_close(&sc->tdev); 697 else 698 printf("cxgb_free: DEVMAP_BIT not set\n"); 699 } else 700 printf("not offloading set\n"); 701 #ifndef IFNET_MULTIQUEUE 702 t3_free_sge_resources(sc); 703 #endif 704 free(sc->filters, M_DEVBUF); 705 t3_sge_free(sc); 706 707 cxgb_offload_exit(); 708 709 if (sc->regs_res != NULL) 710 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 711 sc->regs_res); 712 713 MTX_DESTROY(&sc->mdio_lock); 714 MTX_DESTROY(&sc->sge.reg_lock); 715 MTX_DESTROY(&sc->elmer_lock); 716 ADAPTER_LOCK_DEINIT(sc); 717 } 718 719 /** 720 * setup_sge_qsets - configure SGE Tx/Rx/response queues 721 * @sc: the controller softc 722 * 723 * Determines how many sets of SGE queues to use and initializes them. 724 * We support multiple queue sets per port if we have MSI-X, otherwise 725 * just one queue set per port. 726 */ 727 static int 728 setup_sge_qsets(adapter_t *sc) 729 { 730 int i, j, err, irq_idx = 0, qset_idx = 0; 731 u_int ntxq = SGE_TXQ_PER_SET; 732 733 if ((err = t3_sge_alloc(sc)) != 0) { 734 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 735 return (err); 736 } 737 738 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 739 irq_idx = -1; 740 741 for (i = 0; i < (sc)->params.nports; i++) { 742 struct port_info *pi = &sc->port[i]; 743 744 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 745 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 746 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 747 &sc->params.sge.qset[qset_idx], ntxq, pi); 748 if (err) { 749 t3_free_sge_resources(sc); 750 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", 751 err); 752 return (err); 753 } 754 } 755 } 756 757 return (0); 758 } 759 760 static void 761 cxgb_teardown_msix(adapter_t *sc) 762 { 763 int i, nqsets; 764 765 for (nqsets = i = 0; i < (sc)->params.nports; i++) 766 nqsets += sc->port[i].nqsets; 767 768 for (i = 0; i < nqsets; i++) { 769 if (sc->msix_intr_tag[i] != NULL) { 770 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 771 sc->msix_intr_tag[i]); 772 sc->msix_intr_tag[i] = NULL; 773 } 774 if (sc->msix_irq_res[i] != NULL) { 775 bus_release_resource(sc->dev, SYS_RES_IRQ, 776 sc->msix_irq_rid[i], sc->msix_irq_res[i]); 777 sc->msix_irq_res[i] = NULL; 778 } 779 } 780 } 781 782 static int 783 cxgb_setup_msix(adapter_t *sc, int msix_count) 784 { 785 int i, j, k, nqsets, rid; 786 787 /* The first message indicates link changes and error conditions */ 788 sc->irq_rid = 1; 789 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 790 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 791 device_printf(sc->dev, "Cannot allocate msix interrupt\n"); 792 return (EINVAL); 793 } 794 795 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 796 #ifdef INTR_FILTERS 797 NULL, 798 #endif 799 cxgb_async_intr, sc, &sc->intr_tag)) { 800 device_printf(sc->dev, "Cannot set up interrupt\n"); 801 return (EINVAL); 802 } 803 for (i = k = 0; i < (sc)->params.nports; i++) { 804 nqsets = sc->port[i].nqsets; 805 for (j = 0; j < nqsets; j++, k++) { 806 struct sge_qset *qs = &sc->sge.qs[k]; 807 808 rid = k + 2; 809 if (cxgb_debug) 810 printf("rid=%d ", rid); 811 if ((sc->msix_irq_res[k] = bus_alloc_resource_any( 812 sc->dev, SYS_RES_IRQ, &rid, 813 RF_SHAREABLE | RF_ACTIVE)) == NULL) { 814 device_printf(sc->dev, "Cannot allocate " 815 "interrupt for message %d\n", rid); 816 return (EINVAL); 817 } 818 sc->msix_irq_rid[k] = rid; 819 printf("setting up interrupt for port=%d\n", 820 qs->port->port_id); 821 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k], 822 INTR_MPSAFE|INTR_TYPE_NET, 823 #ifdef INTR_FILTERS 824 NULL, 825 #endif 826 t3_intr_msix, qs, &sc->msix_intr_tag[k])) { 827 device_printf(sc->dev, "Cannot set up " 828 "interrupt for message %d\n", rid); 829 return (EINVAL); 830 } 831 #ifdef IFNET_MULTIQUEUE 832 if (singleq == 0) { 833 int vector = rman_get_start(sc->msix_irq_res[k]); 834 if (bootverbose) 835 device_printf(sc->dev, "binding vector=%d to cpu=%d\n", vector, k % mp_ncpus); 836 intr_bind(vector, k % mp_ncpus); 837 } 838 #endif 839 } 840 } 841 842 return (0); 843 } 844 845 static int 846 cxgb_port_probe(device_t dev) 847 { 848 struct port_info *p; 849 char buf[80]; 850 851 p = device_get_softc(dev); 852 853 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, p->port_type->desc); 854 device_set_desc_copy(dev, buf); 855 return (0); 856 } 857 858 859 static int 860 cxgb_makedev(struct port_info *pi) 861 { 862 863 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 864 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp)); 865 866 if (pi->port_cdev == NULL) 867 return (ENOMEM); 868 869 pi->port_cdev->si_drv1 = (void *)pi; 870 871 return (0); 872 } 873 874 875 #ifdef TSO_SUPPORTED 876 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU) 877 /* Don't enable TSO6 yet */ 878 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU) 879 #else 880 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 881 /* Don't enable TSO6 yet */ 882 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 883 #define IFCAP_TSO4 0x0 884 #define IFCAP_TSO6 0x0 885 #define CSUM_TSO 0x0 886 #endif 887 888 889 static int 890 cxgb_port_attach(device_t dev) 891 { 892 struct port_info *p; 893 struct ifnet *ifp; 894 int err, media_flags; 895 896 p = device_get_softc(dev); 897 898 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 899 device_get_unit(device_get_parent(dev)), p->port_id); 900 PORT_LOCK_INIT(p, p->lockbuf); 901 902 /* Allocate an ifnet object and set it up */ 903 ifp = p->ifp = if_alloc(IFT_ETHER); 904 if (ifp == NULL) { 905 device_printf(dev, "Cannot allocate ifnet\n"); 906 return (ENOMEM); 907 } 908 909 /* 910 * Note that there is currently no watchdog timer. 911 */ 912 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 913 ifp->if_init = cxgb_init; 914 ifp->if_softc = p; 915 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 916 ifp->if_ioctl = cxgb_ioctl; 917 ifp->if_start = cxgb_start; 918 919 #ifdef IFNET_MULTIQUEUE 920 ifp->if_flags |= IFF_MULTIQ; 921 ifp->if_mq_start = cxgb_pcpu_start; 922 #endif 923 924 ifp->if_timer = 0; /* Disable ifnet watchdog */ 925 ifp->if_watchdog = NULL; 926 927 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE; 928 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 929 IFQ_SET_READY(&ifp->if_snd); 930 931 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 932 ifp->if_capabilities |= CXGB_CAP; 933 ifp->if_capenable |= CXGB_CAP_ENABLE; 934 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 935 /* 936 * disable TSO on 4-port - it isn't supported by the firmware yet 937 */ 938 if (p->adapter->params.nports > 2) { 939 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6); 940 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6); 941 ifp->if_hwassist &= ~CSUM_TSO; 942 } 943 944 ether_ifattach(ifp, p->hw_addr); 945 /* 946 * Only default to jumbo frames on 10GigE 947 */ 948 if (p->adapter->params.nports <= 2) 949 ifp->if_mtu = 9000; 950 if ((err = cxgb_makedev(p)) != 0) { 951 printf("makedev failed %d\n", err); 952 return (err); 953 } 954 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 955 cxgb_media_status); 956 957 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) { 958 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX; 959 } else if (!strcmp(p->port_type->desc, "10GBASE-SR")) { 960 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX; 961 } else if (!strcmp(p->port_type->desc, "10GBASE-XR")) { 962 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX; 963 } else if (!strcmp(p->port_type->desc, "10/100/1000BASE-T")) { 964 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL); 965 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX, 966 0, NULL); 967 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX, 968 0, NULL); 969 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 970 0, NULL); 971 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 972 0, NULL); 973 media_flags = 0; 974 } else { 975 printf("unsupported media type %s\n", p->port_type->desc); 976 return (ENXIO); 977 } 978 if (media_flags) { 979 ifmedia_add(&p->media, media_flags, 0, NULL); 980 ifmedia_set(&p->media, media_flags); 981 } else { 982 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 983 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO); 984 } 985 986 987 snprintf(p->taskqbuf, TASKQ_NAME_LEN, "cxgb_port_taskq%d", p->port_id); 988 #ifdef TASKQUEUE_CURRENT 989 /* Create a port for handling TX without starvation */ 990 p->tq = taskqueue_create(p->taskqbuf, M_NOWAIT, 991 taskqueue_thread_enqueue, &p->tq); 992 #else 993 /* Create a port for handling TX without starvation */ 994 p->tq = taskqueue_create_fast(p->taskqbuf, M_NOWAIT, 995 taskqueue_thread_enqueue, &p->tq); 996 #endif 997 #ifndef IFNET_MULTIQUEUE 998 if (p->tq == NULL) { 999 device_printf(dev, "failed to allocate port task queue\n"); 1000 return (ENOMEM); 1001 } 1002 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq", 1003 device_get_nameunit(dev)); 1004 1005 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp); 1006 #endif 1007 t3_sge_init_port(p); 1008 1009 return (0); 1010 } 1011 1012 static int 1013 cxgb_port_detach(device_t dev) 1014 { 1015 struct port_info *p; 1016 1017 p = device_get_softc(dev); 1018 1019 PORT_LOCK(p); 1020 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING) 1021 cxgb_stop_locked(p); 1022 PORT_UNLOCK(p); 1023 1024 if (p->tq != NULL) { 1025 taskqueue_drain(p->tq, &p->start_task); 1026 taskqueue_free(p->tq); 1027 p->tq = NULL; 1028 } 1029 1030 ether_ifdetach(p->ifp); 1031 printf("waiting for callout to stop ..."); 1032 DELAY(1000000); 1033 printf("done\n"); 1034 /* 1035 * the lock may be acquired in ifdetach 1036 */ 1037 PORT_LOCK_DEINIT(p); 1038 if_free(p->ifp); 1039 1040 if (p->port_cdev != NULL) 1041 destroy_dev(p->port_cdev); 1042 1043 return (0); 1044 } 1045 1046 void 1047 t3_fatal_err(struct adapter *sc) 1048 { 1049 u_int fw_status[4]; 1050 1051 if (sc->flags & FULL_INIT_DONE) { 1052 t3_sge_stop(sc); 1053 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1054 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1055 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1056 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1057 t3_intr_disable(sc); 1058 } 1059 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1060 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1061 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1062 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1063 } 1064 1065 int 1066 t3_os_find_pci_capability(adapter_t *sc, int cap) 1067 { 1068 device_t dev; 1069 struct pci_devinfo *dinfo; 1070 pcicfgregs *cfg; 1071 uint32_t status; 1072 uint8_t ptr; 1073 1074 dev = sc->dev; 1075 dinfo = device_get_ivars(dev); 1076 cfg = &dinfo->cfg; 1077 1078 status = pci_read_config(dev, PCIR_STATUS, 2); 1079 if (!(status & PCIM_STATUS_CAPPRESENT)) 1080 return (0); 1081 1082 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1083 case 0: 1084 case 1: 1085 ptr = PCIR_CAP_PTR; 1086 break; 1087 case 2: 1088 ptr = PCIR_CAP_PTR_2; 1089 break; 1090 default: 1091 return (0); 1092 break; 1093 } 1094 ptr = pci_read_config(dev, ptr, 1); 1095 1096 while (ptr != 0) { 1097 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1098 return (ptr); 1099 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1100 } 1101 1102 return (0); 1103 } 1104 1105 int 1106 t3_os_pci_save_state(struct adapter *sc) 1107 { 1108 device_t dev; 1109 struct pci_devinfo *dinfo; 1110 1111 dev = sc->dev; 1112 dinfo = device_get_ivars(dev); 1113 1114 pci_cfg_save(dev, dinfo, 0); 1115 return (0); 1116 } 1117 1118 int 1119 t3_os_pci_restore_state(struct adapter *sc) 1120 { 1121 device_t dev; 1122 struct pci_devinfo *dinfo; 1123 1124 dev = sc->dev; 1125 dinfo = device_get_ivars(dev); 1126 1127 pci_cfg_restore(dev, dinfo); 1128 return (0); 1129 } 1130 1131 /** 1132 * t3_os_link_changed - handle link status changes 1133 * @adapter: the adapter associated with the link change 1134 * @port_id: the port index whose limk status has changed 1135 * @link_stat: the new status of the link 1136 * @speed: the new speed setting 1137 * @duplex: the new duplex setting 1138 * @fc: the new flow-control setting 1139 * 1140 * This is the OS-dependent handler for link status changes. The OS 1141 * neutral handler takes care of most of the processing for these events, 1142 * then calls this handler for any OS-specific processing. 1143 */ 1144 void 1145 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1146 int duplex, int fc) 1147 { 1148 struct port_info *pi = &adapter->port[port_id]; 1149 struct cmac *mac = &adapter->port[port_id].mac; 1150 1151 if ((pi->ifp->if_flags & IFF_UP) == 0) 1152 return; 1153 1154 if (link_status) { 1155 t3_mac_enable(mac, MAC_DIRECTION_RX); 1156 if_link_state_change(pi->ifp, LINK_STATE_UP); 1157 } else { 1158 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 1159 pi->phy.ops->power_down(&pi->phy, 1); 1160 t3_mac_disable(mac, MAC_DIRECTION_RX); 1161 t3_link_start(&pi->phy, mac, &pi->link_config); 1162 } 1163 } 1164 1165 /* 1166 * Interrupt-context handler for external (PHY) interrupts. 1167 */ 1168 void 1169 t3_os_ext_intr_handler(adapter_t *sc) 1170 { 1171 if (cxgb_debug) 1172 printf("t3_os_ext_intr_handler\n"); 1173 /* 1174 * Schedule a task to handle external interrupts as they may be slow 1175 * and we use a mutex to protect MDIO registers. We disable PHY 1176 * interrupts in the meantime and let the task reenable them when 1177 * it's done. 1178 */ 1179 ADAPTER_LOCK(sc); 1180 if (sc->slow_intr_mask) { 1181 sc->slow_intr_mask &= ~F_T3DBG; 1182 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1183 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 1184 } 1185 ADAPTER_UNLOCK(sc); 1186 } 1187 1188 void 1189 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1190 { 1191 1192 /* 1193 * The ifnet might not be allocated before this gets called, 1194 * as this is called early on in attach by t3_prep_adapter 1195 * save the address off in the port structure 1196 */ 1197 if (cxgb_debug) 1198 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1199 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1200 } 1201 1202 /** 1203 * link_start - enable a port 1204 * @p: the port to enable 1205 * 1206 * Performs the MAC and PHY actions needed to enable a port. 1207 */ 1208 static void 1209 cxgb_link_start(struct port_info *p) 1210 { 1211 struct ifnet *ifp; 1212 struct t3_rx_mode rm; 1213 struct cmac *mac = &p->mac; 1214 1215 ifp = p->ifp; 1216 1217 t3_init_rx_mode(&rm, p); 1218 if (!mac->multiport) 1219 t3_mac_reset(mac); 1220 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1221 t3_mac_set_address(mac, 0, p->hw_addr); 1222 t3_mac_set_rx_mode(mac, &rm); 1223 t3_link_start(&p->phy, mac, &p->link_config); 1224 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1225 } 1226 1227 /** 1228 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1229 * @adap: the adapter 1230 * 1231 * Sets up RSS to distribute packets to multiple receive queues. We 1232 * configure the RSS CPU lookup table to distribute to the number of HW 1233 * receive queues, and the response queue lookup table to narrow that 1234 * down to the response queues actually configured for each port. 1235 * We always configure the RSS mapping for two ports since the mapping 1236 * table has plenty of entries. 1237 */ 1238 static void 1239 setup_rss(adapter_t *adap) 1240 { 1241 int i; 1242 u_int nq[2]; 1243 uint8_t cpus[SGE_QSETS + 1]; 1244 uint16_t rspq_map[RSS_TABLE_SIZE]; 1245 1246 for (i = 0; i < SGE_QSETS; ++i) 1247 cpus[i] = i; 1248 cpus[SGE_QSETS] = 0xff; 1249 1250 nq[0] = nq[1] = 0; 1251 for_each_port(adap, i) { 1252 const struct port_info *pi = adap2pinfo(adap, i); 1253 1254 nq[pi->tx_chan] += pi->nqsets; 1255 } 1256 nq[0] = max(nq[0], 1U); 1257 nq[1] = max(nq[1], 1U); 1258 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1259 rspq_map[i] = i % nq[0]; 1260 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq[1]) + nq[0]; 1261 } 1262 /* Calculate the reverse RSS map table */ 1263 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1264 if (adap->rrss_map[rspq_map[i]] == 0xff) 1265 adap->rrss_map[rspq_map[i]] = i; 1266 1267 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1268 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1269 V_RRCPLCPUSIZE(6), cpus, rspq_map); 1270 1271 } 1272 1273 /* 1274 * Sends an mbuf to an offload queue driver 1275 * after dealing with any active network taps. 1276 */ 1277 static inline int 1278 offload_tx(struct t3cdev *tdev, struct mbuf *m) 1279 { 1280 int ret; 1281 1282 ret = t3_offload_tx(tdev, m); 1283 return (ret); 1284 } 1285 1286 static int 1287 write_smt_entry(struct adapter *adapter, int idx) 1288 { 1289 struct port_info *pi = &adapter->port[idx]; 1290 struct cpl_smt_write_req *req; 1291 struct mbuf *m; 1292 1293 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1294 return (ENOMEM); 1295 1296 req = mtod(m, struct cpl_smt_write_req *); 1297 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req); 1298 1299 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1300 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 1301 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 1302 req->iff = idx; 1303 memset(req->src_mac1, 0, sizeof(req->src_mac1)); 1304 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 1305 1306 m_set_priority(m, 1); 1307 1308 offload_tx(&adapter->tdev, m); 1309 1310 return (0); 1311 } 1312 1313 static int 1314 init_smt(struct adapter *adapter) 1315 { 1316 int i; 1317 1318 for_each_port(adapter, i) 1319 write_smt_entry(adapter, i); 1320 return 0; 1321 } 1322 1323 static void 1324 init_port_mtus(adapter_t *adapter) 1325 { 1326 unsigned int mtus = adapter->port[0].ifp->if_mtu; 1327 1328 if (adapter->port[1].ifp) 1329 mtus |= adapter->port[1].ifp->if_mtu << 16; 1330 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); 1331 } 1332 1333 static void 1334 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1335 int hi, int port) 1336 { 1337 struct mbuf *m; 1338 struct mngt_pktsched_wr *req; 1339 1340 m = m_gethdr(M_DONTWAIT, MT_DATA); 1341 if (m) { 1342 req = mtod(m, struct mngt_pktsched_wr *); 1343 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1344 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1345 req->sched = sched; 1346 req->idx = qidx; 1347 req->min = lo; 1348 req->max = hi; 1349 req->binding = port; 1350 m->m_len = m->m_pkthdr.len = sizeof(*req); 1351 t3_mgmt_tx(adap, m); 1352 } 1353 } 1354 1355 static void 1356 bind_qsets(adapter_t *sc) 1357 { 1358 int i, j; 1359 1360 #ifdef IFNET_MULTIQUEUE 1361 cxgb_pcpu_startup_threads(sc); 1362 #endif 1363 1364 for (i = 0; i < (sc)->params.nports; ++i) { 1365 const struct port_info *pi = adap2pinfo(sc, i); 1366 1367 for (j = 0; j < pi->nqsets; ++j) { 1368 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1369 -1, pi->tx_chan); 1370 1371 } 1372 } 1373 } 1374 1375 static void 1376 update_tpeeprom(struct adapter *adap) 1377 { 1378 #ifdef FIRMWARE_LATEST 1379 const struct firmware *tpeeprom; 1380 #else 1381 struct firmware *tpeeprom; 1382 #endif 1383 1384 char buf[64]; 1385 uint32_t version; 1386 unsigned int major, minor; 1387 int ret, len; 1388 char rev; 1389 1390 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1391 1392 major = G_TP_VERSION_MAJOR(version); 1393 minor = G_TP_VERSION_MINOR(version); 1394 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1395 return; 1396 1397 rev = t3rev2char(adap); 1398 1399 snprintf(buf, sizeof(buf), TPEEPROM_NAME, rev, 1400 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1401 1402 tpeeprom = firmware_get(buf); 1403 if (tpeeprom == NULL) { 1404 device_printf(adap->dev, "could not load TP EEPROM: unable to load %s\n", 1405 buf); 1406 return; 1407 } 1408 1409 len = tpeeprom->datasize - 4; 1410 1411 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1412 if (ret) 1413 goto release_tpeeprom; 1414 1415 if (len != TP_SRAM_LEN) { 1416 device_printf(adap->dev, "%s length is wrong len=%d expected=%d\n", buf, len, TP_SRAM_LEN); 1417 return; 1418 } 1419 1420 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1421 TP_SRAM_OFFSET); 1422 1423 if (!ret) { 1424 device_printf(adap->dev, 1425 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1426 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1427 } else 1428 device_printf(adap->dev, "Protocol SRAM image update in EEPROM failed\n"); 1429 1430 release_tpeeprom: 1431 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1432 1433 return; 1434 } 1435 1436 static int 1437 update_tpsram(struct adapter *adap) 1438 { 1439 #ifdef FIRMWARE_LATEST 1440 const struct firmware *tpsram; 1441 #else 1442 struct firmware *tpsram; 1443 #endif 1444 char buf[64]; 1445 int ret; 1446 char rev; 1447 1448 rev = t3rev2char(adap); 1449 if (!rev) 1450 return 0; 1451 1452 update_tpeeprom(adap); 1453 1454 snprintf(buf, sizeof(buf), TPSRAM_NAME, rev, 1455 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1456 1457 tpsram = firmware_get(buf); 1458 if (tpsram == NULL){ 1459 device_printf(adap->dev, "could not load TP SRAM: unable to load %s\n", 1460 buf); 1461 return (EINVAL); 1462 } else 1463 device_printf(adap->dev, "updating TP SRAM with %s\n", buf); 1464 1465 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1466 if (ret) 1467 goto release_tpsram; 1468 1469 ret = t3_set_proto_sram(adap, tpsram->data); 1470 if (ret) 1471 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1472 1473 release_tpsram: 1474 firmware_put(tpsram, FIRMWARE_UNLOAD); 1475 1476 return ret; 1477 } 1478 1479 /** 1480 * cxgb_up - enable the adapter 1481 * @adap: adapter being enabled 1482 * 1483 * Called when the first port is enabled, this function performs the 1484 * actions necessary to make an adapter operational, such as completing 1485 * the initialization of HW modules, and enabling interrupts. 1486 * 1487 */ 1488 static int 1489 cxgb_up(struct adapter *sc) 1490 { 1491 int err = 0; 1492 1493 if ((sc->flags & FULL_INIT_DONE) == 0) { 1494 1495 if ((sc->flags & FW_UPTODATE) == 0) 1496 if ((err = upgrade_fw(sc))) 1497 goto out; 1498 if ((sc->flags & TPS_UPTODATE) == 0) 1499 if ((err = update_tpsram(sc))) 1500 goto out; 1501 err = t3_init_hw(sc, 0); 1502 if (err) 1503 goto out; 1504 1505 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1506 1507 err = setup_sge_qsets(sc); 1508 if (err) 1509 goto out; 1510 1511 setup_rss(sc); 1512 t3_add_configured_sysctls(sc); 1513 sc->flags |= FULL_INIT_DONE; 1514 } 1515 1516 t3_intr_clear(sc); 1517 1518 /* If it's MSI or INTx, allocate a single interrupt for everything */ 1519 if ((sc->flags & USING_MSIX) == 0) { 1520 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 1521 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 1522 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", 1523 sc->irq_rid); 1524 err = EINVAL; 1525 goto out; 1526 } 1527 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res); 1528 1529 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 1530 #ifdef INTR_FILTERS 1531 NULL, 1532 #endif 1533 sc->cxgb_intr, sc, &sc->intr_tag)) { 1534 device_printf(sc->dev, "Cannot set up interrupt\n"); 1535 err = EINVAL; 1536 goto irq_err; 1537 } 1538 } else { 1539 cxgb_setup_msix(sc, sc->msi_count); 1540 } 1541 1542 t3_sge_start(sc); 1543 t3_intr_enable(sc); 1544 1545 if (!(sc->flags & QUEUES_BOUND)) { 1546 printf("bind qsets\n"); 1547 bind_qsets(sc); 1548 sc->flags |= QUEUES_BOUND; 1549 } 1550 out: 1551 return (err); 1552 irq_err: 1553 CH_ERR(sc, "request_irq failed, err %d\n", err); 1554 goto out; 1555 } 1556 1557 1558 /* 1559 * Release resources when all the ports and offloading have been stopped. 1560 */ 1561 static void 1562 cxgb_down_locked(struct adapter *sc) 1563 { 1564 int i; 1565 1566 t3_sge_stop(sc); 1567 t3_intr_disable(sc); 1568 1569 if (sc->intr_tag != NULL) { 1570 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 1571 sc->intr_tag = NULL; 1572 } 1573 if (sc->irq_res != NULL) { 1574 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", 1575 sc->irq_rid, sc->irq_res); 1576 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 1577 sc->irq_res); 1578 sc->irq_res = NULL; 1579 } 1580 1581 if (sc->flags & USING_MSIX) 1582 cxgb_teardown_msix(sc); 1583 ADAPTER_UNLOCK(sc); 1584 1585 callout_stop(&sc->cxgb_tick_ch); 1586 callout_stop(&sc->sge_timer_ch); 1587 callout_drain(&sc->cxgb_tick_ch); 1588 callout_drain(&sc->sge_timer_ch); 1589 1590 if (sc->tq != NULL) { 1591 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1592 for (i = 0; i < sc->params.nports; i++) 1593 taskqueue_drain(sc->tq, &sc->port[i].timer_reclaim_task); 1594 } 1595 } 1596 1597 static int 1598 offload_open(struct port_info *pi) 1599 { 1600 struct adapter *adapter = pi->adapter; 1601 struct t3cdev *tdev = &adapter->tdev; 1602 #ifdef notyet 1603 T3CDEV(pi->ifp); 1604 #endif 1605 int adap_up = adapter->open_device_map & PORT_MASK; 1606 int err = 0; 1607 1608 printf("device_map=0x%x\n", adapter->open_device_map); 1609 if (atomic_cmpset_int(&adapter->open_device_map, 1610 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)), 1611 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0) 1612 return (0); 1613 1614 1615 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1616 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", adapter->open_device_map); 1617 ADAPTER_LOCK(pi->adapter); 1618 if (!adap_up) 1619 err = cxgb_up(adapter); 1620 ADAPTER_UNLOCK(pi->adapter); 1621 if (err) 1622 return (err); 1623 1624 t3_tp_set_offload_mode(adapter, 1); 1625 tdev->lldev = pi->ifp; 1626 err = cxgb_offload_activate(adapter); 1627 if (err) 1628 goto out; 1629 1630 init_port_mtus(adapter); 1631 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, 1632 adapter->params.b_wnd, 1633 adapter->params.rev == 0 ? 1634 adapter->port[0].ifp->if_mtu : 0xffff); 1635 init_smt(adapter); 1636 1637 /* Call back all registered clients */ 1638 cxgb_add_clients(tdev); 1639 1640 out: 1641 /* restore them in case the offload module has changed them */ 1642 if (err) { 1643 t3_tp_set_offload_mode(adapter, 0); 1644 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1645 cxgb_set_dummy_ops(tdev); 1646 } 1647 return (err); 1648 } 1649 1650 static int 1651 offload_close(struct t3cdev *tdev) 1652 { 1653 struct adapter *adapter = tdev2adap(tdev); 1654 1655 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) { 1656 printf("offload_close: DEVMAP_BIT not set\n"); 1657 1658 return (0); 1659 } 1660 1661 /* Call back all registered clients */ 1662 cxgb_remove_clients(tdev); 1663 tdev->lldev = NULL; 1664 cxgb_set_dummy_ops(tdev); 1665 t3_tp_set_offload_mode(adapter, 0); 1666 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1667 1668 ADAPTER_LOCK(adapter); 1669 if (!adapter->open_device_map) 1670 cxgb_down_locked(adapter); 1671 else 1672 ADAPTER_UNLOCK(adapter); 1673 cxgb_offload_deactivate(adapter); 1674 return (0); 1675 } 1676 1677 1678 static void 1679 cxgb_init(void *arg) 1680 { 1681 struct port_info *p = arg; 1682 1683 PORT_LOCK(p); 1684 cxgb_init_locked(p); 1685 PORT_UNLOCK(p); 1686 } 1687 1688 static void 1689 cxgb_init_locked(struct port_info *p) 1690 { 1691 struct ifnet *ifp; 1692 adapter_t *sc = p->adapter; 1693 int err; 1694 1695 PORT_LOCK_ASSERT_OWNED(p); 1696 ifp = p->ifp; 1697 1698 ADAPTER_LOCK(p->adapter); 1699 if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) { 1700 ADAPTER_UNLOCK(p->adapter); 1701 cxgb_stop_locked(p); 1702 return; 1703 } 1704 if (p->adapter->open_device_map == 0) { 1705 t3_intr_clear(sc); 1706 t3_sge_init_adapter(sc); 1707 } 1708 setbit(&p->adapter->open_device_map, p->port_id); 1709 ADAPTER_UNLOCK(p->adapter); 1710 1711 if (is_offload(sc) && !ofld_disable) { 1712 err = offload_open(p); 1713 if (err) 1714 log(LOG_WARNING, 1715 "Could not initialize offload capabilities\n"); 1716 else 1717 printf("offload opened\n"); 1718 } 1719 cxgb_link_start(p); 1720 t3_link_changed(sc, p->port_id); 1721 ifp->if_baudrate = p->link_config.speed * 1000000; 1722 1723 device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id); 1724 t3_port_intr_enable(sc, p->port_id); 1725 1726 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 1727 1728 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1729 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1730 } 1731 1732 static void 1733 cxgb_set_rxmode(struct port_info *p) 1734 { 1735 struct t3_rx_mode rm; 1736 struct cmac *mac = &p->mac; 1737 1738 PORT_LOCK_ASSERT_OWNED(p); 1739 1740 t3_init_rx_mode(&rm, p); 1741 t3_mac_set_rx_mode(mac, &rm); 1742 } 1743 1744 static void 1745 cxgb_stop_locked(struct port_info *p) 1746 { 1747 struct ifnet *ifp; 1748 1749 PORT_LOCK_ASSERT_OWNED(p); 1750 ADAPTER_LOCK_ASSERT_NOTOWNED(p->adapter); 1751 1752 ifp = p->ifp; 1753 t3_port_intr_disable(p->adapter, p->port_id); 1754 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1755 p->phy.ops->power_down(&p->phy, 1); 1756 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1757 1758 ADAPTER_LOCK(p->adapter); 1759 clrbit(&p->adapter->open_device_map, p->port_id); 1760 1761 if (p->adapter->open_device_map == 0) { 1762 cxgb_down_locked(p->adapter); 1763 } else 1764 ADAPTER_UNLOCK(p->adapter); 1765 1766 } 1767 1768 static int 1769 cxgb_set_mtu(struct port_info *p, int mtu) 1770 { 1771 struct ifnet *ifp = p->ifp; 1772 int error = 0; 1773 1774 if ((mtu < ETHERMIN) || (mtu > ETHER_MAX_LEN_JUMBO)) 1775 error = EINVAL; 1776 else if (ifp->if_mtu != mtu) { 1777 PORT_LOCK(p); 1778 ifp->if_mtu = mtu; 1779 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1780 callout_stop(&p->adapter->cxgb_tick_ch); 1781 cxgb_stop_locked(p); 1782 cxgb_init_locked(p); 1783 } 1784 PORT_UNLOCK(p); 1785 } 1786 return (error); 1787 } 1788 1789 static int 1790 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1791 { 1792 struct port_info *p = ifp->if_softc; 1793 struct ifaddr *ifa = (struct ifaddr *)data; 1794 struct ifreq *ifr = (struct ifreq *)data; 1795 int flags, error = 0; 1796 uint32_t mask; 1797 1798 /* 1799 * XXX need to check that we aren't in the middle of an unload 1800 */ 1801 switch (command) { 1802 case SIOCSIFMTU: 1803 error = cxgb_set_mtu(p, ifr->ifr_mtu); 1804 break; 1805 case SIOCSIFADDR: 1806 case SIOCGIFADDR: 1807 PORT_LOCK(p); 1808 if (ifa->ifa_addr->sa_family == AF_INET) { 1809 ifp->if_flags |= IFF_UP; 1810 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1811 cxgb_init_locked(p); 1812 arp_ifinit(ifp, ifa); 1813 } else 1814 error = ether_ioctl(ifp, command, data); 1815 PORT_UNLOCK(p); 1816 break; 1817 case SIOCSIFFLAGS: 1818 callout_drain(&p->adapter->cxgb_tick_ch); 1819 PORT_LOCK(p); 1820 if (ifp->if_flags & IFF_UP) { 1821 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1822 flags = p->if_flags; 1823 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1824 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 1825 cxgb_set_rxmode(p); 1826 } else 1827 cxgb_init_locked(p); 1828 p->if_flags = ifp->if_flags; 1829 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1830 cxgb_stop_locked(p); 1831 1832 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1833 adapter_t *sc = p->adapter; 1834 callout_reset(&sc->cxgb_tick_ch, hz, 1835 cxgb_tick, sc); 1836 } 1837 PORT_UNLOCK(p); 1838 break; 1839 case SIOCSIFMEDIA: 1840 case SIOCGIFMEDIA: 1841 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 1842 break; 1843 case SIOCSIFCAP: 1844 PORT_LOCK(p); 1845 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1846 if (mask & IFCAP_TXCSUM) { 1847 if (IFCAP_TXCSUM & ifp->if_capenable) { 1848 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 1849 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 1850 | CSUM_TSO); 1851 } else { 1852 ifp->if_capenable |= IFCAP_TXCSUM; 1853 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1854 } 1855 } else if (mask & IFCAP_RXCSUM) { 1856 if (IFCAP_RXCSUM & ifp->if_capenable) { 1857 ifp->if_capenable &= ~IFCAP_RXCSUM; 1858 } else { 1859 ifp->if_capenable |= IFCAP_RXCSUM; 1860 } 1861 } 1862 if (mask & IFCAP_TSO4) { 1863 if (IFCAP_TSO4 & ifp->if_capenable) { 1864 ifp->if_capenable &= ~IFCAP_TSO4; 1865 ifp->if_hwassist &= ~CSUM_TSO; 1866 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 1867 ifp->if_capenable |= IFCAP_TSO4; 1868 ifp->if_hwassist |= CSUM_TSO; 1869 } else { 1870 if (cxgb_debug) 1871 printf("cxgb requires tx checksum offload" 1872 " be enabled to use TSO\n"); 1873 error = EINVAL; 1874 } 1875 } 1876 PORT_UNLOCK(p); 1877 break; 1878 default: 1879 error = ether_ioctl(ifp, command, data); 1880 break; 1881 } 1882 return (error); 1883 } 1884 1885 int 1886 cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax) 1887 { 1888 struct sge_txq *txq; 1889 int err, in_use_init, count; 1890 struct mbuf **m_vec; 1891 1892 txq = &qs->txq[TXQ_ETH]; 1893 m_vec = txq->txq_m_vec; 1894 in_use_init = txq->in_use; 1895 err = 0; 1896 while ((txq->in_use - in_use_init < txmax) && 1897 (txq->size > txq->in_use + TX_MAX_DESC)) { 1898 check_pkt_coalesce(qs); 1899 count = cxgb_dequeue_packet(ifp, txq, m_vec); 1900 if (count == 0) 1901 break; 1902 ETHER_BPF_MTAP(ifp, m_vec[0]); 1903 1904 if ((err = t3_encap(qs, m_vec, count)) != 0) 1905 break; 1906 txq->txq_enqueued += count; 1907 } 1908 #ifndef IFNET_MULTIQUEUE 1909 if (__predict_false(err)) { 1910 if (err == ENOMEM) { 1911 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1912 IFQ_LOCK(&ifp->if_snd); 1913 IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]); 1914 IFQ_UNLOCK(&ifp->if_snd); 1915 } 1916 } 1917 if (err == 0 && m_vec[0] == NULL) { 1918 err = ENOBUFS; 1919 } 1920 else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) && 1921 (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { 1922 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1923 err = ENOSPC; 1924 } 1925 #else 1926 if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC)) { 1927 err = ENOSPC; 1928 setbit(&qs->txq_stopped, TXQ_ETH); 1929 } 1930 if (err == ENOMEM) { 1931 int i; 1932 /* 1933 * Sub-optimal :-/ 1934 */ 1935 for (i = 0; i < count; i++) 1936 m_freem(m_vec[i]); 1937 } 1938 #endif 1939 return (err); 1940 } 1941 1942 #ifndef IFNET_MULTIQUEUE 1943 static int 1944 cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) 1945 { 1946 struct sge_qset *qs; 1947 struct sge_txq *txq; 1948 struct port_info *p = ifp->if_softc; 1949 int err; 1950 1951 if (!p->link_config.link_ok) 1952 return (ENXIO); 1953 1954 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 1955 return (ENOBUFS); 1956 } 1957 1958 qs = &p->adapter->sge.qs[p->first_qset]; 1959 txq = &qs->txq[TXQ_ETH]; 1960 err = 0; 1961 1962 if (txq->flags & TXQ_TRANSMITTING) 1963 return (EINPROGRESS); 1964 1965 mtx_lock(&txq->lock); 1966 txq->flags |= TXQ_TRANSMITTING; 1967 cxgb_tx_common(ifp, qs, txmax); 1968 txq->flags &= ~TXQ_TRANSMITTING; 1969 mtx_unlock(&txq->lock); 1970 1971 return (err); 1972 } 1973 1974 static void 1975 cxgb_start_proc(void *arg, int ncount) 1976 { 1977 struct ifnet *ifp = arg; 1978 struct port_info *pi = ifp->if_softc; 1979 struct sge_qset *qs; 1980 struct sge_txq *txq; 1981 int error; 1982 1983 qs = &pi->adapter->sge.qs[pi->first_qset]; 1984 txq = &qs->txq[TXQ_ETH]; 1985 1986 do { 1987 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC >> 2) 1988 taskqueue_enqueue(pi->tq, &txq->qreclaim_task); 1989 1990 error = cxgb_start_tx(ifp, TX_START_MAX_DESC); 1991 } while (error == 0); 1992 } 1993 1994 int 1995 cxgb_dequeue_packet(struct ifnet *ifp, struct sge_txq *unused, struct mbuf **m_vec) 1996 { 1997 1998 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_vec[0]); 1999 return (m_vec[0] ? 1 : 0); 2000 } 2001 2002 void 2003 cxgb_start(struct ifnet *ifp) 2004 { 2005 struct port_info *pi = ifp->if_softc; 2006 struct sge_qset *qs; 2007 struct sge_txq *txq; 2008 int err; 2009 2010 qs = &pi->adapter->sge.qs[pi->first_qset]; 2011 txq = &qs->txq[TXQ_ETH]; 2012 2013 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC >> 2) 2014 taskqueue_enqueue(pi->tq, 2015 &txq->qreclaim_task); 2016 2017 err = cxgb_start_tx(ifp, TX_START_MAX_DESC); 2018 2019 if (err == 0) 2020 taskqueue_enqueue(pi->tq, &pi->start_task); 2021 } 2022 #endif 2023 2024 static int 2025 cxgb_media_change(struct ifnet *ifp) 2026 { 2027 if_printf(ifp, "media change not supported\n"); 2028 return (ENXIO); 2029 } 2030 2031 static void 2032 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2033 { 2034 struct port_info *p = ifp->if_softc; 2035 2036 ifmr->ifm_status = IFM_AVALID; 2037 ifmr->ifm_active = IFM_ETHER; 2038 2039 if (!p->link_config.link_ok) 2040 return; 2041 2042 ifmr->ifm_status |= IFM_ACTIVE; 2043 2044 switch (p->link_config.speed) { 2045 case 10: 2046 ifmr->ifm_active |= IFM_10_T; 2047 break; 2048 case 100: 2049 ifmr->ifm_active |= IFM_100_TX; 2050 break; 2051 case 1000: 2052 ifmr->ifm_active |= IFM_1000_T; 2053 break; 2054 } 2055 2056 if (p->link_config.duplex) 2057 ifmr->ifm_active |= IFM_FDX; 2058 else 2059 ifmr->ifm_active |= IFM_HDX; 2060 } 2061 2062 static void 2063 cxgb_async_intr(void *data) 2064 { 2065 adapter_t *sc = data; 2066 2067 if (cxgb_debug) 2068 device_printf(sc->dev, "cxgb_async_intr\n"); 2069 /* 2070 * May need to sleep - defer to taskqueue 2071 */ 2072 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2073 } 2074 2075 static void 2076 cxgb_ext_intr_handler(void *arg, int count) 2077 { 2078 adapter_t *sc = (adapter_t *)arg; 2079 2080 if (cxgb_debug) 2081 printf("cxgb_ext_intr_handler\n"); 2082 2083 t3_phy_intr_handler(sc); 2084 2085 /* Now reenable external interrupts */ 2086 ADAPTER_LOCK(sc); 2087 if (sc->slow_intr_mask) { 2088 sc->slow_intr_mask |= F_T3DBG; 2089 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 2090 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 2091 } 2092 ADAPTER_UNLOCK(sc); 2093 } 2094 2095 static void 2096 check_link_status(adapter_t *sc) 2097 { 2098 int i; 2099 2100 for (i = 0; i < (sc)->params.nports; ++i) { 2101 struct port_info *p = &sc->port[i]; 2102 2103 if (!(p->port_type->caps & SUPPORTED_IRQ)) 2104 t3_link_changed(sc, i); 2105 p->ifp->if_baudrate = p->link_config.speed * 1000000; 2106 } 2107 } 2108 2109 static void 2110 check_t3b2_mac(struct adapter *adapter) 2111 { 2112 int i; 2113 2114 for_each_port(adapter, i) { 2115 struct port_info *p = &adapter->port[i]; 2116 struct ifnet *ifp = p->ifp; 2117 int status; 2118 2119 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2120 continue; 2121 2122 status = 0; 2123 PORT_LOCK(p); 2124 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 2125 status = t3b2_mac_watchdog_task(&p->mac); 2126 if (status == 1) 2127 p->mac.stats.num_toggled++; 2128 else if (status == 2) { 2129 struct cmac *mac = &p->mac; 2130 2131 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN 2132 + ETHER_VLAN_ENCAP_LEN); 2133 t3_mac_set_address(mac, 0, p->hw_addr); 2134 cxgb_set_rxmode(p); 2135 t3_link_start(&p->phy, mac, &p->link_config); 2136 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2137 t3_port_intr_enable(adapter, p->port_id); 2138 p->mac.stats.num_resets++; 2139 } 2140 PORT_UNLOCK(p); 2141 } 2142 } 2143 2144 static void 2145 cxgb_tick(void *arg) 2146 { 2147 adapter_t *sc = (adapter_t *)arg; 2148 int i, running = 0; 2149 2150 for_each_port(sc, i) { 2151 2152 struct port_info *p = &sc->port[i]; 2153 struct ifnet *ifp = p->ifp; 2154 PORT_LOCK(p); 2155 2156 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 2157 running = 1; 2158 PORT_UNLOCK(p); 2159 } 2160 2161 if (running == 0) 2162 return; 2163 2164 taskqueue_enqueue(sc->tq, &sc->tick_task); 2165 2166 if (sc->open_device_map != 0) 2167 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 2168 } 2169 2170 static void 2171 cxgb_tick_handler(void *arg, int count) 2172 { 2173 adapter_t *sc = (adapter_t *)arg; 2174 const struct adapter_params *p = &sc->params; 2175 2176 ADAPTER_LOCK(sc); 2177 if (p->linkpoll_period) 2178 check_link_status(sc); 2179 2180 /* 2181 * adapter lock can currently only be acquire after the 2182 * port lock 2183 */ 2184 ADAPTER_UNLOCK(sc); 2185 2186 if (p->rev == T3_REV_B2 && p->nports < 4) 2187 check_t3b2_mac(sc); 2188 } 2189 2190 static void 2191 touch_bars(device_t dev) 2192 { 2193 /* 2194 * Don't enable yet 2195 */ 2196 #if !defined(__LP64__) && 0 2197 u32 v; 2198 2199 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2200 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2201 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2202 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2203 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2204 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2205 #endif 2206 } 2207 2208 static int 2209 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2210 { 2211 uint8_t *buf; 2212 int err = 0; 2213 u32 aligned_offset, aligned_len, *p; 2214 struct adapter *adapter = pi->adapter; 2215 2216 2217 aligned_offset = offset & ~3; 2218 aligned_len = (len + (offset & 3) + 3) & ~3; 2219 2220 if (aligned_offset != offset || aligned_len != len) { 2221 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2222 if (!buf) 2223 return (ENOMEM); 2224 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2225 if (!err && aligned_len > 4) 2226 err = t3_seeprom_read(adapter, 2227 aligned_offset + aligned_len - 4, 2228 (u32 *)&buf[aligned_len - 4]); 2229 if (err) 2230 goto out; 2231 memcpy(buf + (offset & 3), data, len); 2232 } else 2233 buf = (uint8_t *)(uintptr_t)data; 2234 2235 err = t3_seeprom_wp(adapter, 0); 2236 if (err) 2237 goto out; 2238 2239 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2240 err = t3_seeprom_write(adapter, aligned_offset, *p); 2241 aligned_offset += 4; 2242 } 2243 2244 if (!err) 2245 err = t3_seeprom_wp(adapter, 1); 2246 out: 2247 if (buf != data) 2248 free(buf, M_DEVBUF); 2249 return err; 2250 } 2251 2252 2253 static int 2254 in_range(int val, int lo, int hi) 2255 { 2256 return val < 0 || (val <= hi && val >= lo); 2257 } 2258 2259 static int 2260 cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td) 2261 { 2262 return (0); 2263 } 2264 2265 static int 2266 cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td) 2267 { 2268 return (0); 2269 } 2270 2271 static int 2272 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2273 int fflag, struct thread *td) 2274 { 2275 int mmd, error = 0; 2276 struct port_info *pi = dev->si_drv1; 2277 adapter_t *sc = pi->adapter; 2278 2279 #ifdef PRIV_SUPPORTED 2280 if (priv_check(td, PRIV_DRIVER)) { 2281 if (cxgb_debug) 2282 printf("user does not have access to privileged ioctls\n"); 2283 return (EPERM); 2284 } 2285 #else 2286 if (suser(td)) { 2287 if (cxgb_debug) 2288 printf("user does not have access to privileged ioctls\n"); 2289 return (EPERM); 2290 } 2291 #endif 2292 2293 switch (cmd) { 2294 case SIOCGMIIREG: { 2295 uint32_t val; 2296 struct cphy *phy = &pi->phy; 2297 struct mii_data *mid = (struct mii_data *)data; 2298 2299 if (!phy->mdio_read) 2300 return (EOPNOTSUPP); 2301 if (is_10G(sc)) { 2302 mmd = mid->phy_id >> 8; 2303 if (!mmd) 2304 mmd = MDIO_DEV_PCS; 2305 else if (mmd > MDIO_DEV_XGXS) 2306 return (EINVAL); 2307 2308 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2309 mid->reg_num, &val); 2310 } else 2311 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2312 mid->reg_num & 0x1f, &val); 2313 if (error == 0) 2314 mid->val_out = val; 2315 break; 2316 } 2317 case SIOCSMIIREG: { 2318 struct cphy *phy = &pi->phy; 2319 struct mii_data *mid = (struct mii_data *)data; 2320 2321 if (!phy->mdio_write) 2322 return (EOPNOTSUPP); 2323 if (is_10G(sc)) { 2324 mmd = mid->phy_id >> 8; 2325 if (!mmd) 2326 mmd = MDIO_DEV_PCS; 2327 else if (mmd > MDIO_DEV_XGXS) 2328 return (EINVAL); 2329 2330 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2331 mmd, mid->reg_num, mid->val_in); 2332 } else 2333 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2334 mid->reg_num & 0x1f, 2335 mid->val_in); 2336 break; 2337 } 2338 case CHELSIO_SETREG: { 2339 struct ch_reg *edata = (struct ch_reg *)data; 2340 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2341 return (EFAULT); 2342 t3_write_reg(sc, edata->addr, edata->val); 2343 break; 2344 } 2345 case CHELSIO_GETREG: { 2346 struct ch_reg *edata = (struct ch_reg *)data; 2347 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2348 return (EFAULT); 2349 edata->val = t3_read_reg(sc, edata->addr); 2350 break; 2351 } 2352 case CHELSIO_GET_SGE_CONTEXT: { 2353 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2354 mtx_lock(&sc->sge.reg_lock); 2355 switch (ecntxt->cntxt_type) { 2356 case CNTXT_TYPE_EGRESS: 2357 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2358 ecntxt->data); 2359 break; 2360 case CNTXT_TYPE_FL: 2361 error = t3_sge_read_fl(sc, ecntxt->cntxt_id, 2362 ecntxt->data); 2363 break; 2364 case CNTXT_TYPE_RSP: 2365 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2366 ecntxt->data); 2367 break; 2368 case CNTXT_TYPE_CQ: 2369 error = t3_sge_read_cq(sc, ecntxt->cntxt_id, 2370 ecntxt->data); 2371 break; 2372 default: 2373 error = EINVAL; 2374 break; 2375 } 2376 mtx_unlock(&sc->sge.reg_lock); 2377 break; 2378 } 2379 case CHELSIO_GET_SGE_DESC: { 2380 struct ch_desc *edesc = (struct ch_desc *)data; 2381 int ret; 2382 if (edesc->queue_num >= SGE_QSETS * 6) 2383 return (EINVAL); 2384 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2385 edesc->queue_num % 6, edesc->idx, edesc->data); 2386 if (ret < 0) 2387 return (EINVAL); 2388 edesc->size = ret; 2389 break; 2390 } 2391 case CHELSIO_SET_QSET_PARAMS: { 2392 struct qset_params *q; 2393 struct ch_qset_params *t = (struct ch_qset_params *)data; 2394 2395 if (t->qset_idx >= SGE_QSETS) 2396 return (EINVAL); 2397 if (!in_range(t->intr_lat, 0, M_NEWTIMER) || 2398 !in_range(t->cong_thres, 0, 255) || 2399 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES, 2400 MAX_TXQ_ENTRIES) || 2401 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES, 2402 MAX_TXQ_ENTRIES) || 2403 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES, 2404 MAX_CTRL_TXQ_ENTRIES) || 2405 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) || 2406 !in_range(t->fl_size[1], MIN_FL_ENTRIES, 2407 MAX_RX_JUMBO_BUFFERS) || 2408 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES)) 2409 return (EINVAL); 2410 if ((sc->flags & FULL_INIT_DONE) && 2411 (t->rspq_size >= 0 || t->fl_size[0] >= 0 || 2412 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 || 2413 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 || 2414 t->polling >= 0 || t->cong_thres >= 0)) 2415 return (EBUSY); 2416 2417 q = &sc->params.sge.qset[t->qset_idx]; 2418 2419 if (t->rspq_size >= 0) 2420 q->rspq_size = t->rspq_size; 2421 if (t->fl_size[0] >= 0) 2422 q->fl_size = t->fl_size[0]; 2423 if (t->fl_size[1] >= 0) 2424 q->jumbo_size = t->fl_size[1]; 2425 if (t->txq_size[0] >= 0) 2426 q->txq_size[0] = t->txq_size[0]; 2427 if (t->txq_size[1] >= 0) 2428 q->txq_size[1] = t->txq_size[1]; 2429 if (t->txq_size[2] >= 0) 2430 q->txq_size[2] = t->txq_size[2]; 2431 if (t->cong_thres >= 0) 2432 q->cong_thres = t->cong_thres; 2433 if (t->intr_lat >= 0) { 2434 struct sge_qset *qs = &sc->sge.qs[t->qset_idx]; 2435 2436 q->coalesce_nsecs = t->intr_lat*1000; 2437 t3_update_qset_coalesce(qs, q); 2438 } 2439 break; 2440 } 2441 case CHELSIO_GET_QSET_PARAMS: { 2442 struct qset_params *q; 2443 struct ch_qset_params *t = (struct ch_qset_params *)data; 2444 2445 if (t->qset_idx >= SGE_QSETS) 2446 return (EINVAL); 2447 2448 q = &(sc)->params.sge.qset[t->qset_idx]; 2449 t->rspq_size = q->rspq_size; 2450 t->txq_size[0] = q->txq_size[0]; 2451 t->txq_size[1] = q->txq_size[1]; 2452 t->txq_size[2] = q->txq_size[2]; 2453 t->fl_size[0] = q->fl_size; 2454 t->fl_size[1] = q->jumbo_size; 2455 t->polling = q->polling; 2456 t->intr_lat = q->coalesce_nsecs / 1000; 2457 t->cong_thres = q->cong_thres; 2458 break; 2459 } 2460 case CHELSIO_SET_QSET_NUM: { 2461 struct ch_reg *edata = (struct ch_reg *)data; 2462 unsigned int port_idx = pi->port_id; 2463 2464 if (sc->flags & FULL_INIT_DONE) 2465 return (EBUSY); 2466 if (edata->val < 1 || 2467 (edata->val > 1 && !(sc->flags & USING_MSIX))) 2468 return (EINVAL); 2469 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS) 2470 return (EINVAL); 2471 sc->port[port_idx].nqsets = edata->val; 2472 sc->port[0].first_qset = 0; 2473 /* 2474 * XXX hardcode ourselves to 2 ports just like LEEENUX 2475 */ 2476 sc->port[1].first_qset = sc->port[0].nqsets; 2477 break; 2478 } 2479 case CHELSIO_GET_QSET_NUM: { 2480 struct ch_reg *edata = (struct ch_reg *)data; 2481 edata->val = pi->nqsets; 2482 break; 2483 } 2484 #ifdef notyet 2485 case CHELSIO_LOAD_FW: 2486 case CHELSIO_GET_PM: 2487 case CHELSIO_SET_PM: 2488 return (EOPNOTSUPP); 2489 break; 2490 #endif 2491 case CHELSIO_SETMTUTAB: { 2492 struct ch_mtus *m = (struct ch_mtus *)data; 2493 int i; 2494 2495 if (!is_offload(sc)) 2496 return (EOPNOTSUPP); 2497 if (offload_running(sc)) 2498 return (EBUSY); 2499 if (m->nmtus != NMTUS) 2500 return (EINVAL); 2501 if (m->mtus[0] < 81) /* accommodate SACK */ 2502 return (EINVAL); 2503 2504 /* 2505 * MTUs must be in ascending order 2506 */ 2507 for (i = 1; i < NMTUS; ++i) 2508 if (m->mtus[i] < m->mtus[i - 1]) 2509 return (EINVAL); 2510 2511 memcpy(sc->params.mtus, m->mtus, 2512 sizeof(sc->params.mtus)); 2513 break; 2514 } 2515 case CHELSIO_GETMTUTAB: { 2516 struct ch_mtus *m = (struct ch_mtus *)data; 2517 2518 if (!is_offload(sc)) 2519 return (EOPNOTSUPP); 2520 2521 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2522 m->nmtus = NMTUS; 2523 break; 2524 } 2525 case CHELSIO_DEVUP: 2526 if (!is_offload(sc)) 2527 return (EOPNOTSUPP); 2528 return offload_open(pi); 2529 break; 2530 case CHELSIO_GET_MEM: { 2531 struct ch_mem_range *t = (struct ch_mem_range *)data; 2532 struct mc7 *mem; 2533 uint8_t *useraddr; 2534 u64 buf[32]; 2535 2536 if (!is_offload(sc)) 2537 return (EOPNOTSUPP); 2538 if (!(sc->flags & FULL_INIT_DONE)) 2539 return (EIO); /* need the memory controllers */ 2540 if ((t->addr & 0x7) || (t->len & 0x7)) 2541 return (EINVAL); 2542 if (t->mem_id == MEM_CM) 2543 mem = &sc->cm; 2544 else if (t->mem_id == MEM_PMRX) 2545 mem = &sc->pmrx; 2546 else if (t->mem_id == MEM_PMTX) 2547 mem = &sc->pmtx; 2548 else 2549 return (EINVAL); 2550 2551 /* 2552 * Version scheme: 2553 * bits 0..9: chip version 2554 * bits 10..15: chip revision 2555 */ 2556 t->version = 3 | (sc->params.rev << 10); 2557 2558 /* 2559 * Read 256 bytes at a time as len can be large and we don't 2560 * want to use huge intermediate buffers. 2561 */ 2562 useraddr = (uint8_t *)t->buf; 2563 while (t->len) { 2564 unsigned int chunk = min(t->len, sizeof(buf)); 2565 2566 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf); 2567 if (error) 2568 return (-error); 2569 if (copyout(buf, useraddr, chunk)) 2570 return (EFAULT); 2571 useraddr += chunk; 2572 t->addr += chunk; 2573 t->len -= chunk; 2574 } 2575 break; 2576 } 2577 case CHELSIO_READ_TCAM_WORD: { 2578 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2579 2580 if (!is_offload(sc)) 2581 return (EOPNOTSUPP); 2582 if (!(sc->flags & FULL_INIT_DONE)) 2583 return (EIO); /* need MC5 */ 2584 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2585 break; 2586 } 2587 case CHELSIO_SET_TRACE_FILTER: { 2588 struct ch_trace *t = (struct ch_trace *)data; 2589 const struct trace_params *tp; 2590 2591 tp = (const struct trace_params *)&t->sip; 2592 if (t->config_tx) 2593 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2594 t->trace_tx); 2595 if (t->config_rx) 2596 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2597 t->trace_rx); 2598 break; 2599 } 2600 case CHELSIO_SET_PKTSCHED: { 2601 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2602 if (sc->open_device_map == 0) 2603 return (EAGAIN); 2604 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2605 p->binding); 2606 break; 2607 } 2608 case CHELSIO_IFCONF_GETREGS: { 2609 struct ifconf_regs *regs = (struct ifconf_regs *)data; 2610 int reglen = cxgb_get_regs_len(); 2611 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT); 2612 if (buf == NULL) { 2613 return (ENOMEM); 2614 } if (regs->len > reglen) 2615 regs->len = reglen; 2616 else if (regs->len < reglen) { 2617 error = E2BIG; 2618 goto done; 2619 } 2620 cxgb_get_regs(sc, regs, buf); 2621 error = copyout(buf, regs->data, reglen); 2622 2623 done: 2624 free(buf, M_DEVBUF); 2625 2626 break; 2627 } 2628 case CHELSIO_SET_HW_SCHED: { 2629 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2630 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2631 2632 if ((sc->flags & FULL_INIT_DONE) == 0) 2633 return (EAGAIN); /* need TP to be initialized */ 2634 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2635 !in_range(t->channel, 0, 1) || 2636 !in_range(t->kbps, 0, 10000000) || 2637 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2638 !in_range(t->flow_ipg, 0, 2639 dack_ticks_to_usec(sc, 0x7ff))) 2640 return (EINVAL); 2641 2642 if (t->kbps >= 0) { 2643 error = t3_config_sched(sc, t->kbps, t->sched); 2644 if (error < 0) 2645 return (-error); 2646 } 2647 if (t->class_ipg >= 0) 2648 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2649 if (t->flow_ipg >= 0) { 2650 t->flow_ipg *= 1000; /* us -> ns */ 2651 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2652 } 2653 if (t->mode >= 0) { 2654 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2655 2656 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2657 bit, t->mode ? bit : 0); 2658 } 2659 if (t->channel >= 0) 2660 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2661 1 << t->sched, t->channel << t->sched); 2662 break; 2663 } 2664 default: 2665 return (EOPNOTSUPP); 2666 break; 2667 } 2668 2669 return (error); 2670 } 2671 2672 static __inline void 2673 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 2674 unsigned int end) 2675 { 2676 uint32_t *p = (uint32_t *)buf + start; 2677 2678 for ( ; start <= end; start += sizeof(uint32_t)) 2679 *p++ = t3_read_reg(ap, start); 2680 } 2681 2682 #define T3_REGMAP_SIZE (3 * 1024) 2683 static int 2684 cxgb_get_regs_len(void) 2685 { 2686 return T3_REGMAP_SIZE; 2687 } 2688 #undef T3_REGMAP_SIZE 2689 2690 static void 2691 cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf) 2692 { 2693 2694 /* 2695 * Version scheme: 2696 * bits 0..9: chip version 2697 * bits 10..15: chip revision 2698 * bit 31: set for PCIe cards 2699 */ 2700 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 2701 2702 /* 2703 * We skip the MAC statistics registers because they are clear-on-read. 2704 * Also reading multi-register stats would need to synchronize with the 2705 * periodic mac stats accumulation. Hard to justify the complexity. 2706 */ 2707 memset(buf, 0, REGDUMP_SIZE); 2708 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 2709 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 2710 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 2711 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 2712 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 2713 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 2714 XGM_REG(A_XGM_SERDES_STAT3, 1)); 2715 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 2716 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 2717 } 2718