1 /************************************************************************** 2 3 Copyright (c) 2007-2008, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 POSSIBILITY OF SUCH DAMAGE. 27 28 ***************************************************************************/ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/bus.h> 37 #include <sys/module.h> 38 #include <sys/pciio.h> 39 #include <sys/conf.h> 40 #include <machine/bus.h> 41 #include <machine/resource.h> 42 #include <sys/bus_dma.h> 43 #include <sys/ktr.h> 44 #include <sys/rman.h> 45 #include <sys/ioccom.h> 46 #include <sys/mbuf.h> 47 #include <sys/linker.h> 48 #include <sys/firmware.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/smp.h> 52 #include <sys/sysctl.h> 53 #include <sys/syslog.h> 54 #include <sys/queue.h> 55 #include <sys/taskqueue.h> 56 #include <sys/proc.h> 57 58 #include <net/bpf.h> 59 #include <net/ethernet.h> 60 #include <net/if.h> 61 #include <net/if_arp.h> 62 #include <net/if_dl.h> 63 #include <net/if_media.h> 64 #include <net/if_types.h> 65 #include <net/if_vlan_var.h> 66 67 #include <netinet/in_systm.h> 68 #include <netinet/in.h> 69 #include <netinet/if_ether.h> 70 #include <netinet/ip.h> 71 #include <netinet/ip.h> 72 #include <netinet/tcp.h> 73 #include <netinet/udp.h> 74 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 #include <dev/pci/pci_private.h> 78 79 #include <cxgb_include.h> 80 81 #ifdef PRIV_SUPPORTED 82 #include <sys/priv.h> 83 #endif 84 85 #ifdef IFNET_MULTIQUEUE 86 #include <machine/intr_machdep.h> 87 #endif 88 89 static int cxgb_setup_msix(adapter_t *, int); 90 static void cxgb_teardown_msix(adapter_t *); 91 static void cxgb_init(void *); 92 static void cxgb_init_locked(struct port_info *); 93 static void cxgb_stop_locked(struct port_info *); 94 static void cxgb_set_rxmode(struct port_info *); 95 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 96 static int cxgb_media_change(struct ifnet *); 97 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 98 static int setup_sge_qsets(adapter_t *); 99 static void cxgb_async_intr(void *); 100 static void cxgb_ext_intr_handler(void *, int); 101 static void cxgb_tick_handler(void *, int); 102 static void cxgb_down_locked(struct adapter *sc); 103 static void cxgb_tick(void *); 104 static void setup_rss(adapter_t *sc); 105 106 /* Attachment glue for the PCI controller end of the device. Each port of 107 * the device is attached separately, as defined later. 108 */ 109 static int cxgb_controller_probe(device_t); 110 static int cxgb_controller_attach(device_t); 111 static int cxgb_controller_detach(device_t); 112 static void cxgb_free(struct adapter *); 113 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 114 unsigned int end); 115 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 116 static int cxgb_get_regs_len(void); 117 static int offload_open(struct port_info *pi); 118 static void touch_bars(device_t dev); 119 static int offload_close(struct t3cdev *tdev); 120 static void cxgb_link_start(struct port_info *p); 121 122 static device_method_t cxgb_controller_methods[] = { 123 DEVMETHOD(device_probe, cxgb_controller_probe), 124 DEVMETHOD(device_attach, cxgb_controller_attach), 125 DEVMETHOD(device_detach, cxgb_controller_detach), 126 127 /* bus interface */ 128 DEVMETHOD(bus_print_child, bus_generic_print_child), 129 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 130 131 { 0, 0 } 132 }; 133 134 static driver_t cxgb_controller_driver = { 135 "cxgbc", 136 cxgb_controller_methods, 137 sizeof(struct adapter) 138 }; 139 140 static devclass_t cxgb_controller_devclass; 141 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 142 143 /* 144 * Attachment glue for the ports. Attachment is done directly to the 145 * controller device. 146 */ 147 static int cxgb_port_probe(device_t); 148 static int cxgb_port_attach(device_t); 149 static int cxgb_port_detach(device_t); 150 151 static device_method_t cxgb_port_methods[] = { 152 DEVMETHOD(device_probe, cxgb_port_probe), 153 DEVMETHOD(device_attach, cxgb_port_attach), 154 DEVMETHOD(device_detach, cxgb_port_detach), 155 { 0, 0 } 156 }; 157 158 static driver_t cxgb_port_driver = { 159 "cxgb", 160 cxgb_port_methods, 161 0 162 }; 163 164 static d_ioctl_t cxgb_extension_ioctl; 165 static d_open_t cxgb_extension_open; 166 static d_close_t cxgb_extension_close; 167 168 static struct cdevsw cxgb_cdevsw = { 169 .d_version = D_VERSION, 170 .d_flags = 0, 171 .d_open = cxgb_extension_open, 172 .d_close = cxgb_extension_close, 173 .d_ioctl = cxgb_extension_ioctl, 174 .d_name = "cxgb", 175 }; 176 177 static devclass_t cxgb_port_devclass; 178 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 179 180 #define SGE_MSIX_COUNT (SGE_QSETS + 1) 181 182 /* 183 * The driver uses the best interrupt scheme available on a platform in the 184 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 185 * of these schemes the driver may consider as follows: 186 * 187 * msi = 2: choose from among all three options 188 * msi = 1 : only consider MSI and pin interrupts 189 * msi = 0: force pin interrupts 190 */ 191 static int msi_allowed = 2; 192 193 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 194 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 195 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 196 "MSI-X, MSI, INTx selector"); 197 198 /* 199 * The driver enables offload as a default. 200 * To disable it, use ofld_disable = 1. 201 */ 202 static int ofld_disable = 0; 203 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable); 204 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0, 205 "disable ULP offload"); 206 207 /* 208 * The driver uses an auto-queue algorithm by default. 209 * To disable it and force a single queue-set per port, use singleq = 1. 210 */ 211 static int singleq = 0; 212 TUNABLE_INT("hw.cxgb.singleq", &singleq); 213 SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0, 214 "use a single queue-set per port"); 215 216 217 /* 218 * The driver uses an auto-queue algorithm by default. 219 * To disable it and force a single queue-set per port, use singleq = 1. 220 */ 221 static int force_fw_update = 0; 222 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update); 223 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 224 "update firmware even if up to date"); 225 226 int cxgb_use_16k_clusters = 1; 227 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters); 228 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 229 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 230 231 enum { 232 MAX_TXQ_ENTRIES = 16384, 233 MAX_CTRL_TXQ_ENTRIES = 1024, 234 MAX_RSPQ_ENTRIES = 16384, 235 MAX_RX_BUFFERS = 16384, 236 MAX_RX_JUMBO_BUFFERS = 16384, 237 MIN_TXQ_ENTRIES = 4, 238 MIN_CTRL_TXQ_ENTRIES = 4, 239 MIN_RSPQ_ENTRIES = 32, 240 MIN_FL_ENTRIES = 32, 241 MIN_FL_JUMBO_ENTRIES = 32 242 }; 243 244 struct filter_info { 245 u32 sip; 246 u32 sip_mask; 247 u32 dip; 248 u16 sport; 249 u16 dport; 250 u32 vlan:12; 251 u32 vlan_prio:3; 252 u32 mac_hit:1; 253 u32 mac_idx:4; 254 u32 mac_vld:1; 255 u32 pkt_type:2; 256 u32 report_filter_id:1; 257 u32 pass:1; 258 u32 rss:1; 259 u32 qset:3; 260 u32 locked:1; 261 u32 valid:1; 262 }; 263 264 enum { FILTER_NO_VLAN_PRI = 7 }; 265 266 #define EEPROM_MAGIC 0x38E2F10C 267 268 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 269 270 /* Table for probing the cards. The desc field isn't actually used */ 271 struct cxgb_ident { 272 uint16_t vendor; 273 uint16_t device; 274 int index; 275 char *desc; 276 } cxgb_identifiers[] = { 277 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 278 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 279 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 280 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 281 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 282 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 283 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 284 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 285 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 286 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 287 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 288 {0, 0, 0, NULL} 289 }; 290 291 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 292 293 294 static __inline char 295 t3rev2char(struct adapter *adapter) 296 { 297 char rev = 'z'; 298 299 switch(adapter->params.rev) { 300 case T3_REV_A: 301 rev = 'a'; 302 break; 303 case T3_REV_B: 304 case T3_REV_B2: 305 rev = 'b'; 306 break; 307 case T3_REV_C: 308 rev = 'c'; 309 break; 310 } 311 return rev; 312 } 313 314 static struct cxgb_ident * 315 cxgb_get_ident(device_t dev) 316 { 317 struct cxgb_ident *id; 318 319 for (id = cxgb_identifiers; id->desc != NULL; id++) { 320 if ((id->vendor == pci_get_vendor(dev)) && 321 (id->device == pci_get_device(dev))) { 322 return (id); 323 } 324 } 325 return (NULL); 326 } 327 328 static const struct adapter_info * 329 cxgb_get_adapter_info(device_t dev) 330 { 331 struct cxgb_ident *id; 332 const struct adapter_info *ai; 333 334 id = cxgb_get_ident(dev); 335 if (id == NULL) 336 return (NULL); 337 338 ai = t3_get_adapter_info(id->index); 339 340 return (ai); 341 } 342 343 static int 344 cxgb_controller_probe(device_t dev) 345 { 346 const struct adapter_info *ai; 347 char *ports, buf[80]; 348 int nports; 349 struct adapter *sc = device_get_softc(dev); 350 351 ai = cxgb_get_adapter_info(dev); 352 if (ai == NULL) 353 return (ENXIO); 354 355 nports = ai->nports0 + ai->nports1; 356 if (nports == 1) 357 ports = "port"; 358 else 359 ports = "ports"; 360 361 snprintf(buf, sizeof(buf), "%s %sNIC, rev: %d nports: %d %s", 362 ai->desc, is_offload(sc) ? "R" : "", 363 sc->params.rev, nports, ports); 364 device_set_desc_copy(dev, buf); 365 return (BUS_PROBE_DEFAULT); 366 } 367 368 #define FW_FNAME "cxgb_t3fw" 369 #define TPEEPROM_NAME "t3b_tp_eeprom" 370 #define TPSRAM_NAME "t3b_protocol_sram" 371 372 static int 373 upgrade_fw(adapter_t *sc) 374 { 375 #ifdef FIRMWARE_LATEST 376 const struct firmware *fw; 377 #else 378 struct firmware *fw; 379 #endif 380 int status; 381 382 if ((fw = firmware_get(FW_FNAME)) == NULL) { 383 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 384 return (ENOENT); 385 } else 386 device_printf(sc->dev, "updating firmware on card\n"); 387 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 388 389 device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status); 390 391 firmware_put(fw, FIRMWARE_UNLOAD); 392 393 return (status); 394 } 395 396 static int 397 cxgb_controller_attach(device_t dev) 398 { 399 device_t child; 400 const struct adapter_info *ai; 401 struct adapter *sc; 402 int i, error = 0; 403 uint32_t vers; 404 int port_qsets = 1; 405 #ifdef MSI_SUPPORTED 406 int msi_needed, reg; 407 #endif 408 int must_load = 0; 409 sc = device_get_softc(dev); 410 sc->dev = dev; 411 sc->msi_count = 0; 412 ai = cxgb_get_adapter_info(dev); 413 414 /* 415 * XXX not really related but a recent addition 416 */ 417 #ifdef MSI_SUPPORTED 418 /* find the PCIe link width and set max read request to 4KB*/ 419 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 420 uint16_t lnk, pectl; 421 lnk = pci_read_config(dev, reg + 0x12, 2); 422 sc->link_width = (lnk >> 4) & 0x3f; 423 424 pectl = pci_read_config(dev, reg + 0x8, 2); 425 pectl = (pectl & ~0x7000) | (5 << 12); 426 pci_write_config(dev, reg + 0x8, pectl, 2); 427 } 428 429 if (sc->link_width != 0 && sc->link_width <= 4 && 430 (ai->nports0 + ai->nports1) <= 2) { 431 device_printf(sc->dev, 432 "PCIe x%d Link, expect reduced performance\n", 433 sc->link_width); 434 } 435 #endif 436 touch_bars(dev); 437 pci_enable_busmaster(dev); 438 /* 439 * Allocate the registers and make them available to the driver. 440 * The registers that we care about for NIC mode are in BAR 0 441 */ 442 sc->regs_rid = PCIR_BAR(0); 443 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 444 &sc->regs_rid, RF_ACTIVE)) == NULL) { 445 device_printf(dev, "Cannot allocate BAR region 0\n"); 446 return (ENXIO); 447 } 448 sc->udbs_rid = PCIR_BAR(2); 449 if ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 450 &sc->udbs_rid, RF_ACTIVE)) == NULL) { 451 device_printf(dev, "Cannot allocate BAR region 1\n"); 452 error = ENXIO; 453 goto out; 454 } 455 456 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 457 device_get_unit(dev)); 458 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 459 460 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 461 device_get_unit(dev)); 462 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 463 device_get_unit(dev)); 464 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 465 device_get_unit(dev)); 466 467 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 468 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 469 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 470 471 sc->bt = rman_get_bustag(sc->regs_res); 472 sc->bh = rman_get_bushandle(sc->regs_res); 473 sc->mmio_len = rman_get_size(sc->regs_res); 474 475 if (t3_prep_adapter(sc, ai, 1) < 0) { 476 printf("prep adapter failed\n"); 477 error = ENODEV; 478 goto out; 479 } 480 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 481 * enough messages for the queue sets. If that fails, try falling 482 * back to MSI. If that fails, then try falling back to the legacy 483 * interrupt pin model. 484 */ 485 #ifdef MSI_SUPPORTED 486 487 sc->msix_regs_rid = 0x20; 488 if ((msi_allowed >= 2) && 489 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 490 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 491 492 msi_needed = sc->msi_count = SGE_MSIX_COUNT; 493 494 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) || 495 (sc->msi_count != msi_needed)) { 496 device_printf(dev, "msix allocation failed - msi_count = %d" 497 " msi_needed=%d will try msi err=%d\n", sc->msi_count, 498 msi_needed, error); 499 sc->msi_count = 0; 500 pci_release_msi(dev); 501 bus_release_resource(dev, SYS_RES_MEMORY, 502 sc->msix_regs_rid, sc->msix_regs_res); 503 sc->msix_regs_res = NULL; 504 } else { 505 sc->flags |= USING_MSIX; 506 sc->cxgb_intr = t3_intr_msix; 507 } 508 } 509 510 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 511 sc->msi_count = 1; 512 if (pci_alloc_msi(dev, &sc->msi_count)) { 513 device_printf(dev, "alloc msi failed - will try INTx\n"); 514 sc->msi_count = 0; 515 pci_release_msi(dev); 516 } else { 517 sc->flags |= USING_MSI; 518 sc->irq_rid = 1; 519 sc->cxgb_intr = t3_intr_msi; 520 } 521 } 522 #endif 523 if (sc->msi_count == 0) { 524 device_printf(dev, "using line interrupts\n"); 525 sc->irq_rid = 0; 526 sc->cxgb_intr = t3b_intr; 527 } 528 529 if ((sc->flags & USING_MSIX) && !singleq) 530 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); 531 532 /* Create a private taskqueue thread for handling driver events */ 533 #ifdef TASKQUEUE_CURRENT 534 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 535 taskqueue_thread_enqueue, &sc->tq); 536 #else 537 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 538 taskqueue_thread_enqueue, &sc->tq); 539 #endif 540 if (sc->tq == NULL) { 541 device_printf(dev, "failed to allocate controller task queue\n"); 542 goto out; 543 } 544 545 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 546 device_get_nameunit(dev)); 547 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 548 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 549 550 551 /* Create a periodic callout for checking adapter status */ 552 callout_init(&sc->cxgb_tick_ch, TRUE); 553 554 if ((t3_check_fw_version(sc, &must_load) != 0 && must_load) || force_fw_update) { 555 /* 556 * Warn user that a firmware update will be attempted in init. 557 */ 558 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 559 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 560 sc->flags &= ~FW_UPTODATE; 561 } else { 562 sc->flags |= FW_UPTODATE; 563 } 564 565 if (t3_check_tpsram_version(sc, &must_load) != 0 && must_load) { 566 /* 567 * Warn user that a firmware update will be attempted in init. 568 */ 569 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 570 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 571 sc->flags &= ~TPS_UPTODATE; 572 } else { 573 sc->flags |= TPS_UPTODATE; 574 } 575 576 /* 577 * Create a child device for each MAC. The ethernet attachment 578 * will be done in these children. 579 */ 580 for (i = 0; i < (sc)->params.nports; i++) { 581 struct port_info *pi; 582 583 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 584 device_printf(dev, "failed to add child port\n"); 585 error = EINVAL; 586 goto out; 587 } 588 pi = &sc->port[i]; 589 pi->adapter = sc; 590 pi->nqsets = port_qsets; 591 pi->first_qset = i*port_qsets; 592 pi->port_id = i; 593 pi->tx_chan = i >= ai->nports0; 594 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 595 sc->rxpkt_map[pi->txpkt_intf] = i; 596 sc->port[i].tx_chan = i >= ai->nports0; 597 sc->portdev[i] = child; 598 device_set_softc(child, pi); 599 } 600 if ((error = bus_generic_attach(dev)) != 0) 601 goto out; 602 603 /* initialize sge private state */ 604 t3_sge_init_adapter(sc); 605 606 t3_led_ready(sc); 607 608 cxgb_offload_init(); 609 if (is_offload(sc)) { 610 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); 611 cxgb_adapter_ofld(sc); 612 } 613 error = t3_get_fw_version(sc, &vers); 614 if (error) 615 goto out; 616 617 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 618 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 619 G_FW_VERSION_MICRO(vers)); 620 621 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 622 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 623 t3_add_attach_sysctls(sc); 624 out: 625 if (error) 626 cxgb_free(sc); 627 628 return (error); 629 } 630 631 static int 632 cxgb_controller_detach(device_t dev) 633 { 634 struct adapter *sc; 635 636 sc = device_get_softc(dev); 637 638 cxgb_free(sc); 639 640 return (0); 641 } 642 643 static void 644 cxgb_free(struct adapter *sc) 645 { 646 int i; 647 648 ADAPTER_LOCK(sc); 649 sc->flags |= CXGB_SHUTDOWN; 650 ADAPTER_UNLOCK(sc); 651 cxgb_pcpu_shutdown_threads(sc); 652 ADAPTER_LOCK(sc); 653 654 /* 655 * drops the lock 656 */ 657 cxgb_down_locked(sc); 658 659 #ifdef MSI_SUPPORTED 660 if (sc->flags & (USING_MSI | USING_MSIX)) { 661 device_printf(sc->dev, "releasing msi message(s)\n"); 662 pci_release_msi(sc->dev); 663 } else { 664 device_printf(sc->dev, "no msi message to release\n"); 665 } 666 #endif 667 if (sc->msix_regs_res != NULL) { 668 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 669 sc->msix_regs_res); 670 } 671 672 t3_sge_deinit_sw(sc); 673 /* 674 * Wait for last callout 675 */ 676 677 DELAY(hz*100); 678 679 for (i = 0; i < (sc)->params.nports; ++i) { 680 if (sc->portdev[i] != NULL) 681 device_delete_child(sc->dev, sc->portdev[i]); 682 } 683 684 bus_generic_detach(sc->dev); 685 if (sc->tq != NULL) { 686 taskqueue_free(sc->tq); 687 sc->tq = NULL; 688 } 689 690 if (is_offload(sc)) { 691 cxgb_adapter_unofld(sc); 692 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT)) 693 offload_close(&sc->tdev); 694 else 695 printf("cxgb_free: DEVMAP_BIT not set\n"); 696 } else 697 printf("not offloading set\n"); 698 #ifdef notyet 699 if (sc->flags & CXGB_OFLD_INIT) 700 cxgb_offload_deactivate(sc); 701 #endif 702 free(sc->filters, M_DEVBUF); 703 t3_sge_free(sc); 704 705 cxgb_offload_exit(); 706 707 if (sc->udbs_res != NULL) 708 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 709 sc->udbs_res); 710 711 if (sc->regs_res != NULL) 712 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 713 sc->regs_res); 714 715 MTX_DESTROY(&sc->mdio_lock); 716 MTX_DESTROY(&sc->sge.reg_lock); 717 MTX_DESTROY(&sc->elmer_lock); 718 ADAPTER_LOCK_DEINIT(sc); 719 } 720 721 /** 722 * setup_sge_qsets - configure SGE Tx/Rx/response queues 723 * @sc: the controller softc 724 * 725 * Determines how many sets of SGE queues to use and initializes them. 726 * We support multiple queue sets per port if we have MSI-X, otherwise 727 * just one queue set per port. 728 */ 729 static int 730 setup_sge_qsets(adapter_t *sc) 731 { 732 int i, j, err, irq_idx = 0, qset_idx = 0; 733 u_int ntxq = SGE_TXQ_PER_SET; 734 735 if ((err = t3_sge_alloc(sc)) != 0) { 736 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 737 return (err); 738 } 739 740 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 741 irq_idx = -1; 742 743 for (i = 0; i < (sc)->params.nports; i++) { 744 struct port_info *pi = &sc->port[i]; 745 746 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 747 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 748 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 749 &sc->params.sge.qset[qset_idx], ntxq, pi); 750 if (err) { 751 t3_free_sge_resources(sc); 752 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", 753 err); 754 return (err); 755 } 756 } 757 } 758 759 return (0); 760 } 761 762 static void 763 cxgb_teardown_msix(adapter_t *sc) 764 { 765 int i, nqsets; 766 767 for (nqsets = i = 0; i < (sc)->params.nports; i++) 768 nqsets += sc->port[i].nqsets; 769 770 for (i = 0; i < nqsets; i++) { 771 if (sc->msix_intr_tag[i] != NULL) { 772 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 773 sc->msix_intr_tag[i]); 774 sc->msix_intr_tag[i] = NULL; 775 } 776 if (sc->msix_irq_res[i] != NULL) { 777 bus_release_resource(sc->dev, SYS_RES_IRQ, 778 sc->msix_irq_rid[i], sc->msix_irq_res[i]); 779 sc->msix_irq_res[i] = NULL; 780 } 781 } 782 } 783 784 static int 785 cxgb_setup_msix(adapter_t *sc, int msix_count) 786 { 787 int i, j, k, nqsets, rid; 788 789 /* The first message indicates link changes and error conditions */ 790 sc->irq_rid = 1; 791 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 792 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 793 device_printf(sc->dev, "Cannot allocate msix interrupt\n"); 794 return (EINVAL); 795 } 796 797 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 798 #ifdef INTR_FILTERS 799 NULL, 800 #endif 801 cxgb_async_intr, sc, &sc->intr_tag)) { 802 device_printf(sc->dev, "Cannot set up interrupt\n"); 803 return (EINVAL); 804 } 805 for (i = k = 0; i < (sc)->params.nports; i++) { 806 nqsets = sc->port[i].nqsets; 807 for (j = 0; j < nqsets; j++, k++) { 808 struct sge_qset *qs = &sc->sge.qs[k]; 809 810 rid = k + 2; 811 if (cxgb_debug) 812 printf("rid=%d ", rid); 813 if ((sc->msix_irq_res[k] = bus_alloc_resource_any( 814 sc->dev, SYS_RES_IRQ, &rid, 815 RF_SHAREABLE | RF_ACTIVE)) == NULL) { 816 device_printf(sc->dev, "Cannot allocate " 817 "interrupt for message %d\n", rid); 818 return (EINVAL); 819 } 820 sc->msix_irq_rid[k] = rid; 821 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k], 822 INTR_MPSAFE|INTR_TYPE_NET, 823 #ifdef INTR_FILTERS 824 NULL, 825 #endif 826 t3_intr_msix, qs, &sc->msix_intr_tag[k])) { 827 device_printf(sc->dev, "Cannot set up " 828 "interrupt for message %d\n", rid); 829 return (EINVAL); 830 } 831 #ifdef IFNET_MULTIQUEUE 832 if (singleq == 0) { 833 int vector = rman_get_start(sc->msix_irq_res[k]); 834 if (bootverbose) 835 device_printf(sc->dev, "binding vector=%d to cpu=%d\n", vector, k % mp_ncpus); 836 intr_bind(vector, k % mp_ncpus); 837 } 838 #endif 839 } 840 } 841 842 return (0); 843 } 844 845 static int 846 cxgb_port_probe(device_t dev) 847 { 848 struct port_info *p; 849 char buf[80]; 850 const char *desc; 851 852 p = device_get_softc(dev); 853 desc = p->phy.desc; 854 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc); 855 device_set_desc_copy(dev, buf); 856 return (0); 857 } 858 859 860 static int 861 cxgb_makedev(struct port_info *pi) 862 { 863 864 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 865 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp)); 866 867 if (pi->port_cdev == NULL) 868 return (ENOMEM); 869 870 pi->port_cdev->si_drv1 = (void *)pi; 871 872 return (0); 873 } 874 875 #ifndef LRO_SUPPORTED 876 #ifdef IFCAP_LRO 877 #undef IFCAP_LRO 878 #endif 879 #define IFCAP_LRO 0x0 880 #endif 881 882 #ifdef TSO_SUPPORTED 883 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO) 884 /* Don't enable TSO6 yet */ 885 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU | IFCAP_LRO) 886 #else 887 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 888 /* Don't enable TSO6 yet */ 889 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 890 #define IFCAP_TSO4 0x0 891 #define IFCAP_TSO6 0x0 892 #define CSUM_TSO 0x0 893 #endif 894 895 896 static int 897 cxgb_port_attach(device_t dev) 898 { 899 struct port_info *p; 900 struct ifnet *ifp; 901 int err, media_flags; 902 struct adapter *sc; 903 904 905 p = device_get_softc(dev); 906 sc = p->adapter; 907 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 908 device_get_unit(device_get_parent(dev)), p->port_id); 909 PORT_LOCK_INIT(p, p->lockbuf); 910 911 /* Allocate an ifnet object and set it up */ 912 ifp = p->ifp = if_alloc(IFT_ETHER); 913 if (ifp == NULL) { 914 device_printf(dev, "Cannot allocate ifnet\n"); 915 return (ENOMEM); 916 } 917 918 /* 919 * Note that there is currently no watchdog timer. 920 */ 921 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 922 ifp->if_init = cxgb_init; 923 ifp->if_softc = p; 924 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 925 ifp->if_ioctl = cxgb_ioctl; 926 ifp->if_start = cxgb_start; 927 928 #if 0 929 #ifdef IFNET_MULTIQUEUE 930 ifp->if_flags |= IFF_MULTIQ; 931 ifp->if_mq_start = cxgb_pcpu_start; 932 #endif 933 #endif 934 ifp->if_timer = 0; /* Disable ifnet watchdog */ 935 ifp->if_watchdog = NULL; 936 937 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 938 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 939 IFQ_SET_READY(&ifp->if_snd); 940 941 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 942 ifp->if_capabilities |= CXGB_CAP; 943 ifp->if_capenable |= CXGB_CAP_ENABLE; 944 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 945 /* 946 * disable TSO on 4-port - it isn't supported by the firmware yet 947 */ 948 if (p->adapter->params.nports > 2) { 949 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6); 950 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6); 951 ifp->if_hwassist &= ~CSUM_TSO; 952 } 953 954 ether_ifattach(ifp, p->hw_addr); 955 /* 956 * Only default to jumbo frames on 10GigE 957 */ 958 if (p->adapter->params.nports <= 2) 959 ifp->if_mtu = ETHERMTU_JUMBO; 960 if ((err = cxgb_makedev(p)) != 0) { 961 printf("makedev failed %d\n", err); 962 return (err); 963 } 964 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 965 cxgb_media_status); 966 967 if (!strcmp(p->phy.desc, "10GBASE-CX4")) { 968 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX; 969 } else if (!strcmp(p->phy.desc, "10GBASE-SR")) { 970 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX; 971 } else if (!strcmp(p->phy.desc, "10GBASE-R")) { 972 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX; 973 } else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) { 974 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL); 975 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX, 976 0, NULL); 977 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX, 978 0, NULL); 979 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 980 0, NULL); 981 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 982 0, NULL); 983 media_flags = 0; 984 } else if (!strcmp(p->phy.desc, "1000BASE-X")) { 985 /* 986 * XXX: This is not very accurate. Fix when common code 987 * returns more specific value - eg 1000BASE-SX, LX, etc. 988 */ 989 media_flags = IFM_ETHER | IFM_1000_SX | IFM_FDX; 990 } else { 991 printf("unsupported media type %s\n", p->phy.desc); 992 return (ENXIO); 993 } 994 if (media_flags) { 995 ifmedia_add(&p->media, media_flags, 0, NULL); 996 ifmedia_set(&p->media, media_flags); 997 } else { 998 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 999 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO); 1000 } 1001 1002 /* Get the latest mac address, User can use a LAA */ 1003 bcopy(IF_LLADDR(p->ifp), p->hw_addr, ETHER_ADDR_LEN); 1004 t3_sge_init_port(p); 1005 #if defined(LINK_ATTACH) 1006 cxgb_link_start(p); 1007 t3_link_changed(sc, p->port_id); 1008 #endif 1009 return (0); 1010 } 1011 1012 static int 1013 cxgb_port_detach(device_t dev) 1014 { 1015 struct port_info *p; 1016 1017 p = device_get_softc(dev); 1018 1019 PORT_LOCK(p); 1020 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING) 1021 cxgb_stop_locked(p); 1022 PORT_UNLOCK(p); 1023 1024 ether_ifdetach(p->ifp); 1025 printf("waiting for callout to stop ..."); 1026 DELAY(1000000); 1027 printf("done\n"); 1028 /* 1029 * the lock may be acquired in ifdetach 1030 */ 1031 PORT_LOCK_DEINIT(p); 1032 if_free(p->ifp); 1033 1034 if (p->port_cdev != NULL) 1035 destroy_dev(p->port_cdev); 1036 1037 return (0); 1038 } 1039 1040 void 1041 t3_fatal_err(struct adapter *sc) 1042 { 1043 u_int fw_status[4]; 1044 1045 if (sc->flags & FULL_INIT_DONE) { 1046 t3_sge_stop(sc); 1047 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1048 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1049 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1050 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1051 t3_intr_disable(sc); 1052 } 1053 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1054 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1055 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1056 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1057 } 1058 1059 int 1060 t3_os_find_pci_capability(adapter_t *sc, int cap) 1061 { 1062 device_t dev; 1063 struct pci_devinfo *dinfo; 1064 pcicfgregs *cfg; 1065 uint32_t status; 1066 uint8_t ptr; 1067 1068 dev = sc->dev; 1069 dinfo = device_get_ivars(dev); 1070 cfg = &dinfo->cfg; 1071 1072 status = pci_read_config(dev, PCIR_STATUS, 2); 1073 if (!(status & PCIM_STATUS_CAPPRESENT)) 1074 return (0); 1075 1076 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1077 case 0: 1078 case 1: 1079 ptr = PCIR_CAP_PTR; 1080 break; 1081 case 2: 1082 ptr = PCIR_CAP_PTR_2; 1083 break; 1084 default: 1085 return (0); 1086 break; 1087 } 1088 ptr = pci_read_config(dev, ptr, 1); 1089 1090 while (ptr != 0) { 1091 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1092 return (ptr); 1093 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1094 } 1095 1096 return (0); 1097 } 1098 1099 int 1100 t3_os_pci_save_state(struct adapter *sc) 1101 { 1102 device_t dev; 1103 struct pci_devinfo *dinfo; 1104 1105 dev = sc->dev; 1106 dinfo = device_get_ivars(dev); 1107 1108 pci_cfg_save(dev, dinfo, 0); 1109 return (0); 1110 } 1111 1112 int 1113 t3_os_pci_restore_state(struct adapter *sc) 1114 { 1115 device_t dev; 1116 struct pci_devinfo *dinfo; 1117 1118 dev = sc->dev; 1119 dinfo = device_get_ivars(dev); 1120 1121 pci_cfg_restore(dev, dinfo); 1122 return (0); 1123 } 1124 1125 /** 1126 * t3_os_link_changed - handle link status changes 1127 * @adapter: the adapter associated with the link change 1128 * @port_id: the port index whose limk status has changed 1129 * @link_status: the new status of the link 1130 * @speed: the new speed setting 1131 * @duplex: the new duplex setting 1132 * @fc: the new flow-control setting 1133 * 1134 * This is the OS-dependent handler for link status changes. The OS 1135 * neutral handler takes care of most of the processing for these events, 1136 * then calls this handler for any OS-specific processing. 1137 */ 1138 void 1139 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1140 int duplex, int fc) 1141 { 1142 struct port_info *pi = &adapter->port[port_id]; 1143 struct cmac *mac = &adapter->port[port_id].mac; 1144 1145 if (link_status) { 1146 DELAY(10); 1147 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1148 /* Clear errors created by MAC enable */ 1149 t3_set_reg_field(adapter, 1150 A_XGM_STAT_CTRL + pi->mac.offset, 1151 F_CLRSTATS, 1); 1152 if_link_state_change(pi->ifp, LINK_STATE_UP); 1153 1154 } else { 1155 pi->phy.ops->power_down(&pi->phy, 1); 1156 t3_mac_disable(mac, MAC_DIRECTION_RX); 1157 t3_link_start(&pi->phy, mac, &pi->link_config); 1158 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1159 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 1160 } 1161 } 1162 1163 /** 1164 * t3_os_phymod_changed - handle PHY module changes 1165 * @phy: the PHY reporting the module change 1166 * @mod_type: new module type 1167 * 1168 * This is the OS-dependent handler for PHY module changes. It is 1169 * invoked when a PHY module is removed or inserted for any OS-specific 1170 * processing. 1171 */ 1172 void t3_os_phymod_changed(struct adapter *adap, int port_id) 1173 { 1174 static const char *mod_str[] = { 1175 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown" 1176 }; 1177 1178 struct port_info *pi = &adap->port[port_id]; 1179 1180 if (pi->phy.modtype == phy_modtype_none) 1181 device_printf(adap->dev, "PHY module unplugged\n"); 1182 else { 1183 KASSERT(pi->phy.modtype < ARRAY_SIZE(mod_str), 1184 ("invalid PHY module type %d", pi->phy.modtype)); 1185 device_printf(adap->dev, "%s PHY module inserted\n", 1186 mod_str[pi->phy.modtype]); 1187 } 1188 } 1189 1190 /* 1191 * Interrupt-context handler for external (PHY) interrupts. 1192 */ 1193 void 1194 t3_os_ext_intr_handler(adapter_t *sc) 1195 { 1196 if (cxgb_debug) 1197 printf("t3_os_ext_intr_handler\n"); 1198 /* 1199 * Schedule a task to handle external interrupts as they may be slow 1200 * and we use a mutex to protect MDIO registers. We disable PHY 1201 * interrupts in the meantime and let the task reenable them when 1202 * it's done. 1203 */ 1204 ADAPTER_LOCK(sc); 1205 if (sc->slow_intr_mask) { 1206 sc->slow_intr_mask &= ~F_T3DBG; 1207 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1208 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 1209 } 1210 ADAPTER_UNLOCK(sc); 1211 } 1212 1213 void 1214 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1215 { 1216 1217 /* 1218 * The ifnet might not be allocated before this gets called, 1219 * as this is called early on in attach by t3_prep_adapter 1220 * save the address off in the port structure 1221 */ 1222 if (cxgb_debug) 1223 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1224 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1225 } 1226 1227 /** 1228 * link_start - enable a port 1229 * @p: the port to enable 1230 * 1231 * Performs the MAC and PHY actions needed to enable a port. 1232 */ 1233 static void 1234 cxgb_link_start(struct port_info *p) 1235 { 1236 struct ifnet *ifp; 1237 struct t3_rx_mode rm; 1238 struct cmac *mac = &p->mac; 1239 int mtu, hwtagging; 1240 1241 ifp = p->ifp; 1242 1243 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); 1244 1245 mtu = ifp->if_mtu; 1246 if (ifp->if_capenable & IFCAP_VLAN_MTU) 1247 mtu += ETHER_VLAN_ENCAP_LEN; 1248 1249 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; 1250 1251 t3_init_rx_mode(&rm, p); 1252 if (!mac->multiport) 1253 t3_mac_reset(mac); 1254 t3_mac_set_mtu(mac, mtu); 1255 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1256 t3_mac_set_address(mac, 0, p->hw_addr); 1257 t3_mac_set_rx_mode(mac, &rm); 1258 t3_link_start(&p->phy, mac, &p->link_config); 1259 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1260 } 1261 1262 1263 static int 1264 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1265 unsigned long n) 1266 { 1267 int attempts = 5; 1268 1269 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1270 if (!--attempts) 1271 return (ETIMEDOUT); 1272 t3_os_sleep(10); 1273 } 1274 return 0; 1275 } 1276 1277 static int 1278 init_tp_parity(struct adapter *adap) 1279 { 1280 int i; 1281 struct mbuf *m; 1282 struct cpl_set_tcb_field *greq; 1283 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1284 1285 t3_tp_set_offload_mode(adap, 1); 1286 1287 for (i = 0; i < 16; i++) { 1288 struct cpl_smt_write_req *req; 1289 1290 m = m_gethdr(M_WAITOK, MT_DATA); 1291 req = mtod(m, struct cpl_smt_write_req *); 1292 m->m_len = m->m_pkthdr.len = sizeof(*req); 1293 memset(req, 0, sizeof(*req)); 1294 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1295 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1296 req->iff = i; 1297 t3_mgmt_tx(adap, m); 1298 } 1299 1300 for (i = 0; i < 2048; i++) { 1301 struct cpl_l2t_write_req *req; 1302 1303 m = m_gethdr(M_WAITOK, MT_DATA); 1304 req = mtod(m, struct cpl_l2t_write_req *); 1305 m->m_len = m->m_pkthdr.len = sizeof(*req); 1306 memset(req, 0, sizeof(*req)); 1307 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1308 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1309 req->params = htonl(V_L2T_W_IDX(i)); 1310 t3_mgmt_tx(adap, m); 1311 } 1312 1313 for (i = 0; i < 2048; i++) { 1314 struct cpl_rte_write_req *req; 1315 1316 m = m_gethdr(M_WAITOK, MT_DATA); 1317 req = mtod(m, struct cpl_rte_write_req *); 1318 m->m_len = m->m_pkthdr.len = sizeof(*req); 1319 memset(req, 0, sizeof(*req)); 1320 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1321 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1322 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1323 t3_mgmt_tx(adap, m); 1324 } 1325 1326 m = m_gethdr(M_WAITOK, MT_DATA); 1327 greq = mtod(m, struct cpl_set_tcb_field *); 1328 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1329 memset(greq, 0, sizeof(*greq)); 1330 greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1331 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1332 greq->mask = htobe64(1); 1333 t3_mgmt_tx(adap, m); 1334 1335 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1336 t3_tp_set_offload_mode(adap, 0); 1337 return (i); 1338 } 1339 1340 /** 1341 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1342 * @adap: the adapter 1343 * 1344 * Sets up RSS to distribute packets to multiple receive queues. We 1345 * configure the RSS CPU lookup table to distribute to the number of HW 1346 * receive queues, and the response queue lookup table to narrow that 1347 * down to the response queues actually configured for each port. 1348 * We always configure the RSS mapping for two ports since the mapping 1349 * table has plenty of entries. 1350 */ 1351 static void 1352 setup_rss(adapter_t *adap) 1353 { 1354 int i; 1355 u_int nq[2]; 1356 uint8_t cpus[SGE_QSETS + 1]; 1357 uint16_t rspq_map[RSS_TABLE_SIZE]; 1358 1359 for (i = 0; i < SGE_QSETS; ++i) 1360 cpus[i] = i; 1361 cpus[SGE_QSETS] = 0xff; 1362 1363 nq[0] = nq[1] = 0; 1364 for_each_port(adap, i) { 1365 const struct port_info *pi = adap2pinfo(adap, i); 1366 1367 nq[pi->tx_chan] += pi->nqsets; 1368 } 1369 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1370 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1371 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1372 } 1373 /* Calculate the reverse RSS map table */ 1374 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1375 if (adap->rrss_map[rspq_map[i]] == 0xff) 1376 adap->rrss_map[rspq_map[i]] = i; 1377 1378 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1379 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1380 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1381 cpus, rspq_map); 1382 1383 } 1384 1385 /* 1386 * Sends an mbuf to an offload queue driver 1387 * after dealing with any active network taps. 1388 */ 1389 static inline int 1390 offload_tx(struct t3cdev *tdev, struct mbuf *m) 1391 { 1392 int ret; 1393 1394 ret = t3_offload_tx(tdev, m); 1395 return (ret); 1396 } 1397 1398 static int 1399 write_smt_entry(struct adapter *adapter, int idx) 1400 { 1401 struct port_info *pi = &adapter->port[idx]; 1402 struct cpl_smt_write_req *req; 1403 struct mbuf *m; 1404 1405 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1406 return (ENOMEM); 1407 1408 req = mtod(m, struct cpl_smt_write_req *); 1409 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req); 1410 1411 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1412 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 1413 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 1414 req->iff = idx; 1415 memset(req->src_mac1, 0, sizeof(req->src_mac1)); 1416 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 1417 1418 m_set_priority(m, 1); 1419 1420 offload_tx(&adapter->tdev, m); 1421 1422 return (0); 1423 } 1424 1425 static int 1426 init_smt(struct adapter *adapter) 1427 { 1428 int i; 1429 1430 for_each_port(adapter, i) 1431 write_smt_entry(adapter, i); 1432 return 0; 1433 } 1434 1435 static void 1436 init_port_mtus(adapter_t *adapter) 1437 { 1438 unsigned int mtus = adapter->port[0].ifp->if_mtu; 1439 1440 if (adapter->port[1].ifp) 1441 mtus |= adapter->port[1].ifp->if_mtu << 16; 1442 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); 1443 } 1444 1445 static void 1446 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1447 int hi, int port) 1448 { 1449 struct mbuf *m; 1450 struct mngt_pktsched_wr *req; 1451 1452 m = m_gethdr(M_DONTWAIT, MT_DATA); 1453 if (m) { 1454 req = mtod(m, struct mngt_pktsched_wr *); 1455 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1456 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1457 req->sched = sched; 1458 req->idx = qidx; 1459 req->min = lo; 1460 req->max = hi; 1461 req->binding = port; 1462 m->m_len = m->m_pkthdr.len = sizeof(*req); 1463 t3_mgmt_tx(adap, m); 1464 } 1465 } 1466 1467 static void 1468 bind_qsets(adapter_t *sc) 1469 { 1470 int i, j; 1471 1472 cxgb_pcpu_startup_threads(sc); 1473 for (i = 0; i < (sc)->params.nports; ++i) { 1474 const struct port_info *pi = adap2pinfo(sc, i); 1475 1476 for (j = 0; j < pi->nqsets; ++j) { 1477 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1478 -1, pi->tx_chan); 1479 1480 } 1481 } 1482 } 1483 1484 static void 1485 update_tpeeprom(struct adapter *adap) 1486 { 1487 #ifdef FIRMWARE_LATEST 1488 const struct firmware *tpeeprom; 1489 #else 1490 struct firmware *tpeeprom; 1491 #endif 1492 1493 uint32_t version; 1494 unsigned int major, minor; 1495 int ret, len; 1496 char rev; 1497 1498 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1499 1500 major = G_TP_VERSION_MAJOR(version); 1501 minor = G_TP_VERSION_MINOR(version); 1502 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1503 return; 1504 1505 rev = t3rev2char(adap); 1506 1507 tpeeprom = firmware_get(TPEEPROM_NAME); 1508 if (tpeeprom == NULL) { 1509 device_printf(adap->dev, "could not load TP EEPROM: unable to load %s\n", 1510 TPEEPROM_NAME); 1511 return; 1512 } 1513 1514 len = tpeeprom->datasize - 4; 1515 1516 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1517 if (ret) 1518 goto release_tpeeprom; 1519 1520 if (len != TP_SRAM_LEN) { 1521 device_printf(adap->dev, "%s length is wrong len=%d expected=%d\n", TPEEPROM_NAME, len, TP_SRAM_LEN); 1522 return; 1523 } 1524 1525 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1526 TP_SRAM_OFFSET); 1527 1528 if (!ret) { 1529 device_printf(adap->dev, 1530 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1531 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1532 } else 1533 device_printf(adap->dev, "Protocol SRAM image update in EEPROM failed\n"); 1534 1535 release_tpeeprom: 1536 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1537 1538 return; 1539 } 1540 1541 static int 1542 update_tpsram(struct adapter *adap) 1543 { 1544 #ifdef FIRMWARE_LATEST 1545 const struct firmware *tpsram; 1546 #else 1547 struct firmware *tpsram; 1548 #endif 1549 int ret; 1550 char rev; 1551 1552 rev = t3rev2char(adap); 1553 if (!rev) 1554 return 0; 1555 1556 update_tpeeprom(adap); 1557 1558 tpsram = firmware_get(TPSRAM_NAME); 1559 if (tpsram == NULL){ 1560 device_printf(adap->dev, "could not load TP SRAM\n"); 1561 return (EINVAL); 1562 } else 1563 device_printf(adap->dev, "updating TP SRAM\n"); 1564 1565 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1566 if (ret) 1567 goto release_tpsram; 1568 1569 ret = t3_set_proto_sram(adap, tpsram->data); 1570 if (ret) 1571 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1572 1573 release_tpsram: 1574 firmware_put(tpsram, FIRMWARE_UNLOAD); 1575 1576 return ret; 1577 } 1578 1579 /** 1580 * cxgb_up - enable the adapter 1581 * @adap: adapter being enabled 1582 * 1583 * Called when the first port is enabled, this function performs the 1584 * actions necessary to make an adapter operational, such as completing 1585 * the initialization of HW modules, and enabling interrupts. 1586 * 1587 */ 1588 static int 1589 cxgb_up(struct adapter *sc) 1590 { 1591 int err = 0; 1592 1593 if ((sc->flags & FULL_INIT_DONE) == 0) { 1594 1595 if ((sc->flags & FW_UPTODATE) == 0) 1596 if ((err = upgrade_fw(sc))) 1597 goto out; 1598 if ((sc->flags & TPS_UPTODATE) == 0) 1599 if ((err = update_tpsram(sc))) 1600 goto out; 1601 err = t3_init_hw(sc, 0); 1602 if (err) 1603 goto out; 1604 1605 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1606 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1607 1608 err = setup_sge_qsets(sc); 1609 if (err) 1610 goto out; 1611 1612 setup_rss(sc); 1613 t3_add_configured_sysctls(sc); 1614 sc->flags |= FULL_INIT_DONE; 1615 } 1616 1617 t3_intr_clear(sc); 1618 1619 /* If it's MSI or INTx, allocate a single interrupt for everything */ 1620 if ((sc->flags & USING_MSIX) == 0) { 1621 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 1622 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 1623 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", 1624 sc->irq_rid); 1625 err = EINVAL; 1626 goto out; 1627 } 1628 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res); 1629 1630 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 1631 #ifdef INTR_FILTERS 1632 NULL, 1633 #endif 1634 sc->cxgb_intr, sc, &sc->intr_tag)) { 1635 device_printf(sc->dev, "Cannot set up interrupt\n"); 1636 err = EINVAL; 1637 goto irq_err; 1638 } 1639 } else { 1640 cxgb_setup_msix(sc, sc->msi_count); 1641 } 1642 1643 t3_sge_start(sc); 1644 t3_intr_enable(sc); 1645 1646 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1647 is_offload(sc) && init_tp_parity(sc) == 0) 1648 sc->flags |= TP_PARITY_INIT; 1649 1650 if (sc->flags & TP_PARITY_INIT) { 1651 t3_write_reg(sc, A_TP_INT_CAUSE, 1652 F_CMCACHEPERR | F_ARPLUTPERR); 1653 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1654 } 1655 1656 1657 if (!(sc->flags & QUEUES_BOUND)) { 1658 bind_qsets(sc); 1659 sc->flags |= QUEUES_BOUND; 1660 } 1661 out: 1662 return (err); 1663 irq_err: 1664 CH_ERR(sc, "request_irq failed, err %d\n", err); 1665 goto out; 1666 } 1667 1668 1669 /* 1670 * Release resources when all the ports and offloading have been stopped. 1671 */ 1672 static void 1673 cxgb_down_locked(struct adapter *sc) 1674 { 1675 1676 t3_sge_stop(sc); 1677 t3_intr_disable(sc); 1678 1679 if (sc->intr_tag != NULL) { 1680 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 1681 sc->intr_tag = NULL; 1682 } 1683 if (sc->irq_res != NULL) { 1684 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", 1685 sc->irq_rid, sc->irq_res); 1686 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 1687 sc->irq_res); 1688 sc->irq_res = NULL; 1689 } 1690 1691 if (sc->flags & USING_MSIX) 1692 cxgb_teardown_msix(sc); 1693 1694 callout_stop(&sc->cxgb_tick_ch); 1695 callout_stop(&sc->sge_timer_ch); 1696 callout_drain(&sc->cxgb_tick_ch); 1697 callout_drain(&sc->sge_timer_ch); 1698 1699 if (sc->tq != NULL) { 1700 printf("draining slow intr\n"); 1701 1702 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1703 printf("draining ext intr\n"); 1704 taskqueue_drain(sc->tq, &sc->ext_intr_task); 1705 printf("draining tick task\n"); 1706 taskqueue_drain(sc->tq, &sc->tick_task); 1707 } 1708 ADAPTER_UNLOCK(sc); 1709 } 1710 1711 static int 1712 offload_open(struct port_info *pi) 1713 { 1714 struct adapter *adapter = pi->adapter; 1715 struct t3cdev *tdev = &adapter->tdev; 1716 1717 int adap_up = adapter->open_device_map & PORT_MASK; 1718 int err = 0; 1719 1720 if (atomic_cmpset_int(&adapter->open_device_map, 1721 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)), 1722 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0) 1723 return (0); 1724 1725 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1726 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", 1727 adapter->open_device_map); 1728 ADAPTER_LOCK(pi->adapter); 1729 if (!adap_up) 1730 err = cxgb_up(adapter); 1731 ADAPTER_UNLOCK(pi->adapter); 1732 if (err) 1733 return (err); 1734 1735 t3_tp_set_offload_mode(adapter, 1); 1736 tdev->lldev = pi->ifp; 1737 1738 init_port_mtus(adapter); 1739 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, 1740 adapter->params.b_wnd, 1741 adapter->params.rev == 0 ? 1742 adapter->port[0].ifp->if_mtu : 0xffff); 1743 init_smt(adapter); 1744 /* Call back all registered clients */ 1745 cxgb_add_clients(tdev); 1746 1747 /* restore them in case the offload module has changed them */ 1748 if (err) { 1749 t3_tp_set_offload_mode(adapter, 0); 1750 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1751 cxgb_set_dummy_ops(tdev); 1752 } 1753 return (err); 1754 } 1755 1756 static int 1757 offload_close(struct t3cdev *tdev) 1758 { 1759 struct adapter *adapter = tdev2adap(tdev); 1760 1761 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1762 return (0); 1763 1764 /* Call back all registered clients */ 1765 cxgb_remove_clients(tdev); 1766 1767 tdev->lldev = NULL; 1768 cxgb_set_dummy_ops(tdev); 1769 t3_tp_set_offload_mode(adapter, 0); 1770 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1771 1772 ADAPTER_LOCK(adapter); 1773 if (!adapter->open_device_map) 1774 cxgb_down_locked(adapter); 1775 else 1776 ADAPTER_UNLOCK(adapter); 1777 return (0); 1778 } 1779 1780 1781 static void 1782 cxgb_init(void *arg) 1783 { 1784 struct port_info *p = arg; 1785 1786 PORT_LOCK(p); 1787 cxgb_init_locked(p); 1788 PORT_UNLOCK(p); 1789 } 1790 1791 static void 1792 cxgb_init_locked(struct port_info *p) 1793 { 1794 struct ifnet *ifp; 1795 adapter_t *sc = p->adapter; 1796 int err; 1797 1798 PORT_LOCK_ASSERT_OWNED(p); 1799 ifp = p->ifp; 1800 1801 ADAPTER_LOCK(p->adapter); 1802 if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) { 1803 ADAPTER_UNLOCK(p->adapter); 1804 cxgb_stop_locked(p); 1805 return; 1806 } 1807 if (p->adapter->open_device_map == 0) { 1808 t3_intr_clear(sc); 1809 } 1810 setbit(&p->adapter->open_device_map, p->port_id); 1811 ADAPTER_UNLOCK(p->adapter); 1812 1813 if (is_offload(sc) && !ofld_disable) { 1814 err = offload_open(p); 1815 if (err) 1816 log(LOG_WARNING, 1817 "Could not initialize offload capabilities\n"); 1818 } 1819 #if !defined(LINK_ATTACH) 1820 cxgb_link_start(p); 1821 t3_link_changed(sc, p->port_id); 1822 #endif 1823 ifp->if_baudrate = p->link_config.speed * 1000000; 1824 1825 device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id); 1826 t3_port_intr_enable(sc, p->port_id); 1827 1828 t3_sge_reset_adapter(sc); 1829 1830 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1831 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1832 } 1833 1834 static void 1835 cxgb_set_rxmode(struct port_info *p) 1836 { 1837 struct t3_rx_mode rm; 1838 struct cmac *mac = &p->mac; 1839 1840 t3_init_rx_mode(&rm, p); 1841 mtx_lock(&p->adapter->mdio_lock); 1842 t3_mac_set_rx_mode(mac, &rm); 1843 mtx_unlock(&p->adapter->mdio_lock); 1844 } 1845 1846 static void 1847 cxgb_stop_locked(struct port_info *pi) 1848 { 1849 struct ifnet *ifp; 1850 1851 PORT_LOCK_ASSERT_OWNED(pi); 1852 ADAPTER_LOCK_ASSERT_NOTOWNED(pi->adapter); 1853 1854 ifp = pi->ifp; 1855 t3_port_intr_disable(pi->adapter, pi->port_id); 1856 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1857 1858 /* disable pause frames */ 1859 t3_set_reg_field(pi->adapter, A_XGM_TX_CFG + pi->mac.offset, 1860 F_TXPAUSEEN, 0); 1861 1862 /* Reset RX FIFO HWM */ 1863 t3_set_reg_field(pi->adapter, A_XGM_RXFIFO_CFG + pi->mac.offset, 1864 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1865 1866 1867 ADAPTER_LOCK(pi->adapter); 1868 clrbit(&pi->adapter->open_device_map, pi->port_id); 1869 1870 if (pi->adapter->open_device_map == 0) { 1871 cxgb_down_locked(pi->adapter); 1872 } else 1873 ADAPTER_UNLOCK(pi->adapter); 1874 1875 #if !defined(LINK_ATTACH) 1876 DELAY(100); 1877 1878 /* Wait for TXFIFO empty */ 1879 t3_wait_op_done(pi->adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 1880 F_TXFIFO_EMPTY, 1, 20, 5); 1881 1882 DELAY(100); 1883 t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1884 1885 pi->phy.ops->power_down(&pi->phy, 1); 1886 #endif 1887 1888 } 1889 1890 static int 1891 cxgb_set_mtu(struct port_info *p, int mtu) 1892 { 1893 struct ifnet *ifp = p->ifp; 1894 int error = 0; 1895 1896 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1897 error = EINVAL; 1898 else if (ifp->if_mtu != mtu) { 1899 PORT_LOCK(p); 1900 ifp->if_mtu = mtu; 1901 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1902 cxgb_stop_locked(p); 1903 cxgb_init_locked(p); 1904 } 1905 PORT_UNLOCK(p); 1906 } 1907 return (error); 1908 } 1909 1910 #ifdef LRO_SUPPORTED 1911 /* 1912 * Mark lro enabled or disabled in all qsets for this port 1913 */ 1914 static int 1915 cxgb_set_lro(struct port_info *p, int enabled) 1916 { 1917 int i; 1918 struct adapter *adp = p->adapter; 1919 struct sge_qset *q; 1920 1921 PORT_LOCK_ASSERT_OWNED(p); 1922 for (i = 0; i < p->nqsets; i++) { 1923 q = &adp->sge.qs[p->first_qset + i]; 1924 q->lro.enabled = (enabled != 0); 1925 } 1926 return (0); 1927 } 1928 #endif 1929 1930 static int 1931 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1932 { 1933 struct port_info *p = ifp->if_softc; 1934 struct ifaddr *ifa = (struct ifaddr *)data; 1935 struct ifreq *ifr = (struct ifreq *)data; 1936 int flags, error = 0, reinit = 0; 1937 uint32_t mask; 1938 1939 /* 1940 * XXX need to check that we aren't in the middle of an unload 1941 */ 1942 switch (command) { 1943 case SIOCSIFMTU: 1944 error = cxgb_set_mtu(p, ifr->ifr_mtu); 1945 break; 1946 case SIOCSIFADDR: 1947 if (ifa->ifa_addr->sa_family == AF_INET) { 1948 ifp->if_flags |= IFF_UP; 1949 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1950 PORT_LOCK(p); 1951 cxgb_init_locked(p); 1952 PORT_UNLOCK(p); 1953 } 1954 arp_ifinit(ifp, ifa); 1955 } else 1956 error = ether_ioctl(ifp, command, data); 1957 break; 1958 case SIOCSIFFLAGS: 1959 PORT_LOCK(p); 1960 if (ifp->if_flags & IFF_UP) { 1961 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1962 flags = p->if_flags; 1963 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1964 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 1965 cxgb_set_rxmode(p); 1966 } else 1967 cxgb_init_locked(p); 1968 p->if_flags = ifp->if_flags; 1969 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1970 cxgb_stop_locked(p); 1971 1972 PORT_UNLOCK(p); 1973 break; 1974 case SIOCADDMULTI: 1975 case SIOCDELMULTI: 1976 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1977 cxgb_set_rxmode(p); 1978 } 1979 break; 1980 case SIOCSIFMEDIA: 1981 case SIOCGIFMEDIA: 1982 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 1983 break; 1984 case SIOCSIFCAP: 1985 PORT_LOCK(p); 1986 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1987 if (mask & IFCAP_TXCSUM) { 1988 if (IFCAP_TXCSUM & ifp->if_capenable) { 1989 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 1990 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 1991 | CSUM_IP | CSUM_TSO); 1992 } else { 1993 ifp->if_capenable |= IFCAP_TXCSUM; 1994 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP 1995 | CSUM_IP); 1996 } 1997 } 1998 if (mask & IFCAP_RXCSUM) { 1999 ifp->if_capenable ^= IFCAP_RXCSUM; 2000 } 2001 if (mask & IFCAP_TSO4) { 2002 if (IFCAP_TSO4 & ifp->if_capenable) { 2003 ifp->if_capenable &= ~IFCAP_TSO4; 2004 ifp->if_hwassist &= ~CSUM_TSO; 2005 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 2006 ifp->if_capenable |= IFCAP_TSO4; 2007 ifp->if_hwassist |= CSUM_TSO; 2008 } else { 2009 if (cxgb_debug) 2010 printf("cxgb requires tx checksum offload" 2011 " be enabled to use TSO\n"); 2012 error = EINVAL; 2013 } 2014 } 2015 #ifdef LRO_SUPPORTED 2016 if (mask & IFCAP_LRO) { 2017 ifp->if_capenable ^= IFCAP_LRO; 2018 2019 /* Safe to do this even if cxgb_up not called yet */ 2020 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO); 2021 } 2022 #endif 2023 if (mask & IFCAP_VLAN_HWTAGGING) { 2024 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2025 reinit = ifp->if_drv_flags & IFF_DRV_RUNNING; 2026 } 2027 if (mask & IFCAP_VLAN_MTU) { 2028 ifp->if_capenable ^= IFCAP_VLAN_MTU; 2029 reinit = ifp->if_drv_flags & IFF_DRV_RUNNING; 2030 } 2031 if (mask & IFCAP_VLAN_HWCSUM) { 2032 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 2033 } 2034 if (reinit) { 2035 cxgb_stop_locked(p); 2036 cxgb_init_locked(p); 2037 } 2038 PORT_UNLOCK(p); 2039 2040 #ifdef VLAN_CAPABILITIES 2041 VLAN_CAPABILITIES(ifp); 2042 #endif 2043 break; 2044 default: 2045 error = ether_ioctl(ifp, command, data); 2046 break; 2047 } 2048 return (error); 2049 } 2050 2051 static int 2052 cxgb_media_change(struct ifnet *ifp) 2053 { 2054 if_printf(ifp, "media change not supported\n"); 2055 return (ENXIO); 2056 } 2057 2058 static void 2059 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2060 { 2061 struct port_info *p = ifp->if_softc; 2062 2063 ifmr->ifm_status = IFM_AVALID; 2064 ifmr->ifm_active = IFM_ETHER; 2065 2066 if (!p->link_config.link_ok) 2067 return; 2068 2069 ifmr->ifm_status |= IFM_ACTIVE; 2070 2071 switch (p->link_config.speed) { 2072 case 10: 2073 ifmr->ifm_active |= IFM_10_T; 2074 break; 2075 case 100: 2076 ifmr->ifm_active |= IFM_100_TX; 2077 break; 2078 case 1000: 2079 ifmr->ifm_active |= IFM_1000_T; 2080 break; 2081 } 2082 2083 if (p->link_config.duplex) 2084 ifmr->ifm_active |= IFM_FDX; 2085 else 2086 ifmr->ifm_active |= IFM_HDX; 2087 } 2088 2089 static void 2090 cxgb_async_intr(void *data) 2091 { 2092 adapter_t *sc = data; 2093 2094 if (cxgb_debug) 2095 device_printf(sc->dev, "cxgb_async_intr\n"); 2096 /* 2097 * May need to sleep - defer to taskqueue 2098 */ 2099 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2100 } 2101 2102 static void 2103 cxgb_ext_intr_handler(void *arg, int count) 2104 { 2105 adapter_t *sc = (adapter_t *)arg; 2106 2107 if (cxgb_debug) 2108 printf("cxgb_ext_intr_handler\n"); 2109 2110 t3_phy_intr_handler(sc); 2111 2112 /* Now reenable external interrupts */ 2113 ADAPTER_LOCK(sc); 2114 if (sc->slow_intr_mask) { 2115 sc->slow_intr_mask |= F_T3DBG; 2116 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 2117 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 2118 } 2119 ADAPTER_UNLOCK(sc); 2120 } 2121 2122 static void 2123 check_link_status(adapter_t *sc) 2124 { 2125 int i; 2126 2127 for (i = 0; i < (sc)->params.nports; ++i) { 2128 struct port_info *p = &sc->port[i]; 2129 2130 if (!(p->phy.caps & SUPPORTED_IRQ)) 2131 t3_link_changed(sc, i); 2132 p->ifp->if_baudrate = p->link_config.speed * 1000000; 2133 } 2134 } 2135 2136 static void 2137 check_t3b2_mac(struct adapter *adapter) 2138 { 2139 int i; 2140 2141 if(adapter->flags & CXGB_SHUTDOWN) 2142 return; 2143 2144 for_each_port(adapter, i) { 2145 struct port_info *p = &adapter->port[i]; 2146 struct ifnet *ifp = p->ifp; 2147 int status; 2148 2149 if(adapter->flags & CXGB_SHUTDOWN) 2150 return; 2151 2152 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2153 continue; 2154 2155 status = 0; 2156 PORT_LOCK(p); 2157 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 2158 status = t3b2_mac_watchdog_task(&p->mac); 2159 if (status == 1) 2160 p->mac.stats.num_toggled++; 2161 else if (status == 2) { 2162 struct cmac *mac = &p->mac; 2163 int mtu = ifp->if_mtu; 2164 2165 if (ifp->if_capenable & IFCAP_VLAN_MTU) 2166 mtu += ETHER_VLAN_ENCAP_LEN; 2167 t3_mac_set_mtu(mac, mtu); 2168 t3_mac_set_address(mac, 0, p->hw_addr); 2169 cxgb_set_rxmode(p); 2170 t3_link_start(&p->phy, mac, &p->link_config); 2171 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2172 t3_port_intr_enable(adapter, p->port_id); 2173 p->mac.stats.num_resets++; 2174 } 2175 PORT_UNLOCK(p); 2176 } 2177 } 2178 2179 static void 2180 cxgb_tick(void *arg) 2181 { 2182 adapter_t *sc = (adapter_t *)arg; 2183 2184 if(sc->flags & CXGB_SHUTDOWN) 2185 return; 2186 2187 taskqueue_enqueue(sc->tq, &sc->tick_task); 2188 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 2189 } 2190 2191 static void 2192 cxgb_tick_handler(void *arg, int count) 2193 { 2194 adapter_t *sc = (adapter_t *)arg; 2195 const struct adapter_params *p = &sc->params; 2196 int i; 2197 2198 if(sc->flags & CXGB_SHUTDOWN) 2199 return; 2200 2201 ADAPTER_LOCK(sc); 2202 if (p->linkpoll_period) 2203 check_link_status(sc); 2204 2205 sc->check_task_cnt++; 2206 2207 /* 2208 * adapter lock can currently only be acquired after the 2209 * port lock 2210 */ 2211 ADAPTER_UNLOCK(sc); 2212 2213 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2214 check_t3b2_mac(sc); 2215 2216 /* Update MAC stats if it's time to do so */ 2217 if (!p->linkpoll_period || 2218 (sc->check_task_cnt * p->linkpoll_period) / 10 >= 2219 p->stats_update_period) { 2220 for_each_port(sc, i) { 2221 struct port_info *port = &sc->port[i]; 2222 PORT_LOCK(port); 2223 t3_mac_update_stats(&port->mac); 2224 PORT_UNLOCK(port); 2225 } 2226 sc->check_task_cnt = 0; 2227 } 2228 } 2229 2230 static void 2231 touch_bars(device_t dev) 2232 { 2233 /* 2234 * Don't enable yet 2235 */ 2236 #if !defined(__LP64__) && 0 2237 u32 v; 2238 2239 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2240 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2241 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2242 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2243 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2244 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2245 #endif 2246 } 2247 2248 static int 2249 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2250 { 2251 uint8_t *buf; 2252 int err = 0; 2253 u32 aligned_offset, aligned_len, *p; 2254 struct adapter *adapter = pi->adapter; 2255 2256 2257 aligned_offset = offset & ~3; 2258 aligned_len = (len + (offset & 3) + 3) & ~3; 2259 2260 if (aligned_offset != offset || aligned_len != len) { 2261 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2262 if (!buf) 2263 return (ENOMEM); 2264 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2265 if (!err && aligned_len > 4) 2266 err = t3_seeprom_read(adapter, 2267 aligned_offset + aligned_len - 4, 2268 (u32 *)&buf[aligned_len - 4]); 2269 if (err) 2270 goto out; 2271 memcpy(buf + (offset & 3), data, len); 2272 } else 2273 buf = (uint8_t *)(uintptr_t)data; 2274 2275 err = t3_seeprom_wp(adapter, 0); 2276 if (err) 2277 goto out; 2278 2279 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2280 err = t3_seeprom_write(adapter, aligned_offset, *p); 2281 aligned_offset += 4; 2282 } 2283 2284 if (!err) 2285 err = t3_seeprom_wp(adapter, 1); 2286 out: 2287 if (buf != data) 2288 free(buf, M_DEVBUF); 2289 return err; 2290 } 2291 2292 2293 static int 2294 in_range(int val, int lo, int hi) 2295 { 2296 return val < 0 || (val <= hi && val >= lo); 2297 } 2298 2299 static int 2300 cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td) 2301 { 2302 return (0); 2303 } 2304 2305 static int 2306 cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td) 2307 { 2308 return (0); 2309 } 2310 2311 static int 2312 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2313 int fflag, struct thread *td) 2314 { 2315 int mmd, error = 0; 2316 struct port_info *pi = dev->si_drv1; 2317 adapter_t *sc = pi->adapter; 2318 2319 #ifdef PRIV_SUPPORTED 2320 if (priv_check(td, PRIV_DRIVER)) { 2321 if (cxgb_debug) 2322 printf("user does not have access to privileged ioctls\n"); 2323 return (EPERM); 2324 } 2325 #else 2326 if (suser(td)) { 2327 if (cxgb_debug) 2328 printf("user does not have access to privileged ioctls\n"); 2329 return (EPERM); 2330 } 2331 #endif 2332 2333 switch (cmd) { 2334 case CHELSIO_GET_MIIREG: { 2335 uint32_t val; 2336 struct cphy *phy = &pi->phy; 2337 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2338 2339 if (!phy->mdio_read) 2340 return (EOPNOTSUPP); 2341 if (is_10G(sc)) { 2342 mmd = mid->phy_id >> 8; 2343 if (!mmd) 2344 mmd = MDIO_DEV_PCS; 2345 else if (mmd > MDIO_DEV_XGXS) 2346 return (EINVAL); 2347 2348 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2349 mid->reg_num, &val); 2350 } else 2351 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2352 mid->reg_num & 0x1f, &val); 2353 if (error == 0) 2354 mid->val_out = val; 2355 break; 2356 } 2357 case CHELSIO_SET_MIIREG: { 2358 struct cphy *phy = &pi->phy; 2359 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2360 2361 if (!phy->mdio_write) 2362 return (EOPNOTSUPP); 2363 if (is_10G(sc)) { 2364 mmd = mid->phy_id >> 8; 2365 if (!mmd) 2366 mmd = MDIO_DEV_PCS; 2367 else if (mmd > MDIO_DEV_XGXS) 2368 return (EINVAL); 2369 2370 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2371 mmd, mid->reg_num, mid->val_in); 2372 } else 2373 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2374 mid->reg_num & 0x1f, 2375 mid->val_in); 2376 break; 2377 } 2378 case CHELSIO_SETREG: { 2379 struct ch_reg *edata = (struct ch_reg *)data; 2380 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2381 return (EFAULT); 2382 t3_write_reg(sc, edata->addr, edata->val); 2383 break; 2384 } 2385 case CHELSIO_GETREG: { 2386 struct ch_reg *edata = (struct ch_reg *)data; 2387 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2388 return (EFAULT); 2389 edata->val = t3_read_reg(sc, edata->addr); 2390 break; 2391 } 2392 case CHELSIO_GET_SGE_CONTEXT: { 2393 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2394 mtx_lock_spin(&sc->sge.reg_lock); 2395 switch (ecntxt->cntxt_type) { 2396 case CNTXT_TYPE_EGRESS: 2397 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2398 ecntxt->data); 2399 break; 2400 case CNTXT_TYPE_FL: 2401 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2402 ecntxt->data); 2403 break; 2404 case CNTXT_TYPE_RSP: 2405 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2406 ecntxt->data); 2407 break; 2408 case CNTXT_TYPE_CQ: 2409 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2410 ecntxt->data); 2411 break; 2412 default: 2413 error = EINVAL; 2414 break; 2415 } 2416 mtx_unlock_spin(&sc->sge.reg_lock); 2417 break; 2418 } 2419 case CHELSIO_GET_SGE_DESC: { 2420 struct ch_desc *edesc = (struct ch_desc *)data; 2421 int ret; 2422 if (edesc->queue_num >= SGE_QSETS * 6) 2423 return (EINVAL); 2424 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2425 edesc->queue_num % 6, edesc->idx, edesc->data); 2426 if (ret < 0) 2427 return (EINVAL); 2428 edesc->size = ret; 2429 break; 2430 } 2431 case CHELSIO_GET_QSET_PARAMS: { 2432 struct qset_params *q; 2433 struct ch_qset_params *t = (struct ch_qset_params *)data; 2434 int q1 = pi->first_qset; 2435 int nqsets = pi->nqsets; 2436 int i; 2437 2438 if (t->qset_idx >= nqsets) 2439 return EINVAL; 2440 2441 i = q1 + t->qset_idx; 2442 q = &sc->params.sge.qset[i]; 2443 t->rspq_size = q->rspq_size; 2444 t->txq_size[0] = q->txq_size[0]; 2445 t->txq_size[1] = q->txq_size[1]; 2446 t->txq_size[2] = q->txq_size[2]; 2447 t->fl_size[0] = q->fl_size; 2448 t->fl_size[1] = q->jumbo_size; 2449 t->polling = q->polling; 2450 t->lro = q->lro; 2451 t->intr_lat = q->coalesce_usecs; 2452 t->cong_thres = q->cong_thres; 2453 t->qnum = i; 2454 2455 if (sc->flags & USING_MSIX) 2456 t->vector = rman_get_start(sc->msix_irq_res[i]); 2457 else 2458 t->vector = rman_get_start(sc->irq_res); 2459 2460 break; 2461 } 2462 case CHELSIO_GET_QSET_NUM: { 2463 struct ch_reg *edata = (struct ch_reg *)data; 2464 edata->val = pi->nqsets; 2465 break; 2466 } 2467 case CHELSIO_LOAD_FW: { 2468 uint8_t *fw_data; 2469 uint32_t vers; 2470 struct ch_mem_range *t = (struct ch_mem_range *)data; 2471 2472 /* 2473 * You're allowed to load a firmware only before FULL_INIT_DONE 2474 * 2475 * FW_UPTODATE is also set so the rest of the initialization 2476 * will not overwrite what was loaded here. This gives you the 2477 * flexibility to load any firmware (and maybe shoot yourself in 2478 * the foot). 2479 */ 2480 2481 ADAPTER_LOCK(sc); 2482 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2483 ADAPTER_UNLOCK(sc); 2484 return (EBUSY); 2485 } 2486 2487 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2488 if (!fw_data) 2489 error = ENOMEM; 2490 else 2491 error = copyin(t->buf, fw_data, t->len); 2492 2493 if (!error) 2494 error = -t3_load_fw(sc, fw_data, t->len); 2495 2496 if (t3_get_fw_version(sc, &vers) == 0) { 2497 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2498 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2499 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2500 } 2501 2502 if (!error) 2503 sc->flags |= FW_UPTODATE; 2504 2505 free(fw_data, M_DEVBUF); 2506 ADAPTER_UNLOCK(sc); 2507 break; 2508 } 2509 case CHELSIO_LOAD_BOOT: { 2510 uint8_t *boot_data; 2511 struct ch_mem_range *t = (struct ch_mem_range *)data; 2512 2513 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2514 if (!boot_data) 2515 return ENOMEM; 2516 2517 error = copyin(t->buf, boot_data, t->len); 2518 if (!error) 2519 error = -t3_load_boot(sc, boot_data, t->len); 2520 2521 free(boot_data, M_DEVBUF); 2522 break; 2523 } 2524 case CHELSIO_GET_PM: { 2525 struct ch_pm *m = (struct ch_pm *)data; 2526 struct tp_params *p = &sc->params.tp; 2527 2528 if (!is_offload(sc)) 2529 return (EOPNOTSUPP); 2530 2531 m->tx_pg_sz = p->tx_pg_size; 2532 m->tx_num_pg = p->tx_num_pgs; 2533 m->rx_pg_sz = p->rx_pg_size; 2534 m->rx_num_pg = p->rx_num_pgs; 2535 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2536 2537 break; 2538 } 2539 case CHELSIO_SET_PM: { 2540 struct ch_pm *m = (struct ch_pm *)data; 2541 struct tp_params *p = &sc->params.tp; 2542 2543 if (!is_offload(sc)) 2544 return (EOPNOTSUPP); 2545 if (sc->flags & FULL_INIT_DONE) 2546 return (EBUSY); 2547 2548 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2549 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2550 return (EINVAL); /* not power of 2 */ 2551 if (!(m->rx_pg_sz & 0x14000)) 2552 return (EINVAL); /* not 16KB or 64KB */ 2553 if (!(m->tx_pg_sz & 0x1554000)) 2554 return (EINVAL); 2555 if (m->tx_num_pg == -1) 2556 m->tx_num_pg = p->tx_num_pgs; 2557 if (m->rx_num_pg == -1) 2558 m->rx_num_pg = p->rx_num_pgs; 2559 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2560 return (EINVAL); 2561 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2562 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2563 return (EINVAL); 2564 2565 p->rx_pg_size = m->rx_pg_sz; 2566 p->tx_pg_size = m->tx_pg_sz; 2567 p->rx_num_pgs = m->rx_num_pg; 2568 p->tx_num_pgs = m->tx_num_pg; 2569 break; 2570 } 2571 case CHELSIO_SETMTUTAB: { 2572 struct ch_mtus *m = (struct ch_mtus *)data; 2573 int i; 2574 2575 if (!is_offload(sc)) 2576 return (EOPNOTSUPP); 2577 if (offload_running(sc)) 2578 return (EBUSY); 2579 if (m->nmtus != NMTUS) 2580 return (EINVAL); 2581 if (m->mtus[0] < 81) /* accommodate SACK */ 2582 return (EINVAL); 2583 2584 /* 2585 * MTUs must be in ascending order 2586 */ 2587 for (i = 1; i < NMTUS; ++i) 2588 if (m->mtus[i] < m->mtus[i - 1]) 2589 return (EINVAL); 2590 2591 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2592 break; 2593 } 2594 case CHELSIO_GETMTUTAB: { 2595 struct ch_mtus *m = (struct ch_mtus *)data; 2596 2597 if (!is_offload(sc)) 2598 return (EOPNOTSUPP); 2599 2600 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2601 m->nmtus = NMTUS; 2602 break; 2603 } 2604 case CHELSIO_GET_MEM: { 2605 struct ch_mem_range *t = (struct ch_mem_range *)data; 2606 struct mc7 *mem; 2607 uint8_t *useraddr; 2608 u64 buf[32]; 2609 2610 /* 2611 * Use these to avoid modifying len/addr in the the return 2612 * struct 2613 */ 2614 uint32_t len = t->len, addr = t->addr; 2615 2616 if (!is_offload(sc)) 2617 return (EOPNOTSUPP); 2618 if (!(sc->flags & FULL_INIT_DONE)) 2619 return (EIO); /* need the memory controllers */ 2620 if ((addr & 0x7) || (len & 0x7)) 2621 return (EINVAL); 2622 if (t->mem_id == MEM_CM) 2623 mem = &sc->cm; 2624 else if (t->mem_id == MEM_PMRX) 2625 mem = &sc->pmrx; 2626 else if (t->mem_id == MEM_PMTX) 2627 mem = &sc->pmtx; 2628 else 2629 return (EINVAL); 2630 2631 /* 2632 * Version scheme: 2633 * bits 0..9: chip version 2634 * bits 10..15: chip revision 2635 */ 2636 t->version = 3 | (sc->params.rev << 10); 2637 2638 /* 2639 * Read 256 bytes at a time as len can be large and we don't 2640 * want to use huge intermediate buffers. 2641 */ 2642 useraddr = (uint8_t *)t->buf; 2643 while (len) { 2644 unsigned int chunk = min(len, sizeof(buf)); 2645 2646 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2647 if (error) 2648 return (-error); 2649 if (copyout(buf, useraddr, chunk)) 2650 return (EFAULT); 2651 useraddr += chunk; 2652 addr += chunk; 2653 len -= chunk; 2654 } 2655 break; 2656 } 2657 case CHELSIO_READ_TCAM_WORD: { 2658 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2659 2660 if (!is_offload(sc)) 2661 return (EOPNOTSUPP); 2662 if (!(sc->flags & FULL_INIT_DONE)) 2663 return (EIO); /* need MC5 */ 2664 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2665 break; 2666 } 2667 case CHELSIO_SET_TRACE_FILTER: { 2668 struct ch_trace *t = (struct ch_trace *)data; 2669 const struct trace_params *tp; 2670 2671 tp = (const struct trace_params *)&t->sip; 2672 if (t->config_tx) 2673 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2674 t->trace_tx); 2675 if (t->config_rx) 2676 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2677 t->trace_rx); 2678 break; 2679 } 2680 case CHELSIO_SET_PKTSCHED: { 2681 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2682 if (sc->open_device_map == 0) 2683 return (EAGAIN); 2684 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2685 p->binding); 2686 break; 2687 } 2688 case CHELSIO_IFCONF_GETREGS: { 2689 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2690 int reglen = cxgb_get_regs_len(); 2691 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2692 if (buf == NULL) { 2693 return (ENOMEM); 2694 } 2695 if (regs->len > reglen) 2696 regs->len = reglen; 2697 else if (regs->len < reglen) 2698 error = E2BIG; 2699 2700 if (!error) { 2701 cxgb_get_regs(sc, regs, buf); 2702 error = copyout(buf, regs->data, reglen); 2703 } 2704 free(buf, M_DEVBUF); 2705 2706 break; 2707 } 2708 case CHELSIO_SET_HW_SCHED: { 2709 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2710 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2711 2712 if ((sc->flags & FULL_INIT_DONE) == 0) 2713 return (EAGAIN); /* need TP to be initialized */ 2714 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2715 !in_range(t->channel, 0, 1) || 2716 !in_range(t->kbps, 0, 10000000) || 2717 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2718 !in_range(t->flow_ipg, 0, 2719 dack_ticks_to_usec(sc, 0x7ff))) 2720 return (EINVAL); 2721 2722 if (t->kbps >= 0) { 2723 error = t3_config_sched(sc, t->kbps, t->sched); 2724 if (error < 0) 2725 return (-error); 2726 } 2727 if (t->class_ipg >= 0) 2728 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2729 if (t->flow_ipg >= 0) { 2730 t->flow_ipg *= 1000; /* us -> ns */ 2731 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2732 } 2733 if (t->mode >= 0) { 2734 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2735 2736 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2737 bit, t->mode ? bit : 0); 2738 } 2739 if (t->channel >= 0) 2740 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2741 1 << t->sched, t->channel << t->sched); 2742 break; 2743 } 2744 case CHELSIO_GET_EEPROM: { 2745 int i; 2746 struct ch_eeprom *e = (struct ch_eeprom *)data; 2747 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2748 2749 if (buf == NULL) { 2750 return (ENOMEM); 2751 } 2752 e->magic = EEPROM_MAGIC; 2753 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2754 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2755 2756 if (!error) 2757 error = copyout(buf + e->offset, e->data, e->len); 2758 2759 free(buf, M_DEVBUF); 2760 break; 2761 } 2762 case CHELSIO_CLEAR_STATS: { 2763 if (!(sc->flags & FULL_INIT_DONE)) 2764 return EAGAIN; 2765 2766 PORT_LOCK(pi); 2767 t3_mac_update_stats(&pi->mac); 2768 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 2769 PORT_UNLOCK(pi); 2770 break; 2771 } 2772 default: 2773 return (EOPNOTSUPP); 2774 break; 2775 } 2776 2777 return (error); 2778 } 2779 2780 static __inline void 2781 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 2782 unsigned int end) 2783 { 2784 uint32_t *p = (uint32_t *)(buf + start); 2785 2786 for ( ; start <= end; start += sizeof(uint32_t)) 2787 *p++ = t3_read_reg(ap, start); 2788 } 2789 2790 #define T3_REGMAP_SIZE (3 * 1024) 2791 static int 2792 cxgb_get_regs_len(void) 2793 { 2794 return T3_REGMAP_SIZE; 2795 } 2796 2797 static void 2798 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 2799 { 2800 2801 /* 2802 * Version scheme: 2803 * bits 0..9: chip version 2804 * bits 10..15: chip revision 2805 * bit 31: set for PCIe cards 2806 */ 2807 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 2808 2809 /* 2810 * We skip the MAC statistics registers because they are clear-on-read. 2811 * Also reading multi-register stats would need to synchronize with the 2812 * periodic mac stats accumulation. Hard to justify the complexity. 2813 */ 2814 memset(buf, 0, cxgb_get_regs_len()); 2815 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 2816 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 2817 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 2818 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 2819 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 2820 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 2821 XGM_REG(A_XGM_SERDES_STAT3, 1)); 2822 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 2823 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 2824 } 2825 2826 2827 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1); 2828