1 /************************************************************************** 2 3 Copyright (c) 2007-2009, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 POSSIBILITY OF SUCH DAMAGE. 27 28 ***************************************************************************/ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/bus.h> 37 #include <sys/module.h> 38 #include <sys/pciio.h> 39 #include <sys/conf.h> 40 #include <machine/bus.h> 41 #include <machine/resource.h> 42 #include <sys/bus_dma.h> 43 #include <sys/ktr.h> 44 #include <sys/rman.h> 45 #include <sys/ioccom.h> 46 #include <sys/mbuf.h> 47 #include <sys/linker.h> 48 #include <sys/firmware.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/smp.h> 52 #include <sys/sysctl.h> 53 #include <sys/syslog.h> 54 #include <sys/queue.h> 55 #include <sys/taskqueue.h> 56 #include <sys/proc.h> 57 58 #include <net/bpf.h> 59 #include <net/ethernet.h> 60 #include <net/if.h> 61 #include <net/if_arp.h> 62 #include <net/if_dl.h> 63 #include <net/if_media.h> 64 #include <net/if_types.h> 65 #include <net/if_vlan_var.h> 66 67 #include <netinet/in_systm.h> 68 #include <netinet/in.h> 69 #include <netinet/if_ether.h> 70 #include <netinet/ip.h> 71 #include <netinet/ip.h> 72 #include <netinet/tcp.h> 73 #include <netinet/udp.h> 74 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 #include <dev/pci/pci_private.h> 78 79 #include <cxgb_include.h> 80 81 #ifdef PRIV_SUPPORTED 82 #include <sys/priv.h> 83 #endif 84 85 static int cxgb_setup_msix(adapter_t *, int); 86 static void cxgb_teardown_msix(adapter_t *); 87 static void cxgb_init(void *); 88 static void cxgb_init_locked(struct port_info *); 89 static void cxgb_stop_locked(struct port_info *); 90 static void cxgb_set_rxmode(struct port_info *); 91 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 92 static int cxgb_media_change(struct ifnet *); 93 static int cxgb_ifm_type(int); 94 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 95 static int setup_sge_qsets(adapter_t *); 96 static void cxgb_async_intr(void *); 97 static void cxgb_ext_intr_handler(void *, int); 98 static void cxgb_tick_handler(void *, int); 99 static void cxgb_down_locked(struct adapter *sc); 100 static void cxgb_tick(void *); 101 static void setup_rss(adapter_t *sc); 102 103 /* Attachment glue for the PCI controller end of the device. Each port of 104 * the device is attached separately, as defined later. 105 */ 106 static int cxgb_controller_probe(device_t); 107 static int cxgb_controller_attach(device_t); 108 static int cxgb_controller_detach(device_t); 109 static void cxgb_free(struct adapter *); 110 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 111 unsigned int end); 112 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 113 static int cxgb_get_regs_len(void); 114 static int offload_open(struct port_info *pi); 115 static void touch_bars(device_t dev); 116 static int offload_close(struct t3cdev *tdev); 117 static void cxgb_link_start(struct port_info *p); 118 static void cxgb_link_fault(void *arg, int ncount); 119 120 static device_method_t cxgb_controller_methods[] = { 121 DEVMETHOD(device_probe, cxgb_controller_probe), 122 DEVMETHOD(device_attach, cxgb_controller_attach), 123 DEVMETHOD(device_detach, cxgb_controller_detach), 124 125 /* bus interface */ 126 DEVMETHOD(bus_print_child, bus_generic_print_child), 127 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 128 129 { 0, 0 } 130 }; 131 132 static driver_t cxgb_controller_driver = { 133 "cxgbc", 134 cxgb_controller_methods, 135 sizeof(struct adapter) 136 }; 137 138 static devclass_t cxgb_controller_devclass; 139 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 140 141 /* 142 * Attachment glue for the ports. Attachment is done directly to the 143 * controller device. 144 */ 145 static int cxgb_port_probe(device_t); 146 static int cxgb_port_attach(device_t); 147 static int cxgb_port_detach(device_t); 148 149 static device_method_t cxgb_port_methods[] = { 150 DEVMETHOD(device_probe, cxgb_port_probe), 151 DEVMETHOD(device_attach, cxgb_port_attach), 152 DEVMETHOD(device_detach, cxgb_port_detach), 153 { 0, 0 } 154 }; 155 156 static driver_t cxgb_port_driver = { 157 "cxgb", 158 cxgb_port_methods, 159 0 160 }; 161 162 static d_ioctl_t cxgb_extension_ioctl; 163 static d_open_t cxgb_extension_open; 164 static d_close_t cxgb_extension_close; 165 166 static struct cdevsw cxgb_cdevsw = { 167 .d_version = D_VERSION, 168 .d_flags = 0, 169 .d_open = cxgb_extension_open, 170 .d_close = cxgb_extension_close, 171 .d_ioctl = cxgb_extension_ioctl, 172 .d_name = "cxgb", 173 }; 174 175 static devclass_t cxgb_port_devclass; 176 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 177 178 #define SGE_MSIX_COUNT (SGE_QSETS + 1) 179 180 /* 181 * The driver uses the best interrupt scheme available on a platform in the 182 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 183 * of these schemes the driver may consider as follows: 184 * 185 * msi = 2: choose from among all three options 186 * msi = 1 : only consider MSI and pin interrupts 187 * msi = 0: force pin interrupts 188 */ 189 static int msi_allowed = 2; 190 191 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 192 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 193 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 194 "MSI-X, MSI, INTx selector"); 195 196 /* 197 * The driver enables offload as a default. 198 * To disable it, use ofld_disable = 1. 199 */ 200 static int ofld_disable = 0; 201 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable); 202 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0, 203 "disable ULP offload"); 204 205 /* 206 * The driver uses an auto-queue algorithm by default. 207 * To disable it and force a single queue-set per port, use multiq = 0 208 */ 209 static int multiq = 1; 210 TUNABLE_INT("hw.cxgb.multiq", &multiq); 211 SYSCTL_UINT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0, 212 "use min(ncpus/ports, 8) queue-sets per port"); 213 214 /* 215 * By default the driver will not update the firmware unless 216 * it was compiled against a newer version 217 * 218 */ 219 static int force_fw_update = 0; 220 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update); 221 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 222 "update firmware even if up to date"); 223 224 int cxgb_use_16k_clusters = 1; 225 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters); 226 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 227 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 228 229 enum { 230 MAX_TXQ_ENTRIES = 16384, 231 MAX_CTRL_TXQ_ENTRIES = 1024, 232 MAX_RSPQ_ENTRIES = 16384, 233 MAX_RX_BUFFERS = 16384, 234 MAX_RX_JUMBO_BUFFERS = 16384, 235 MIN_TXQ_ENTRIES = 4, 236 MIN_CTRL_TXQ_ENTRIES = 4, 237 MIN_RSPQ_ENTRIES = 32, 238 MIN_FL_ENTRIES = 32, 239 MIN_FL_JUMBO_ENTRIES = 32 240 }; 241 242 struct filter_info { 243 u32 sip; 244 u32 sip_mask; 245 u32 dip; 246 u16 sport; 247 u16 dport; 248 u32 vlan:12; 249 u32 vlan_prio:3; 250 u32 mac_hit:1; 251 u32 mac_idx:4; 252 u32 mac_vld:1; 253 u32 pkt_type:2; 254 u32 report_filter_id:1; 255 u32 pass:1; 256 u32 rss:1; 257 u32 qset:3; 258 u32 locked:1; 259 u32 valid:1; 260 }; 261 262 enum { FILTER_NO_VLAN_PRI = 7 }; 263 264 #define EEPROM_MAGIC 0x38E2F10C 265 266 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 267 268 /* Table for probing the cards. The desc field isn't actually used */ 269 struct cxgb_ident { 270 uint16_t vendor; 271 uint16_t device; 272 int index; 273 char *desc; 274 } cxgb_identifiers[] = { 275 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 276 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 277 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 278 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 279 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 280 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 281 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 282 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 283 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 284 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 285 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 286 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "N310E"}, 287 {0, 0, 0, NULL} 288 }; 289 290 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 291 292 293 static __inline char 294 t3rev2char(struct adapter *adapter) 295 { 296 char rev = 'z'; 297 298 switch(adapter->params.rev) { 299 case T3_REV_A: 300 rev = 'a'; 301 break; 302 case T3_REV_B: 303 case T3_REV_B2: 304 rev = 'b'; 305 break; 306 case T3_REV_C: 307 rev = 'c'; 308 break; 309 } 310 return rev; 311 } 312 313 static struct cxgb_ident * 314 cxgb_get_ident(device_t dev) 315 { 316 struct cxgb_ident *id; 317 318 for (id = cxgb_identifiers; id->desc != NULL; id++) { 319 if ((id->vendor == pci_get_vendor(dev)) && 320 (id->device == pci_get_device(dev))) { 321 return (id); 322 } 323 } 324 return (NULL); 325 } 326 327 static const struct adapter_info * 328 cxgb_get_adapter_info(device_t dev) 329 { 330 struct cxgb_ident *id; 331 const struct adapter_info *ai; 332 333 id = cxgb_get_ident(dev); 334 if (id == NULL) 335 return (NULL); 336 337 ai = t3_get_adapter_info(id->index); 338 339 return (ai); 340 } 341 342 static int 343 cxgb_controller_probe(device_t dev) 344 { 345 const struct adapter_info *ai; 346 char *ports, buf[80]; 347 int nports; 348 struct adapter *sc = device_get_softc(dev); 349 350 ai = cxgb_get_adapter_info(dev); 351 if (ai == NULL) 352 return (ENXIO); 353 354 nports = ai->nports0 + ai->nports1; 355 if (nports == 1) 356 ports = "port"; 357 else 358 ports = "ports"; 359 360 snprintf(buf, sizeof(buf), "%s %sNIC, rev: %d nports: %d %s", 361 ai->desc, is_offload(sc) ? "R" : "", 362 sc->params.rev, nports, ports); 363 device_set_desc_copy(dev, buf); 364 return (BUS_PROBE_DEFAULT); 365 } 366 367 #define FW_FNAME "cxgb_t3fw" 368 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom" 369 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram" 370 371 static int 372 upgrade_fw(adapter_t *sc) 373 { 374 #ifdef FIRMWARE_LATEST 375 const struct firmware *fw; 376 #else 377 struct firmware *fw; 378 #endif 379 int status; 380 381 if ((fw = firmware_get(FW_FNAME)) == NULL) { 382 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 383 return (ENOENT); 384 } else 385 device_printf(sc->dev, "updating firmware on card\n"); 386 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 387 388 device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status); 389 390 firmware_put(fw, FIRMWARE_UNLOAD); 391 392 return (status); 393 } 394 395 static int 396 cxgb_controller_attach(device_t dev) 397 { 398 device_t child; 399 const struct adapter_info *ai; 400 struct adapter *sc; 401 int i, error = 0; 402 uint32_t vers; 403 int port_qsets = 1; 404 #ifdef MSI_SUPPORTED 405 int msi_needed, reg; 406 #endif 407 char buf[80]; 408 409 sc = device_get_softc(dev); 410 sc->dev = dev; 411 sc->msi_count = 0; 412 ai = cxgb_get_adapter_info(dev); 413 414 /* 415 * XXX not really related but a recent addition 416 */ 417 #ifdef MSI_SUPPORTED 418 /* find the PCIe link width and set max read request to 4KB*/ 419 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 420 uint16_t lnk, pectl; 421 lnk = pci_read_config(dev, reg + 0x12, 2); 422 sc->link_width = (lnk >> 4) & 0x3f; 423 424 pectl = pci_read_config(dev, reg + 0x8, 2); 425 pectl = (pectl & ~0x7000) | (5 << 12); 426 pci_write_config(dev, reg + 0x8, pectl, 2); 427 } 428 429 if (sc->link_width != 0 && sc->link_width <= 4 && 430 (ai->nports0 + ai->nports1) <= 2) { 431 device_printf(sc->dev, 432 "PCIe x%d Link, expect reduced performance\n", 433 sc->link_width); 434 } 435 #endif 436 touch_bars(dev); 437 pci_enable_busmaster(dev); 438 /* 439 * Allocate the registers and make them available to the driver. 440 * The registers that we care about for NIC mode are in BAR 0 441 */ 442 sc->regs_rid = PCIR_BAR(0); 443 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 444 &sc->regs_rid, RF_ACTIVE)) == NULL) { 445 device_printf(dev, "Cannot allocate BAR region 0\n"); 446 return (ENXIO); 447 } 448 sc->udbs_rid = PCIR_BAR(2); 449 sc->udbs_res = NULL; 450 if (is_offload(sc) && 451 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 452 &sc->udbs_rid, RF_ACTIVE)) == NULL)) { 453 device_printf(dev, "Cannot allocate BAR region 1\n"); 454 error = ENXIO; 455 goto out; 456 } 457 458 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 459 device_get_unit(dev)); 460 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 461 462 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 463 device_get_unit(dev)); 464 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 465 device_get_unit(dev)); 466 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 467 device_get_unit(dev)); 468 469 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 470 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 471 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 472 473 sc->bt = rman_get_bustag(sc->regs_res); 474 sc->bh = rman_get_bushandle(sc->regs_res); 475 sc->mmio_len = rman_get_size(sc->regs_res); 476 477 if (t3_prep_adapter(sc, ai, 1) < 0) { 478 printf("prep adapter failed\n"); 479 error = ENODEV; 480 goto out; 481 } 482 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 483 * enough messages for the queue sets. If that fails, try falling 484 * back to MSI. If that fails, then try falling back to the legacy 485 * interrupt pin model. 486 */ 487 #ifdef MSI_SUPPORTED 488 489 sc->msix_regs_rid = 0x20; 490 if ((msi_allowed >= 2) && 491 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 492 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 493 494 msi_needed = sc->msi_count = SGE_MSIX_COUNT; 495 496 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) || 497 (sc->msi_count != msi_needed)) { 498 device_printf(dev, "msix allocation failed - msi_count = %d" 499 " msi_needed=%d will try msi err=%d\n", sc->msi_count, 500 msi_needed, error); 501 sc->msi_count = 0; 502 pci_release_msi(dev); 503 bus_release_resource(dev, SYS_RES_MEMORY, 504 sc->msix_regs_rid, sc->msix_regs_res); 505 sc->msix_regs_res = NULL; 506 } else { 507 sc->flags |= USING_MSIX; 508 sc->cxgb_intr = t3_intr_msix; 509 } 510 } 511 512 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 513 sc->msi_count = 1; 514 if (pci_alloc_msi(dev, &sc->msi_count)) { 515 device_printf(dev, "alloc msi failed - will try INTx\n"); 516 sc->msi_count = 0; 517 pci_release_msi(dev); 518 } else { 519 sc->flags |= USING_MSI; 520 sc->irq_rid = 1; 521 sc->cxgb_intr = t3_intr_msi; 522 } 523 } 524 #endif 525 if (sc->msi_count == 0) { 526 device_printf(dev, "using line interrupts\n"); 527 sc->irq_rid = 0; 528 sc->cxgb_intr = t3b_intr; 529 } 530 531 if ((sc->flags & USING_MSIX) && multiq) 532 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); 533 534 /* Create a private taskqueue thread for handling driver events */ 535 #ifdef TASKQUEUE_CURRENT 536 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 537 taskqueue_thread_enqueue, &sc->tq); 538 #else 539 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 540 taskqueue_thread_enqueue, &sc->tq); 541 #endif 542 if (sc->tq == NULL) { 543 device_printf(dev, "failed to allocate controller task queue\n"); 544 goto out; 545 } 546 547 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 548 device_get_nameunit(dev)); 549 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 550 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 551 552 553 /* Create a periodic callout for checking adapter status */ 554 callout_init(&sc->cxgb_tick_ch, TRUE); 555 556 if (t3_check_fw_version(sc) < 0 || force_fw_update) { 557 /* 558 * Warn user that a firmware update will be attempted in init. 559 */ 560 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 561 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 562 sc->flags &= ~FW_UPTODATE; 563 } else { 564 sc->flags |= FW_UPTODATE; 565 } 566 567 if (t3_check_tpsram_version(sc) < 0) { 568 /* 569 * Warn user that a firmware update will be attempted in init. 570 */ 571 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 572 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 573 sc->flags &= ~TPS_UPTODATE; 574 } else { 575 sc->flags |= TPS_UPTODATE; 576 } 577 578 /* 579 * Create a child device for each MAC. The ethernet attachment 580 * will be done in these children. 581 */ 582 for (i = 0; i < (sc)->params.nports; i++) { 583 struct port_info *pi; 584 585 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 586 device_printf(dev, "failed to add child port\n"); 587 error = EINVAL; 588 goto out; 589 } 590 pi = &sc->port[i]; 591 pi->adapter = sc; 592 pi->nqsets = port_qsets; 593 pi->first_qset = i*port_qsets; 594 pi->port_id = i; 595 pi->tx_chan = i >= ai->nports0; 596 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 597 sc->rxpkt_map[pi->txpkt_intf] = i; 598 sc->port[i].tx_chan = i >= ai->nports0; 599 sc->portdev[i] = child; 600 device_set_softc(child, pi); 601 } 602 if ((error = bus_generic_attach(dev)) != 0) 603 goto out; 604 605 /* initialize sge private state */ 606 t3_sge_init_adapter(sc); 607 608 t3_led_ready(sc); 609 610 cxgb_offload_init(); 611 if (is_offload(sc)) { 612 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); 613 cxgb_adapter_ofld(sc); 614 } 615 error = t3_get_fw_version(sc, &vers); 616 if (error) 617 goto out; 618 619 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 620 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 621 G_FW_VERSION_MICRO(vers)); 622 623 snprintf(buf, sizeof(buf), "%s\t E/C: %s S/N: %s", 624 ai->desc, 625 sc->params.vpd.ec, sc->params.vpd.sn); 626 device_set_desc_copy(dev, buf); 627 628 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 629 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 630 t3_add_attach_sysctls(sc); 631 out: 632 if (error) 633 cxgb_free(sc); 634 635 return (error); 636 } 637 638 static int 639 cxgb_controller_detach(device_t dev) 640 { 641 struct adapter *sc; 642 643 sc = device_get_softc(dev); 644 645 cxgb_free(sc); 646 647 return (0); 648 } 649 650 static void 651 cxgb_free(struct adapter *sc) 652 { 653 int i; 654 655 ADAPTER_LOCK(sc); 656 sc->flags |= CXGB_SHUTDOWN; 657 ADAPTER_UNLOCK(sc); 658 cxgb_pcpu_shutdown_threads(sc); 659 ADAPTER_LOCK(sc); 660 661 /* 662 * drops the lock 663 */ 664 cxgb_down_locked(sc); 665 666 #ifdef MSI_SUPPORTED 667 if (sc->flags & (USING_MSI | USING_MSIX)) { 668 device_printf(sc->dev, "releasing msi message(s)\n"); 669 pci_release_msi(sc->dev); 670 } else { 671 device_printf(sc->dev, "no msi message to release\n"); 672 } 673 #endif 674 if (sc->msix_regs_res != NULL) { 675 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 676 sc->msix_regs_res); 677 } 678 679 t3_sge_deinit_sw(sc); 680 /* 681 * Wait for last callout 682 */ 683 684 DELAY(hz*100); 685 686 for (i = 0; i < (sc)->params.nports; ++i) { 687 if (sc->portdev[i] != NULL) 688 device_delete_child(sc->dev, sc->portdev[i]); 689 } 690 691 bus_generic_detach(sc->dev); 692 if (sc->tq != NULL) { 693 taskqueue_free(sc->tq); 694 sc->tq = NULL; 695 } 696 697 if (is_offload(sc)) { 698 cxgb_adapter_unofld(sc); 699 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT)) 700 offload_close(&sc->tdev); 701 else 702 printf("cxgb_free: DEVMAP_BIT not set\n"); 703 } else 704 printf("not offloading set\n"); 705 #ifdef notyet 706 if (sc->flags & CXGB_OFLD_INIT) 707 cxgb_offload_deactivate(sc); 708 #endif 709 free(sc->filters, M_DEVBUF); 710 t3_sge_free(sc); 711 712 cxgb_offload_exit(); 713 714 if (sc->udbs_res != NULL) 715 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 716 sc->udbs_res); 717 718 if (sc->regs_res != NULL) 719 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 720 sc->regs_res); 721 722 MTX_DESTROY(&sc->mdio_lock); 723 MTX_DESTROY(&sc->sge.reg_lock); 724 MTX_DESTROY(&sc->elmer_lock); 725 ADAPTER_LOCK_DEINIT(sc); 726 } 727 728 /** 729 * setup_sge_qsets - configure SGE Tx/Rx/response queues 730 * @sc: the controller softc 731 * 732 * Determines how many sets of SGE queues to use and initializes them. 733 * We support multiple queue sets per port if we have MSI-X, otherwise 734 * just one queue set per port. 735 */ 736 static int 737 setup_sge_qsets(adapter_t *sc) 738 { 739 int i, j, err, irq_idx = 0, qset_idx = 0; 740 u_int ntxq = SGE_TXQ_PER_SET; 741 742 if ((err = t3_sge_alloc(sc)) != 0) { 743 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 744 return (err); 745 } 746 747 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 748 irq_idx = -1; 749 750 for (i = 0; i < (sc)->params.nports; i++) { 751 struct port_info *pi = &sc->port[i]; 752 753 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 754 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 755 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 756 &sc->params.sge.qset[qset_idx], ntxq, pi); 757 if (err) { 758 t3_free_sge_resources(sc); 759 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", 760 err); 761 return (err); 762 } 763 } 764 } 765 766 return (0); 767 } 768 769 static void 770 cxgb_teardown_msix(adapter_t *sc) 771 { 772 int i, nqsets; 773 774 for (nqsets = i = 0; i < (sc)->params.nports; i++) 775 nqsets += sc->port[i].nqsets; 776 777 for (i = 0; i < nqsets; i++) { 778 if (sc->msix_intr_tag[i] != NULL) { 779 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 780 sc->msix_intr_tag[i]); 781 sc->msix_intr_tag[i] = NULL; 782 } 783 if (sc->msix_irq_res[i] != NULL) { 784 bus_release_resource(sc->dev, SYS_RES_IRQ, 785 sc->msix_irq_rid[i], sc->msix_irq_res[i]); 786 sc->msix_irq_res[i] = NULL; 787 } 788 } 789 } 790 791 static int 792 cxgb_setup_msix(adapter_t *sc, int msix_count) 793 { 794 int i, j, k, nqsets, rid; 795 796 /* The first message indicates link changes and error conditions */ 797 sc->irq_rid = 1; 798 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 799 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 800 device_printf(sc->dev, "Cannot allocate msix interrupt\n"); 801 return (EINVAL); 802 } 803 804 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 805 #ifdef INTR_FILTERS 806 NULL, 807 #endif 808 cxgb_async_intr, sc, &sc->intr_tag)) { 809 device_printf(sc->dev, "Cannot set up interrupt\n"); 810 return (EINVAL); 811 } 812 for (i = k = 0; i < (sc)->params.nports; i++) { 813 nqsets = sc->port[i].nqsets; 814 for (j = 0; j < nqsets; j++, k++) { 815 struct sge_qset *qs = &sc->sge.qs[k]; 816 817 rid = k + 2; 818 if (cxgb_debug) 819 printf("rid=%d ", rid); 820 if ((sc->msix_irq_res[k] = bus_alloc_resource_any( 821 sc->dev, SYS_RES_IRQ, &rid, 822 RF_SHAREABLE | RF_ACTIVE)) == NULL) { 823 device_printf(sc->dev, "Cannot allocate " 824 "interrupt for message %d\n", rid); 825 return (EINVAL); 826 } 827 sc->msix_irq_rid[k] = rid; 828 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k], 829 INTR_MPSAFE|INTR_TYPE_NET, 830 #ifdef INTR_FILTERS 831 NULL, 832 #endif 833 t3_intr_msix, qs, &sc->msix_intr_tag[k])) { 834 device_printf(sc->dev, "Cannot set up " 835 "interrupt for message %d\n", rid); 836 return (EINVAL); 837 838 } 839 #if 0 840 #ifdef IFNET_MULTIQUEUE 841 if (multiq) { 842 int vector = rman_get_start(sc->msix_irq_res[k]); 843 if (bootverbose) 844 device_printf(sc->dev, "binding vector=%d to cpu=%d\n", vector, k % mp_ncpus); 845 intr_bind(vector, k % mp_ncpus); 846 } 847 #endif 848 #endif 849 } 850 } 851 852 return (0); 853 } 854 855 static int 856 cxgb_port_probe(device_t dev) 857 { 858 struct port_info *p; 859 char buf[80]; 860 const char *desc; 861 862 p = device_get_softc(dev); 863 desc = p->phy.desc; 864 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc); 865 device_set_desc_copy(dev, buf); 866 return (0); 867 } 868 869 870 static int 871 cxgb_makedev(struct port_info *pi) 872 { 873 874 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 875 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp)); 876 877 if (pi->port_cdev == NULL) 878 return (ENOMEM); 879 880 pi->port_cdev->si_drv1 = (void *)pi; 881 882 return (0); 883 } 884 885 #ifndef LRO_SUPPORTED 886 #ifdef IFCAP_LRO 887 #undef IFCAP_LRO 888 #endif 889 #define IFCAP_LRO 0x0 890 #endif 891 892 #ifdef TSO_SUPPORTED 893 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO) 894 /* Don't enable TSO6 yet */ 895 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU | IFCAP_LRO) 896 #else 897 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 898 /* Don't enable TSO6 yet */ 899 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 900 #define IFCAP_TSO4 0x0 901 #define IFCAP_TSO6 0x0 902 #define CSUM_TSO 0x0 903 #endif 904 905 906 static int 907 cxgb_port_attach(device_t dev) 908 { 909 struct port_info *p; 910 struct ifnet *ifp; 911 int err, media_flags; 912 struct adapter *sc; 913 914 915 p = device_get_softc(dev); 916 sc = p->adapter; 917 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 918 device_get_unit(device_get_parent(dev)), p->port_id); 919 PORT_LOCK_INIT(p, p->lockbuf); 920 921 /* Allocate an ifnet object and set it up */ 922 ifp = p->ifp = if_alloc(IFT_ETHER); 923 if (ifp == NULL) { 924 device_printf(dev, "Cannot allocate ifnet\n"); 925 return (ENOMEM); 926 } 927 928 /* 929 * Note that there is currently no watchdog timer. 930 */ 931 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 932 ifp->if_init = cxgb_init; 933 ifp->if_softc = p; 934 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 935 ifp->if_ioctl = cxgb_ioctl; 936 ifp->if_start = cxgb_start; 937 938 939 ifp->if_timer = 0; /* Disable ifnet watchdog */ 940 ifp->if_watchdog = NULL; 941 942 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 943 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 944 IFQ_SET_READY(&ifp->if_snd); 945 946 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 947 ifp->if_capabilities |= CXGB_CAP; 948 ifp->if_capenable |= CXGB_CAP_ENABLE; 949 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 950 /* 951 * disable TSO on 4-port - it isn't supported by the firmware yet 952 */ 953 if (p->adapter->params.nports > 2) { 954 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6); 955 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6); 956 ifp->if_hwassist &= ~CSUM_TSO; 957 } 958 959 ether_ifattach(ifp, p->hw_addr); 960 #ifdef IFNET_MULTIQUEUE 961 ifp->if_transmit = cxgb_pcpu_transmit; 962 #endif 963 /* 964 * Only default to jumbo frames on 10GigE 965 */ 966 if (p->adapter->params.nports <= 2) 967 ifp->if_mtu = ETHERMTU_JUMBO; 968 if ((err = cxgb_makedev(p)) != 0) { 969 printf("makedev failed %d\n", err); 970 return (err); 971 } 972 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 973 cxgb_media_status); 974 975 if (!strcmp(p->phy.desc, "10GBASE-CX4")) { 976 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX; 977 } else if (!strcmp(p->phy.desc, "10GBASE-SR")) { 978 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX; 979 } else if (!strcmp(p->phy.desc, "10GBASE-R")) { 980 media_flags = cxgb_ifm_type(p->phy.modtype); 981 } else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) { 982 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL); 983 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX, 984 0, NULL); 985 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX, 986 0, NULL); 987 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 988 0, NULL); 989 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 990 0, NULL); 991 media_flags = 0; 992 } else if (!strcmp(p->phy.desc, "1000BASE-X")) { 993 /* 994 * XXX: This is not very accurate. Fix when common code 995 * returns more specific value - eg 1000BASE-SX, LX, etc. 996 * 997 * XXX: In the meantime, don't lie. Consider setting IFM_AUTO 998 * instead of SX. 999 */ 1000 media_flags = IFM_ETHER | IFM_1000_SX | IFM_FDX; 1001 } else { 1002 printf("unsupported media type %s\n", p->phy.desc); 1003 return (ENXIO); 1004 } 1005 if (media_flags) { 1006 /* 1007 * Note the modtype on which we based our flags. If modtype 1008 * changes, we'll redo the ifmedia for this ifp. modtype may 1009 * change when transceivers are plugged in/out, and in other 1010 * situations. 1011 */ 1012 ifmedia_add(&p->media, media_flags, p->phy.modtype, NULL); 1013 ifmedia_set(&p->media, media_flags); 1014 } else { 1015 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1016 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO); 1017 } 1018 1019 /* Get the latest mac address, User can use a LAA */ 1020 bcopy(IF_LLADDR(p->ifp), p->hw_addr, ETHER_ADDR_LEN); 1021 t3_sge_init_port(p); 1022 1023 TASK_INIT(&p->link_fault_task, 0, cxgb_link_fault, p); 1024 1025 #if defined(LINK_ATTACH) 1026 cxgb_link_start(p); 1027 t3_link_changed(sc, p->port_id); 1028 #endif 1029 return (0); 1030 } 1031 1032 static int 1033 cxgb_port_detach(device_t dev) 1034 { 1035 struct port_info *p; 1036 1037 p = device_get_softc(dev); 1038 1039 PORT_LOCK(p); 1040 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING) 1041 cxgb_stop_locked(p); 1042 PORT_UNLOCK(p); 1043 1044 ether_ifdetach(p->ifp); 1045 printf("waiting for callout to stop ..."); 1046 DELAY(1000000); 1047 printf("done\n"); 1048 /* 1049 * the lock may be acquired in ifdetach 1050 */ 1051 PORT_LOCK_DEINIT(p); 1052 if_free(p->ifp); 1053 1054 if (p->port_cdev != NULL) 1055 destroy_dev(p->port_cdev); 1056 1057 return (0); 1058 } 1059 1060 void 1061 t3_fatal_err(struct adapter *sc) 1062 { 1063 u_int fw_status[4]; 1064 1065 if (sc->flags & FULL_INIT_DONE) { 1066 t3_sge_stop(sc); 1067 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1068 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1069 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1070 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1071 t3_intr_disable(sc); 1072 } 1073 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1074 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1075 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1076 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1077 } 1078 1079 int 1080 t3_os_find_pci_capability(adapter_t *sc, int cap) 1081 { 1082 device_t dev; 1083 struct pci_devinfo *dinfo; 1084 pcicfgregs *cfg; 1085 uint32_t status; 1086 uint8_t ptr; 1087 1088 dev = sc->dev; 1089 dinfo = device_get_ivars(dev); 1090 cfg = &dinfo->cfg; 1091 1092 status = pci_read_config(dev, PCIR_STATUS, 2); 1093 if (!(status & PCIM_STATUS_CAPPRESENT)) 1094 return (0); 1095 1096 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1097 case 0: 1098 case 1: 1099 ptr = PCIR_CAP_PTR; 1100 break; 1101 case 2: 1102 ptr = PCIR_CAP_PTR_2; 1103 break; 1104 default: 1105 return (0); 1106 break; 1107 } 1108 ptr = pci_read_config(dev, ptr, 1); 1109 1110 while (ptr != 0) { 1111 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1112 return (ptr); 1113 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1114 } 1115 1116 return (0); 1117 } 1118 1119 int 1120 t3_os_pci_save_state(struct adapter *sc) 1121 { 1122 device_t dev; 1123 struct pci_devinfo *dinfo; 1124 1125 dev = sc->dev; 1126 dinfo = device_get_ivars(dev); 1127 1128 pci_cfg_save(dev, dinfo, 0); 1129 return (0); 1130 } 1131 1132 int 1133 t3_os_pci_restore_state(struct adapter *sc) 1134 { 1135 device_t dev; 1136 struct pci_devinfo *dinfo; 1137 1138 dev = sc->dev; 1139 dinfo = device_get_ivars(dev); 1140 1141 pci_cfg_restore(dev, dinfo); 1142 return (0); 1143 } 1144 1145 void t3_os_link_fault(struct adapter *adap, int port_id, int state) 1146 { 1147 struct port_info *pi = &adap->port[port_id]; 1148 1149 if (!state) { 1150 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 1151 return; 1152 } 1153 1154 if (adap->params.nports <= 2) { 1155 struct cmac *mac = &pi->mac; 1156 1157 /* Clear local faults */ 1158 t3_xgm_intr_disable(adap, port_id); 1159 t3_read_reg(adap, A_XGM_INT_STATUS + pi->mac.offset); 1160 t3_write_reg(adap, A_XGM_INT_CAUSE + pi->mac.offset, F_XGM_INT); 1161 1162 t3_set_reg_field(adap, A_XGM_INT_ENABLE + pi->mac.offset, 1163 F_XGM_INT, F_XGM_INT); 1164 t3_xgm_intr_enable(adap, pi->port_id); 1165 t3_mac_enable(mac, MAC_DIRECTION_TX); 1166 } 1167 1168 if_link_state_change(pi->ifp, LINK_STATE_UP); 1169 } 1170 1171 /** 1172 * t3_os_link_changed - handle link status changes 1173 * @adapter: the adapter associated with the link change 1174 * @port_id: the port index whose limk status has changed 1175 * @link_status: the new status of the link 1176 * @speed: the new speed setting 1177 * @duplex: the new duplex setting 1178 * @fc: the new flow-control setting 1179 * 1180 * This is the OS-dependent handler for link status changes. The OS 1181 * neutral handler takes care of most of the processing for these events, 1182 * then calls this handler for any OS-specific processing. 1183 */ 1184 void 1185 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1186 int duplex, int fc) 1187 { 1188 struct port_info *pi = &adapter->port[port_id]; 1189 struct cmac *mac = &adapter->port[port_id].mac; 1190 1191 if (link_status) { 1192 DELAY(10); 1193 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1194 /* Clear errors created by MAC enable */ 1195 t3_set_reg_field(adapter, A_XGM_STAT_CTRL + pi->mac.offset, 1196 F_CLRSTATS, 1); 1197 1198 if (adapter->params.nports <= 2) { 1199 /* Clear local faults */ 1200 t3_xgm_intr_disable(adapter, pi->port_id); 1201 t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset); 1202 t3_write_reg(adapter, A_XGM_INT_CAUSE + pi->mac.offset, 1203 F_XGM_INT); 1204 1205 t3_set_reg_field(adapter, 1206 A_XGM_INT_ENABLE + pi->mac.offset, 1207 F_XGM_INT, F_XGM_INT); 1208 t3_xgm_intr_enable(adapter, pi->port_id); 1209 } 1210 1211 if_link_state_change(pi->ifp, LINK_STATE_UP); 1212 } else { 1213 t3_xgm_intr_disable(adapter, pi->port_id); 1214 t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset); 1215 if (adapter->params.nports <= 2) { 1216 t3_set_reg_field(adapter, 1217 A_XGM_INT_ENABLE + pi->mac.offset, 1218 F_XGM_INT, 0); 1219 } 1220 1221 /* PR 5666. We shouldn't power down 1G phys */ 1222 if (is_10G(adapter)) 1223 pi->phy.ops->power_down(&pi->phy, 1); 1224 1225 t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset); 1226 t3_mac_disable(mac, MAC_DIRECTION_RX); 1227 t3_link_start(&pi->phy, mac, &pi->link_config); 1228 1229 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 1230 } 1231 } 1232 1233 /** 1234 * t3_os_phymod_changed - handle PHY module changes 1235 * @phy: the PHY reporting the module change 1236 * @mod_type: new module type 1237 * 1238 * This is the OS-dependent handler for PHY module changes. It is 1239 * invoked when a PHY module is removed or inserted for any OS-specific 1240 * processing. 1241 */ 1242 void t3_os_phymod_changed(struct adapter *adap, int port_id) 1243 { 1244 static const char *mod_str[] = { 1245 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown" 1246 }; 1247 1248 struct port_info *pi = &adap->port[port_id]; 1249 1250 if (pi->phy.modtype == phy_modtype_none) 1251 device_printf(adap->dev, "PHY module unplugged\n"); 1252 else { 1253 KASSERT(pi->phy.modtype < ARRAY_SIZE(mod_str), 1254 ("invalid PHY module type %d", pi->phy.modtype)); 1255 device_printf(adap->dev, "%s PHY module inserted\n", 1256 mod_str[pi->phy.modtype]); 1257 } 1258 } 1259 1260 /* 1261 * Interrupt-context handler for external (PHY) interrupts. 1262 */ 1263 void 1264 t3_os_ext_intr_handler(adapter_t *sc) 1265 { 1266 if (cxgb_debug) 1267 printf("t3_os_ext_intr_handler\n"); 1268 /* 1269 * Schedule a task to handle external interrupts as they may be slow 1270 * and we use a mutex to protect MDIO registers. We disable PHY 1271 * interrupts in the meantime and let the task reenable them when 1272 * it's done. 1273 */ 1274 ADAPTER_LOCK(sc); 1275 if (sc->slow_intr_mask) { 1276 sc->slow_intr_mask &= ~F_T3DBG; 1277 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1278 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 1279 } 1280 ADAPTER_UNLOCK(sc); 1281 } 1282 1283 static void 1284 cxgb_link_fault(void *arg, int ncount) 1285 { 1286 struct port_info *pi = arg; 1287 1288 t3_link_fault(pi->adapter, pi->port_id); 1289 } 1290 1291 void t3_os_link_fault_handler(struct adapter *sc, int port_id) 1292 { 1293 struct port_info *pi = &sc->port[port_id]; 1294 1295 pi->link_fault = 1; 1296 taskqueue_enqueue(sc->tq, &pi->link_fault_task); 1297 } 1298 1299 void 1300 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1301 { 1302 1303 /* 1304 * The ifnet might not be allocated before this gets called, 1305 * as this is called early on in attach by t3_prep_adapter 1306 * save the address off in the port structure 1307 */ 1308 if (cxgb_debug) 1309 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1310 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1311 } 1312 1313 /** 1314 * link_start - enable a port 1315 * @p: the port to enable 1316 * 1317 * Performs the MAC and PHY actions needed to enable a port. 1318 */ 1319 static void 1320 cxgb_link_start(struct port_info *p) 1321 { 1322 struct ifnet *ifp; 1323 struct t3_rx_mode rm; 1324 struct cmac *mac = &p->mac; 1325 int mtu, hwtagging; 1326 1327 ifp = p->ifp; 1328 1329 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); 1330 1331 mtu = ifp->if_mtu; 1332 if (ifp->if_capenable & IFCAP_VLAN_MTU) 1333 mtu += ETHER_VLAN_ENCAP_LEN; 1334 1335 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; 1336 1337 t3_init_rx_mode(&rm, p); 1338 if (!mac->multiport) 1339 t3_mac_reset(mac); 1340 t3_mac_set_mtu(mac, mtu); 1341 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1342 t3_mac_set_address(mac, 0, p->hw_addr); 1343 t3_mac_set_rx_mode(mac, &rm); 1344 t3_link_start(&p->phy, mac, &p->link_config); 1345 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1346 } 1347 1348 1349 static int 1350 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1351 unsigned long n) 1352 { 1353 int attempts = 5; 1354 1355 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1356 if (!--attempts) 1357 return (ETIMEDOUT); 1358 t3_os_sleep(10); 1359 } 1360 return 0; 1361 } 1362 1363 static int 1364 init_tp_parity(struct adapter *adap) 1365 { 1366 int i; 1367 struct mbuf *m; 1368 struct cpl_set_tcb_field *greq; 1369 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1370 1371 t3_tp_set_offload_mode(adap, 1); 1372 1373 for (i = 0; i < 16; i++) { 1374 struct cpl_smt_write_req *req; 1375 1376 m = m_gethdr(M_WAITOK, MT_DATA); 1377 req = mtod(m, struct cpl_smt_write_req *); 1378 m->m_len = m->m_pkthdr.len = sizeof(*req); 1379 memset(req, 0, sizeof(*req)); 1380 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1381 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1382 req->iff = i; 1383 t3_mgmt_tx(adap, m); 1384 } 1385 1386 for (i = 0; i < 2048; i++) { 1387 struct cpl_l2t_write_req *req; 1388 1389 m = m_gethdr(M_WAITOK, MT_DATA); 1390 req = mtod(m, struct cpl_l2t_write_req *); 1391 m->m_len = m->m_pkthdr.len = sizeof(*req); 1392 memset(req, 0, sizeof(*req)); 1393 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1394 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1395 req->params = htonl(V_L2T_W_IDX(i)); 1396 t3_mgmt_tx(adap, m); 1397 } 1398 1399 for (i = 0; i < 2048; i++) { 1400 struct cpl_rte_write_req *req; 1401 1402 m = m_gethdr(M_WAITOK, MT_DATA); 1403 req = mtod(m, struct cpl_rte_write_req *); 1404 m->m_len = m->m_pkthdr.len = sizeof(*req); 1405 memset(req, 0, sizeof(*req)); 1406 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1407 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1408 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1409 t3_mgmt_tx(adap, m); 1410 } 1411 1412 m = m_gethdr(M_WAITOK, MT_DATA); 1413 greq = mtod(m, struct cpl_set_tcb_field *); 1414 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1415 memset(greq, 0, sizeof(*greq)); 1416 greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1417 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1418 greq->mask = htobe64(1); 1419 t3_mgmt_tx(adap, m); 1420 1421 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1422 t3_tp_set_offload_mode(adap, 0); 1423 return (i); 1424 } 1425 1426 /** 1427 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1428 * @adap: the adapter 1429 * 1430 * Sets up RSS to distribute packets to multiple receive queues. We 1431 * configure the RSS CPU lookup table to distribute to the number of HW 1432 * receive queues, and the response queue lookup table to narrow that 1433 * down to the response queues actually configured for each port. 1434 * We always configure the RSS mapping for two ports since the mapping 1435 * table has plenty of entries. 1436 */ 1437 static void 1438 setup_rss(adapter_t *adap) 1439 { 1440 int i; 1441 u_int nq[2]; 1442 uint8_t cpus[SGE_QSETS + 1]; 1443 uint16_t rspq_map[RSS_TABLE_SIZE]; 1444 1445 for (i = 0; i < SGE_QSETS; ++i) 1446 cpus[i] = i; 1447 cpus[SGE_QSETS] = 0xff; 1448 1449 nq[0] = nq[1] = 0; 1450 for_each_port(adap, i) { 1451 const struct port_info *pi = adap2pinfo(adap, i); 1452 1453 nq[pi->tx_chan] += pi->nqsets; 1454 } 1455 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1456 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1457 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1458 } 1459 /* Calculate the reverse RSS map table */ 1460 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1461 if (adap->rrss_map[rspq_map[i]] == 0xff) 1462 adap->rrss_map[rspq_map[i]] = i; 1463 1464 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1465 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1466 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1467 cpus, rspq_map); 1468 1469 } 1470 1471 /* 1472 * Sends an mbuf to an offload queue driver 1473 * after dealing with any active network taps. 1474 */ 1475 static inline int 1476 offload_tx(struct t3cdev *tdev, struct mbuf *m) 1477 { 1478 int ret; 1479 1480 ret = t3_offload_tx(tdev, m); 1481 return (ret); 1482 } 1483 1484 static int 1485 write_smt_entry(struct adapter *adapter, int idx) 1486 { 1487 struct port_info *pi = &adapter->port[idx]; 1488 struct cpl_smt_write_req *req; 1489 struct mbuf *m; 1490 1491 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1492 return (ENOMEM); 1493 1494 req = mtod(m, struct cpl_smt_write_req *); 1495 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req); 1496 1497 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1498 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 1499 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 1500 req->iff = idx; 1501 memset(req->src_mac1, 0, sizeof(req->src_mac1)); 1502 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 1503 1504 m_set_priority(m, 1); 1505 1506 offload_tx(&adapter->tdev, m); 1507 1508 return (0); 1509 } 1510 1511 static int 1512 init_smt(struct adapter *adapter) 1513 { 1514 int i; 1515 1516 for_each_port(adapter, i) 1517 write_smt_entry(adapter, i); 1518 return 0; 1519 } 1520 1521 static void 1522 init_port_mtus(adapter_t *adapter) 1523 { 1524 unsigned int mtus = adapter->port[0].ifp->if_mtu; 1525 1526 if (adapter->port[1].ifp) 1527 mtus |= adapter->port[1].ifp->if_mtu << 16; 1528 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); 1529 } 1530 1531 static void 1532 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1533 int hi, int port) 1534 { 1535 struct mbuf *m; 1536 struct mngt_pktsched_wr *req; 1537 1538 m = m_gethdr(M_DONTWAIT, MT_DATA); 1539 if (m) { 1540 req = mtod(m, struct mngt_pktsched_wr *); 1541 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1542 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1543 req->sched = sched; 1544 req->idx = qidx; 1545 req->min = lo; 1546 req->max = hi; 1547 req->binding = port; 1548 m->m_len = m->m_pkthdr.len = sizeof(*req); 1549 t3_mgmt_tx(adap, m); 1550 } 1551 } 1552 1553 static void 1554 bind_qsets(adapter_t *sc) 1555 { 1556 int i, j; 1557 1558 cxgb_pcpu_startup_threads(sc); 1559 for (i = 0; i < (sc)->params.nports; ++i) { 1560 const struct port_info *pi = adap2pinfo(sc, i); 1561 1562 for (j = 0; j < pi->nqsets; ++j) { 1563 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1564 -1, pi->tx_chan); 1565 1566 } 1567 } 1568 } 1569 1570 static void 1571 update_tpeeprom(struct adapter *adap) 1572 { 1573 #ifdef FIRMWARE_LATEST 1574 const struct firmware *tpeeprom; 1575 #else 1576 struct firmware *tpeeprom; 1577 #endif 1578 1579 uint32_t version; 1580 unsigned int major, minor; 1581 int ret, len; 1582 char rev, name[32]; 1583 1584 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1585 1586 major = G_TP_VERSION_MAJOR(version); 1587 minor = G_TP_VERSION_MINOR(version); 1588 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1589 return; 1590 1591 rev = t3rev2char(adap); 1592 snprintf(name, sizeof(name), TPEEPROM_NAME, rev); 1593 1594 tpeeprom = firmware_get(name); 1595 if (tpeeprom == NULL) { 1596 device_printf(adap->dev, 1597 "could not load TP EEPROM: unable to load %s\n", 1598 name); 1599 return; 1600 } 1601 1602 len = tpeeprom->datasize - 4; 1603 1604 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1605 if (ret) 1606 goto release_tpeeprom; 1607 1608 if (len != TP_SRAM_LEN) { 1609 device_printf(adap->dev, 1610 "%s length is wrong len=%d expected=%d\n", name, 1611 len, TP_SRAM_LEN); 1612 return; 1613 } 1614 1615 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1616 TP_SRAM_OFFSET); 1617 1618 if (!ret) { 1619 device_printf(adap->dev, 1620 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1621 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1622 } else 1623 device_printf(adap->dev, 1624 "Protocol SRAM image update in EEPROM failed\n"); 1625 1626 release_tpeeprom: 1627 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1628 1629 return; 1630 } 1631 1632 static int 1633 update_tpsram(struct adapter *adap) 1634 { 1635 #ifdef FIRMWARE_LATEST 1636 const struct firmware *tpsram; 1637 #else 1638 struct firmware *tpsram; 1639 #endif 1640 int ret; 1641 char rev, name[32]; 1642 1643 rev = t3rev2char(adap); 1644 snprintf(name, sizeof(name), TPSRAM_NAME, rev); 1645 1646 update_tpeeprom(adap); 1647 1648 tpsram = firmware_get(name); 1649 if (tpsram == NULL){ 1650 device_printf(adap->dev, "could not load TP SRAM\n"); 1651 return (EINVAL); 1652 } else 1653 device_printf(adap->dev, "updating TP SRAM\n"); 1654 1655 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1656 if (ret) 1657 goto release_tpsram; 1658 1659 ret = t3_set_proto_sram(adap, tpsram->data); 1660 if (ret) 1661 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1662 1663 release_tpsram: 1664 firmware_put(tpsram, FIRMWARE_UNLOAD); 1665 1666 return ret; 1667 } 1668 1669 /** 1670 * cxgb_up - enable the adapter 1671 * @adap: adapter being enabled 1672 * 1673 * Called when the first port is enabled, this function performs the 1674 * actions necessary to make an adapter operational, such as completing 1675 * the initialization of HW modules, and enabling interrupts. 1676 */ 1677 static int 1678 cxgb_up(struct adapter *sc) 1679 { 1680 int err = 0; 1681 1682 if ((sc->flags & FULL_INIT_DONE) == 0) { 1683 1684 if ((sc->flags & FW_UPTODATE) == 0) 1685 if ((err = upgrade_fw(sc))) 1686 goto out; 1687 if ((sc->flags & TPS_UPTODATE) == 0) 1688 if ((err = update_tpsram(sc))) 1689 goto out; 1690 err = t3_init_hw(sc, 0); 1691 if (err) 1692 goto out; 1693 1694 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1695 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1696 1697 err = setup_sge_qsets(sc); 1698 if (err) 1699 goto out; 1700 1701 setup_rss(sc); 1702 t3_add_configured_sysctls(sc); 1703 sc->flags |= FULL_INIT_DONE; 1704 } 1705 1706 t3_intr_clear(sc); 1707 1708 /* If it's MSI or INTx, allocate a single interrupt for everything */ 1709 if ((sc->flags & USING_MSIX) == 0) { 1710 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 1711 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 1712 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", 1713 sc->irq_rid); 1714 err = EINVAL; 1715 goto out; 1716 } 1717 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res); 1718 1719 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 1720 #ifdef INTR_FILTERS 1721 NULL, 1722 #endif 1723 sc->cxgb_intr, sc, &sc->intr_tag)) { 1724 device_printf(sc->dev, "Cannot set up interrupt\n"); 1725 err = EINVAL; 1726 goto irq_err; 1727 } 1728 } else { 1729 cxgb_setup_msix(sc, sc->msi_count); 1730 } 1731 1732 t3_sge_start(sc); 1733 t3_intr_enable(sc); 1734 1735 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1736 is_offload(sc) && init_tp_parity(sc) == 0) 1737 sc->flags |= TP_PARITY_INIT; 1738 1739 if (sc->flags & TP_PARITY_INIT) { 1740 t3_write_reg(sc, A_TP_INT_CAUSE, 1741 F_CMCACHEPERR | F_ARPLUTPERR); 1742 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1743 } 1744 1745 1746 if (!(sc->flags & QUEUES_BOUND)) { 1747 bind_qsets(sc); 1748 sc->flags |= QUEUES_BOUND; 1749 } 1750 out: 1751 return (err); 1752 irq_err: 1753 CH_ERR(sc, "request_irq failed, err %d\n", err); 1754 goto out; 1755 } 1756 1757 1758 /* 1759 * Release resources when all the ports and offloading have been stopped. 1760 */ 1761 static void 1762 cxgb_down_locked(struct adapter *sc) 1763 { 1764 1765 t3_sge_stop(sc); 1766 t3_intr_disable(sc); 1767 1768 if (sc->intr_tag != NULL) { 1769 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 1770 sc->intr_tag = NULL; 1771 } 1772 if (sc->irq_res != NULL) { 1773 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", 1774 sc->irq_rid, sc->irq_res); 1775 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 1776 sc->irq_res); 1777 sc->irq_res = NULL; 1778 } 1779 1780 if (sc->flags & USING_MSIX) 1781 cxgb_teardown_msix(sc); 1782 1783 callout_stop(&sc->cxgb_tick_ch); 1784 callout_stop(&sc->sge_timer_ch); 1785 callout_drain(&sc->cxgb_tick_ch); 1786 callout_drain(&sc->sge_timer_ch); 1787 1788 if (sc->tq != NULL) { 1789 printf("draining slow intr\n"); 1790 1791 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1792 printf("draining ext intr\n"); 1793 taskqueue_drain(sc->tq, &sc->ext_intr_task); 1794 printf("draining tick task\n"); 1795 taskqueue_drain(sc->tq, &sc->tick_task); 1796 } 1797 ADAPTER_UNLOCK(sc); 1798 } 1799 1800 static int 1801 offload_open(struct port_info *pi) 1802 { 1803 struct adapter *adapter = pi->adapter; 1804 struct t3cdev *tdev = &adapter->tdev; 1805 1806 int adap_up = adapter->open_device_map & PORT_MASK; 1807 int err = 0; 1808 1809 if (atomic_cmpset_int(&adapter->open_device_map, 1810 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)), 1811 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0) 1812 return (0); 1813 1814 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1815 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", 1816 adapter->open_device_map); 1817 ADAPTER_LOCK(pi->adapter); 1818 if (!adap_up) 1819 err = cxgb_up(adapter); 1820 ADAPTER_UNLOCK(pi->adapter); 1821 if (err) 1822 return (err); 1823 1824 t3_tp_set_offload_mode(adapter, 1); 1825 tdev->lldev = pi->ifp; 1826 1827 init_port_mtus(adapter); 1828 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, 1829 adapter->params.b_wnd, 1830 adapter->params.rev == 0 ? 1831 adapter->port[0].ifp->if_mtu : 0xffff); 1832 init_smt(adapter); 1833 /* Call back all registered clients */ 1834 cxgb_add_clients(tdev); 1835 1836 /* restore them in case the offload module has changed them */ 1837 if (err) { 1838 t3_tp_set_offload_mode(adapter, 0); 1839 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1840 cxgb_set_dummy_ops(tdev); 1841 } 1842 return (err); 1843 } 1844 1845 static int 1846 offload_close(struct t3cdev *tdev) 1847 { 1848 struct adapter *adapter = tdev2adap(tdev); 1849 1850 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1851 return (0); 1852 1853 /* Call back all registered clients */ 1854 cxgb_remove_clients(tdev); 1855 1856 tdev->lldev = NULL; 1857 cxgb_set_dummy_ops(tdev); 1858 t3_tp_set_offload_mode(adapter, 0); 1859 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1860 1861 ADAPTER_LOCK(adapter); 1862 if (!adapter->open_device_map) 1863 cxgb_down_locked(adapter); 1864 else 1865 ADAPTER_UNLOCK(adapter); 1866 return (0); 1867 } 1868 1869 1870 static void 1871 cxgb_init(void *arg) 1872 { 1873 struct port_info *p = arg; 1874 1875 PORT_LOCK(p); 1876 cxgb_init_locked(p); 1877 PORT_UNLOCK(p); 1878 } 1879 1880 static void 1881 cxgb_init_locked(struct port_info *p) 1882 { 1883 struct ifnet *ifp; 1884 adapter_t *sc = p->adapter; 1885 int err; 1886 1887 PORT_LOCK_ASSERT_OWNED(p); 1888 ifp = p->ifp; 1889 1890 ADAPTER_LOCK(p->adapter); 1891 if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) { 1892 ADAPTER_UNLOCK(p->adapter); 1893 cxgb_stop_locked(p); 1894 return; 1895 } 1896 if (p->adapter->open_device_map == 0) { 1897 t3_intr_clear(sc); 1898 } 1899 setbit(&p->adapter->open_device_map, p->port_id); 1900 ADAPTER_UNLOCK(p->adapter); 1901 1902 if (is_offload(sc) && !ofld_disable) { 1903 err = offload_open(p); 1904 if (err) 1905 log(LOG_WARNING, 1906 "Could not initialize offload capabilities\n"); 1907 } 1908 #if !defined(LINK_ATTACH) 1909 cxgb_link_start(p); 1910 t3_link_changed(sc, p->port_id); 1911 #endif 1912 ifp->if_baudrate = IF_Mbps(p->link_config.speed); 1913 1914 device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id); 1915 t3_port_intr_enable(sc, p->port_id); 1916 1917 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 1918 t3_sge_reset_adapter(sc); 1919 1920 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1921 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1922 } 1923 1924 static void 1925 cxgb_set_rxmode(struct port_info *p) 1926 { 1927 struct t3_rx_mode rm; 1928 struct cmac *mac = &p->mac; 1929 1930 t3_init_rx_mode(&rm, p); 1931 mtx_lock(&p->adapter->mdio_lock); 1932 t3_mac_set_rx_mode(mac, &rm); 1933 mtx_unlock(&p->adapter->mdio_lock); 1934 } 1935 1936 static void 1937 cxgb_stop_locked(struct port_info *pi) 1938 { 1939 struct ifnet *ifp; 1940 1941 PORT_LOCK_ASSERT_OWNED(pi); 1942 ADAPTER_LOCK_ASSERT_NOTOWNED(pi->adapter); 1943 1944 ifp = pi->ifp; 1945 t3_port_intr_disable(pi->adapter, pi->port_id); 1946 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1947 1948 /* disable pause frames */ 1949 t3_set_reg_field(pi->adapter, A_XGM_TX_CFG + pi->mac.offset, 1950 F_TXPAUSEEN, 0); 1951 1952 /* Reset RX FIFO HWM */ 1953 t3_set_reg_field(pi->adapter, A_XGM_RXFIFO_CFG + pi->mac.offset, 1954 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1955 1956 1957 ADAPTER_LOCK(pi->adapter); 1958 clrbit(&pi->adapter->open_device_map, pi->port_id); 1959 1960 if (pi->adapter->open_device_map == 0) { 1961 cxgb_down_locked(pi->adapter); 1962 } else 1963 ADAPTER_UNLOCK(pi->adapter); 1964 1965 #if !defined(LINK_ATTACH) 1966 DELAY(100); 1967 1968 /* Wait for TXFIFO empty */ 1969 t3_wait_op_done(pi->adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 1970 F_TXFIFO_EMPTY, 1, 20, 5); 1971 1972 DELAY(100); 1973 t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1974 1975 pi->phy.ops->power_down(&pi->phy, 1); 1976 #endif 1977 1978 } 1979 1980 static int 1981 cxgb_set_mtu(struct port_info *p, int mtu) 1982 { 1983 struct ifnet *ifp = p->ifp; 1984 int error = 0; 1985 1986 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1987 error = EINVAL; 1988 else if (ifp->if_mtu != mtu) { 1989 PORT_LOCK(p); 1990 ifp->if_mtu = mtu; 1991 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1992 cxgb_stop_locked(p); 1993 cxgb_init_locked(p); 1994 } 1995 PORT_UNLOCK(p); 1996 } 1997 return (error); 1998 } 1999 2000 #ifdef LRO_SUPPORTED 2001 /* 2002 * Mark lro enabled or disabled in all qsets for this port 2003 */ 2004 static int 2005 cxgb_set_lro(struct port_info *p, int enabled) 2006 { 2007 int i; 2008 struct adapter *adp = p->adapter; 2009 struct sge_qset *q; 2010 2011 PORT_LOCK_ASSERT_OWNED(p); 2012 for (i = 0; i < p->nqsets; i++) { 2013 q = &adp->sge.qs[p->first_qset + i]; 2014 q->lro.enabled = (enabled != 0); 2015 } 2016 return (0); 2017 } 2018 #endif 2019 2020 static int 2021 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 2022 { 2023 struct port_info *p = ifp->if_softc; 2024 #ifdef INET 2025 struct ifaddr *ifa = (struct ifaddr *)data; 2026 #endif 2027 struct ifreq *ifr = (struct ifreq *)data; 2028 int flags, error = 0, reinit = 0; 2029 uint32_t mask; 2030 2031 /* 2032 * XXX need to check that we aren't in the middle of an unload 2033 */ 2034 switch (command) { 2035 case SIOCSIFMTU: 2036 error = cxgb_set_mtu(p, ifr->ifr_mtu); 2037 break; 2038 case SIOCSIFADDR: 2039 #ifdef INET 2040 if (ifa->ifa_addr->sa_family == AF_INET) { 2041 ifp->if_flags |= IFF_UP; 2042 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2043 PORT_LOCK(p); 2044 cxgb_init_locked(p); 2045 PORT_UNLOCK(p); 2046 } 2047 arp_ifinit(ifp, ifa); 2048 } else 2049 #endif 2050 error = ether_ioctl(ifp, command, data); 2051 break; 2052 case SIOCSIFFLAGS: 2053 PORT_LOCK(p); 2054 if (ifp->if_flags & IFF_UP) { 2055 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2056 flags = p->if_flags; 2057 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 2058 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 2059 cxgb_set_rxmode(p); 2060 } else 2061 cxgb_init_locked(p); 2062 p->if_flags = ifp->if_flags; 2063 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2064 cxgb_stop_locked(p); 2065 2066 PORT_UNLOCK(p); 2067 break; 2068 case SIOCADDMULTI: 2069 case SIOCDELMULTI: 2070 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2071 cxgb_set_rxmode(p); 2072 } 2073 break; 2074 case SIOCSIFMEDIA: 2075 case SIOCGIFMEDIA: 2076 PORT_LOCK(p); 2077 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 2078 PORT_UNLOCK(p); 2079 break; 2080 case SIOCSIFCAP: 2081 PORT_LOCK(p); 2082 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 2083 if (mask & IFCAP_TXCSUM) { 2084 if (IFCAP_TXCSUM & ifp->if_capenable) { 2085 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 2086 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 2087 | CSUM_IP | CSUM_TSO); 2088 } else { 2089 ifp->if_capenable |= IFCAP_TXCSUM; 2090 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP 2091 | CSUM_IP); 2092 } 2093 } 2094 if (mask & IFCAP_RXCSUM) { 2095 ifp->if_capenable ^= IFCAP_RXCSUM; 2096 } 2097 if (mask & IFCAP_TSO4) { 2098 if (IFCAP_TSO4 & ifp->if_capenable) { 2099 ifp->if_capenable &= ~IFCAP_TSO4; 2100 ifp->if_hwassist &= ~CSUM_TSO; 2101 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 2102 ifp->if_capenable |= IFCAP_TSO4; 2103 ifp->if_hwassist |= CSUM_TSO; 2104 } else { 2105 if (cxgb_debug) 2106 printf("cxgb requires tx checksum offload" 2107 " be enabled to use TSO\n"); 2108 error = EINVAL; 2109 } 2110 } 2111 #ifdef LRO_SUPPORTED 2112 if (mask & IFCAP_LRO) { 2113 ifp->if_capenable ^= IFCAP_LRO; 2114 2115 /* Safe to do this even if cxgb_up not called yet */ 2116 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO); 2117 } 2118 #endif 2119 if (mask & IFCAP_VLAN_HWTAGGING) { 2120 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2121 reinit = ifp->if_drv_flags & IFF_DRV_RUNNING; 2122 } 2123 if (mask & IFCAP_VLAN_MTU) { 2124 ifp->if_capenable ^= IFCAP_VLAN_MTU; 2125 reinit = ifp->if_drv_flags & IFF_DRV_RUNNING; 2126 } 2127 if (mask & IFCAP_VLAN_HWCSUM) { 2128 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 2129 } 2130 if (reinit) { 2131 cxgb_stop_locked(p); 2132 cxgb_init_locked(p); 2133 } 2134 PORT_UNLOCK(p); 2135 2136 #ifdef VLAN_CAPABILITIES 2137 VLAN_CAPABILITIES(ifp); 2138 #endif 2139 break; 2140 default: 2141 error = ether_ioctl(ifp, command, data); 2142 break; 2143 } 2144 return (error); 2145 } 2146 2147 static int 2148 cxgb_media_change(struct ifnet *ifp) 2149 { 2150 if_printf(ifp, "media change not supported\n"); 2151 return (ENXIO); 2152 } 2153 2154 /* 2155 * Translates from phy->modtype to IFM_TYPE. 2156 */ 2157 static int 2158 cxgb_ifm_type(int phymod) 2159 { 2160 int rc = IFM_ETHER | IFM_FDX; 2161 2162 switch (phymod) { 2163 case phy_modtype_sr: 2164 rc |= IFM_10G_SR; 2165 break; 2166 case phy_modtype_lr: 2167 rc |= IFM_10G_LR; 2168 break; 2169 case phy_modtype_lrm: 2170 #ifdef IFM_10G_LRM 2171 rc |= IFM_10G_LRM; 2172 #endif 2173 break; 2174 case phy_modtype_twinax: 2175 #ifdef IFM_10G_TWINAX 2176 rc |= IFM_10G_TWINAX; 2177 #endif 2178 break; 2179 case phy_modtype_twinax_long: 2180 #ifdef IFM_10G_TWINAX_LONG 2181 rc |= IFM_10G_TWINAX_LONG; 2182 #endif 2183 break; 2184 case phy_modtype_none: 2185 rc = IFM_ETHER | IFM_NONE; 2186 break; 2187 case phy_modtype_unknown: 2188 break; 2189 } 2190 2191 return (rc); 2192 } 2193 2194 static void 2195 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2196 { 2197 struct port_info *p = ifp->if_softc; 2198 struct ifmedia_entry *cur = p->media.ifm_cur; 2199 int m; 2200 2201 if (cur->ifm_data != p->phy.modtype) { 2202 /* p->media about to be rebuilt, must hold lock */ 2203 PORT_LOCK_ASSERT_OWNED(p); 2204 2205 m = cxgb_ifm_type(p->phy.modtype); 2206 ifmedia_removeall(&p->media); 2207 ifmedia_add(&p->media, m, p->phy.modtype, NULL); 2208 ifmedia_set(&p->media, m); 2209 cur = p->media.ifm_cur; /* ifmedia_set modified ifm_cur */ 2210 ifmr->ifm_current = m; 2211 } 2212 2213 ifmr->ifm_status = IFM_AVALID; 2214 ifmr->ifm_active = IFM_ETHER; 2215 2216 if (!p->link_config.link_ok) 2217 return; 2218 2219 ifmr->ifm_status |= IFM_ACTIVE; 2220 2221 switch (p->link_config.speed) { 2222 case 10: 2223 ifmr->ifm_active |= IFM_10_T; 2224 break; 2225 case 100: 2226 ifmr->ifm_active |= IFM_100_TX; 2227 break; 2228 case 1000: 2229 ifmr->ifm_active |= IFM_1000_T; 2230 break; 2231 case 10000: 2232 ifmr->ifm_active |= IFM_SUBTYPE(cur->ifm_media); 2233 break; 2234 } 2235 2236 if (p->link_config.duplex) 2237 ifmr->ifm_active |= IFM_FDX; 2238 else 2239 ifmr->ifm_active |= IFM_HDX; 2240 } 2241 2242 static void 2243 cxgb_async_intr(void *data) 2244 { 2245 adapter_t *sc = data; 2246 2247 if (cxgb_debug) 2248 device_printf(sc->dev, "cxgb_async_intr\n"); 2249 /* 2250 * May need to sleep - defer to taskqueue 2251 */ 2252 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2253 } 2254 2255 static void 2256 cxgb_ext_intr_handler(void *arg, int count) 2257 { 2258 adapter_t *sc = (adapter_t *)arg; 2259 2260 if (cxgb_debug) 2261 printf("cxgb_ext_intr_handler\n"); 2262 2263 t3_phy_intr_handler(sc); 2264 2265 /* Now reenable external interrupts */ 2266 ADAPTER_LOCK(sc); 2267 if (sc->slow_intr_mask) { 2268 sc->slow_intr_mask |= F_T3DBG; 2269 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 2270 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 2271 } 2272 ADAPTER_UNLOCK(sc); 2273 } 2274 2275 static void 2276 check_link_status(adapter_t *sc) 2277 { 2278 int i; 2279 2280 for (i = 0; i < (sc)->params.nports; ++i) { 2281 struct port_info *p = &sc->port[i]; 2282 2283 if (!(p->phy.caps & SUPPORTED_IRQ)) 2284 t3_link_changed(sc, i); 2285 p->ifp->if_baudrate = IF_Mbps(p->link_config.speed); 2286 } 2287 } 2288 2289 static void 2290 check_t3b2_mac(struct adapter *adapter) 2291 { 2292 int i; 2293 2294 if(adapter->flags & CXGB_SHUTDOWN) 2295 return; 2296 2297 for_each_port(adapter, i) { 2298 struct port_info *p = &adapter->port[i]; 2299 struct ifnet *ifp = p->ifp; 2300 int status; 2301 2302 if(adapter->flags & CXGB_SHUTDOWN) 2303 return; 2304 2305 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2306 continue; 2307 2308 status = 0; 2309 PORT_LOCK(p); 2310 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 2311 status = t3b2_mac_watchdog_task(&p->mac); 2312 if (status == 1) 2313 p->mac.stats.num_toggled++; 2314 else if (status == 2) { 2315 struct cmac *mac = &p->mac; 2316 int mtu = ifp->if_mtu; 2317 2318 if (ifp->if_capenable & IFCAP_VLAN_MTU) 2319 mtu += ETHER_VLAN_ENCAP_LEN; 2320 t3_mac_set_mtu(mac, mtu); 2321 t3_mac_set_address(mac, 0, p->hw_addr); 2322 cxgb_set_rxmode(p); 2323 t3_link_start(&p->phy, mac, &p->link_config); 2324 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2325 t3_port_intr_enable(adapter, p->port_id); 2326 p->mac.stats.num_resets++; 2327 } 2328 PORT_UNLOCK(p); 2329 } 2330 } 2331 2332 static void 2333 cxgb_tick(void *arg) 2334 { 2335 adapter_t *sc = (adapter_t *)arg; 2336 2337 if(sc->flags & CXGB_SHUTDOWN) 2338 return; 2339 2340 taskqueue_enqueue(sc->tq, &sc->tick_task); 2341 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 2342 } 2343 2344 static void 2345 cxgb_tick_handler(void *arg, int count) 2346 { 2347 adapter_t *sc = (adapter_t *)arg; 2348 const struct adapter_params *p = &sc->params; 2349 int i; 2350 uint32_t cause, reset; 2351 2352 if(sc->flags & CXGB_SHUTDOWN) 2353 return; 2354 2355 ADAPTER_LOCK(sc); 2356 if (p->linkpoll_period) 2357 check_link_status(sc); 2358 2359 sc->check_task_cnt++; 2360 2361 /* 2362 * adapter lock can currently only be acquired after the 2363 * port lock 2364 */ 2365 ADAPTER_UNLOCK(sc); 2366 2367 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2368 check_t3b2_mac(sc); 2369 2370 cause = t3_read_reg(sc, A_SG_INT_CAUSE); 2371 reset = 0; 2372 if (cause & F_FLEMPTY) { 2373 struct sge_qset *qs = &sc->sge.qs[0]; 2374 2375 i = 0; 2376 reset |= F_FLEMPTY; 2377 2378 cause = (t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) >> 2379 S_FL0EMPTY) & 0xffff; 2380 while (cause) { 2381 qs->fl[i].empty += (cause & 1); 2382 if (i) 2383 qs++; 2384 i ^= 1; 2385 cause >>= 1; 2386 } 2387 } 2388 t3_write_reg(sc, A_SG_INT_CAUSE, reset); 2389 2390 for (i = 0; i < sc->params.nports; i++) { 2391 struct port_info *pi = &sc->port[i]; 2392 struct ifnet *ifp = pi->ifp; 2393 struct cmac *mac = &pi->mac; 2394 struct mac_stats *mstats = &mac->stats; 2395 PORT_LOCK(pi); 2396 t3_mac_update_stats(mac); 2397 PORT_UNLOCK(pi); 2398 2399 if (pi->link_fault) 2400 taskqueue_enqueue(sc->tq, &pi->link_fault_task); 2401 2402 ifp->if_opackets = 2403 mstats->tx_frames_64 + 2404 mstats->tx_frames_65_127 + 2405 mstats->tx_frames_128_255 + 2406 mstats->tx_frames_256_511 + 2407 mstats->tx_frames_512_1023 + 2408 mstats->tx_frames_1024_1518 + 2409 mstats->tx_frames_1519_max; 2410 2411 ifp->if_ipackets = 2412 mstats->rx_frames_64 + 2413 mstats->rx_frames_65_127 + 2414 mstats->rx_frames_128_255 + 2415 mstats->rx_frames_256_511 + 2416 mstats->rx_frames_512_1023 + 2417 mstats->rx_frames_1024_1518 + 2418 mstats->rx_frames_1519_max; 2419 2420 ifp->if_obytes = mstats->tx_octets; 2421 ifp->if_ibytes = mstats->rx_octets; 2422 ifp->if_omcasts = mstats->tx_mcast_frames; 2423 ifp->if_imcasts = mstats->rx_mcast_frames; 2424 2425 ifp->if_collisions = 2426 mstats->tx_total_collisions; 2427 2428 ifp->if_iqdrops = mstats->rx_cong_drops; 2429 2430 ifp->if_oerrors = 2431 mstats->tx_excess_collisions + 2432 mstats->tx_underrun + 2433 mstats->tx_len_errs + 2434 mstats->tx_mac_internal_errs + 2435 mstats->tx_excess_deferral + 2436 mstats->tx_fcs_errs; 2437 ifp->if_ierrors = 2438 mstats->rx_jabber + 2439 mstats->rx_data_errs + 2440 mstats->rx_sequence_errs + 2441 mstats->rx_runt + 2442 mstats->rx_too_long + 2443 mstats->rx_mac_internal_errs + 2444 mstats->rx_short + 2445 mstats->rx_fcs_errs; 2446 2447 if (mac->multiport) 2448 continue; 2449 2450 /* Count rx fifo overflows, once per second */ 2451 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset); 2452 reset = 0; 2453 if (cause & F_RXFIFO_OVERFLOW) { 2454 mac->stats.rx_fifo_ovfl++; 2455 reset |= F_RXFIFO_OVERFLOW; 2456 } 2457 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset); 2458 } 2459 } 2460 2461 static void 2462 touch_bars(device_t dev) 2463 { 2464 /* 2465 * Don't enable yet 2466 */ 2467 #if !defined(__LP64__) && 0 2468 u32 v; 2469 2470 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2471 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2472 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2473 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2474 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2475 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2476 #endif 2477 } 2478 2479 static int 2480 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2481 { 2482 uint8_t *buf; 2483 int err = 0; 2484 u32 aligned_offset, aligned_len, *p; 2485 struct adapter *adapter = pi->adapter; 2486 2487 2488 aligned_offset = offset & ~3; 2489 aligned_len = (len + (offset & 3) + 3) & ~3; 2490 2491 if (aligned_offset != offset || aligned_len != len) { 2492 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2493 if (!buf) 2494 return (ENOMEM); 2495 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2496 if (!err && aligned_len > 4) 2497 err = t3_seeprom_read(adapter, 2498 aligned_offset + aligned_len - 4, 2499 (u32 *)&buf[aligned_len - 4]); 2500 if (err) 2501 goto out; 2502 memcpy(buf + (offset & 3), data, len); 2503 } else 2504 buf = (uint8_t *)(uintptr_t)data; 2505 2506 err = t3_seeprom_wp(adapter, 0); 2507 if (err) 2508 goto out; 2509 2510 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2511 err = t3_seeprom_write(adapter, aligned_offset, *p); 2512 aligned_offset += 4; 2513 } 2514 2515 if (!err) 2516 err = t3_seeprom_wp(adapter, 1); 2517 out: 2518 if (buf != data) 2519 free(buf, M_DEVBUF); 2520 return err; 2521 } 2522 2523 2524 static int 2525 in_range(int val, int lo, int hi) 2526 { 2527 return val < 0 || (val <= hi && val >= lo); 2528 } 2529 2530 static int 2531 cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td) 2532 { 2533 return (0); 2534 } 2535 2536 static int 2537 cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td) 2538 { 2539 return (0); 2540 } 2541 2542 static int 2543 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2544 int fflag, struct thread *td) 2545 { 2546 int mmd, error = 0; 2547 struct port_info *pi = dev->si_drv1; 2548 adapter_t *sc = pi->adapter; 2549 2550 #ifdef PRIV_SUPPORTED 2551 if (priv_check(td, PRIV_DRIVER)) { 2552 if (cxgb_debug) 2553 printf("user does not have access to privileged ioctls\n"); 2554 return (EPERM); 2555 } 2556 #else 2557 if (suser(td)) { 2558 if (cxgb_debug) 2559 printf("user does not have access to privileged ioctls\n"); 2560 return (EPERM); 2561 } 2562 #endif 2563 2564 switch (cmd) { 2565 case CHELSIO_GET_MIIREG: { 2566 uint32_t val; 2567 struct cphy *phy = &pi->phy; 2568 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2569 2570 if (!phy->mdio_read) 2571 return (EOPNOTSUPP); 2572 if (is_10G(sc)) { 2573 mmd = mid->phy_id >> 8; 2574 if (!mmd) 2575 mmd = MDIO_DEV_PCS; 2576 else if (mmd > MDIO_DEV_VEND2) 2577 return (EINVAL); 2578 2579 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2580 mid->reg_num, &val); 2581 } else 2582 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2583 mid->reg_num & 0x1f, &val); 2584 if (error == 0) 2585 mid->val_out = val; 2586 break; 2587 } 2588 case CHELSIO_SET_MIIREG: { 2589 struct cphy *phy = &pi->phy; 2590 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2591 2592 if (!phy->mdio_write) 2593 return (EOPNOTSUPP); 2594 if (is_10G(sc)) { 2595 mmd = mid->phy_id >> 8; 2596 if (!mmd) 2597 mmd = MDIO_DEV_PCS; 2598 else if (mmd > MDIO_DEV_VEND2) 2599 return (EINVAL); 2600 2601 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2602 mmd, mid->reg_num, mid->val_in); 2603 } else 2604 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2605 mid->reg_num & 0x1f, 2606 mid->val_in); 2607 break; 2608 } 2609 case CHELSIO_SETREG: { 2610 struct ch_reg *edata = (struct ch_reg *)data; 2611 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2612 return (EFAULT); 2613 t3_write_reg(sc, edata->addr, edata->val); 2614 break; 2615 } 2616 case CHELSIO_GETREG: { 2617 struct ch_reg *edata = (struct ch_reg *)data; 2618 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2619 return (EFAULT); 2620 edata->val = t3_read_reg(sc, edata->addr); 2621 break; 2622 } 2623 case CHELSIO_GET_SGE_CONTEXT: { 2624 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2625 mtx_lock_spin(&sc->sge.reg_lock); 2626 switch (ecntxt->cntxt_type) { 2627 case CNTXT_TYPE_EGRESS: 2628 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2629 ecntxt->data); 2630 break; 2631 case CNTXT_TYPE_FL: 2632 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2633 ecntxt->data); 2634 break; 2635 case CNTXT_TYPE_RSP: 2636 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2637 ecntxt->data); 2638 break; 2639 case CNTXT_TYPE_CQ: 2640 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2641 ecntxt->data); 2642 break; 2643 default: 2644 error = EINVAL; 2645 break; 2646 } 2647 mtx_unlock_spin(&sc->sge.reg_lock); 2648 break; 2649 } 2650 case CHELSIO_GET_SGE_DESC: { 2651 struct ch_desc *edesc = (struct ch_desc *)data; 2652 int ret; 2653 if (edesc->queue_num >= SGE_QSETS * 6) 2654 return (EINVAL); 2655 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2656 edesc->queue_num % 6, edesc->idx, edesc->data); 2657 if (ret < 0) 2658 return (EINVAL); 2659 edesc->size = ret; 2660 break; 2661 } 2662 case CHELSIO_GET_QSET_PARAMS: { 2663 struct qset_params *q; 2664 struct ch_qset_params *t = (struct ch_qset_params *)data; 2665 int q1 = pi->first_qset; 2666 int nqsets = pi->nqsets; 2667 int i; 2668 2669 if (t->qset_idx >= nqsets) 2670 return EINVAL; 2671 2672 i = q1 + t->qset_idx; 2673 q = &sc->params.sge.qset[i]; 2674 t->rspq_size = q->rspq_size; 2675 t->txq_size[0] = q->txq_size[0]; 2676 t->txq_size[1] = q->txq_size[1]; 2677 t->txq_size[2] = q->txq_size[2]; 2678 t->fl_size[0] = q->fl_size; 2679 t->fl_size[1] = q->jumbo_size; 2680 t->polling = q->polling; 2681 t->lro = q->lro; 2682 t->intr_lat = q->coalesce_usecs; 2683 t->cong_thres = q->cong_thres; 2684 t->qnum = i; 2685 2686 if (sc->flags & USING_MSIX) 2687 t->vector = rman_get_start(sc->msix_irq_res[i]); 2688 else 2689 t->vector = rman_get_start(sc->irq_res); 2690 2691 break; 2692 } 2693 case CHELSIO_GET_QSET_NUM: { 2694 struct ch_reg *edata = (struct ch_reg *)data; 2695 edata->val = pi->nqsets; 2696 break; 2697 } 2698 case CHELSIO_LOAD_FW: { 2699 uint8_t *fw_data; 2700 uint32_t vers; 2701 struct ch_mem_range *t = (struct ch_mem_range *)data; 2702 2703 /* 2704 * You're allowed to load a firmware only before FULL_INIT_DONE 2705 * 2706 * FW_UPTODATE is also set so the rest of the initialization 2707 * will not overwrite what was loaded here. This gives you the 2708 * flexibility to load any firmware (and maybe shoot yourself in 2709 * the foot). 2710 */ 2711 2712 ADAPTER_LOCK(sc); 2713 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2714 ADAPTER_UNLOCK(sc); 2715 return (EBUSY); 2716 } 2717 2718 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2719 if (!fw_data) 2720 error = ENOMEM; 2721 else 2722 error = copyin(t->buf, fw_data, t->len); 2723 2724 if (!error) 2725 error = -t3_load_fw(sc, fw_data, t->len); 2726 2727 if (t3_get_fw_version(sc, &vers) == 0) { 2728 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2729 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2730 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2731 } 2732 2733 if (!error) 2734 sc->flags |= FW_UPTODATE; 2735 2736 free(fw_data, M_DEVBUF); 2737 ADAPTER_UNLOCK(sc); 2738 break; 2739 } 2740 case CHELSIO_LOAD_BOOT: { 2741 uint8_t *boot_data; 2742 struct ch_mem_range *t = (struct ch_mem_range *)data; 2743 2744 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2745 if (!boot_data) 2746 return ENOMEM; 2747 2748 error = copyin(t->buf, boot_data, t->len); 2749 if (!error) 2750 error = -t3_load_boot(sc, boot_data, t->len); 2751 2752 free(boot_data, M_DEVBUF); 2753 break; 2754 } 2755 case CHELSIO_GET_PM: { 2756 struct ch_pm *m = (struct ch_pm *)data; 2757 struct tp_params *p = &sc->params.tp; 2758 2759 if (!is_offload(sc)) 2760 return (EOPNOTSUPP); 2761 2762 m->tx_pg_sz = p->tx_pg_size; 2763 m->tx_num_pg = p->tx_num_pgs; 2764 m->rx_pg_sz = p->rx_pg_size; 2765 m->rx_num_pg = p->rx_num_pgs; 2766 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2767 2768 break; 2769 } 2770 case CHELSIO_SET_PM: { 2771 struct ch_pm *m = (struct ch_pm *)data; 2772 struct tp_params *p = &sc->params.tp; 2773 2774 if (!is_offload(sc)) 2775 return (EOPNOTSUPP); 2776 if (sc->flags & FULL_INIT_DONE) 2777 return (EBUSY); 2778 2779 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2780 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2781 return (EINVAL); /* not power of 2 */ 2782 if (!(m->rx_pg_sz & 0x14000)) 2783 return (EINVAL); /* not 16KB or 64KB */ 2784 if (!(m->tx_pg_sz & 0x1554000)) 2785 return (EINVAL); 2786 if (m->tx_num_pg == -1) 2787 m->tx_num_pg = p->tx_num_pgs; 2788 if (m->rx_num_pg == -1) 2789 m->rx_num_pg = p->rx_num_pgs; 2790 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2791 return (EINVAL); 2792 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2793 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2794 return (EINVAL); 2795 2796 p->rx_pg_size = m->rx_pg_sz; 2797 p->tx_pg_size = m->tx_pg_sz; 2798 p->rx_num_pgs = m->rx_num_pg; 2799 p->tx_num_pgs = m->tx_num_pg; 2800 break; 2801 } 2802 case CHELSIO_SETMTUTAB: { 2803 struct ch_mtus *m = (struct ch_mtus *)data; 2804 int i; 2805 2806 if (!is_offload(sc)) 2807 return (EOPNOTSUPP); 2808 if (offload_running(sc)) 2809 return (EBUSY); 2810 if (m->nmtus != NMTUS) 2811 return (EINVAL); 2812 if (m->mtus[0] < 81) /* accommodate SACK */ 2813 return (EINVAL); 2814 2815 /* 2816 * MTUs must be in ascending order 2817 */ 2818 for (i = 1; i < NMTUS; ++i) 2819 if (m->mtus[i] < m->mtus[i - 1]) 2820 return (EINVAL); 2821 2822 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2823 break; 2824 } 2825 case CHELSIO_GETMTUTAB: { 2826 struct ch_mtus *m = (struct ch_mtus *)data; 2827 2828 if (!is_offload(sc)) 2829 return (EOPNOTSUPP); 2830 2831 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2832 m->nmtus = NMTUS; 2833 break; 2834 } 2835 case CHELSIO_GET_MEM: { 2836 struct ch_mem_range *t = (struct ch_mem_range *)data; 2837 struct mc7 *mem; 2838 uint8_t *useraddr; 2839 u64 buf[32]; 2840 2841 /* 2842 * Use these to avoid modifying len/addr in the the return 2843 * struct 2844 */ 2845 uint32_t len = t->len, addr = t->addr; 2846 2847 if (!is_offload(sc)) 2848 return (EOPNOTSUPP); 2849 if (!(sc->flags & FULL_INIT_DONE)) 2850 return (EIO); /* need the memory controllers */ 2851 if ((addr & 0x7) || (len & 0x7)) 2852 return (EINVAL); 2853 if (t->mem_id == MEM_CM) 2854 mem = &sc->cm; 2855 else if (t->mem_id == MEM_PMRX) 2856 mem = &sc->pmrx; 2857 else if (t->mem_id == MEM_PMTX) 2858 mem = &sc->pmtx; 2859 else 2860 return (EINVAL); 2861 2862 /* 2863 * Version scheme: 2864 * bits 0..9: chip version 2865 * bits 10..15: chip revision 2866 */ 2867 t->version = 3 | (sc->params.rev << 10); 2868 2869 /* 2870 * Read 256 bytes at a time as len can be large and we don't 2871 * want to use huge intermediate buffers. 2872 */ 2873 useraddr = (uint8_t *)t->buf; 2874 while (len) { 2875 unsigned int chunk = min(len, sizeof(buf)); 2876 2877 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2878 if (error) 2879 return (-error); 2880 if (copyout(buf, useraddr, chunk)) 2881 return (EFAULT); 2882 useraddr += chunk; 2883 addr += chunk; 2884 len -= chunk; 2885 } 2886 break; 2887 } 2888 case CHELSIO_READ_TCAM_WORD: { 2889 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2890 2891 if (!is_offload(sc)) 2892 return (EOPNOTSUPP); 2893 if (!(sc->flags & FULL_INIT_DONE)) 2894 return (EIO); /* need MC5 */ 2895 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2896 break; 2897 } 2898 case CHELSIO_SET_TRACE_FILTER: { 2899 struct ch_trace *t = (struct ch_trace *)data; 2900 const struct trace_params *tp; 2901 2902 tp = (const struct trace_params *)&t->sip; 2903 if (t->config_tx) 2904 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2905 t->trace_tx); 2906 if (t->config_rx) 2907 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2908 t->trace_rx); 2909 break; 2910 } 2911 case CHELSIO_SET_PKTSCHED: { 2912 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2913 if (sc->open_device_map == 0) 2914 return (EAGAIN); 2915 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2916 p->binding); 2917 break; 2918 } 2919 case CHELSIO_IFCONF_GETREGS: { 2920 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2921 int reglen = cxgb_get_regs_len(); 2922 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2923 if (buf == NULL) { 2924 return (ENOMEM); 2925 } 2926 if (regs->len > reglen) 2927 regs->len = reglen; 2928 else if (regs->len < reglen) 2929 error = ENOBUFS; 2930 2931 if (!error) { 2932 cxgb_get_regs(sc, regs, buf); 2933 error = copyout(buf, regs->data, reglen); 2934 } 2935 free(buf, M_DEVBUF); 2936 2937 break; 2938 } 2939 case CHELSIO_SET_HW_SCHED: { 2940 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2941 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2942 2943 if ((sc->flags & FULL_INIT_DONE) == 0) 2944 return (EAGAIN); /* need TP to be initialized */ 2945 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2946 !in_range(t->channel, 0, 1) || 2947 !in_range(t->kbps, 0, 10000000) || 2948 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2949 !in_range(t->flow_ipg, 0, 2950 dack_ticks_to_usec(sc, 0x7ff))) 2951 return (EINVAL); 2952 2953 if (t->kbps >= 0) { 2954 error = t3_config_sched(sc, t->kbps, t->sched); 2955 if (error < 0) 2956 return (-error); 2957 } 2958 if (t->class_ipg >= 0) 2959 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2960 if (t->flow_ipg >= 0) { 2961 t->flow_ipg *= 1000; /* us -> ns */ 2962 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2963 } 2964 if (t->mode >= 0) { 2965 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2966 2967 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2968 bit, t->mode ? bit : 0); 2969 } 2970 if (t->channel >= 0) 2971 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2972 1 << t->sched, t->channel << t->sched); 2973 break; 2974 } 2975 case CHELSIO_GET_EEPROM: { 2976 int i; 2977 struct ch_eeprom *e = (struct ch_eeprom *)data; 2978 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2979 2980 if (buf == NULL) { 2981 return (ENOMEM); 2982 } 2983 e->magic = EEPROM_MAGIC; 2984 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2985 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2986 2987 if (!error) 2988 error = copyout(buf + e->offset, e->data, e->len); 2989 2990 free(buf, M_DEVBUF); 2991 break; 2992 } 2993 case CHELSIO_CLEAR_STATS: { 2994 if (!(sc->flags & FULL_INIT_DONE)) 2995 return EAGAIN; 2996 2997 PORT_LOCK(pi); 2998 t3_mac_update_stats(&pi->mac); 2999 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 3000 PORT_UNLOCK(pi); 3001 break; 3002 } 3003 case CHELSIO_GET_UP_LA: { 3004 struct ch_up_la *la = (struct ch_up_la *)data; 3005 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT); 3006 if (buf == NULL) { 3007 return (ENOMEM); 3008 } 3009 if (la->bufsize < LA_BUFSIZE) 3010 error = ENOBUFS; 3011 3012 if (!error) 3013 error = -t3_get_up_la(sc, &la->stopped, &la->idx, 3014 &la->bufsize, buf); 3015 if (!error) 3016 error = copyout(buf, la->data, la->bufsize); 3017 3018 free(buf, M_DEVBUF); 3019 break; 3020 } 3021 case CHELSIO_GET_UP_IOQS: { 3022 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data; 3023 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT); 3024 uint32_t *v; 3025 3026 if (buf == NULL) { 3027 return (ENOMEM); 3028 } 3029 if (ioqs->bufsize < IOQS_BUFSIZE) 3030 error = ENOBUFS; 3031 3032 if (!error) 3033 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf); 3034 3035 if (!error) { 3036 v = (uint32_t *)buf; 3037 3038 ioqs->bufsize -= 4 * sizeof(uint32_t); 3039 ioqs->ioq_rx_enable = *v++; 3040 ioqs->ioq_tx_enable = *v++; 3041 ioqs->ioq_rx_status = *v++; 3042 ioqs->ioq_tx_status = *v++; 3043 3044 error = copyout(v, ioqs->data, ioqs->bufsize); 3045 } 3046 3047 free(buf, M_DEVBUF); 3048 break; 3049 } 3050 default: 3051 return (EOPNOTSUPP); 3052 break; 3053 } 3054 3055 return (error); 3056 } 3057 3058 static __inline void 3059 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 3060 unsigned int end) 3061 { 3062 uint32_t *p = (uint32_t *)(buf + start); 3063 3064 for ( ; start <= end; start += sizeof(uint32_t)) 3065 *p++ = t3_read_reg(ap, start); 3066 } 3067 3068 #define T3_REGMAP_SIZE (3 * 1024) 3069 static int 3070 cxgb_get_regs_len(void) 3071 { 3072 return T3_REGMAP_SIZE; 3073 } 3074 3075 static void 3076 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 3077 { 3078 3079 /* 3080 * Version scheme: 3081 * bits 0..9: chip version 3082 * bits 10..15: chip revision 3083 * bit 31: set for PCIe cards 3084 */ 3085 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 3086 3087 /* 3088 * We skip the MAC statistics registers because they are clear-on-read. 3089 * Also reading multi-register stats would need to synchronize with the 3090 * periodic mac stats accumulation. Hard to justify the complexity. 3091 */ 3092 memset(buf, 0, cxgb_get_regs_len()); 3093 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 3094 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 3095 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 3096 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 3097 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 3098 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 3099 XGM_REG(A_XGM_SERDES_STAT3, 1)); 3100 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 3101 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 3102 } 3103 3104 3105 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1); 3106