1 /************************************************************************** 2 3 Copyright (c) 2007-2009, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 POSSIBILITY OF SUCH DAMAGE. 27 28 ***************************************************************************/ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/bus.h> 37 #include <sys/module.h> 38 #include <sys/pciio.h> 39 #include <sys/conf.h> 40 #include <machine/bus.h> 41 #include <machine/resource.h> 42 #include <sys/bus_dma.h> 43 #include <sys/ktr.h> 44 #include <sys/rman.h> 45 #include <sys/ioccom.h> 46 #include <sys/mbuf.h> 47 #include <sys/linker.h> 48 #include <sys/firmware.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/smp.h> 52 #include <sys/sysctl.h> 53 #include <sys/syslog.h> 54 #include <sys/queue.h> 55 #include <sys/taskqueue.h> 56 #include <sys/proc.h> 57 58 #include <net/bpf.h> 59 #include <net/ethernet.h> 60 #include <net/if.h> 61 #include <net/if_arp.h> 62 #include <net/if_dl.h> 63 #include <net/if_media.h> 64 #include <net/if_types.h> 65 #include <net/if_vlan_var.h> 66 67 #include <netinet/in_systm.h> 68 #include <netinet/in.h> 69 #include <netinet/if_ether.h> 70 #include <netinet/ip.h> 71 #include <netinet/ip.h> 72 #include <netinet/tcp.h> 73 #include <netinet/udp.h> 74 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 #include <dev/pci/pci_private.h> 78 79 #include <cxgb_include.h> 80 81 #ifdef PRIV_SUPPORTED 82 #include <sys/priv.h> 83 #endif 84 85 static int cxgb_setup_interrupts(adapter_t *); 86 static void cxgb_teardown_interrupts(adapter_t *); 87 static void cxgb_init(void *); 88 static void cxgb_init_locked(struct port_info *); 89 static void cxgb_stop_locked(struct port_info *); 90 static void cxgb_set_rxmode(struct port_info *); 91 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 92 static int cxgb_media_change(struct ifnet *); 93 static int cxgb_ifm_type(int); 94 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 95 static int setup_sge_qsets(adapter_t *); 96 static void cxgb_async_intr(void *); 97 static void cxgb_ext_intr_handler(void *, int); 98 static void cxgb_tick_handler(void *, int); 99 static void cxgb_down_locked(struct adapter *sc); 100 static void cxgb_tick(void *); 101 static void setup_rss(adapter_t *sc); 102 103 /* Attachment glue for the PCI controller end of the device. Each port of 104 * the device is attached separately, as defined later. 105 */ 106 static int cxgb_controller_probe(device_t); 107 static int cxgb_controller_attach(device_t); 108 static int cxgb_controller_detach(device_t); 109 static void cxgb_free(struct adapter *); 110 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 111 unsigned int end); 112 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 113 static int cxgb_get_regs_len(void); 114 static int offload_open(struct port_info *pi); 115 static void touch_bars(device_t dev); 116 static int offload_close(struct t3cdev *tdev); 117 static void cxgb_link_start(struct port_info *p); 118 int t3_detect_link_fault(adapter_t *adapter, int port_id); 119 120 static device_method_t cxgb_controller_methods[] = { 121 DEVMETHOD(device_probe, cxgb_controller_probe), 122 DEVMETHOD(device_attach, cxgb_controller_attach), 123 DEVMETHOD(device_detach, cxgb_controller_detach), 124 125 /* bus interface */ 126 DEVMETHOD(bus_print_child, bus_generic_print_child), 127 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 128 129 { 0, 0 } 130 }; 131 132 static driver_t cxgb_controller_driver = { 133 "cxgbc", 134 cxgb_controller_methods, 135 sizeof(struct adapter) 136 }; 137 138 static devclass_t cxgb_controller_devclass; 139 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 140 141 /* 142 * Attachment glue for the ports. Attachment is done directly to the 143 * controller device. 144 */ 145 static int cxgb_port_probe(device_t); 146 static int cxgb_port_attach(device_t); 147 static int cxgb_port_detach(device_t); 148 149 static device_method_t cxgb_port_methods[] = { 150 DEVMETHOD(device_probe, cxgb_port_probe), 151 DEVMETHOD(device_attach, cxgb_port_attach), 152 DEVMETHOD(device_detach, cxgb_port_detach), 153 { 0, 0 } 154 }; 155 156 static driver_t cxgb_port_driver = { 157 "cxgb", 158 cxgb_port_methods, 159 0 160 }; 161 162 static d_ioctl_t cxgb_extension_ioctl; 163 static d_open_t cxgb_extension_open; 164 static d_close_t cxgb_extension_close; 165 166 static struct cdevsw cxgb_cdevsw = { 167 .d_version = D_VERSION, 168 .d_flags = 0, 169 .d_open = cxgb_extension_open, 170 .d_close = cxgb_extension_close, 171 .d_ioctl = cxgb_extension_ioctl, 172 .d_name = "cxgb", 173 }; 174 175 static devclass_t cxgb_port_devclass; 176 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 177 178 /* 179 * The driver uses the best interrupt scheme available on a platform in the 180 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 181 * of these schemes the driver may consider as follows: 182 * 183 * msi = 2: choose from among all three options 184 * msi = 1 : only consider MSI and pin interrupts 185 * msi = 0: force pin interrupts 186 */ 187 static int msi_allowed = 2; 188 189 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 190 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 191 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 192 "MSI-X, MSI, INTx selector"); 193 194 /* 195 * The driver enables offload as a default. 196 * To disable it, use ofld_disable = 1. 197 */ 198 static int ofld_disable = 0; 199 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable); 200 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0, 201 "disable ULP offload"); 202 203 /* 204 * The driver uses an auto-queue algorithm by default. 205 * To disable it and force a single queue-set per port, use multiq = 0 206 */ 207 static int multiq = 1; 208 TUNABLE_INT("hw.cxgb.multiq", &multiq); 209 SYSCTL_UINT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0, 210 "use min(ncpus/ports, 8) queue-sets per port"); 211 212 /* 213 * By default the driver will not update the firmware unless 214 * it was compiled against a newer version 215 * 216 */ 217 static int force_fw_update = 0; 218 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update); 219 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 220 "update firmware even if up to date"); 221 222 int cxgb_use_16k_clusters = 1; 223 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters); 224 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 225 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 226 227 enum { 228 MAX_TXQ_ENTRIES = 16384, 229 MAX_CTRL_TXQ_ENTRIES = 1024, 230 MAX_RSPQ_ENTRIES = 16384, 231 MAX_RX_BUFFERS = 16384, 232 MAX_RX_JUMBO_BUFFERS = 16384, 233 MIN_TXQ_ENTRIES = 4, 234 MIN_CTRL_TXQ_ENTRIES = 4, 235 MIN_RSPQ_ENTRIES = 32, 236 MIN_FL_ENTRIES = 32, 237 MIN_FL_JUMBO_ENTRIES = 32 238 }; 239 240 struct filter_info { 241 u32 sip; 242 u32 sip_mask; 243 u32 dip; 244 u16 sport; 245 u16 dport; 246 u32 vlan:12; 247 u32 vlan_prio:3; 248 u32 mac_hit:1; 249 u32 mac_idx:4; 250 u32 mac_vld:1; 251 u32 pkt_type:2; 252 u32 report_filter_id:1; 253 u32 pass:1; 254 u32 rss:1; 255 u32 qset:3; 256 u32 locked:1; 257 u32 valid:1; 258 }; 259 260 enum { FILTER_NO_VLAN_PRI = 7 }; 261 262 #define EEPROM_MAGIC 0x38E2F10C 263 264 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 265 266 /* Table for probing the cards. The desc field isn't actually used */ 267 struct cxgb_ident { 268 uint16_t vendor; 269 uint16_t device; 270 int index; 271 char *desc; 272 } cxgb_identifiers[] = { 273 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 274 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 275 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 276 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 277 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 278 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 279 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 280 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 281 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 282 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 283 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 284 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "N310E"}, 285 {0, 0, 0, NULL} 286 }; 287 288 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 289 290 291 static __inline char 292 t3rev2char(struct adapter *adapter) 293 { 294 char rev = 'z'; 295 296 switch(adapter->params.rev) { 297 case T3_REV_A: 298 rev = 'a'; 299 break; 300 case T3_REV_B: 301 case T3_REV_B2: 302 rev = 'b'; 303 break; 304 case T3_REV_C: 305 rev = 'c'; 306 break; 307 } 308 return rev; 309 } 310 311 static struct cxgb_ident * 312 cxgb_get_ident(device_t dev) 313 { 314 struct cxgb_ident *id; 315 316 for (id = cxgb_identifiers; id->desc != NULL; id++) { 317 if ((id->vendor == pci_get_vendor(dev)) && 318 (id->device == pci_get_device(dev))) { 319 return (id); 320 } 321 } 322 return (NULL); 323 } 324 325 static const struct adapter_info * 326 cxgb_get_adapter_info(device_t dev) 327 { 328 struct cxgb_ident *id; 329 const struct adapter_info *ai; 330 331 id = cxgb_get_ident(dev); 332 if (id == NULL) 333 return (NULL); 334 335 ai = t3_get_adapter_info(id->index); 336 337 return (ai); 338 } 339 340 static int 341 cxgb_controller_probe(device_t dev) 342 { 343 const struct adapter_info *ai; 344 char *ports, buf[80]; 345 int nports; 346 struct adapter *sc = device_get_softc(dev); 347 348 ai = cxgb_get_adapter_info(dev); 349 if (ai == NULL) 350 return (ENXIO); 351 352 nports = ai->nports0 + ai->nports1; 353 if (nports == 1) 354 ports = "port"; 355 else 356 ports = "ports"; 357 358 snprintf(buf, sizeof(buf), "%s %sNIC, rev: %d nports: %d %s", 359 ai->desc, is_offload(sc) ? "R" : "", 360 sc->params.rev, nports, ports); 361 device_set_desc_copy(dev, buf); 362 return (BUS_PROBE_DEFAULT); 363 } 364 365 #define FW_FNAME "cxgb_t3fw" 366 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom" 367 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram" 368 369 static int 370 upgrade_fw(adapter_t *sc) 371 { 372 #ifdef FIRMWARE_LATEST 373 const struct firmware *fw; 374 #else 375 struct firmware *fw; 376 #endif 377 int status; 378 379 if ((fw = firmware_get(FW_FNAME)) == NULL) { 380 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 381 return (ENOENT); 382 } else 383 device_printf(sc->dev, "updating firmware on card\n"); 384 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 385 386 device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status); 387 388 firmware_put(fw, FIRMWARE_UNLOAD); 389 390 return (status); 391 } 392 393 /* 394 * The cxgb_controller_attach function is responsible for the initial 395 * bringup of the device. Its responsibilities include: 396 * 397 * 1. Determine if the device supports MSI or MSI-X. 398 * 2. Allocate bus resources so that we can access the Base Address Register 399 * 3. Create and initialize mutexes for the controller and its control 400 * logic such as SGE and MDIO. 401 * 4. Call hardware specific setup routine for the adapter as a whole. 402 * 5. Allocate the BAR for doing MSI-X. 403 * 6. Setup the line interrupt iff MSI-X is not supported. 404 * 7. Create the driver's taskq. 405 * 8. Start one task queue service thread. 406 * 9. Check if the firmware and SRAM are up-to-date. They will be 407 * auto-updated later (before FULL_INIT_DONE), if required. 408 * 10. Create a child device for each MAC (port) 409 * 11. Initialize T3 private state. 410 * 12. Trigger the LED 411 * 13. Setup offload iff supported. 412 * 14. Reset/restart the tick callout. 413 * 15. Attach sysctls 414 * 415 * NOTE: Any modification or deviation from this list MUST be reflected in 416 * the above comment. Failure to do so will result in problems on various 417 * error conditions including link flapping. 418 */ 419 static int 420 cxgb_controller_attach(device_t dev) 421 { 422 device_t child; 423 const struct adapter_info *ai; 424 struct adapter *sc; 425 int i, error = 0; 426 uint32_t vers; 427 int port_qsets = 1; 428 #ifdef MSI_SUPPORTED 429 int msi_needed, reg; 430 #endif 431 char buf[80]; 432 433 sc = device_get_softc(dev); 434 sc->dev = dev; 435 sc->msi_count = 0; 436 ai = cxgb_get_adapter_info(dev); 437 438 /* 439 * XXX not really related but a recent addition 440 */ 441 #ifdef MSI_SUPPORTED 442 /* find the PCIe link width and set max read request to 4KB*/ 443 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 444 uint16_t lnk, pectl; 445 lnk = pci_read_config(dev, reg + 0x12, 2); 446 sc->link_width = (lnk >> 4) & 0x3f; 447 448 pectl = pci_read_config(dev, reg + 0x8, 2); 449 pectl = (pectl & ~0x7000) | (5 << 12); 450 pci_write_config(dev, reg + 0x8, pectl, 2); 451 } 452 453 if (sc->link_width != 0 && sc->link_width <= 4 && 454 (ai->nports0 + ai->nports1) <= 2) { 455 device_printf(sc->dev, 456 "PCIe x%d Link, expect reduced performance\n", 457 sc->link_width); 458 } 459 #endif 460 touch_bars(dev); 461 pci_enable_busmaster(dev); 462 /* 463 * Allocate the registers and make them available to the driver. 464 * The registers that we care about for NIC mode are in BAR 0 465 */ 466 sc->regs_rid = PCIR_BAR(0); 467 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 468 &sc->regs_rid, RF_ACTIVE)) == NULL) { 469 device_printf(dev, "Cannot allocate BAR region 0\n"); 470 return (ENXIO); 471 } 472 sc->udbs_rid = PCIR_BAR(2); 473 sc->udbs_res = NULL; 474 if (is_offload(sc) && 475 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 476 &sc->udbs_rid, RF_ACTIVE)) == NULL)) { 477 device_printf(dev, "Cannot allocate BAR region 1\n"); 478 error = ENXIO; 479 goto out; 480 } 481 482 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 483 device_get_unit(dev)); 484 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 485 486 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 487 device_get_unit(dev)); 488 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 489 device_get_unit(dev)); 490 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 491 device_get_unit(dev)); 492 493 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 494 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 495 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 496 497 sc->bt = rman_get_bustag(sc->regs_res); 498 sc->bh = rman_get_bushandle(sc->regs_res); 499 sc->mmio_len = rman_get_size(sc->regs_res); 500 501 if (t3_prep_adapter(sc, ai, 1) < 0) { 502 printf("prep adapter failed\n"); 503 error = ENODEV; 504 goto out; 505 } 506 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 507 * enough messages for the queue sets. If that fails, try falling 508 * back to MSI. If that fails, then try falling back to the legacy 509 * interrupt pin model. 510 */ 511 #ifdef MSI_SUPPORTED 512 513 sc->msix_regs_rid = 0x20; 514 if ((msi_allowed >= 2) && 515 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 516 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 517 518 if (multiq) 519 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus); 520 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1; 521 522 if (pci_msix_count(dev) == 0 || 523 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 || 524 sc->msi_count != msi_needed) { 525 device_printf(dev, "alloc msix failed - " 526 "msi_count=%d, msi_needed=%d, err=%d; " 527 "will try MSI\n", sc->msi_count, 528 msi_needed, error); 529 sc->msi_count = 0; 530 port_qsets = 1; 531 pci_release_msi(dev); 532 bus_release_resource(dev, SYS_RES_MEMORY, 533 sc->msix_regs_rid, sc->msix_regs_res); 534 sc->msix_regs_res = NULL; 535 } else { 536 sc->flags |= USING_MSIX; 537 sc->cxgb_intr = cxgb_async_intr; 538 device_printf(dev, 539 "using MSI-X interrupts (%u vectors)\n", 540 sc->msi_count); 541 } 542 } 543 544 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 545 sc->msi_count = 1; 546 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) { 547 device_printf(dev, "alloc msi failed - " 548 "err=%d; will try INTx\n", error); 549 sc->msi_count = 0; 550 port_qsets = 1; 551 pci_release_msi(dev); 552 } else { 553 sc->flags |= USING_MSI; 554 sc->cxgb_intr = t3_intr_msi; 555 device_printf(dev, "using MSI interrupts\n"); 556 } 557 } 558 #endif 559 if (sc->msi_count == 0) { 560 device_printf(dev, "using line interrupts\n"); 561 sc->cxgb_intr = t3b_intr; 562 } 563 564 /* Create a private taskqueue thread for handling driver events */ 565 #ifdef TASKQUEUE_CURRENT 566 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 567 taskqueue_thread_enqueue, &sc->tq); 568 #else 569 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 570 taskqueue_thread_enqueue, &sc->tq); 571 #endif 572 if (sc->tq == NULL) { 573 device_printf(dev, "failed to allocate controller task queue\n"); 574 goto out; 575 } 576 577 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 578 device_get_nameunit(dev)); 579 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 580 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 581 582 583 /* Create a periodic callout for checking adapter status */ 584 callout_init(&sc->cxgb_tick_ch, TRUE); 585 586 if (t3_check_fw_version(sc) < 0 || force_fw_update) { 587 /* 588 * Warn user that a firmware update will be attempted in init. 589 */ 590 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 591 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 592 sc->flags &= ~FW_UPTODATE; 593 } else { 594 sc->flags |= FW_UPTODATE; 595 } 596 597 if (t3_check_tpsram_version(sc) < 0) { 598 /* 599 * Warn user that a firmware update will be attempted in init. 600 */ 601 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 602 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 603 sc->flags &= ~TPS_UPTODATE; 604 } else { 605 sc->flags |= TPS_UPTODATE; 606 } 607 608 /* 609 * Create a child device for each MAC. The ethernet attachment 610 * will be done in these children. 611 */ 612 for (i = 0; i < (sc)->params.nports; i++) { 613 struct port_info *pi; 614 615 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 616 device_printf(dev, "failed to add child port\n"); 617 error = EINVAL; 618 goto out; 619 } 620 pi = &sc->port[i]; 621 pi->adapter = sc; 622 pi->nqsets = port_qsets; 623 pi->first_qset = i*port_qsets; 624 pi->port_id = i; 625 pi->tx_chan = i >= ai->nports0; 626 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 627 sc->rxpkt_map[pi->txpkt_intf] = i; 628 sc->port[i].tx_chan = i >= ai->nports0; 629 sc->portdev[i] = child; 630 device_set_softc(child, pi); 631 } 632 if ((error = bus_generic_attach(dev)) != 0) 633 goto out; 634 635 /* initialize sge private state */ 636 t3_sge_init_adapter(sc); 637 638 t3_led_ready(sc); 639 640 cxgb_offload_init(); 641 if (is_offload(sc)) { 642 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); 643 cxgb_adapter_ofld(sc); 644 } 645 error = t3_get_fw_version(sc, &vers); 646 if (error) 647 goto out; 648 649 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 650 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 651 G_FW_VERSION_MICRO(vers)); 652 653 snprintf(buf, sizeof(buf), "%s\t E/C: %s S/N: %s", 654 ai->desc, 655 sc->params.vpd.ec, sc->params.vpd.sn); 656 device_set_desc_copy(dev, buf); 657 658 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x", 659 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1], 660 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]); 661 662 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 663 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 664 t3_add_attach_sysctls(sc); 665 out: 666 if (error) 667 cxgb_free(sc); 668 669 return (error); 670 } 671 672 /* 673 * The cxgb_controller_detach routine is called with the device is 674 * unloaded from the system. 675 */ 676 677 static int 678 cxgb_controller_detach(device_t dev) 679 { 680 struct adapter *sc; 681 682 sc = device_get_softc(dev); 683 684 cxgb_free(sc); 685 686 return (0); 687 } 688 689 /* 690 * The cxgb_free() is called by the cxgb_controller_detach() routine 691 * to tear down the structures that were built up in 692 * cxgb_controller_attach(), and should be the final piece of work 693 * done when fully unloading the driver. 694 * 695 * 696 * 1. Shutting down the threads started by the cxgb_controller_attach() 697 * routine. 698 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()). 699 * 3. Detaching all of the port devices created during the 700 * cxgb_controller_attach() routine. 701 * 4. Removing the device children created via cxgb_controller_attach(). 702 * 5. Releasing PCI resources associated with the device. 703 * 6. Turning off the offload support, iff it was turned on. 704 * 7. Destroying the mutexes created in cxgb_controller_attach(). 705 * 706 */ 707 static void 708 cxgb_free(struct adapter *sc) 709 { 710 int i; 711 712 ADAPTER_LOCK(sc); 713 sc->flags |= CXGB_SHUTDOWN; 714 ADAPTER_UNLOCK(sc); 715 716 cxgb_pcpu_shutdown_threads(sc); 717 718 ADAPTER_LOCK(sc); 719 cxgb_down_locked(sc); 720 ADAPTER_UNLOCK(sc); 721 722 t3_sge_deinit_sw(sc); 723 /* 724 * Wait for last callout 725 */ 726 727 DELAY(hz*100); 728 729 bus_generic_detach(sc->dev); 730 731 for (i = 0; i < (sc)->params.nports; i++) { 732 if (sc->portdev[i] && 733 device_delete_child(sc->dev, sc->portdev[i]) != 0) 734 device_printf(sc->dev, "failed to delete child port\n"); 735 } 736 737 cxgb_teardown_interrupts(sc); 738 739 #ifdef MSI_SUPPORTED 740 if (sc->flags & (USING_MSI | USING_MSIX)) { 741 device_printf(sc->dev, "releasing msi message(s)\n"); 742 pci_release_msi(sc->dev); 743 } else { 744 device_printf(sc->dev, "no msi message to release\n"); 745 } 746 747 if (sc->msix_regs_res != NULL) { 748 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 749 sc->msix_regs_res); 750 } 751 #endif 752 753 if (sc->tq != NULL) { 754 taskqueue_free(sc->tq); 755 sc->tq = NULL; 756 } 757 758 if (is_offload(sc)) { 759 cxgb_adapter_unofld(sc); 760 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT)) 761 offload_close(&sc->tdev); 762 else 763 printf("cxgb_free: DEVMAP_BIT not set\n"); 764 } else 765 printf("not offloading set\n"); 766 #ifdef notyet 767 if (sc->flags & CXGB_OFLD_INIT) 768 cxgb_offload_deactivate(sc); 769 #endif 770 free(sc->filters, M_DEVBUF); 771 t3_sge_free(sc); 772 773 cxgb_offload_exit(); 774 775 if (sc->udbs_res != NULL) 776 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 777 sc->udbs_res); 778 779 if (sc->regs_res != NULL) 780 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 781 sc->regs_res); 782 783 MTX_DESTROY(&sc->mdio_lock); 784 MTX_DESTROY(&sc->sge.reg_lock); 785 MTX_DESTROY(&sc->elmer_lock); 786 ADAPTER_LOCK_DEINIT(sc); 787 } 788 789 /** 790 * setup_sge_qsets - configure SGE Tx/Rx/response queues 791 * @sc: the controller softc 792 * 793 * Determines how many sets of SGE queues to use and initializes them. 794 * We support multiple queue sets per port if we have MSI-X, otherwise 795 * just one queue set per port. 796 */ 797 static int 798 setup_sge_qsets(adapter_t *sc) 799 { 800 int i, j, err, irq_idx = 0, qset_idx = 0; 801 u_int ntxq = SGE_TXQ_PER_SET; 802 803 if ((err = t3_sge_alloc(sc)) != 0) { 804 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 805 return (err); 806 } 807 808 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 809 irq_idx = -1; 810 811 for (i = 0; i < (sc)->params.nports; i++) { 812 struct port_info *pi = &sc->port[i]; 813 814 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 815 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 816 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 817 &sc->params.sge.qset[qset_idx], ntxq, pi); 818 if (err) { 819 t3_free_sge_resources(sc); 820 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", 821 err); 822 return (err); 823 } 824 } 825 } 826 827 return (0); 828 } 829 830 static void 831 cxgb_teardown_interrupts(adapter_t *sc) 832 { 833 int i; 834 835 for (i = 0; i < SGE_QSETS; i++) { 836 if (sc->msix_intr_tag[i] == NULL) { 837 838 /* Should have been setup fully or not at all */ 839 KASSERT(sc->msix_irq_res[i] == NULL && 840 sc->msix_irq_rid[i] == 0, 841 ("%s: half-done interrupt (%d).", __func__, i)); 842 843 continue; 844 } 845 846 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 847 sc->msix_intr_tag[i]); 848 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i], 849 sc->msix_irq_res[i]); 850 851 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL; 852 sc->msix_irq_rid[i] = 0; 853 } 854 855 if (sc->intr_tag) { 856 KASSERT(sc->irq_res != NULL, 857 ("%s: half-done interrupt.", __func__)); 858 859 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 860 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 861 sc->irq_res); 862 863 sc->irq_res = sc->intr_tag = NULL; 864 sc->irq_rid = 0; 865 } 866 } 867 868 static int 869 cxgb_setup_interrupts(adapter_t *sc) 870 { 871 struct resource *res; 872 void *tag; 873 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX); 874 875 sc->irq_rid = intr_flag ? 1 : 0; 876 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, 877 RF_SHAREABLE | RF_ACTIVE); 878 if (sc->irq_res == NULL) { 879 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n", 880 intr_flag, sc->irq_rid); 881 err = EINVAL; 882 sc->irq_rid = 0; 883 } else { 884 err = bus_setup_intr(sc->dev, sc->irq_res, 885 INTR_MPSAFE | INTR_TYPE_NET, 886 #ifdef INTR_FILTERS 887 NULL, 888 #endif 889 sc->cxgb_intr, sc, &sc->intr_tag); 890 891 if (err) { 892 device_printf(sc->dev, 893 "Cannot set up interrupt (%x, %u, %d)\n", 894 intr_flag, sc->irq_rid, err); 895 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 896 sc->irq_res); 897 sc->irq_res = sc->intr_tag = NULL; 898 sc->irq_rid = 0; 899 } 900 } 901 902 /* That's all for INTx or MSI */ 903 if (!(intr_flag & USING_MSIX) || err) 904 return (err); 905 906 for (i = 0; i < sc->msi_count - 1; i++) { 907 rid = i + 2; 908 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, 909 RF_SHAREABLE | RF_ACTIVE); 910 if (res == NULL) { 911 device_printf(sc->dev, "Cannot allocate interrupt " 912 "for message %d\n", rid); 913 err = EINVAL; 914 break; 915 } 916 917 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET, 918 #ifdef INTR_FILTERS 919 NULL, 920 #endif 921 t3_intr_msix, &sc->sge.qs[i], &tag); 922 if (err) { 923 device_printf(sc->dev, "Cannot set up interrupt " 924 "for message %d (%d)\n", rid, err); 925 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res); 926 break; 927 } 928 929 sc->msix_irq_rid[i] = rid; 930 sc->msix_irq_res[i] = res; 931 sc->msix_intr_tag[i] = tag; 932 } 933 934 if (err) 935 cxgb_teardown_interrupts(sc); 936 937 return (err); 938 } 939 940 941 static int 942 cxgb_port_probe(device_t dev) 943 { 944 struct port_info *p; 945 char buf[80]; 946 const char *desc; 947 948 p = device_get_softc(dev); 949 desc = p->phy.desc; 950 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc); 951 device_set_desc_copy(dev, buf); 952 return (0); 953 } 954 955 956 static int 957 cxgb_makedev(struct port_info *pi) 958 { 959 960 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 961 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp)); 962 963 if (pi->port_cdev == NULL) 964 return (ENOMEM); 965 966 pi->port_cdev->si_drv1 = (void *)pi; 967 968 return (0); 969 } 970 971 #ifndef LRO_SUPPORTED 972 #ifdef IFCAP_LRO 973 #undef IFCAP_LRO 974 #endif 975 #define IFCAP_LRO 0x0 976 #endif 977 978 #ifdef TSO_SUPPORTED 979 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO) 980 /* Don't enable TSO6 yet */ 981 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU | IFCAP_LRO) 982 #else 983 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 984 /* Don't enable TSO6 yet */ 985 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 986 #define IFCAP_TSO4 0x0 987 #define IFCAP_TSO6 0x0 988 #define CSUM_TSO 0x0 989 #endif 990 991 992 static int 993 cxgb_port_attach(device_t dev) 994 { 995 struct port_info *p; 996 struct ifnet *ifp; 997 int err, media_flags; 998 struct adapter *sc; 999 1000 1001 p = device_get_softc(dev); 1002 sc = p->adapter; 1003 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 1004 device_get_unit(device_get_parent(dev)), p->port_id); 1005 PORT_LOCK_INIT(p, p->lockbuf); 1006 1007 /* Allocate an ifnet object and set it up */ 1008 ifp = p->ifp = if_alloc(IFT_ETHER); 1009 if (ifp == NULL) { 1010 device_printf(dev, "Cannot allocate ifnet\n"); 1011 return (ENOMEM); 1012 } 1013 1014 /* 1015 * Note that there is currently no watchdog timer. 1016 */ 1017 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1018 ifp->if_init = cxgb_init; 1019 ifp->if_softc = p; 1020 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1021 ifp->if_ioctl = cxgb_ioctl; 1022 ifp->if_start = cxgb_start; 1023 1024 1025 ifp->if_timer = 0; /* Disable ifnet watchdog */ 1026 ifp->if_watchdog = NULL; 1027 1028 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 1029 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 1030 IFQ_SET_READY(&ifp->if_snd); 1031 1032 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 1033 ifp->if_capabilities |= CXGB_CAP; 1034 ifp->if_capenable |= CXGB_CAP_ENABLE; 1035 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 1036 /* 1037 * disable TSO on 4-port - it isn't supported by the firmware yet 1038 */ 1039 if (p->adapter->params.nports > 2) { 1040 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6); 1041 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6); 1042 ifp->if_hwassist &= ~CSUM_TSO; 1043 } 1044 1045 ether_ifattach(ifp, p->hw_addr); 1046 1047 #ifdef IFNET_MULTIQUEUE 1048 ifp->if_transmit = cxgb_pcpu_transmit; 1049 #endif 1050 /* 1051 * Only default to jumbo frames on 10GigE 1052 */ 1053 if (p->adapter->params.nports <= 2) 1054 ifp->if_mtu = ETHERMTU_JUMBO; 1055 if ((err = cxgb_makedev(p)) != 0) { 1056 printf("makedev failed %d\n", err); 1057 return (err); 1058 } 1059 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 1060 cxgb_media_status); 1061 1062 if (!strcmp(p->phy.desc, "10GBASE-CX4")) { 1063 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX; 1064 } else if (!strcmp(p->phy.desc, "10GBASE-SR")) { 1065 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX; 1066 } else if (!strcmp(p->phy.desc, "10GBASE-R")) { 1067 media_flags = cxgb_ifm_type(p->phy.modtype); 1068 } else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) { 1069 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL); 1070 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX, 1071 0, NULL); 1072 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX, 1073 0, NULL); 1074 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 1075 0, NULL); 1076 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 1077 0, NULL); 1078 media_flags = 0; 1079 } else if (!strcmp(p->phy.desc, "1000BASE-X")) { 1080 /* 1081 * XXX: This is not very accurate. Fix when common code 1082 * returns more specific value - eg 1000BASE-SX, LX, etc. 1083 * 1084 * XXX: In the meantime, don't lie. Consider setting IFM_AUTO 1085 * instead of SX. 1086 */ 1087 media_flags = IFM_ETHER | IFM_1000_SX | IFM_FDX; 1088 } else { 1089 printf("unsupported media type %s\n", p->phy.desc); 1090 return (ENXIO); 1091 } 1092 if (media_flags) { 1093 /* 1094 * Note the modtype on which we based our flags. If modtype 1095 * changes, we'll redo the ifmedia for this ifp. modtype may 1096 * change when transceivers are plugged in/out, and in other 1097 * situations. 1098 */ 1099 ifmedia_add(&p->media, media_flags, p->phy.modtype, NULL); 1100 ifmedia_set(&p->media, media_flags); 1101 } else { 1102 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1103 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO); 1104 } 1105 1106 /* Get the latest mac address, User can use a LAA */ 1107 bcopy(IF_LLADDR(p->ifp), p->hw_addr, ETHER_ADDR_LEN); 1108 t3_sge_init_port(p); 1109 1110 #if defined(LINK_ATTACH) 1111 cxgb_link_start(p); 1112 t3_link_changed(sc, p->port_id); 1113 #endif 1114 1115 return (err); 1116 } 1117 1118 /* 1119 * cxgb_port_detach() is called via the device_detach methods when 1120 * cxgb_free() calls the bus_generic_detach. It is responsible for 1121 * removing the device from the view of the kernel, i.e. from all 1122 * interfaces lists etc. This routine is only called when the driver is 1123 * being unloaded, not when the link goes down. 1124 * 1125 */ 1126 static int 1127 cxgb_port_detach(device_t dev) 1128 { 1129 struct port_info *p; 1130 struct adapter *sc; 1131 1132 p = device_get_softc(dev); 1133 sc = p->adapter; 1134 1135 if (p->port_cdev != NULL) 1136 destroy_dev(p->port_cdev); 1137 1138 ether_ifdetach(p->ifp); 1139 1140 PORT_LOCK(p); 1141 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING) 1142 cxgb_stop_locked(p); 1143 PORT_UNLOCK(p); 1144 1145 callout_drain(&sc->cxgb_tick_ch); 1146 callout_drain(&sc->sge_timer_ch); 1147 1148 if (sc->tq != NULL) { 1149 printf("draining slow intr\n"); 1150 1151 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1152 printf("draining ext intr\n"); 1153 taskqueue_drain(sc->tq, &sc->ext_intr_task); 1154 printf("draining tick task\n"); 1155 taskqueue_drain(sc->tq, &sc->tick_task); 1156 } 1157 1158 /* 1159 * the lock may be acquired in ifdetach 1160 */ 1161 PORT_LOCK_DEINIT(p); 1162 if_free(p->ifp); 1163 1164 return (0); 1165 } 1166 1167 void 1168 t3_fatal_err(struct adapter *sc) 1169 { 1170 u_int fw_status[4]; 1171 1172 if (sc->flags & FULL_INIT_DONE) { 1173 t3_sge_stop(sc); 1174 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1175 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1176 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1177 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1178 t3_intr_disable(sc); 1179 } 1180 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1181 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1182 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1183 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1184 } 1185 1186 int 1187 t3_os_find_pci_capability(adapter_t *sc, int cap) 1188 { 1189 device_t dev; 1190 struct pci_devinfo *dinfo; 1191 pcicfgregs *cfg; 1192 uint32_t status; 1193 uint8_t ptr; 1194 1195 dev = sc->dev; 1196 dinfo = device_get_ivars(dev); 1197 cfg = &dinfo->cfg; 1198 1199 status = pci_read_config(dev, PCIR_STATUS, 2); 1200 if (!(status & PCIM_STATUS_CAPPRESENT)) 1201 return (0); 1202 1203 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1204 case 0: 1205 case 1: 1206 ptr = PCIR_CAP_PTR; 1207 break; 1208 case 2: 1209 ptr = PCIR_CAP_PTR_2; 1210 break; 1211 default: 1212 return (0); 1213 break; 1214 } 1215 ptr = pci_read_config(dev, ptr, 1); 1216 1217 while (ptr != 0) { 1218 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1219 return (ptr); 1220 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1221 } 1222 1223 return (0); 1224 } 1225 1226 int 1227 t3_os_pci_save_state(struct adapter *sc) 1228 { 1229 device_t dev; 1230 struct pci_devinfo *dinfo; 1231 1232 dev = sc->dev; 1233 dinfo = device_get_ivars(dev); 1234 1235 pci_cfg_save(dev, dinfo, 0); 1236 return (0); 1237 } 1238 1239 int 1240 t3_os_pci_restore_state(struct adapter *sc) 1241 { 1242 device_t dev; 1243 struct pci_devinfo *dinfo; 1244 1245 dev = sc->dev; 1246 dinfo = device_get_ivars(dev); 1247 1248 pci_cfg_restore(dev, dinfo); 1249 return (0); 1250 } 1251 1252 /** 1253 * t3_os_link_changed - handle link status changes 1254 * @adapter: the adapter associated with the link change 1255 * @port_id: the port index whose limk status has changed 1256 * @link_status: the new status of the link 1257 * @speed: the new speed setting 1258 * @duplex: the new duplex setting 1259 * @fc: the new flow-control setting 1260 * 1261 * This is the OS-dependent handler for link status changes. The OS 1262 * neutral handler takes care of most of the processing for these events, 1263 * then calls this handler for any OS-specific processing. 1264 */ 1265 void 1266 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1267 int duplex, int fc) 1268 { 1269 struct port_info *pi = &adapter->port[port_id]; 1270 1271 if (link_status) { 1272 pi->ifp->if_baudrate = IF_Mbps(speed); 1273 if_link_state_change(pi->ifp, LINK_STATE_UP); 1274 } else 1275 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 1276 } 1277 1278 /** 1279 * t3_os_phymod_changed - handle PHY module changes 1280 * @phy: the PHY reporting the module change 1281 * @mod_type: new module type 1282 * 1283 * This is the OS-dependent handler for PHY module changes. It is 1284 * invoked when a PHY module is removed or inserted for any OS-specific 1285 * processing. 1286 */ 1287 void t3_os_phymod_changed(struct adapter *adap, int port_id) 1288 { 1289 static const char *mod_str[] = { 1290 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown" 1291 }; 1292 1293 struct port_info *pi = &adap->port[port_id]; 1294 1295 if (pi->phy.modtype == phy_modtype_none) 1296 device_printf(adap->dev, "PHY module unplugged\n"); 1297 else { 1298 KASSERT(pi->phy.modtype < ARRAY_SIZE(mod_str), 1299 ("invalid PHY module type %d", pi->phy.modtype)); 1300 device_printf(adap->dev, "%s PHY module inserted\n", 1301 mod_str[pi->phy.modtype]); 1302 } 1303 } 1304 1305 /* 1306 * Interrupt-context handler for external (PHY) interrupts. 1307 */ 1308 void 1309 t3_os_ext_intr_handler(adapter_t *sc) 1310 { 1311 if (cxgb_debug) 1312 printf("t3_os_ext_intr_handler\n"); 1313 /* 1314 * Schedule a task to handle external interrupts as they may be slow 1315 * and we use a mutex to protect MDIO registers. We disable PHY 1316 * interrupts in the meantime and let the task reenable them when 1317 * it's done. 1318 */ 1319 ADAPTER_LOCK(sc); 1320 if (sc->slow_intr_mask) { 1321 sc->slow_intr_mask &= ~F_T3DBG; 1322 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1323 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 1324 } 1325 ADAPTER_UNLOCK(sc); 1326 } 1327 1328 void 1329 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1330 { 1331 1332 /* 1333 * The ifnet might not be allocated before this gets called, 1334 * as this is called early on in attach by t3_prep_adapter 1335 * save the address off in the port structure 1336 */ 1337 if (cxgb_debug) 1338 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1339 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1340 } 1341 1342 /** 1343 * link_start - enable a port 1344 * @p: the port to enable 1345 * 1346 * Performs the MAC and PHY actions needed to enable a port. 1347 */ 1348 static void 1349 cxgb_link_start(struct port_info *p) 1350 { 1351 struct ifnet *ifp; 1352 struct t3_rx_mode rm; 1353 struct cmac *mac = &p->mac; 1354 int mtu, hwtagging; 1355 1356 ifp = p->ifp; 1357 1358 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); 1359 1360 mtu = ifp->if_mtu; 1361 if (ifp->if_capenable & IFCAP_VLAN_MTU) 1362 mtu += ETHER_VLAN_ENCAP_LEN; 1363 1364 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; 1365 1366 t3_init_rx_mode(&rm, p); 1367 if (!mac->multiport) 1368 t3_mac_reset(mac); 1369 t3_mac_set_mtu(mac, mtu); 1370 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1371 t3_mac_set_address(mac, 0, p->hw_addr); 1372 t3_mac_set_rx_mode(mac, &rm); 1373 t3_link_start(&p->phy, mac, &p->link_config); 1374 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1375 } 1376 1377 1378 static int 1379 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1380 unsigned long n) 1381 { 1382 int attempts = 5; 1383 1384 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1385 if (!--attempts) 1386 return (ETIMEDOUT); 1387 t3_os_sleep(10); 1388 } 1389 return 0; 1390 } 1391 1392 static int 1393 init_tp_parity(struct adapter *adap) 1394 { 1395 int i; 1396 struct mbuf *m; 1397 struct cpl_set_tcb_field *greq; 1398 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1399 1400 t3_tp_set_offload_mode(adap, 1); 1401 1402 for (i = 0; i < 16; i++) { 1403 struct cpl_smt_write_req *req; 1404 1405 m = m_gethdr(M_WAITOK, MT_DATA); 1406 req = mtod(m, struct cpl_smt_write_req *); 1407 m->m_len = m->m_pkthdr.len = sizeof(*req); 1408 memset(req, 0, sizeof(*req)); 1409 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1410 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1411 req->iff = i; 1412 t3_mgmt_tx(adap, m); 1413 } 1414 1415 for (i = 0; i < 2048; i++) { 1416 struct cpl_l2t_write_req *req; 1417 1418 m = m_gethdr(M_WAITOK, MT_DATA); 1419 req = mtod(m, struct cpl_l2t_write_req *); 1420 m->m_len = m->m_pkthdr.len = sizeof(*req); 1421 memset(req, 0, sizeof(*req)); 1422 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1423 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1424 req->params = htonl(V_L2T_W_IDX(i)); 1425 t3_mgmt_tx(adap, m); 1426 } 1427 1428 for (i = 0; i < 2048; i++) { 1429 struct cpl_rte_write_req *req; 1430 1431 m = m_gethdr(M_WAITOK, MT_DATA); 1432 req = mtod(m, struct cpl_rte_write_req *); 1433 m->m_len = m->m_pkthdr.len = sizeof(*req); 1434 memset(req, 0, sizeof(*req)); 1435 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1436 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1437 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1438 t3_mgmt_tx(adap, m); 1439 } 1440 1441 m = m_gethdr(M_WAITOK, MT_DATA); 1442 greq = mtod(m, struct cpl_set_tcb_field *); 1443 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1444 memset(greq, 0, sizeof(*greq)); 1445 greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1446 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1447 greq->mask = htobe64(1); 1448 t3_mgmt_tx(adap, m); 1449 1450 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1451 t3_tp_set_offload_mode(adap, 0); 1452 return (i); 1453 } 1454 1455 /** 1456 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1457 * @adap: the adapter 1458 * 1459 * Sets up RSS to distribute packets to multiple receive queues. We 1460 * configure the RSS CPU lookup table to distribute to the number of HW 1461 * receive queues, and the response queue lookup table to narrow that 1462 * down to the response queues actually configured for each port. 1463 * We always configure the RSS mapping for two ports since the mapping 1464 * table has plenty of entries. 1465 */ 1466 static void 1467 setup_rss(adapter_t *adap) 1468 { 1469 int i; 1470 u_int nq[2]; 1471 uint8_t cpus[SGE_QSETS + 1]; 1472 uint16_t rspq_map[RSS_TABLE_SIZE]; 1473 1474 for (i = 0; i < SGE_QSETS; ++i) 1475 cpus[i] = i; 1476 cpus[SGE_QSETS] = 0xff; 1477 1478 nq[0] = nq[1] = 0; 1479 for_each_port(adap, i) { 1480 const struct port_info *pi = adap2pinfo(adap, i); 1481 1482 nq[pi->tx_chan] += pi->nqsets; 1483 } 1484 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1485 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1486 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1487 } 1488 /* Calculate the reverse RSS map table */ 1489 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1490 if (adap->rrss_map[rspq_map[i]] == 0xff) 1491 adap->rrss_map[rspq_map[i]] = i; 1492 1493 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1494 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1495 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1496 cpus, rspq_map); 1497 1498 } 1499 1500 /* 1501 * Sends an mbuf to an offload queue driver 1502 * after dealing with any active network taps. 1503 */ 1504 static inline int 1505 offload_tx(struct t3cdev *tdev, struct mbuf *m) 1506 { 1507 int ret; 1508 1509 ret = t3_offload_tx(tdev, m); 1510 return (ret); 1511 } 1512 1513 static int 1514 write_smt_entry(struct adapter *adapter, int idx) 1515 { 1516 struct port_info *pi = &adapter->port[idx]; 1517 struct cpl_smt_write_req *req; 1518 struct mbuf *m; 1519 1520 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1521 return (ENOMEM); 1522 1523 req = mtod(m, struct cpl_smt_write_req *); 1524 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req); 1525 1526 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1527 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 1528 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 1529 req->iff = idx; 1530 memset(req->src_mac1, 0, sizeof(req->src_mac1)); 1531 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 1532 1533 m_set_priority(m, 1); 1534 1535 offload_tx(&adapter->tdev, m); 1536 1537 return (0); 1538 } 1539 1540 static int 1541 init_smt(struct adapter *adapter) 1542 { 1543 int i; 1544 1545 for_each_port(adapter, i) 1546 write_smt_entry(adapter, i); 1547 return 0; 1548 } 1549 1550 static void 1551 init_port_mtus(adapter_t *adapter) 1552 { 1553 unsigned int mtus = adapter->port[0].ifp->if_mtu; 1554 1555 if (adapter->port[1].ifp) 1556 mtus |= adapter->port[1].ifp->if_mtu << 16; 1557 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); 1558 } 1559 1560 static void 1561 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1562 int hi, int port) 1563 { 1564 struct mbuf *m; 1565 struct mngt_pktsched_wr *req; 1566 1567 m = m_gethdr(M_DONTWAIT, MT_DATA); 1568 if (m) { 1569 req = mtod(m, struct mngt_pktsched_wr *); 1570 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1571 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1572 req->sched = sched; 1573 req->idx = qidx; 1574 req->min = lo; 1575 req->max = hi; 1576 req->binding = port; 1577 m->m_len = m->m_pkthdr.len = sizeof(*req); 1578 t3_mgmt_tx(adap, m); 1579 } 1580 } 1581 1582 static void 1583 bind_qsets(adapter_t *sc) 1584 { 1585 int i, j; 1586 1587 cxgb_pcpu_startup_threads(sc); 1588 for (i = 0; i < (sc)->params.nports; ++i) { 1589 const struct port_info *pi = adap2pinfo(sc, i); 1590 1591 for (j = 0; j < pi->nqsets; ++j) { 1592 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1593 -1, pi->tx_chan); 1594 1595 } 1596 } 1597 } 1598 1599 static void 1600 update_tpeeprom(struct adapter *adap) 1601 { 1602 #ifdef FIRMWARE_LATEST 1603 const struct firmware *tpeeprom; 1604 #else 1605 struct firmware *tpeeprom; 1606 #endif 1607 1608 uint32_t version; 1609 unsigned int major, minor; 1610 int ret, len; 1611 char rev, name[32]; 1612 1613 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1614 1615 major = G_TP_VERSION_MAJOR(version); 1616 minor = G_TP_VERSION_MINOR(version); 1617 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1618 return; 1619 1620 rev = t3rev2char(adap); 1621 snprintf(name, sizeof(name), TPEEPROM_NAME, rev); 1622 1623 tpeeprom = firmware_get(name); 1624 if (tpeeprom == NULL) { 1625 device_printf(adap->dev, 1626 "could not load TP EEPROM: unable to load %s\n", 1627 name); 1628 return; 1629 } 1630 1631 len = tpeeprom->datasize - 4; 1632 1633 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1634 if (ret) 1635 goto release_tpeeprom; 1636 1637 if (len != TP_SRAM_LEN) { 1638 device_printf(adap->dev, 1639 "%s length is wrong len=%d expected=%d\n", name, 1640 len, TP_SRAM_LEN); 1641 return; 1642 } 1643 1644 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1645 TP_SRAM_OFFSET); 1646 1647 if (!ret) { 1648 device_printf(adap->dev, 1649 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1650 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1651 } else 1652 device_printf(adap->dev, 1653 "Protocol SRAM image update in EEPROM failed\n"); 1654 1655 release_tpeeprom: 1656 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1657 1658 return; 1659 } 1660 1661 static int 1662 update_tpsram(struct adapter *adap) 1663 { 1664 #ifdef FIRMWARE_LATEST 1665 const struct firmware *tpsram; 1666 #else 1667 struct firmware *tpsram; 1668 #endif 1669 int ret; 1670 char rev, name[32]; 1671 1672 rev = t3rev2char(adap); 1673 snprintf(name, sizeof(name), TPSRAM_NAME, rev); 1674 1675 update_tpeeprom(adap); 1676 1677 tpsram = firmware_get(name); 1678 if (tpsram == NULL){ 1679 device_printf(adap->dev, "could not load TP SRAM\n"); 1680 return (EINVAL); 1681 } else 1682 device_printf(adap->dev, "updating TP SRAM\n"); 1683 1684 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1685 if (ret) 1686 goto release_tpsram; 1687 1688 ret = t3_set_proto_sram(adap, tpsram->data); 1689 if (ret) 1690 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1691 1692 release_tpsram: 1693 firmware_put(tpsram, FIRMWARE_UNLOAD); 1694 1695 return ret; 1696 } 1697 1698 /** 1699 * cxgb_up - enable the adapter 1700 * @adap: adapter being enabled 1701 * 1702 * Called when the first port is enabled, this function performs the 1703 * actions necessary to make an adapter operational, such as completing 1704 * the initialization of HW modules, and enabling interrupts. 1705 */ 1706 static int 1707 cxgb_up(struct adapter *sc) 1708 { 1709 int err = 0; 1710 1711 if ((sc->flags & FULL_INIT_DONE) == 0) { 1712 1713 if ((sc->flags & FW_UPTODATE) == 0) 1714 if ((err = upgrade_fw(sc))) 1715 goto out; 1716 if ((sc->flags & TPS_UPTODATE) == 0) 1717 if ((err = update_tpsram(sc))) 1718 goto out; 1719 err = t3_init_hw(sc, 0); 1720 if (err) 1721 goto out; 1722 1723 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1724 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1725 1726 err = setup_sge_qsets(sc); 1727 if (err) 1728 goto out; 1729 1730 setup_rss(sc); 1731 1732 t3_intr_clear(sc); 1733 err = cxgb_setup_interrupts(sc); 1734 if (err) 1735 goto out; 1736 1737 t3_add_configured_sysctls(sc); 1738 sc->flags |= FULL_INIT_DONE; 1739 } 1740 1741 t3_intr_clear(sc); 1742 t3_sge_start(sc); 1743 t3_intr_enable(sc); 1744 1745 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1746 is_offload(sc) && init_tp_parity(sc) == 0) 1747 sc->flags |= TP_PARITY_INIT; 1748 1749 if (sc->flags & TP_PARITY_INIT) { 1750 t3_write_reg(sc, A_TP_INT_CAUSE, 1751 F_CMCACHEPERR | F_ARPLUTPERR); 1752 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1753 } 1754 1755 1756 if (!(sc->flags & QUEUES_BOUND)) { 1757 bind_qsets(sc); 1758 sc->flags |= QUEUES_BOUND; 1759 } 1760 out: 1761 return (err); 1762 } 1763 1764 1765 /* 1766 * Bring down the interface but do not free any resources. 1767 */ 1768 static void 1769 cxgb_down_locked(struct adapter *sc) 1770 { 1771 1772 t3_sge_stop(sc); 1773 t3_intr_disable(sc); 1774 1775 callout_stop(&sc->cxgb_tick_ch); 1776 callout_stop(&sc->sge_timer_ch); 1777 } 1778 1779 static int 1780 offload_open(struct port_info *pi) 1781 { 1782 struct adapter *adapter = pi->adapter; 1783 struct t3cdev *tdev = &adapter->tdev; 1784 1785 int adap_up = adapter->open_device_map & PORT_MASK; 1786 int err = 0; 1787 1788 if (atomic_cmpset_int(&adapter->open_device_map, 1789 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)), 1790 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0) 1791 return (0); 1792 1793 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1794 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", 1795 adapter->open_device_map); 1796 ADAPTER_LOCK(pi->adapter); 1797 if (!adap_up) 1798 err = cxgb_up(adapter); 1799 ADAPTER_UNLOCK(pi->adapter); 1800 if (err) 1801 return (err); 1802 1803 t3_tp_set_offload_mode(adapter, 1); 1804 tdev->lldev = pi->ifp; 1805 1806 init_port_mtus(adapter); 1807 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, 1808 adapter->params.b_wnd, 1809 adapter->params.rev == 0 ? 1810 adapter->port[0].ifp->if_mtu : 0xffff); 1811 init_smt(adapter); 1812 /* Call back all registered clients */ 1813 cxgb_add_clients(tdev); 1814 1815 /* restore them in case the offload module has changed them */ 1816 if (err) { 1817 t3_tp_set_offload_mode(adapter, 0); 1818 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1819 cxgb_set_dummy_ops(tdev); 1820 } 1821 return (err); 1822 } 1823 1824 static int 1825 offload_close(struct t3cdev *tdev) 1826 { 1827 struct adapter *adapter = tdev2adap(tdev); 1828 1829 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1830 return (0); 1831 1832 /* Call back all registered clients */ 1833 cxgb_remove_clients(tdev); 1834 1835 tdev->lldev = NULL; 1836 cxgb_set_dummy_ops(tdev); 1837 t3_tp_set_offload_mode(adapter, 0); 1838 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1839 1840 ADAPTER_LOCK(adapter); 1841 if (!adapter->open_device_map) 1842 cxgb_down_locked(adapter); 1843 1844 ADAPTER_UNLOCK(adapter); 1845 1846 return (0); 1847 } 1848 1849 1850 static void 1851 cxgb_init(void *arg) 1852 { 1853 struct port_info *p = arg; 1854 1855 PORT_LOCK(p); 1856 cxgb_init_locked(p); 1857 PORT_UNLOCK(p); 1858 } 1859 1860 static void 1861 cxgb_init_locked(struct port_info *p) 1862 { 1863 struct ifnet *ifp; 1864 adapter_t *sc = p->adapter; 1865 int err; 1866 1867 PORT_LOCK_ASSERT_OWNED(p); 1868 ifp = p->ifp; 1869 1870 ADAPTER_LOCK(p->adapter); 1871 if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) { 1872 ADAPTER_UNLOCK(p->adapter); 1873 cxgb_stop_locked(p); 1874 return; 1875 } 1876 if (p->adapter->open_device_map == 0) { 1877 t3_intr_clear(sc); 1878 } 1879 setbit(&p->adapter->open_device_map, p->port_id); 1880 ADAPTER_UNLOCK(p->adapter); 1881 1882 if (is_offload(sc) && !ofld_disable) { 1883 err = offload_open(p); 1884 if (err) 1885 log(LOG_WARNING, 1886 "Could not initialize offload capabilities\n"); 1887 } 1888 1889 device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id); 1890 t3_port_intr_enable(sc, p->port_id); 1891 1892 #if !defined(LINK_ATTACH) 1893 cxgb_link_start(p); 1894 t3_link_changed(sc, p->port_id); 1895 #endif 1896 ifp->if_baudrate = IF_Mbps(p->link_config.speed); 1897 1898 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 1899 t3_sge_reset_adapter(sc); 1900 1901 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1902 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1903 } 1904 1905 static void 1906 cxgb_set_rxmode(struct port_info *p) 1907 { 1908 struct t3_rx_mode rm; 1909 struct cmac *mac = &p->mac; 1910 1911 t3_init_rx_mode(&rm, p); 1912 mtx_lock(&p->adapter->mdio_lock); 1913 t3_mac_set_rx_mode(mac, &rm); 1914 mtx_unlock(&p->adapter->mdio_lock); 1915 } 1916 1917 static void 1918 cxgb_stop_locked(struct port_info *pi) 1919 { 1920 struct ifnet *ifp; 1921 1922 PORT_LOCK_ASSERT_OWNED(pi); 1923 ADAPTER_LOCK_ASSERT_NOTOWNED(pi->adapter); 1924 1925 ifp = pi->ifp; 1926 t3_port_intr_disable(pi->adapter, pi->port_id); 1927 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1928 1929 /* disable pause frames */ 1930 t3_set_reg_field(pi->adapter, A_XGM_TX_CFG + pi->mac.offset, 1931 F_TXPAUSEEN, 0); 1932 1933 /* Reset RX FIFO HWM */ 1934 t3_set_reg_field(pi->adapter, A_XGM_RXFIFO_CFG + pi->mac.offset, 1935 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1936 1937 1938 ADAPTER_LOCK(pi->adapter); 1939 clrbit(&pi->adapter->open_device_map, pi->port_id); 1940 1941 if (pi->adapter->open_device_map == 0) 1942 cxgb_down_locked(pi->adapter); 1943 1944 ADAPTER_UNLOCK(pi->adapter); 1945 1946 #if !defined(LINK_ATTACH) 1947 DELAY(100); 1948 1949 /* Wait for TXFIFO empty */ 1950 t3_wait_op_done(pi->adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 1951 F_TXFIFO_EMPTY, 1, 20, 5); 1952 1953 DELAY(100); 1954 t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1955 1956 pi->phy.ops->power_down(&pi->phy, 1); 1957 #endif 1958 1959 } 1960 1961 static int 1962 cxgb_set_mtu(struct port_info *p, int mtu) 1963 { 1964 struct ifnet *ifp = p->ifp; 1965 int error = 0; 1966 1967 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1968 error = EINVAL; 1969 else if (ifp->if_mtu != mtu) { 1970 PORT_LOCK(p); 1971 ifp->if_mtu = mtu; 1972 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1973 cxgb_stop_locked(p); 1974 cxgb_init_locked(p); 1975 } 1976 PORT_UNLOCK(p); 1977 } 1978 return (error); 1979 } 1980 1981 #ifdef LRO_SUPPORTED 1982 /* 1983 * Mark lro enabled or disabled in all qsets for this port 1984 */ 1985 static int 1986 cxgb_set_lro(struct port_info *p, int enabled) 1987 { 1988 int i; 1989 struct adapter *adp = p->adapter; 1990 struct sge_qset *q; 1991 1992 PORT_LOCK_ASSERT_OWNED(p); 1993 for (i = 0; i < p->nqsets; i++) { 1994 q = &adp->sge.qs[p->first_qset + i]; 1995 q->lro.enabled = (enabled != 0); 1996 } 1997 return (0); 1998 } 1999 #endif 2000 2001 static int 2002 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 2003 { 2004 struct port_info *p = ifp->if_softc; 2005 #ifdef INET 2006 struct ifaddr *ifa = (struct ifaddr *)data; 2007 #endif 2008 struct ifreq *ifr = (struct ifreq *)data; 2009 int flags, error = 0, reinit = 0; 2010 uint32_t mask; 2011 2012 /* 2013 * XXX need to check that we aren't in the middle of an unload 2014 */ 2015 switch (command) { 2016 case SIOCSIFMTU: 2017 error = cxgb_set_mtu(p, ifr->ifr_mtu); 2018 break; 2019 case SIOCSIFADDR: 2020 #ifdef INET 2021 if (ifa->ifa_addr->sa_family == AF_INET) { 2022 ifp->if_flags |= IFF_UP; 2023 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2024 PORT_LOCK(p); 2025 cxgb_init_locked(p); 2026 PORT_UNLOCK(p); 2027 } 2028 arp_ifinit(ifp, ifa); 2029 } else 2030 #endif 2031 error = ether_ioctl(ifp, command, data); 2032 break; 2033 case SIOCSIFFLAGS: 2034 PORT_LOCK(p); 2035 if (ifp->if_flags & IFF_UP) { 2036 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2037 flags = p->if_flags; 2038 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 2039 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 2040 cxgb_set_rxmode(p); 2041 } else 2042 cxgb_init_locked(p); 2043 p->if_flags = ifp->if_flags; 2044 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2045 cxgb_stop_locked(p); 2046 2047 PORT_UNLOCK(p); 2048 break; 2049 case SIOCADDMULTI: 2050 case SIOCDELMULTI: 2051 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2052 cxgb_set_rxmode(p); 2053 } 2054 break; 2055 case SIOCSIFMEDIA: 2056 case SIOCGIFMEDIA: 2057 PORT_LOCK(p); 2058 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 2059 PORT_UNLOCK(p); 2060 break; 2061 case SIOCSIFCAP: 2062 PORT_LOCK(p); 2063 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 2064 if (mask & IFCAP_TXCSUM) { 2065 if (IFCAP_TXCSUM & ifp->if_capenable) { 2066 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 2067 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 2068 | CSUM_IP | CSUM_TSO); 2069 } else { 2070 ifp->if_capenable |= IFCAP_TXCSUM; 2071 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP 2072 | CSUM_IP); 2073 } 2074 } 2075 if (mask & IFCAP_RXCSUM) { 2076 ifp->if_capenable ^= IFCAP_RXCSUM; 2077 } 2078 if (mask & IFCAP_TSO4) { 2079 if (IFCAP_TSO4 & ifp->if_capenable) { 2080 ifp->if_capenable &= ~IFCAP_TSO4; 2081 ifp->if_hwassist &= ~CSUM_TSO; 2082 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 2083 ifp->if_capenable |= IFCAP_TSO4; 2084 ifp->if_hwassist |= CSUM_TSO; 2085 } else { 2086 if (cxgb_debug) 2087 printf("cxgb requires tx checksum offload" 2088 " be enabled to use TSO\n"); 2089 error = EINVAL; 2090 } 2091 } 2092 #ifdef LRO_SUPPORTED 2093 if (mask & IFCAP_LRO) { 2094 ifp->if_capenable ^= IFCAP_LRO; 2095 2096 /* Safe to do this even if cxgb_up not called yet */ 2097 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO); 2098 } 2099 #endif 2100 if (mask & IFCAP_VLAN_HWTAGGING) { 2101 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2102 reinit = ifp->if_drv_flags & IFF_DRV_RUNNING; 2103 } 2104 if (mask & IFCAP_VLAN_MTU) { 2105 ifp->if_capenable ^= IFCAP_VLAN_MTU; 2106 reinit = ifp->if_drv_flags & IFF_DRV_RUNNING; 2107 } 2108 if (mask & IFCAP_VLAN_HWCSUM) { 2109 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 2110 } 2111 if (reinit) { 2112 cxgb_stop_locked(p); 2113 cxgb_init_locked(p); 2114 } 2115 PORT_UNLOCK(p); 2116 2117 #ifdef VLAN_CAPABILITIES 2118 VLAN_CAPABILITIES(ifp); 2119 #endif 2120 break; 2121 default: 2122 error = ether_ioctl(ifp, command, data); 2123 break; 2124 } 2125 return (error); 2126 } 2127 2128 static int 2129 cxgb_media_change(struct ifnet *ifp) 2130 { 2131 if_printf(ifp, "media change not supported\n"); 2132 return (ENXIO); 2133 } 2134 2135 /* 2136 * Translates from phy->modtype to IFM_TYPE. 2137 */ 2138 static int 2139 cxgb_ifm_type(int phymod) 2140 { 2141 int rc = IFM_ETHER | IFM_FDX; 2142 2143 switch (phymod) { 2144 case phy_modtype_sr: 2145 rc |= IFM_10G_SR; 2146 break; 2147 case phy_modtype_lr: 2148 rc |= IFM_10G_LR; 2149 break; 2150 case phy_modtype_lrm: 2151 #ifdef IFM_10G_LRM 2152 rc |= IFM_10G_LRM; 2153 #endif 2154 break; 2155 case phy_modtype_twinax: 2156 #ifdef IFM_10G_TWINAX 2157 rc |= IFM_10G_TWINAX; 2158 #endif 2159 break; 2160 case phy_modtype_twinax_long: 2161 #ifdef IFM_10G_TWINAX_LONG 2162 rc |= IFM_10G_TWINAX_LONG; 2163 #endif 2164 break; 2165 case phy_modtype_none: 2166 rc = IFM_ETHER | IFM_NONE; 2167 break; 2168 case phy_modtype_unknown: 2169 break; 2170 } 2171 2172 return (rc); 2173 } 2174 2175 static void 2176 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2177 { 2178 struct port_info *p = ifp->if_softc; 2179 struct ifmedia_entry *cur = p->media.ifm_cur; 2180 int m; 2181 2182 if (cur->ifm_data != p->phy.modtype) { 2183 /* p->media about to be rebuilt, must hold lock */ 2184 PORT_LOCK_ASSERT_OWNED(p); 2185 2186 m = cxgb_ifm_type(p->phy.modtype); 2187 ifmedia_removeall(&p->media); 2188 ifmedia_add(&p->media, m, p->phy.modtype, NULL); 2189 ifmedia_set(&p->media, m); 2190 cur = p->media.ifm_cur; /* ifmedia_set modified ifm_cur */ 2191 ifmr->ifm_current = m; 2192 } 2193 2194 ifmr->ifm_status = IFM_AVALID; 2195 ifmr->ifm_active = IFM_ETHER; 2196 2197 if (!p->link_config.link_ok) 2198 return; 2199 2200 ifmr->ifm_status |= IFM_ACTIVE; 2201 2202 switch (p->link_config.speed) { 2203 case 10: 2204 ifmr->ifm_active |= IFM_10_T; 2205 break; 2206 case 100: 2207 ifmr->ifm_active |= IFM_100_TX; 2208 break; 2209 case 1000: 2210 ifmr->ifm_active |= IFM_1000_T; 2211 break; 2212 case 10000: 2213 ifmr->ifm_active |= IFM_SUBTYPE(cur->ifm_media); 2214 break; 2215 } 2216 2217 if (p->link_config.duplex) 2218 ifmr->ifm_active |= IFM_FDX; 2219 else 2220 ifmr->ifm_active |= IFM_HDX; 2221 } 2222 2223 static void 2224 cxgb_async_intr(void *data) 2225 { 2226 adapter_t *sc = data; 2227 2228 if (cxgb_debug) 2229 device_printf(sc->dev, "cxgb_async_intr\n"); 2230 /* 2231 * May need to sleep - defer to taskqueue 2232 */ 2233 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2234 } 2235 2236 static void 2237 cxgb_ext_intr_handler(void *arg, int count) 2238 { 2239 adapter_t *sc = (adapter_t *)arg; 2240 2241 if (cxgb_debug) 2242 printf("cxgb_ext_intr_handler\n"); 2243 2244 t3_phy_intr_handler(sc); 2245 2246 /* Now reenable external interrupts */ 2247 ADAPTER_LOCK(sc); 2248 if (sc->slow_intr_mask) { 2249 sc->slow_intr_mask |= F_T3DBG; 2250 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 2251 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 2252 } 2253 ADAPTER_UNLOCK(sc); 2254 } 2255 2256 static void 2257 check_link_status(adapter_t *sc) 2258 { 2259 int i; 2260 2261 /* For synchronized access to open_device_map */ 2262 ADAPTER_LOCK_ASSERT_OWNED(sc); 2263 2264 for (i = 0; i < (sc)->params.nports; ++i) { 2265 struct port_info *p = &sc->port[i]; 2266 struct link_config *lc = &p->link_config; 2267 2268 if (!isset(&sc->open_device_map, p->port_id)) { 2269 /* 2270 * port is down, report link down too. Note 2271 * that we do this for IRQ based PHYs too. 2272 */ 2273 lc->link_ok = 0; 2274 t3_os_link_changed(sc, i, lc->link_ok, lc->speed, 2275 lc->duplex, lc->fc); 2276 } else if (p->link_fault || !(p->phy.caps & SUPPORTED_IRQ)) 2277 t3_link_changed(sc, i); 2278 } 2279 } 2280 2281 static void 2282 check_t3b2_mac(struct adapter *adapter) 2283 { 2284 int i; 2285 2286 if(adapter->flags & CXGB_SHUTDOWN) 2287 return; 2288 2289 for_each_port(adapter, i) { 2290 struct port_info *p = &adapter->port[i]; 2291 struct ifnet *ifp = p->ifp; 2292 int status; 2293 2294 if(adapter->flags & CXGB_SHUTDOWN) 2295 return; 2296 2297 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2298 continue; 2299 2300 status = 0; 2301 PORT_LOCK(p); 2302 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 2303 status = t3b2_mac_watchdog_task(&p->mac); 2304 if (status == 1) 2305 p->mac.stats.num_toggled++; 2306 else if (status == 2) { 2307 struct cmac *mac = &p->mac; 2308 int mtu = ifp->if_mtu; 2309 2310 if (ifp->if_capenable & IFCAP_VLAN_MTU) 2311 mtu += ETHER_VLAN_ENCAP_LEN; 2312 t3_mac_set_mtu(mac, mtu); 2313 t3_mac_set_address(mac, 0, p->hw_addr); 2314 cxgb_set_rxmode(p); 2315 t3_link_start(&p->phy, mac, &p->link_config); 2316 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2317 t3_port_intr_enable(adapter, p->port_id); 2318 p->mac.stats.num_resets++; 2319 } 2320 PORT_UNLOCK(p); 2321 } 2322 } 2323 2324 static void 2325 cxgb_tick(void *arg) 2326 { 2327 adapter_t *sc = (adapter_t *)arg; 2328 2329 if(sc->flags & CXGB_SHUTDOWN) 2330 return; 2331 2332 taskqueue_enqueue(sc->tq, &sc->tick_task); 2333 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 2334 } 2335 2336 static void 2337 cxgb_tick_handler(void *arg, int count) 2338 { 2339 adapter_t *sc = (adapter_t *)arg; 2340 const struct adapter_params *p = &sc->params; 2341 int i; 2342 uint32_t cause, reset; 2343 2344 if(sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE)) 2345 return; 2346 2347 ADAPTER_LOCK(sc); 2348 2349 check_link_status(sc); 2350 2351 sc->check_task_cnt++; 2352 2353 /* 2354 * adapter lock can currently only be acquired after the 2355 * port lock 2356 */ 2357 ADAPTER_UNLOCK(sc); 2358 2359 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2360 check_t3b2_mac(sc); 2361 2362 cause = t3_read_reg(sc, A_SG_INT_CAUSE); 2363 reset = 0; 2364 if (cause & F_FLEMPTY) { 2365 struct sge_qset *qs = &sc->sge.qs[0]; 2366 2367 i = 0; 2368 reset |= F_FLEMPTY; 2369 2370 cause = (t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) >> 2371 S_FL0EMPTY) & 0xffff; 2372 while (cause) { 2373 qs->fl[i].empty += (cause & 1); 2374 if (i) 2375 qs++; 2376 i ^= 1; 2377 cause >>= 1; 2378 } 2379 } 2380 t3_write_reg(sc, A_SG_INT_CAUSE, reset); 2381 2382 for (i = 0; i < sc->params.nports; i++) { 2383 struct port_info *pi = &sc->port[i]; 2384 struct ifnet *ifp = pi->ifp; 2385 struct cmac *mac = &pi->mac; 2386 struct mac_stats *mstats = &mac->stats; 2387 PORT_LOCK(pi); 2388 t3_mac_update_stats(mac); 2389 PORT_UNLOCK(pi); 2390 2391 ifp->if_opackets = 2392 mstats->tx_frames_64 + 2393 mstats->tx_frames_65_127 + 2394 mstats->tx_frames_128_255 + 2395 mstats->tx_frames_256_511 + 2396 mstats->tx_frames_512_1023 + 2397 mstats->tx_frames_1024_1518 + 2398 mstats->tx_frames_1519_max; 2399 2400 ifp->if_ipackets = 2401 mstats->rx_frames_64 + 2402 mstats->rx_frames_65_127 + 2403 mstats->rx_frames_128_255 + 2404 mstats->rx_frames_256_511 + 2405 mstats->rx_frames_512_1023 + 2406 mstats->rx_frames_1024_1518 + 2407 mstats->rx_frames_1519_max; 2408 2409 ifp->if_obytes = mstats->tx_octets; 2410 ifp->if_ibytes = mstats->rx_octets; 2411 ifp->if_omcasts = mstats->tx_mcast_frames; 2412 ifp->if_imcasts = mstats->rx_mcast_frames; 2413 2414 ifp->if_collisions = 2415 mstats->tx_total_collisions; 2416 2417 ifp->if_iqdrops = mstats->rx_cong_drops; 2418 2419 ifp->if_oerrors = 2420 mstats->tx_excess_collisions + 2421 mstats->tx_underrun + 2422 mstats->tx_len_errs + 2423 mstats->tx_mac_internal_errs + 2424 mstats->tx_excess_deferral + 2425 mstats->tx_fcs_errs; 2426 ifp->if_ierrors = 2427 mstats->rx_jabber + 2428 mstats->rx_data_errs + 2429 mstats->rx_sequence_errs + 2430 mstats->rx_runt + 2431 mstats->rx_too_long + 2432 mstats->rx_mac_internal_errs + 2433 mstats->rx_short + 2434 mstats->rx_fcs_errs; 2435 2436 if (mac->multiport) 2437 continue; 2438 2439 /* Count rx fifo overflows, once per second */ 2440 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset); 2441 reset = 0; 2442 if (cause & F_RXFIFO_OVERFLOW) { 2443 mac->stats.rx_fifo_ovfl++; 2444 reset |= F_RXFIFO_OVERFLOW; 2445 } 2446 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset); 2447 } 2448 } 2449 2450 static void 2451 touch_bars(device_t dev) 2452 { 2453 /* 2454 * Don't enable yet 2455 */ 2456 #if !defined(__LP64__) && 0 2457 u32 v; 2458 2459 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2460 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2461 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2462 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2463 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2464 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2465 #endif 2466 } 2467 2468 static int 2469 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2470 { 2471 uint8_t *buf; 2472 int err = 0; 2473 u32 aligned_offset, aligned_len, *p; 2474 struct adapter *adapter = pi->adapter; 2475 2476 2477 aligned_offset = offset & ~3; 2478 aligned_len = (len + (offset & 3) + 3) & ~3; 2479 2480 if (aligned_offset != offset || aligned_len != len) { 2481 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2482 if (!buf) 2483 return (ENOMEM); 2484 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2485 if (!err && aligned_len > 4) 2486 err = t3_seeprom_read(adapter, 2487 aligned_offset + aligned_len - 4, 2488 (u32 *)&buf[aligned_len - 4]); 2489 if (err) 2490 goto out; 2491 memcpy(buf + (offset & 3), data, len); 2492 } else 2493 buf = (uint8_t *)(uintptr_t)data; 2494 2495 err = t3_seeprom_wp(adapter, 0); 2496 if (err) 2497 goto out; 2498 2499 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2500 err = t3_seeprom_write(adapter, aligned_offset, *p); 2501 aligned_offset += 4; 2502 } 2503 2504 if (!err) 2505 err = t3_seeprom_wp(adapter, 1); 2506 out: 2507 if (buf != data) 2508 free(buf, M_DEVBUF); 2509 return err; 2510 } 2511 2512 2513 static int 2514 in_range(int val, int lo, int hi) 2515 { 2516 return val < 0 || (val <= hi && val >= lo); 2517 } 2518 2519 static int 2520 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td) 2521 { 2522 return (0); 2523 } 2524 2525 static int 2526 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td) 2527 { 2528 return (0); 2529 } 2530 2531 static int 2532 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2533 int fflag, struct thread *td) 2534 { 2535 int mmd, error = 0; 2536 struct port_info *pi = dev->si_drv1; 2537 adapter_t *sc = pi->adapter; 2538 2539 #ifdef PRIV_SUPPORTED 2540 if (priv_check(td, PRIV_DRIVER)) { 2541 if (cxgb_debug) 2542 printf("user does not have access to privileged ioctls\n"); 2543 return (EPERM); 2544 } 2545 #else 2546 if (suser(td)) { 2547 if (cxgb_debug) 2548 printf("user does not have access to privileged ioctls\n"); 2549 return (EPERM); 2550 } 2551 #endif 2552 2553 switch (cmd) { 2554 case CHELSIO_GET_MIIREG: { 2555 uint32_t val; 2556 struct cphy *phy = &pi->phy; 2557 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2558 2559 if (!phy->mdio_read) 2560 return (EOPNOTSUPP); 2561 if (is_10G(sc)) { 2562 mmd = mid->phy_id >> 8; 2563 if (!mmd) 2564 mmd = MDIO_DEV_PCS; 2565 else if (mmd > MDIO_DEV_VEND2) 2566 return (EINVAL); 2567 2568 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2569 mid->reg_num, &val); 2570 } else 2571 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2572 mid->reg_num & 0x1f, &val); 2573 if (error == 0) 2574 mid->val_out = val; 2575 break; 2576 } 2577 case CHELSIO_SET_MIIREG: { 2578 struct cphy *phy = &pi->phy; 2579 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2580 2581 if (!phy->mdio_write) 2582 return (EOPNOTSUPP); 2583 if (is_10G(sc)) { 2584 mmd = mid->phy_id >> 8; 2585 if (!mmd) 2586 mmd = MDIO_DEV_PCS; 2587 else if (mmd > MDIO_DEV_VEND2) 2588 return (EINVAL); 2589 2590 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2591 mmd, mid->reg_num, mid->val_in); 2592 } else 2593 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2594 mid->reg_num & 0x1f, 2595 mid->val_in); 2596 break; 2597 } 2598 case CHELSIO_SETREG: { 2599 struct ch_reg *edata = (struct ch_reg *)data; 2600 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2601 return (EFAULT); 2602 t3_write_reg(sc, edata->addr, edata->val); 2603 break; 2604 } 2605 case CHELSIO_GETREG: { 2606 struct ch_reg *edata = (struct ch_reg *)data; 2607 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2608 return (EFAULT); 2609 edata->val = t3_read_reg(sc, edata->addr); 2610 break; 2611 } 2612 case CHELSIO_GET_SGE_CONTEXT: { 2613 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2614 mtx_lock_spin(&sc->sge.reg_lock); 2615 switch (ecntxt->cntxt_type) { 2616 case CNTXT_TYPE_EGRESS: 2617 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2618 ecntxt->data); 2619 break; 2620 case CNTXT_TYPE_FL: 2621 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2622 ecntxt->data); 2623 break; 2624 case CNTXT_TYPE_RSP: 2625 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2626 ecntxt->data); 2627 break; 2628 case CNTXT_TYPE_CQ: 2629 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2630 ecntxt->data); 2631 break; 2632 default: 2633 error = EINVAL; 2634 break; 2635 } 2636 mtx_unlock_spin(&sc->sge.reg_lock); 2637 break; 2638 } 2639 case CHELSIO_GET_SGE_DESC: { 2640 struct ch_desc *edesc = (struct ch_desc *)data; 2641 int ret; 2642 if (edesc->queue_num >= SGE_QSETS * 6) 2643 return (EINVAL); 2644 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2645 edesc->queue_num % 6, edesc->idx, edesc->data); 2646 if (ret < 0) 2647 return (EINVAL); 2648 edesc->size = ret; 2649 break; 2650 } 2651 case CHELSIO_GET_QSET_PARAMS: { 2652 struct qset_params *q; 2653 struct ch_qset_params *t = (struct ch_qset_params *)data; 2654 int q1 = pi->first_qset; 2655 int nqsets = pi->nqsets; 2656 int i; 2657 2658 if (t->qset_idx >= nqsets) 2659 return EINVAL; 2660 2661 i = q1 + t->qset_idx; 2662 q = &sc->params.sge.qset[i]; 2663 t->rspq_size = q->rspq_size; 2664 t->txq_size[0] = q->txq_size[0]; 2665 t->txq_size[1] = q->txq_size[1]; 2666 t->txq_size[2] = q->txq_size[2]; 2667 t->fl_size[0] = q->fl_size; 2668 t->fl_size[1] = q->jumbo_size; 2669 t->polling = q->polling; 2670 t->lro = q->lro; 2671 t->intr_lat = q->coalesce_usecs; 2672 t->cong_thres = q->cong_thres; 2673 t->qnum = i; 2674 2675 if (sc->flags & USING_MSIX) 2676 t->vector = rman_get_start(sc->msix_irq_res[i]); 2677 else 2678 t->vector = rman_get_start(sc->irq_res); 2679 2680 break; 2681 } 2682 case CHELSIO_GET_QSET_NUM: { 2683 struct ch_reg *edata = (struct ch_reg *)data; 2684 edata->val = pi->nqsets; 2685 break; 2686 } 2687 case CHELSIO_LOAD_FW: { 2688 uint8_t *fw_data; 2689 uint32_t vers; 2690 struct ch_mem_range *t = (struct ch_mem_range *)data; 2691 2692 /* 2693 * You're allowed to load a firmware only before FULL_INIT_DONE 2694 * 2695 * FW_UPTODATE is also set so the rest of the initialization 2696 * will not overwrite what was loaded here. This gives you the 2697 * flexibility to load any firmware (and maybe shoot yourself in 2698 * the foot). 2699 */ 2700 2701 ADAPTER_LOCK(sc); 2702 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2703 ADAPTER_UNLOCK(sc); 2704 return (EBUSY); 2705 } 2706 2707 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2708 if (!fw_data) 2709 error = ENOMEM; 2710 else 2711 error = copyin(t->buf, fw_data, t->len); 2712 2713 if (!error) 2714 error = -t3_load_fw(sc, fw_data, t->len); 2715 2716 if (t3_get_fw_version(sc, &vers) == 0) { 2717 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2718 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2719 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2720 } 2721 2722 if (!error) 2723 sc->flags |= FW_UPTODATE; 2724 2725 free(fw_data, M_DEVBUF); 2726 ADAPTER_UNLOCK(sc); 2727 break; 2728 } 2729 case CHELSIO_LOAD_BOOT: { 2730 uint8_t *boot_data; 2731 struct ch_mem_range *t = (struct ch_mem_range *)data; 2732 2733 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2734 if (!boot_data) 2735 return ENOMEM; 2736 2737 error = copyin(t->buf, boot_data, t->len); 2738 if (!error) 2739 error = -t3_load_boot(sc, boot_data, t->len); 2740 2741 free(boot_data, M_DEVBUF); 2742 break; 2743 } 2744 case CHELSIO_GET_PM: { 2745 struct ch_pm *m = (struct ch_pm *)data; 2746 struct tp_params *p = &sc->params.tp; 2747 2748 if (!is_offload(sc)) 2749 return (EOPNOTSUPP); 2750 2751 m->tx_pg_sz = p->tx_pg_size; 2752 m->tx_num_pg = p->tx_num_pgs; 2753 m->rx_pg_sz = p->rx_pg_size; 2754 m->rx_num_pg = p->rx_num_pgs; 2755 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2756 2757 break; 2758 } 2759 case CHELSIO_SET_PM: { 2760 struct ch_pm *m = (struct ch_pm *)data; 2761 struct tp_params *p = &sc->params.tp; 2762 2763 if (!is_offload(sc)) 2764 return (EOPNOTSUPP); 2765 if (sc->flags & FULL_INIT_DONE) 2766 return (EBUSY); 2767 2768 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2769 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2770 return (EINVAL); /* not power of 2 */ 2771 if (!(m->rx_pg_sz & 0x14000)) 2772 return (EINVAL); /* not 16KB or 64KB */ 2773 if (!(m->tx_pg_sz & 0x1554000)) 2774 return (EINVAL); 2775 if (m->tx_num_pg == -1) 2776 m->tx_num_pg = p->tx_num_pgs; 2777 if (m->rx_num_pg == -1) 2778 m->rx_num_pg = p->rx_num_pgs; 2779 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2780 return (EINVAL); 2781 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2782 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2783 return (EINVAL); 2784 2785 p->rx_pg_size = m->rx_pg_sz; 2786 p->tx_pg_size = m->tx_pg_sz; 2787 p->rx_num_pgs = m->rx_num_pg; 2788 p->tx_num_pgs = m->tx_num_pg; 2789 break; 2790 } 2791 case CHELSIO_SETMTUTAB: { 2792 struct ch_mtus *m = (struct ch_mtus *)data; 2793 int i; 2794 2795 if (!is_offload(sc)) 2796 return (EOPNOTSUPP); 2797 if (offload_running(sc)) 2798 return (EBUSY); 2799 if (m->nmtus != NMTUS) 2800 return (EINVAL); 2801 if (m->mtus[0] < 81) /* accommodate SACK */ 2802 return (EINVAL); 2803 2804 /* 2805 * MTUs must be in ascending order 2806 */ 2807 for (i = 1; i < NMTUS; ++i) 2808 if (m->mtus[i] < m->mtus[i - 1]) 2809 return (EINVAL); 2810 2811 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2812 break; 2813 } 2814 case CHELSIO_GETMTUTAB: { 2815 struct ch_mtus *m = (struct ch_mtus *)data; 2816 2817 if (!is_offload(sc)) 2818 return (EOPNOTSUPP); 2819 2820 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2821 m->nmtus = NMTUS; 2822 break; 2823 } 2824 case CHELSIO_GET_MEM: { 2825 struct ch_mem_range *t = (struct ch_mem_range *)data; 2826 struct mc7 *mem; 2827 uint8_t *useraddr; 2828 u64 buf[32]; 2829 2830 /* 2831 * Use these to avoid modifying len/addr in the the return 2832 * struct 2833 */ 2834 uint32_t len = t->len, addr = t->addr; 2835 2836 if (!is_offload(sc)) 2837 return (EOPNOTSUPP); 2838 if (!(sc->flags & FULL_INIT_DONE)) 2839 return (EIO); /* need the memory controllers */ 2840 if ((addr & 0x7) || (len & 0x7)) 2841 return (EINVAL); 2842 if (t->mem_id == MEM_CM) 2843 mem = &sc->cm; 2844 else if (t->mem_id == MEM_PMRX) 2845 mem = &sc->pmrx; 2846 else if (t->mem_id == MEM_PMTX) 2847 mem = &sc->pmtx; 2848 else 2849 return (EINVAL); 2850 2851 /* 2852 * Version scheme: 2853 * bits 0..9: chip version 2854 * bits 10..15: chip revision 2855 */ 2856 t->version = 3 | (sc->params.rev << 10); 2857 2858 /* 2859 * Read 256 bytes at a time as len can be large and we don't 2860 * want to use huge intermediate buffers. 2861 */ 2862 useraddr = (uint8_t *)t->buf; 2863 while (len) { 2864 unsigned int chunk = min(len, sizeof(buf)); 2865 2866 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2867 if (error) 2868 return (-error); 2869 if (copyout(buf, useraddr, chunk)) 2870 return (EFAULT); 2871 useraddr += chunk; 2872 addr += chunk; 2873 len -= chunk; 2874 } 2875 break; 2876 } 2877 case CHELSIO_READ_TCAM_WORD: { 2878 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2879 2880 if (!is_offload(sc)) 2881 return (EOPNOTSUPP); 2882 if (!(sc->flags & FULL_INIT_DONE)) 2883 return (EIO); /* need MC5 */ 2884 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2885 break; 2886 } 2887 case CHELSIO_SET_TRACE_FILTER: { 2888 struct ch_trace *t = (struct ch_trace *)data; 2889 const struct trace_params *tp; 2890 2891 tp = (const struct trace_params *)&t->sip; 2892 if (t->config_tx) 2893 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2894 t->trace_tx); 2895 if (t->config_rx) 2896 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2897 t->trace_rx); 2898 break; 2899 } 2900 case CHELSIO_SET_PKTSCHED: { 2901 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2902 if (sc->open_device_map == 0) 2903 return (EAGAIN); 2904 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2905 p->binding); 2906 break; 2907 } 2908 case CHELSIO_IFCONF_GETREGS: { 2909 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2910 int reglen = cxgb_get_regs_len(); 2911 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2912 if (buf == NULL) { 2913 return (ENOMEM); 2914 } 2915 if (regs->len > reglen) 2916 regs->len = reglen; 2917 else if (regs->len < reglen) 2918 error = ENOBUFS; 2919 2920 if (!error) { 2921 cxgb_get_regs(sc, regs, buf); 2922 error = copyout(buf, regs->data, reglen); 2923 } 2924 free(buf, M_DEVBUF); 2925 2926 break; 2927 } 2928 case CHELSIO_SET_HW_SCHED: { 2929 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2930 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2931 2932 if ((sc->flags & FULL_INIT_DONE) == 0) 2933 return (EAGAIN); /* need TP to be initialized */ 2934 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2935 !in_range(t->channel, 0, 1) || 2936 !in_range(t->kbps, 0, 10000000) || 2937 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2938 !in_range(t->flow_ipg, 0, 2939 dack_ticks_to_usec(sc, 0x7ff))) 2940 return (EINVAL); 2941 2942 if (t->kbps >= 0) { 2943 error = t3_config_sched(sc, t->kbps, t->sched); 2944 if (error < 0) 2945 return (-error); 2946 } 2947 if (t->class_ipg >= 0) 2948 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2949 if (t->flow_ipg >= 0) { 2950 t->flow_ipg *= 1000; /* us -> ns */ 2951 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2952 } 2953 if (t->mode >= 0) { 2954 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2955 2956 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2957 bit, t->mode ? bit : 0); 2958 } 2959 if (t->channel >= 0) 2960 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2961 1 << t->sched, t->channel << t->sched); 2962 break; 2963 } 2964 case CHELSIO_GET_EEPROM: { 2965 int i; 2966 struct ch_eeprom *e = (struct ch_eeprom *)data; 2967 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2968 2969 if (buf == NULL) { 2970 return (ENOMEM); 2971 } 2972 e->magic = EEPROM_MAGIC; 2973 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2974 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2975 2976 if (!error) 2977 error = copyout(buf + e->offset, e->data, e->len); 2978 2979 free(buf, M_DEVBUF); 2980 break; 2981 } 2982 case CHELSIO_CLEAR_STATS: { 2983 if (!(sc->flags & FULL_INIT_DONE)) 2984 return EAGAIN; 2985 2986 PORT_LOCK(pi); 2987 t3_mac_update_stats(&pi->mac); 2988 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 2989 PORT_UNLOCK(pi); 2990 break; 2991 } 2992 case CHELSIO_GET_UP_LA: { 2993 struct ch_up_la *la = (struct ch_up_la *)data; 2994 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT); 2995 if (buf == NULL) { 2996 return (ENOMEM); 2997 } 2998 if (la->bufsize < LA_BUFSIZE) 2999 error = ENOBUFS; 3000 3001 if (!error) 3002 error = -t3_get_up_la(sc, &la->stopped, &la->idx, 3003 &la->bufsize, buf); 3004 if (!error) 3005 error = copyout(buf, la->data, la->bufsize); 3006 3007 free(buf, M_DEVBUF); 3008 break; 3009 } 3010 case CHELSIO_GET_UP_IOQS: { 3011 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data; 3012 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT); 3013 uint32_t *v; 3014 3015 if (buf == NULL) { 3016 return (ENOMEM); 3017 } 3018 if (ioqs->bufsize < IOQS_BUFSIZE) 3019 error = ENOBUFS; 3020 3021 if (!error) 3022 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf); 3023 3024 if (!error) { 3025 v = (uint32_t *)buf; 3026 3027 ioqs->bufsize -= 4 * sizeof(uint32_t); 3028 ioqs->ioq_rx_enable = *v++; 3029 ioqs->ioq_tx_enable = *v++; 3030 ioqs->ioq_rx_status = *v++; 3031 ioqs->ioq_tx_status = *v++; 3032 3033 error = copyout(v, ioqs->data, ioqs->bufsize); 3034 } 3035 3036 free(buf, M_DEVBUF); 3037 break; 3038 } 3039 default: 3040 return (EOPNOTSUPP); 3041 break; 3042 } 3043 3044 return (error); 3045 } 3046 3047 static __inline void 3048 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 3049 unsigned int end) 3050 { 3051 uint32_t *p = (uint32_t *)(buf + start); 3052 3053 for ( ; start <= end; start += sizeof(uint32_t)) 3054 *p++ = t3_read_reg(ap, start); 3055 } 3056 3057 #define T3_REGMAP_SIZE (3 * 1024) 3058 static int 3059 cxgb_get_regs_len(void) 3060 { 3061 return T3_REGMAP_SIZE; 3062 } 3063 3064 static void 3065 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 3066 { 3067 3068 /* 3069 * Version scheme: 3070 * bits 0..9: chip version 3071 * bits 10..15: chip revision 3072 * bit 31: set for PCIe cards 3073 */ 3074 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 3075 3076 /* 3077 * We skip the MAC statistics registers because they are clear-on-read. 3078 * Also reading multi-register stats would need to synchronize with the 3079 * periodic mac stats accumulation. Hard to justify the complexity. 3080 */ 3081 memset(buf, 0, cxgb_get_regs_len()); 3082 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 3083 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 3084 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 3085 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 3086 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 3087 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 3088 XGM_REG(A_XGM_SERDES_STAT3, 1)); 3089 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 3090 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 3091 } 3092 3093 3094 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1); 3095