1 /************************************************************************** 2 SPDX-License-Identifier: BSD-2-Clause 3 4 Copyright (c) 2007-2009, Chelsio Inc. 5 All rights reserved. 6 7 Redistribution and use in source and binary forms, with or without 8 modification, are permitted provided that the following conditions are met: 9 10 1. Redistributions of source code must retain the above copyright notice, 11 this list of conditions and the following disclaimer. 12 13 2. Neither the name of the Chelsio Corporation nor the names of its 14 contributors may be used to endorse or promote products derived from 15 this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 POSSIBILITY OF SUCH DAMAGE. 28 29 ***************************************************************************/ 30 31 #include <sys/cdefs.h> 32 #include "opt_inet.h" 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/bus.h> 38 #include <sys/module.h> 39 #include <sys/pciio.h> 40 #include <sys/conf.h> 41 #include <machine/bus.h> 42 #include <machine/resource.h> 43 #include <sys/ktr.h> 44 #include <sys/rman.h> 45 #include <sys/ioccom.h> 46 #include <sys/mbuf.h> 47 #include <sys/linker.h> 48 #include <sys/firmware.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/smp.h> 52 #include <sys/sysctl.h> 53 #include <sys/syslog.h> 54 #include <sys/queue.h> 55 #include <sys/taskqueue.h> 56 #include <sys/proc.h> 57 58 #include <net/bpf.h> 59 #include <net/debugnet.h> 60 #include <net/ethernet.h> 61 #include <net/if.h> 62 #include <net/if_var.h> 63 #include <net/if_arp.h> 64 #include <net/if_dl.h> 65 #include <net/if_media.h> 66 #include <net/if_types.h> 67 #include <net/if_vlan_var.h> 68 69 #include <netinet/in_systm.h> 70 #include <netinet/in.h> 71 #include <netinet/if_ether.h> 72 #include <netinet/ip.h> 73 #include <netinet/ip.h> 74 #include <netinet/tcp.h> 75 #include <netinet/udp.h> 76 77 #include <dev/pci/pcireg.h> 78 #include <dev/pci/pcivar.h> 79 #include <dev/pci/pci_private.h> 80 81 #include <cxgb_include.h> 82 83 #ifdef PRIV_SUPPORTED 84 #include <sys/priv.h> 85 #endif 86 87 static int cxgb_setup_interrupts(adapter_t *); 88 static void cxgb_teardown_interrupts(adapter_t *); 89 static void cxgb_init(void *); 90 static int cxgb_init_locked(struct port_info *); 91 static int cxgb_uninit_locked(struct port_info *); 92 static int cxgb_uninit_synchronized(struct port_info *); 93 static int cxgb_ioctl(if_t, unsigned long, caddr_t); 94 static int cxgb_media_change(if_t); 95 static int cxgb_ifm_type(int); 96 static void cxgb_build_medialist(struct port_info *); 97 static void cxgb_media_status(if_t, struct ifmediareq *); 98 static uint64_t cxgb_get_counter(if_t, ift_counter); 99 static int setup_sge_qsets(adapter_t *); 100 static void cxgb_async_intr(void *); 101 static void cxgb_tick_handler(void *, int); 102 static void cxgb_tick(void *); 103 static void link_check_callout(void *); 104 static void check_link_status(void *, int); 105 static void setup_rss(adapter_t *sc); 106 static int alloc_filters(struct adapter *); 107 static int setup_hw_filters(struct adapter *); 108 static int set_filter(struct adapter *, int, const struct filter_info *); 109 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int, 110 unsigned int, u64, u64); 111 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int, 112 unsigned int, u64, u64); 113 #ifdef TCP_OFFLOAD 114 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *); 115 #endif 116 117 /* Attachment glue for the PCI controller end of the device. Each port of 118 * the device is attached separately, as defined later. 119 */ 120 static int cxgb_controller_probe(device_t); 121 static int cxgb_controller_attach(device_t); 122 static int cxgb_controller_detach(device_t); 123 static void cxgb_free(struct adapter *); 124 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 125 unsigned int end); 126 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 127 static int cxgb_get_regs_len(void); 128 static void touch_bars(device_t dev); 129 static void cxgb_update_mac_settings(struct port_info *p); 130 #ifdef TCP_OFFLOAD 131 static int toe_capability(struct port_info *, int); 132 #endif 133 134 /* Table for probing the cards. The desc field isn't actually used */ 135 struct cxgb_ident { 136 uint16_t vendor; 137 uint16_t device; 138 int index; 139 char *desc; 140 } cxgb_identifiers[] = { 141 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 142 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 143 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 144 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 145 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 146 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 147 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 148 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 149 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 150 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 151 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 152 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"}, 153 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"}, 154 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"}, 155 {0, 0, 0, NULL} 156 }; 157 158 static device_method_t cxgb_controller_methods[] = { 159 DEVMETHOD(device_probe, cxgb_controller_probe), 160 DEVMETHOD(device_attach, cxgb_controller_attach), 161 DEVMETHOD(device_detach, cxgb_controller_detach), 162 163 DEVMETHOD_END 164 }; 165 166 static driver_t cxgb_controller_driver = { 167 "cxgbc", 168 cxgb_controller_methods, 169 sizeof(struct adapter) 170 }; 171 172 static int cxgbc_mod_event(module_t, int, void *); 173 174 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgbc_mod_event, NULL); 175 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers, 176 nitems(cxgb_identifiers) - 1); 177 MODULE_VERSION(cxgbc, 1); 178 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1); 179 180 /* 181 * Attachment glue for the ports. Attachment is done directly to the 182 * controller device. 183 */ 184 static int cxgb_port_probe(device_t); 185 static int cxgb_port_attach(device_t); 186 static int cxgb_port_detach(device_t); 187 188 static device_method_t cxgb_port_methods[] = { 189 DEVMETHOD(device_probe, cxgb_port_probe), 190 DEVMETHOD(device_attach, cxgb_port_attach), 191 DEVMETHOD(device_detach, cxgb_port_detach), 192 { 0, 0 } 193 }; 194 195 static driver_t cxgb_port_driver = { 196 "cxgb", 197 cxgb_port_methods, 198 0 199 }; 200 201 static d_ioctl_t cxgb_extension_ioctl; 202 static d_open_t cxgb_extension_open; 203 static d_close_t cxgb_extension_close; 204 205 static struct cdevsw cxgb_cdevsw = { 206 .d_version = D_VERSION, 207 .d_flags = 0, 208 .d_open = cxgb_extension_open, 209 .d_close = cxgb_extension_close, 210 .d_ioctl = cxgb_extension_ioctl, 211 .d_name = "cxgb", 212 }; 213 214 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, 0, 0); 215 MODULE_VERSION(cxgb, 1); 216 217 DEBUGNET_DEFINE(cxgb); 218 219 static struct mtx t3_list_lock; 220 static SLIST_HEAD(, adapter) t3_list; 221 #ifdef TCP_OFFLOAD 222 static struct mtx t3_uld_list_lock; 223 static SLIST_HEAD(, uld_info) t3_uld_list; 224 #endif 225 226 /* 227 * The driver uses the best interrupt scheme available on a platform in the 228 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 229 * of these schemes the driver may consider as follows: 230 * 231 * msi = 2: choose from among all three options 232 * msi = 1 : only consider MSI and pin interrupts 233 * msi = 0: force pin interrupts 234 */ 235 static int msi_allowed = 2; 236 237 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 238 "CXGB driver parameters"); 239 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 240 "MSI-X, MSI, INTx selector"); 241 242 /* 243 * The driver uses an auto-queue algorithm by default. 244 * To disable it and force a single queue-set per port, use multiq = 0 245 */ 246 static int multiq = 1; 247 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0, 248 "use min(ncpus/ports, 8) queue-sets per port"); 249 250 /* 251 * By default the driver will not update the firmware unless 252 * it was compiled against a newer version 253 * 254 */ 255 static int force_fw_update = 0; 256 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 257 "update firmware even if up to date"); 258 259 int cxgb_use_16k_clusters = -1; 260 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 261 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 262 263 static int nfilters = -1; 264 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN, 265 &nfilters, 0, "max number of entries in the filter table"); 266 267 enum { 268 MAX_TXQ_ENTRIES = 16384, 269 MAX_CTRL_TXQ_ENTRIES = 1024, 270 MAX_RSPQ_ENTRIES = 16384, 271 MAX_RX_BUFFERS = 16384, 272 MAX_RX_JUMBO_BUFFERS = 16384, 273 MIN_TXQ_ENTRIES = 4, 274 MIN_CTRL_TXQ_ENTRIES = 4, 275 MIN_RSPQ_ENTRIES = 32, 276 MIN_FL_ENTRIES = 32, 277 MIN_FL_JUMBO_ENTRIES = 32 278 }; 279 280 struct filter_info { 281 u32 sip; 282 u32 sip_mask; 283 u32 dip; 284 u16 sport; 285 u16 dport; 286 u32 vlan:12; 287 u32 vlan_prio:3; 288 u32 mac_hit:1; 289 u32 mac_idx:4; 290 u32 mac_vld:1; 291 u32 pkt_type:2; 292 u32 report_filter_id:1; 293 u32 pass:1; 294 u32 rss:1; 295 u32 qset:3; 296 u32 locked:1; 297 u32 valid:1; 298 }; 299 300 enum { FILTER_NO_VLAN_PRI = 7 }; 301 302 #define EEPROM_MAGIC 0x38E2F10C 303 304 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 305 306 307 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 308 309 310 static __inline char 311 t3rev2char(struct adapter *adapter) 312 { 313 char rev = 'z'; 314 315 switch(adapter->params.rev) { 316 case T3_REV_A: 317 rev = 'a'; 318 break; 319 case T3_REV_B: 320 case T3_REV_B2: 321 rev = 'b'; 322 break; 323 case T3_REV_C: 324 rev = 'c'; 325 break; 326 } 327 return rev; 328 } 329 330 static struct cxgb_ident * 331 cxgb_get_ident(device_t dev) 332 { 333 struct cxgb_ident *id; 334 335 for (id = cxgb_identifiers; id->desc != NULL; id++) { 336 if ((id->vendor == pci_get_vendor(dev)) && 337 (id->device == pci_get_device(dev))) { 338 return (id); 339 } 340 } 341 return (NULL); 342 } 343 344 static const struct adapter_info * 345 cxgb_get_adapter_info(device_t dev) 346 { 347 struct cxgb_ident *id; 348 const struct adapter_info *ai; 349 350 id = cxgb_get_ident(dev); 351 if (id == NULL) 352 return (NULL); 353 354 ai = t3_get_adapter_info(id->index); 355 356 return (ai); 357 } 358 359 static int 360 cxgb_controller_probe(device_t dev) 361 { 362 const struct adapter_info *ai; 363 const char *ports; 364 int nports; 365 366 ai = cxgb_get_adapter_info(dev); 367 if (ai == NULL) 368 return (ENXIO); 369 370 nports = ai->nports0 + ai->nports1; 371 if (nports == 1) 372 ports = "port"; 373 else 374 ports = "ports"; 375 376 device_set_descf(dev, "%s, %d %s", ai->desc, nports, ports); 377 return (BUS_PROBE_DEFAULT); 378 } 379 380 #define FW_FNAME "cxgb_t3fw" 381 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom" 382 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram" 383 384 static int 385 upgrade_fw(adapter_t *sc) 386 { 387 const struct firmware *fw; 388 int status; 389 u32 vers; 390 391 if ((fw = firmware_get(FW_FNAME)) == NULL) { 392 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 393 return (ENOENT); 394 } else 395 device_printf(sc->dev, "installing firmware on card\n"); 396 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 397 398 if (status != 0) { 399 device_printf(sc->dev, "failed to install firmware: %d\n", 400 status); 401 } else { 402 t3_get_fw_version(sc, &vers); 403 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 404 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 405 G_FW_VERSION_MICRO(vers)); 406 } 407 408 firmware_put(fw, FIRMWARE_UNLOAD); 409 410 return (status); 411 } 412 413 /* 414 * The cxgb_controller_attach function is responsible for the initial 415 * bringup of the device. Its responsibilities include: 416 * 417 * 1. Determine if the device supports MSI or MSI-X. 418 * 2. Allocate bus resources so that we can access the Base Address Register 419 * 3. Create and initialize mutexes for the controller and its control 420 * logic such as SGE and MDIO. 421 * 4. Call hardware specific setup routine for the adapter as a whole. 422 * 5. Allocate the BAR for doing MSI-X. 423 * 6. Setup the line interrupt iff MSI-X is not supported. 424 * 7. Create the driver's taskq. 425 * 8. Start one task queue service thread. 426 * 9. Check if the firmware and SRAM are up-to-date. They will be 427 * auto-updated later (before FULL_INIT_DONE), if required. 428 * 10. Create a child device for each MAC (port) 429 * 11. Initialize T3 private state. 430 * 12. Trigger the LED 431 * 13. Setup offload iff supported. 432 * 14. Reset/restart the tick callout. 433 * 15. Attach sysctls 434 * 435 * NOTE: Any modification or deviation from this list MUST be reflected in 436 * the above comment. Failure to do so will result in problems on various 437 * error conditions including link flapping. 438 */ 439 static int 440 cxgb_controller_attach(device_t dev) 441 { 442 device_t child; 443 const struct adapter_info *ai; 444 struct adapter *sc; 445 int i, error = 0; 446 uint32_t vers; 447 int port_qsets = 1; 448 int msi_needed, reg; 449 450 sc = device_get_softc(dev); 451 sc->dev = dev; 452 sc->msi_count = 0; 453 ai = cxgb_get_adapter_info(dev); 454 455 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 456 device_get_unit(dev)); 457 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 458 459 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 460 device_get_unit(dev)); 461 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 462 device_get_unit(dev)); 463 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 464 device_get_unit(dev)); 465 466 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 467 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 468 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 469 470 mtx_lock(&t3_list_lock); 471 SLIST_INSERT_HEAD(&t3_list, sc, link); 472 mtx_unlock(&t3_list_lock); 473 474 /* find the PCIe link width and set max read request to 4KB*/ 475 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 476 uint16_t lnk; 477 478 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2); 479 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4; 480 if (sc->link_width < 8 && 481 (ai->caps & SUPPORTED_10000baseT_Full)) { 482 device_printf(sc->dev, 483 "PCIe x%d Link, expect reduced performance\n", 484 sc->link_width); 485 } 486 487 pci_set_max_read_req(dev, 4096); 488 } 489 490 touch_bars(dev); 491 pci_enable_busmaster(dev); 492 /* 493 * Allocate the registers and make them available to the driver. 494 * The registers that we care about for NIC mode are in BAR 0 495 */ 496 sc->regs_rid = PCIR_BAR(0); 497 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 498 &sc->regs_rid, RF_ACTIVE)) == NULL) { 499 device_printf(dev, "Cannot allocate BAR region 0\n"); 500 error = ENXIO; 501 goto out; 502 } 503 504 sc->bt = rman_get_bustag(sc->regs_res); 505 sc->bh = rman_get_bushandle(sc->regs_res); 506 sc->mmio_len = rman_get_size(sc->regs_res); 507 508 for (i = 0; i < MAX_NPORTS; i++) 509 sc->port[i].adapter = sc; 510 511 if (t3_prep_adapter(sc, ai, 1) < 0) { 512 printf("prep adapter failed\n"); 513 error = ENODEV; 514 goto out; 515 } 516 517 sc->udbs_rid = PCIR_BAR(2); 518 sc->udbs_res = NULL; 519 if (is_offload(sc) && 520 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 521 &sc->udbs_rid, RF_ACTIVE)) == NULL)) { 522 device_printf(dev, "Cannot allocate BAR region 1\n"); 523 error = ENXIO; 524 goto out; 525 } 526 527 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 528 * enough messages for the queue sets. If that fails, try falling 529 * back to MSI. If that fails, then try falling back to the legacy 530 * interrupt pin model. 531 */ 532 sc->msix_regs_rid = 0x20; 533 if ((msi_allowed >= 2) && 534 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 535 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 536 537 if (multiq) 538 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus); 539 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1; 540 541 if (pci_msix_count(dev) == 0 || 542 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 || 543 sc->msi_count != msi_needed) { 544 device_printf(dev, "alloc msix failed - " 545 "msi_count=%d, msi_needed=%d, err=%d; " 546 "will try MSI\n", sc->msi_count, 547 msi_needed, error); 548 sc->msi_count = 0; 549 port_qsets = 1; 550 pci_release_msi(dev); 551 bus_release_resource(dev, SYS_RES_MEMORY, 552 sc->msix_regs_rid, sc->msix_regs_res); 553 sc->msix_regs_res = NULL; 554 } else { 555 sc->flags |= USING_MSIX; 556 sc->cxgb_intr = cxgb_async_intr; 557 device_printf(dev, 558 "using MSI-X interrupts (%u vectors)\n", 559 sc->msi_count); 560 } 561 } 562 563 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 564 sc->msi_count = 1; 565 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) { 566 device_printf(dev, "alloc msi failed - " 567 "err=%d; will try INTx\n", error); 568 sc->msi_count = 0; 569 port_qsets = 1; 570 pci_release_msi(dev); 571 } else { 572 sc->flags |= USING_MSI; 573 sc->cxgb_intr = t3_intr_msi; 574 device_printf(dev, "using MSI interrupts\n"); 575 } 576 } 577 if (sc->msi_count == 0) { 578 device_printf(dev, "using line interrupts\n"); 579 sc->cxgb_intr = t3b_intr; 580 } 581 582 /* Create a private taskqueue thread for handling driver events */ 583 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 584 taskqueue_thread_enqueue, &sc->tq); 585 if (sc->tq == NULL) { 586 device_printf(dev, "failed to allocate controller task queue\n"); 587 goto out; 588 } 589 590 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 591 device_get_nameunit(dev)); 592 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 593 594 595 /* Create a periodic callout for checking adapter status */ 596 callout_init(&sc->cxgb_tick_ch, 1); 597 598 if (t3_check_fw_version(sc) < 0 || force_fw_update) { 599 /* 600 * Warn user that a firmware update will be attempted in init. 601 */ 602 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 603 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 604 sc->flags &= ~FW_UPTODATE; 605 } else { 606 sc->flags |= FW_UPTODATE; 607 } 608 609 if (t3_check_tpsram_version(sc) < 0) { 610 /* 611 * Warn user that a firmware update will be attempted in init. 612 */ 613 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 614 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 615 sc->flags &= ~TPS_UPTODATE; 616 } else { 617 sc->flags |= TPS_UPTODATE; 618 } 619 620 /* 621 * Create a child device for each MAC. The ethernet attachment 622 * will be done in these children. 623 */ 624 for (i = 0; i < (sc)->params.nports; i++) { 625 struct port_info *pi; 626 627 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 628 device_printf(dev, "failed to add child port\n"); 629 error = EINVAL; 630 goto out; 631 } 632 pi = &sc->port[i]; 633 pi->adapter = sc; 634 pi->nqsets = port_qsets; 635 pi->first_qset = i*port_qsets; 636 pi->port_id = i; 637 pi->tx_chan = i >= ai->nports0; 638 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 639 sc->rxpkt_map[pi->txpkt_intf] = i; 640 sc->port[i].tx_chan = i >= ai->nports0; 641 sc->portdev[i] = child; 642 device_set_softc(child, pi); 643 } 644 if ((error = bus_generic_attach(dev)) != 0) 645 goto out; 646 647 /* initialize sge private state */ 648 t3_sge_init_adapter(sc); 649 650 t3_led_ready(sc); 651 652 error = t3_get_fw_version(sc, &vers); 653 if (error) 654 goto out; 655 656 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 657 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 658 G_FW_VERSION_MICRO(vers)); 659 660 device_set_descf(dev, "%s %sNIC\t E/C: %s S/N: %s", 661 ai->desc, is_offload(sc) ? "R" : "", 662 sc->params.vpd.ec, sc->params.vpd.sn); 663 664 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x", 665 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1], 666 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]); 667 668 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 669 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 670 t3_add_attach_sysctls(sc); 671 672 #ifdef TCP_OFFLOAD 673 for (i = 0; i < NUM_CPL_HANDLERS; i++) 674 sc->cpl_handler[i] = cpl_not_handled; 675 #endif 676 677 t3_intr_clear(sc); 678 error = cxgb_setup_interrupts(sc); 679 out: 680 if (error) 681 cxgb_free(sc); 682 683 return (error); 684 } 685 686 /* 687 * The cxgb_controller_detach routine is called with the device is 688 * unloaded from the system. 689 */ 690 691 static int 692 cxgb_controller_detach(device_t dev) 693 { 694 struct adapter *sc; 695 696 sc = device_get_softc(dev); 697 698 cxgb_free(sc); 699 700 return (0); 701 } 702 703 /* 704 * The cxgb_free() is called by the cxgb_controller_detach() routine 705 * to tear down the structures that were built up in 706 * cxgb_controller_attach(), and should be the final piece of work 707 * done when fully unloading the driver. 708 * 709 * 710 * 1. Shutting down the threads started by the cxgb_controller_attach() 711 * routine. 712 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()). 713 * 3. Detaching all of the port devices created during the 714 * cxgb_controller_attach() routine. 715 * 4. Removing the device children created via cxgb_controller_attach(). 716 * 5. Releasing PCI resources associated with the device. 717 * 6. Turning off the offload support, iff it was turned on. 718 * 7. Destroying the mutexes created in cxgb_controller_attach(). 719 * 720 */ 721 static void 722 cxgb_free(struct adapter *sc) 723 { 724 int i, nqsets = 0; 725 726 ADAPTER_LOCK(sc); 727 sc->flags |= CXGB_SHUTDOWN; 728 ADAPTER_UNLOCK(sc); 729 730 /* 731 * Make sure all child devices are gone. 732 */ 733 bus_generic_detach(sc->dev); 734 for (i = 0; i < (sc)->params.nports; i++) { 735 if (sc->portdev[i] && 736 device_delete_child(sc->dev, sc->portdev[i]) != 0) 737 device_printf(sc->dev, "failed to delete child port\n"); 738 nqsets += sc->port[i].nqsets; 739 } 740 741 /* 742 * At this point, it is as if cxgb_port_detach has run on all ports, and 743 * cxgb_down has run on the adapter. All interrupts have been silenced, 744 * all open devices have been closed. 745 */ 746 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)", 747 __func__, sc->open_device_map)); 748 for (i = 0; i < sc->params.nports; i++) { 749 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!", 750 __func__, i)); 751 } 752 753 /* 754 * Finish off the adapter's callouts. 755 */ 756 callout_drain(&sc->cxgb_tick_ch); 757 callout_drain(&sc->sge_timer_ch); 758 759 /* 760 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The 761 * sysctls are cleaned up by the kernel linker. 762 */ 763 if (sc->flags & FULL_INIT_DONE) { 764 t3_free_sge_resources(sc, nqsets); 765 sc->flags &= ~FULL_INIT_DONE; 766 } 767 768 /* 769 * Release all interrupt resources. 770 */ 771 cxgb_teardown_interrupts(sc); 772 if (sc->flags & (USING_MSI | USING_MSIX)) { 773 device_printf(sc->dev, "releasing msi message(s)\n"); 774 pci_release_msi(sc->dev); 775 } else { 776 device_printf(sc->dev, "no msi message to release\n"); 777 } 778 779 if (sc->msix_regs_res != NULL) { 780 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 781 sc->msix_regs_res); 782 } 783 784 /* 785 * Free the adapter's taskqueue. 786 */ 787 if (sc->tq != NULL) { 788 taskqueue_free(sc->tq); 789 sc->tq = NULL; 790 } 791 792 free(sc->filters, M_DEVBUF); 793 t3_sge_free(sc); 794 795 if (sc->udbs_res != NULL) 796 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 797 sc->udbs_res); 798 799 if (sc->regs_res != NULL) 800 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 801 sc->regs_res); 802 803 MTX_DESTROY(&sc->mdio_lock); 804 MTX_DESTROY(&sc->sge.reg_lock); 805 MTX_DESTROY(&sc->elmer_lock); 806 mtx_lock(&t3_list_lock); 807 SLIST_REMOVE(&t3_list, sc, adapter, link); 808 mtx_unlock(&t3_list_lock); 809 ADAPTER_LOCK_DEINIT(sc); 810 } 811 812 /** 813 * setup_sge_qsets - configure SGE Tx/Rx/response queues 814 * @sc: the controller softc 815 * 816 * Determines how many sets of SGE queues to use and initializes them. 817 * We support multiple queue sets per port if we have MSI-X, otherwise 818 * just one queue set per port. 819 */ 820 static int 821 setup_sge_qsets(adapter_t *sc) 822 { 823 int i, j, err, irq_idx = 0, qset_idx = 0; 824 u_int ntxq = SGE_TXQ_PER_SET; 825 826 if ((err = t3_sge_alloc(sc)) != 0) { 827 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 828 return (err); 829 } 830 831 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 832 irq_idx = -1; 833 834 for (i = 0; i < (sc)->params.nports; i++) { 835 struct port_info *pi = &sc->port[i]; 836 837 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 838 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 839 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 840 &sc->params.sge.qset[qset_idx], ntxq, pi); 841 if (err) { 842 t3_free_sge_resources(sc, qset_idx); 843 device_printf(sc->dev, 844 "t3_sge_alloc_qset failed with %d\n", err); 845 return (err); 846 } 847 } 848 } 849 850 sc->nqsets = qset_idx; 851 852 return (0); 853 } 854 855 static void 856 cxgb_teardown_interrupts(adapter_t *sc) 857 { 858 int i; 859 860 for (i = 0; i < SGE_QSETS; i++) { 861 if (sc->msix_intr_tag[i] == NULL) { 862 863 /* Should have been setup fully or not at all */ 864 KASSERT(sc->msix_irq_res[i] == NULL && 865 sc->msix_irq_rid[i] == 0, 866 ("%s: half-done interrupt (%d).", __func__, i)); 867 868 continue; 869 } 870 871 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 872 sc->msix_intr_tag[i]); 873 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i], 874 sc->msix_irq_res[i]); 875 876 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL; 877 sc->msix_irq_rid[i] = 0; 878 } 879 880 if (sc->intr_tag) { 881 KASSERT(sc->irq_res != NULL, 882 ("%s: half-done interrupt.", __func__)); 883 884 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 885 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 886 sc->irq_res); 887 888 sc->irq_res = sc->intr_tag = NULL; 889 sc->irq_rid = 0; 890 } 891 } 892 893 static int 894 cxgb_setup_interrupts(adapter_t *sc) 895 { 896 struct resource *res; 897 void *tag; 898 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX); 899 900 sc->irq_rid = intr_flag ? 1 : 0; 901 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, 902 RF_SHAREABLE | RF_ACTIVE); 903 if (sc->irq_res == NULL) { 904 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n", 905 intr_flag, sc->irq_rid); 906 err = EINVAL; 907 sc->irq_rid = 0; 908 } else { 909 err = bus_setup_intr(sc->dev, sc->irq_res, 910 INTR_MPSAFE | INTR_TYPE_NET, NULL, 911 sc->cxgb_intr, sc, &sc->intr_tag); 912 913 if (err) { 914 device_printf(sc->dev, 915 "Cannot set up interrupt (%x, %u, %d)\n", 916 intr_flag, sc->irq_rid, err); 917 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 918 sc->irq_res); 919 sc->irq_res = sc->intr_tag = NULL; 920 sc->irq_rid = 0; 921 } 922 } 923 924 /* That's all for INTx or MSI */ 925 if (!(intr_flag & USING_MSIX) || err) 926 return (err); 927 928 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err"); 929 for (i = 0; i < sc->msi_count - 1; i++) { 930 rid = i + 2; 931 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, 932 RF_SHAREABLE | RF_ACTIVE); 933 if (res == NULL) { 934 device_printf(sc->dev, "Cannot allocate interrupt " 935 "for message %d\n", rid); 936 err = EINVAL; 937 break; 938 } 939 940 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET, 941 NULL, t3_intr_msix, &sc->sge.qs[i], &tag); 942 if (err) { 943 device_printf(sc->dev, "Cannot set up interrupt " 944 "for message %d (%d)\n", rid, err); 945 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res); 946 break; 947 } 948 949 sc->msix_irq_rid[i] = rid; 950 sc->msix_irq_res[i] = res; 951 sc->msix_intr_tag[i] = tag; 952 bus_describe_intr(sc->dev, res, tag, "qs%d", i); 953 } 954 955 if (err) 956 cxgb_teardown_interrupts(sc); 957 958 return (err); 959 } 960 961 962 static int 963 cxgb_port_probe(device_t dev) 964 { 965 struct port_info *p; 966 const char *desc; 967 968 p = device_get_softc(dev); 969 desc = p->phy.desc; 970 device_set_descf(dev, "Port %d %s", p->port_id, desc); 971 return (0); 972 } 973 974 975 static int 976 cxgb_makedev(struct port_info *pi) 977 { 978 979 pi->port_cdev = make_dev(&cxgb_cdevsw, if_getdunit(pi->ifp), 980 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp)); 981 982 if (pi->port_cdev == NULL) 983 return (ENOMEM); 984 985 pi->port_cdev->si_drv1 = (void *)pi; 986 987 return (0); 988 } 989 990 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 991 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 992 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 993 #define CXGB_CAP_ENABLE CXGB_CAP 994 995 static int 996 cxgb_port_attach(device_t dev) 997 { 998 struct port_info *p; 999 if_t ifp; 1000 int err; 1001 struct adapter *sc; 1002 1003 p = device_get_softc(dev); 1004 sc = p->adapter; 1005 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 1006 device_get_unit(device_get_parent(dev)), p->port_id); 1007 PORT_LOCK_INIT(p, p->lockbuf); 1008 1009 callout_init(&p->link_check_ch, 1); 1010 TASK_INIT(&p->link_check_task, 0, check_link_status, p); 1011 1012 /* Allocate an ifnet object and set it up */ 1013 ifp = p->ifp = if_alloc(IFT_ETHER); 1014 if (ifp == NULL) { 1015 device_printf(dev, "Cannot allocate ifnet\n"); 1016 return (ENOMEM); 1017 } 1018 1019 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1020 if_setinitfn(ifp, cxgb_init); 1021 if_setsoftc(ifp, p); 1022 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 1023 if_setioctlfn(ifp, cxgb_ioctl); 1024 if_settransmitfn(ifp, cxgb_transmit); 1025 if_setqflushfn(ifp, cxgb_qflush); 1026 if_setgetcounterfn(ifp, cxgb_get_counter); 1027 1028 if_setcapabilities(ifp, CXGB_CAP); 1029 #ifdef TCP_OFFLOAD 1030 if (is_offload(sc)) 1031 if_setcapabilitiesbit(ifp, IFCAP_TOE4, 0); 1032 #endif 1033 if_setcapenable(ifp, CXGB_CAP_ENABLE); 1034 if_sethwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1035 CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1036 if_sethwtsomax(ifp, IP_MAXPACKET); 1037 if_sethwtsomaxsegcount(ifp, 36); 1038 if_sethwtsomaxsegsize(ifp, 65536); 1039 1040 /* 1041 * Disable TSO on 4-port - it isn't supported by the firmware. 1042 */ 1043 if (sc->params.nports > 2) { 1044 if_setcapabilitiesbit(ifp, 0, IFCAP_TSO | IFCAP_VLAN_HWTSO); 1045 if_setcapenablebit(ifp, 0, IFCAP_TSO | IFCAP_VLAN_HWTSO); 1046 if_sethwassistbits(ifp, 0, CSUM_TSO); 1047 } 1048 1049 ether_ifattach(ifp, p->hw_addr); 1050 1051 /* Attach driver debugnet methods. */ 1052 DEBUGNET_SET(ifp, cxgb); 1053 1054 #ifdef DEFAULT_JUMBO 1055 if (sc->params.nports <= 2) 1056 if_setmtu(ifp, ETHERMTU_JUMBO); 1057 #endif 1058 if ((err = cxgb_makedev(p)) != 0) { 1059 printf("makedev failed %d\n", err); 1060 return (err); 1061 } 1062 1063 /* Create a list of media supported by this port */ 1064 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 1065 cxgb_media_status); 1066 cxgb_build_medialist(p); 1067 1068 t3_sge_init_port(p); 1069 1070 return (err); 1071 } 1072 1073 /* 1074 * cxgb_port_detach() is called via the device_detach methods when 1075 * cxgb_free() calls the bus_generic_detach. It is responsible for 1076 * removing the device from the view of the kernel, i.e. from all 1077 * interfaces lists etc. This routine is only called when the driver is 1078 * being unloaded, not when the link goes down. 1079 */ 1080 static int 1081 cxgb_port_detach(device_t dev) 1082 { 1083 struct port_info *p; 1084 struct adapter *sc; 1085 int i; 1086 1087 p = device_get_softc(dev); 1088 sc = p->adapter; 1089 1090 /* Tell cxgb_ioctl and if_init that the port is going away */ 1091 ADAPTER_LOCK(sc); 1092 SET_DOOMED(p); 1093 wakeup(&sc->flags); 1094 while (IS_BUSY(sc)) 1095 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0); 1096 SET_BUSY(sc); 1097 ADAPTER_UNLOCK(sc); 1098 1099 if (p->port_cdev != NULL) 1100 destroy_dev(p->port_cdev); 1101 1102 cxgb_uninit_synchronized(p); 1103 ether_ifdetach(p->ifp); 1104 1105 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1106 struct sge_qset *qs = &sc->sge.qs[i]; 1107 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1108 1109 callout_drain(&txq->txq_watchdog); 1110 callout_drain(&txq->txq_timer); 1111 } 1112 1113 PORT_LOCK_DEINIT(p); 1114 if_free(p->ifp); 1115 p->ifp = NULL; 1116 1117 ADAPTER_LOCK(sc); 1118 CLR_BUSY(sc); 1119 wakeup_one(&sc->flags); 1120 ADAPTER_UNLOCK(sc); 1121 return (0); 1122 } 1123 1124 void 1125 t3_fatal_err(struct adapter *sc) 1126 { 1127 u_int fw_status[4]; 1128 1129 if (sc->flags & FULL_INIT_DONE) { 1130 t3_sge_stop(sc); 1131 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1132 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1133 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1134 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1135 t3_intr_disable(sc); 1136 } 1137 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1138 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1139 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1140 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1141 } 1142 1143 int 1144 t3_os_find_pci_capability(adapter_t *sc, int cap) 1145 { 1146 device_t dev; 1147 struct pci_devinfo *dinfo; 1148 pcicfgregs *cfg; 1149 uint32_t status; 1150 uint8_t ptr; 1151 1152 dev = sc->dev; 1153 dinfo = device_get_ivars(dev); 1154 cfg = &dinfo->cfg; 1155 1156 status = pci_read_config(dev, PCIR_STATUS, 2); 1157 if (!(status & PCIM_STATUS_CAPPRESENT)) 1158 return (0); 1159 1160 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1161 case 0: 1162 case 1: 1163 ptr = PCIR_CAP_PTR; 1164 break; 1165 case 2: 1166 ptr = PCIR_CAP_PTR_2; 1167 break; 1168 default: 1169 return (0); 1170 break; 1171 } 1172 ptr = pci_read_config(dev, ptr, 1); 1173 1174 while (ptr != 0) { 1175 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1176 return (ptr); 1177 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1178 } 1179 1180 return (0); 1181 } 1182 1183 int 1184 t3_os_pci_save_state(struct adapter *sc) 1185 { 1186 device_t dev; 1187 struct pci_devinfo *dinfo; 1188 1189 dev = sc->dev; 1190 dinfo = device_get_ivars(dev); 1191 1192 pci_cfg_save(dev, dinfo, 0); 1193 return (0); 1194 } 1195 1196 int 1197 t3_os_pci_restore_state(struct adapter *sc) 1198 { 1199 device_t dev; 1200 struct pci_devinfo *dinfo; 1201 1202 dev = sc->dev; 1203 dinfo = device_get_ivars(dev); 1204 1205 pci_cfg_restore(dev, dinfo); 1206 return (0); 1207 } 1208 1209 /** 1210 * t3_os_link_changed - handle link status changes 1211 * @sc: the adapter associated with the link change 1212 * @port_id: the port index whose link status has changed 1213 * @link_status: the new status of the link 1214 * @speed: the new speed setting 1215 * @duplex: the new duplex setting 1216 * @fc: the new flow-control setting 1217 * 1218 * This is the OS-dependent handler for link status changes. The OS 1219 * neutral handler takes care of most of the processing for these events, 1220 * then calls this handler for any OS-specific processing. 1221 */ 1222 void 1223 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1224 int duplex, int fc, int mac_was_reset) 1225 { 1226 struct port_info *pi = &adapter->port[port_id]; 1227 if_t ifp = pi->ifp; 1228 1229 /* no race with detach, so ifp should always be good */ 1230 KASSERT(ifp, ("%s: if detached.", __func__)); 1231 1232 /* Reapply mac settings if they were lost due to a reset */ 1233 if (mac_was_reset) { 1234 PORT_LOCK(pi); 1235 cxgb_update_mac_settings(pi); 1236 PORT_UNLOCK(pi); 1237 } 1238 1239 if (link_status) { 1240 if_setbaudrate(ifp, IF_Mbps(speed)); 1241 if_link_state_change(ifp, LINK_STATE_UP); 1242 } else 1243 if_link_state_change(ifp, LINK_STATE_DOWN); 1244 } 1245 1246 /** 1247 * t3_os_phymod_changed - handle PHY module changes 1248 * @phy: the PHY reporting the module change 1249 * @mod_type: new module type 1250 * 1251 * This is the OS-dependent handler for PHY module changes. It is 1252 * invoked when a PHY module is removed or inserted for any OS-specific 1253 * processing. 1254 */ 1255 void t3_os_phymod_changed(struct adapter *adap, int port_id) 1256 { 1257 static const char *mod_str[] = { 1258 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown" 1259 }; 1260 struct port_info *pi = &adap->port[port_id]; 1261 int mod = pi->phy.modtype; 1262 1263 if (mod != pi->media.ifm_cur->ifm_data) 1264 cxgb_build_medialist(pi); 1265 1266 if (mod == phy_modtype_none) 1267 if_printf(pi->ifp, "PHY module unplugged\n"); 1268 else { 1269 KASSERT(mod < ARRAY_SIZE(mod_str), 1270 ("invalid PHY module type %d", mod)); 1271 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]); 1272 } 1273 } 1274 1275 void 1276 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1277 { 1278 1279 /* 1280 * The ifnet might not be allocated before this gets called, 1281 * as this is called early on in attach by t3_prep_adapter 1282 * save the address off in the port structure 1283 */ 1284 if (cxgb_debug) 1285 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1286 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1287 } 1288 1289 /* 1290 * Programs the XGMAC based on the settings in the ifnet. These settings 1291 * include MTU, MAC address, mcast addresses, etc. 1292 */ 1293 static void 1294 cxgb_update_mac_settings(struct port_info *p) 1295 { 1296 if_t ifp = p->ifp; 1297 struct t3_rx_mode rm; 1298 struct cmac *mac = &p->mac; 1299 int mtu, hwtagging; 1300 1301 PORT_LOCK_ASSERT_OWNED(p); 1302 1303 bcopy(if_getlladdr(ifp), p->hw_addr, ETHER_ADDR_LEN); 1304 1305 mtu = if_getmtu(ifp); 1306 if (if_getcapenable(ifp) & IFCAP_VLAN_MTU) 1307 mtu += ETHER_VLAN_ENCAP_LEN; 1308 1309 hwtagging = (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) != 0; 1310 1311 t3_mac_set_mtu(mac, mtu); 1312 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1313 t3_mac_set_address(mac, 0, p->hw_addr); 1314 t3_init_rx_mode(&rm, p); 1315 t3_mac_set_rx_mode(mac, &rm); 1316 } 1317 1318 1319 static int 1320 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1321 unsigned long n) 1322 { 1323 int attempts = 5; 1324 1325 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1326 if (!--attempts) 1327 return (ETIMEDOUT); 1328 t3_os_sleep(10); 1329 } 1330 return 0; 1331 } 1332 1333 static int 1334 init_tp_parity(struct adapter *adap) 1335 { 1336 int i; 1337 struct mbuf *m; 1338 struct cpl_set_tcb_field *greq; 1339 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1340 1341 t3_tp_set_offload_mode(adap, 1); 1342 1343 for (i = 0; i < 16; i++) { 1344 struct cpl_smt_write_req *req; 1345 1346 m = m_gethdr(M_WAITOK, MT_DATA); 1347 req = mtod(m, struct cpl_smt_write_req *); 1348 m->m_len = m->m_pkthdr.len = sizeof(*req); 1349 memset(req, 0, sizeof(*req)); 1350 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1351 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1352 req->iff = i; 1353 t3_mgmt_tx(adap, m); 1354 } 1355 1356 for (i = 0; i < 2048; i++) { 1357 struct cpl_l2t_write_req *req; 1358 1359 m = m_gethdr(M_WAITOK, MT_DATA); 1360 req = mtod(m, struct cpl_l2t_write_req *); 1361 m->m_len = m->m_pkthdr.len = sizeof(*req); 1362 memset(req, 0, sizeof(*req)); 1363 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1364 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1365 req->params = htonl(V_L2T_W_IDX(i)); 1366 t3_mgmt_tx(adap, m); 1367 } 1368 1369 for (i = 0; i < 2048; i++) { 1370 struct cpl_rte_write_req *req; 1371 1372 m = m_gethdr(M_WAITOK, MT_DATA); 1373 req = mtod(m, struct cpl_rte_write_req *); 1374 m->m_len = m->m_pkthdr.len = sizeof(*req); 1375 memset(req, 0, sizeof(*req)); 1376 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1377 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1378 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1379 t3_mgmt_tx(adap, m); 1380 } 1381 1382 m = m_gethdr(M_WAITOK, MT_DATA); 1383 greq = mtod(m, struct cpl_set_tcb_field *); 1384 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1385 memset(greq, 0, sizeof(*greq)); 1386 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1387 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1388 greq->mask = htobe64(1); 1389 t3_mgmt_tx(adap, m); 1390 1391 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1392 t3_tp_set_offload_mode(adap, 0); 1393 return (i); 1394 } 1395 1396 /** 1397 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1398 * @adap: the adapter 1399 * 1400 * Sets up RSS to distribute packets to multiple receive queues. We 1401 * configure the RSS CPU lookup table to distribute to the number of HW 1402 * receive queues, and the response queue lookup table to narrow that 1403 * down to the response queues actually configured for each port. 1404 * We always configure the RSS mapping for two ports since the mapping 1405 * table has plenty of entries. 1406 */ 1407 static void 1408 setup_rss(adapter_t *adap) 1409 { 1410 int i; 1411 u_int nq[2]; 1412 uint8_t cpus[SGE_QSETS + 1]; 1413 uint16_t rspq_map[RSS_TABLE_SIZE]; 1414 1415 for (i = 0; i < SGE_QSETS; ++i) 1416 cpus[i] = i; 1417 cpus[SGE_QSETS] = 0xff; 1418 1419 nq[0] = nq[1] = 0; 1420 for_each_port(adap, i) { 1421 const struct port_info *pi = adap2pinfo(adap, i); 1422 1423 nq[pi->tx_chan] += pi->nqsets; 1424 } 1425 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1426 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1427 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1428 } 1429 1430 /* Calculate the reverse RSS map table */ 1431 for (i = 0; i < SGE_QSETS; ++i) 1432 adap->rrss_map[i] = 0xff; 1433 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1434 if (adap->rrss_map[rspq_map[i]] == 0xff) 1435 adap->rrss_map[rspq_map[i]] = i; 1436 1437 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1438 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1439 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1440 cpus, rspq_map); 1441 1442 } 1443 static void 1444 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1445 int hi, int port) 1446 { 1447 struct mbuf *m; 1448 struct mngt_pktsched_wr *req; 1449 1450 m = m_gethdr(M_NOWAIT, MT_DATA); 1451 if (m) { 1452 req = mtod(m, struct mngt_pktsched_wr *); 1453 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1454 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1455 req->sched = sched; 1456 req->idx = qidx; 1457 req->min = lo; 1458 req->max = hi; 1459 req->binding = port; 1460 m->m_len = m->m_pkthdr.len = sizeof(*req); 1461 t3_mgmt_tx(adap, m); 1462 } 1463 } 1464 1465 static void 1466 bind_qsets(adapter_t *sc) 1467 { 1468 int i, j; 1469 1470 for (i = 0; i < (sc)->params.nports; ++i) { 1471 const struct port_info *pi = adap2pinfo(sc, i); 1472 1473 for (j = 0; j < pi->nqsets; ++j) { 1474 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1475 -1, pi->tx_chan); 1476 1477 } 1478 } 1479 } 1480 1481 static void 1482 update_tpeeprom(struct adapter *adap) 1483 { 1484 const struct firmware *tpeeprom; 1485 1486 uint32_t version; 1487 unsigned int major, minor; 1488 int ret, len; 1489 char rev, name[32]; 1490 1491 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1492 1493 major = G_TP_VERSION_MAJOR(version); 1494 minor = G_TP_VERSION_MINOR(version); 1495 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1496 return; 1497 1498 rev = t3rev2char(adap); 1499 snprintf(name, sizeof(name), TPEEPROM_NAME, rev); 1500 1501 tpeeprom = firmware_get(name); 1502 if (tpeeprom == NULL) { 1503 device_printf(adap->dev, 1504 "could not load TP EEPROM: unable to load %s\n", 1505 name); 1506 return; 1507 } 1508 1509 len = tpeeprom->datasize - 4; 1510 1511 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1512 if (ret) 1513 goto release_tpeeprom; 1514 1515 if (len != TP_SRAM_LEN) { 1516 device_printf(adap->dev, 1517 "%s length is wrong len=%d expected=%d\n", name, 1518 len, TP_SRAM_LEN); 1519 return; 1520 } 1521 1522 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1523 TP_SRAM_OFFSET); 1524 1525 if (!ret) { 1526 device_printf(adap->dev, 1527 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1528 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1529 } else 1530 device_printf(adap->dev, 1531 "Protocol SRAM image update in EEPROM failed\n"); 1532 1533 release_tpeeprom: 1534 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1535 1536 return; 1537 } 1538 1539 static int 1540 update_tpsram(struct adapter *adap) 1541 { 1542 const struct firmware *tpsram; 1543 int ret; 1544 char rev, name[32]; 1545 1546 rev = t3rev2char(adap); 1547 snprintf(name, sizeof(name), TPSRAM_NAME, rev); 1548 1549 update_tpeeprom(adap); 1550 1551 tpsram = firmware_get(name); 1552 if (tpsram == NULL){ 1553 device_printf(adap->dev, "could not load TP SRAM\n"); 1554 return (EINVAL); 1555 } else 1556 device_printf(adap->dev, "updating TP SRAM\n"); 1557 1558 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1559 if (ret) 1560 goto release_tpsram; 1561 1562 ret = t3_set_proto_sram(adap, tpsram->data); 1563 if (ret) 1564 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1565 1566 release_tpsram: 1567 firmware_put(tpsram, FIRMWARE_UNLOAD); 1568 1569 return ret; 1570 } 1571 1572 /** 1573 * cxgb_up - enable the adapter 1574 * @adap: adapter being enabled 1575 * 1576 * Called when the first port is enabled, this function performs the 1577 * actions necessary to make an adapter operational, such as completing 1578 * the initialization of HW modules, and enabling interrupts. 1579 */ 1580 static int 1581 cxgb_up(struct adapter *sc) 1582 { 1583 int err = 0; 1584 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS; 1585 1586 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)", 1587 __func__, sc->open_device_map)); 1588 1589 if ((sc->flags & FULL_INIT_DONE) == 0) { 1590 1591 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1592 1593 if ((sc->flags & FW_UPTODATE) == 0) 1594 if ((err = upgrade_fw(sc))) 1595 goto out; 1596 1597 if ((sc->flags & TPS_UPTODATE) == 0) 1598 if ((err = update_tpsram(sc))) 1599 goto out; 1600 1601 if (is_offload(sc) && nfilters != 0) { 1602 sc->params.mc5.nservers = 0; 1603 1604 if (nfilters < 0) 1605 sc->params.mc5.nfilters = mxf; 1606 else 1607 sc->params.mc5.nfilters = min(nfilters, mxf); 1608 } 1609 1610 err = t3_init_hw(sc, 0); 1611 if (err) 1612 goto out; 1613 1614 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1615 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1616 1617 err = setup_sge_qsets(sc); 1618 if (err) 1619 goto out; 1620 1621 alloc_filters(sc); 1622 setup_rss(sc); 1623 1624 t3_add_configured_sysctls(sc); 1625 sc->flags |= FULL_INIT_DONE; 1626 } 1627 1628 t3_intr_clear(sc); 1629 t3_sge_start(sc); 1630 t3_intr_enable(sc); 1631 1632 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1633 is_offload(sc) && init_tp_parity(sc) == 0) 1634 sc->flags |= TP_PARITY_INIT; 1635 1636 if (sc->flags & TP_PARITY_INIT) { 1637 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR); 1638 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1639 } 1640 1641 if (!(sc->flags & QUEUES_BOUND)) { 1642 bind_qsets(sc); 1643 setup_hw_filters(sc); 1644 sc->flags |= QUEUES_BOUND; 1645 } 1646 1647 t3_sge_reset_adapter(sc); 1648 out: 1649 return (err); 1650 } 1651 1652 /* 1653 * Called when the last open device is closed. Does NOT undo all of cxgb_up's 1654 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released 1655 * during controller_detach, not here. 1656 */ 1657 static void 1658 cxgb_down(struct adapter *sc) 1659 { 1660 t3_sge_stop(sc); 1661 t3_intr_disable(sc); 1662 } 1663 1664 /* 1665 * if_init for cxgb ports. 1666 */ 1667 static void 1668 cxgb_init(void *arg) 1669 { 1670 struct port_info *p = arg; 1671 struct adapter *sc = p->adapter; 1672 1673 ADAPTER_LOCK(sc); 1674 cxgb_init_locked(p); /* releases adapter lock */ 1675 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1676 } 1677 1678 static int 1679 cxgb_init_locked(struct port_info *p) 1680 { 1681 struct adapter *sc = p->adapter; 1682 if_t ifp = p->ifp; 1683 struct cmac *mac = &p->mac; 1684 int i, rc = 0, may_sleep = 0, gave_up_lock = 0; 1685 1686 ADAPTER_LOCK_ASSERT_OWNED(sc); 1687 1688 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1689 gave_up_lock = 1; 1690 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) { 1691 rc = EINTR; 1692 goto done; 1693 } 1694 } 1695 if (IS_DOOMED(p)) { 1696 rc = ENXIO; 1697 goto done; 1698 } 1699 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1700 1701 /* 1702 * The code that runs during one-time adapter initialization can sleep 1703 * so it's important not to hold any locks across it. 1704 */ 1705 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1; 1706 1707 if (may_sleep) { 1708 SET_BUSY(sc); 1709 gave_up_lock = 1; 1710 ADAPTER_UNLOCK(sc); 1711 } 1712 1713 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0)) 1714 goto done; 1715 1716 PORT_LOCK(p); 1717 if (isset(&sc->open_device_map, p->port_id) && 1718 (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { 1719 PORT_UNLOCK(p); 1720 goto done; 1721 } 1722 t3_port_intr_enable(sc, p->port_id); 1723 if (!mac->multiport) 1724 t3_mac_init(mac); 1725 cxgb_update_mac_settings(p); 1726 t3_link_start(&p->phy, mac, &p->link_config); 1727 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1728 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 1729 PORT_UNLOCK(p); 1730 1731 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1732 struct sge_qset *qs = &sc->sge.qs[i]; 1733 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1734 1735 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs, 1736 txq->txq_watchdog.c_cpu); 1737 } 1738 1739 /* all ok */ 1740 setbit(&sc->open_device_map, p->port_id); 1741 callout_reset(&p->link_check_ch, 1742 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4, 1743 link_check_callout, p); 1744 1745 done: 1746 if (may_sleep) { 1747 ADAPTER_LOCK(sc); 1748 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1749 CLR_BUSY(sc); 1750 } 1751 if (gave_up_lock) 1752 wakeup_one(&sc->flags); 1753 ADAPTER_UNLOCK(sc); 1754 return (rc); 1755 } 1756 1757 static int 1758 cxgb_uninit_locked(struct port_info *p) 1759 { 1760 struct adapter *sc = p->adapter; 1761 int rc; 1762 1763 ADAPTER_LOCK_ASSERT_OWNED(sc); 1764 1765 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1766 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) { 1767 rc = EINTR; 1768 goto done; 1769 } 1770 } 1771 if (IS_DOOMED(p)) { 1772 rc = ENXIO; 1773 goto done; 1774 } 1775 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1776 SET_BUSY(sc); 1777 ADAPTER_UNLOCK(sc); 1778 1779 rc = cxgb_uninit_synchronized(p); 1780 1781 ADAPTER_LOCK(sc); 1782 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1783 CLR_BUSY(sc); 1784 wakeup_one(&sc->flags); 1785 done: 1786 ADAPTER_UNLOCK(sc); 1787 return (rc); 1788 } 1789 1790 /* 1791 * Called on "ifconfig down", and from port_detach 1792 */ 1793 static int 1794 cxgb_uninit_synchronized(struct port_info *pi) 1795 { 1796 struct adapter *sc = pi->adapter; 1797 if_t ifp = pi->ifp; 1798 1799 /* 1800 * taskqueue_drain may cause a deadlock if the adapter lock is held. 1801 */ 1802 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1803 1804 /* 1805 * Clear this port's bit from the open device map, and then drain all 1806 * the tasks that can access/manipulate this port's port_info or ifp. 1807 * We disable this port's interrupts here and so the slow/ext 1808 * interrupt tasks won't be enqueued. The tick task will continue to 1809 * be enqueued every second but the runs after this drain will not see 1810 * this port in the open device map. 1811 * 1812 * A well behaved task must take open_device_map into account and ignore 1813 * ports that are not open. 1814 */ 1815 clrbit(&sc->open_device_map, pi->port_id); 1816 t3_port_intr_disable(sc, pi->port_id); 1817 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1818 taskqueue_drain(sc->tq, &sc->tick_task); 1819 1820 callout_drain(&pi->link_check_ch); 1821 taskqueue_drain(sc->tq, &pi->link_check_task); 1822 1823 PORT_LOCK(pi); 1824 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1825 1826 /* disable pause frames */ 1827 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0); 1828 1829 /* Reset RX FIFO HWM */ 1830 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset, 1831 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1832 1833 DELAY(100 * 1000); 1834 1835 /* Wait for TXFIFO empty */ 1836 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset, 1837 F_TXFIFO_EMPTY, 1, 20, 5); 1838 1839 DELAY(100 * 1000); 1840 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX); 1841 1842 pi->phy.ops->power_down(&pi->phy, 1); 1843 1844 PORT_UNLOCK(pi); 1845 1846 pi->link_config.link_ok = 0; 1847 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0); 1848 1849 if (sc->open_device_map == 0) 1850 cxgb_down(pi->adapter); 1851 1852 return (0); 1853 } 1854 1855 /* 1856 * Mark lro enabled or disabled in all qsets for this port 1857 */ 1858 static int 1859 cxgb_set_lro(struct port_info *p, int enabled) 1860 { 1861 int i; 1862 struct adapter *adp = p->adapter; 1863 struct sge_qset *q; 1864 1865 for (i = 0; i < p->nqsets; i++) { 1866 q = &adp->sge.qs[p->first_qset + i]; 1867 q->lro.enabled = (enabled != 0); 1868 } 1869 return (0); 1870 } 1871 1872 static int 1873 cxgb_ioctl(if_t ifp, unsigned long command, caddr_t data) 1874 { 1875 struct port_info *p = if_getsoftc(ifp); 1876 struct adapter *sc = p->adapter; 1877 struct ifreq *ifr = (struct ifreq *)data; 1878 int flags, error = 0, mtu; 1879 uint32_t mask; 1880 1881 switch (command) { 1882 case SIOCSIFMTU: 1883 ADAPTER_LOCK(sc); 1884 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1885 if (error) { 1886 fail: 1887 ADAPTER_UNLOCK(sc); 1888 return (error); 1889 } 1890 1891 mtu = ifr->ifr_mtu; 1892 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 1893 error = EINVAL; 1894 } else { 1895 if_setmtu(ifp, mtu); 1896 PORT_LOCK(p); 1897 cxgb_update_mac_settings(p); 1898 PORT_UNLOCK(p); 1899 } 1900 ADAPTER_UNLOCK(sc); 1901 break; 1902 case SIOCSIFFLAGS: 1903 ADAPTER_LOCK(sc); 1904 if (IS_DOOMED(p)) { 1905 error = ENXIO; 1906 goto fail; 1907 } 1908 if (if_getflags(ifp) & IFF_UP) { 1909 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1910 flags = p->if_flags; 1911 if (((if_getflags(ifp) ^ flags) & IFF_PROMISC) || 1912 ((if_getflags(ifp) ^ flags) & IFF_ALLMULTI)) { 1913 if (IS_BUSY(sc)) { 1914 error = EBUSY; 1915 goto fail; 1916 } 1917 PORT_LOCK(p); 1918 cxgb_update_mac_settings(p); 1919 PORT_UNLOCK(p); 1920 } 1921 ADAPTER_UNLOCK(sc); 1922 } else 1923 error = cxgb_init_locked(p); 1924 p->if_flags = if_getflags(ifp); 1925 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 1926 error = cxgb_uninit_locked(p); 1927 else 1928 ADAPTER_UNLOCK(sc); 1929 1930 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1931 break; 1932 case SIOCADDMULTI: 1933 case SIOCDELMULTI: 1934 ADAPTER_LOCK(sc); 1935 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1936 if (error) 1937 goto fail; 1938 1939 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1940 PORT_LOCK(p); 1941 cxgb_update_mac_settings(p); 1942 PORT_UNLOCK(p); 1943 } 1944 ADAPTER_UNLOCK(sc); 1945 1946 break; 1947 case SIOCSIFCAP: 1948 ADAPTER_LOCK(sc); 1949 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1950 if (error) 1951 goto fail; 1952 1953 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); 1954 if (mask & IFCAP_TXCSUM) { 1955 if_togglecapenable(ifp, IFCAP_TXCSUM); 1956 if_togglehwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP); 1957 1958 if (IFCAP_TSO4 & if_getcapenable(ifp) && 1959 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 1960 mask &= ~IFCAP_TSO4; 1961 if_setcapenablebit(ifp, 0, IFCAP_TSO4); 1962 if_printf(ifp, 1963 "tso4 disabled due to -txcsum.\n"); 1964 } 1965 } 1966 if (mask & IFCAP_TXCSUM_IPV6) { 1967 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); 1968 if_togglehwassist(ifp, CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1969 1970 if (IFCAP_TSO6 & if_getcapenable(ifp) && 1971 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { 1972 mask &= ~IFCAP_TSO6; 1973 if_setcapenablebit(ifp, 0, IFCAP_TSO6); 1974 if_printf(ifp, 1975 "tso6 disabled due to -txcsum6.\n"); 1976 } 1977 } 1978 if (mask & IFCAP_RXCSUM) 1979 if_togglecapenable(ifp, IFCAP_RXCSUM); 1980 if (mask & IFCAP_RXCSUM_IPV6) 1981 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); 1982 1983 /* 1984 * Note that we leave CSUM_TSO alone (it is always set). The 1985 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1986 * sending a TSO request our way, so it's sufficient to toggle 1987 * IFCAP_TSOx only. 1988 */ 1989 if (mask & IFCAP_TSO4) { 1990 if (!(IFCAP_TSO4 & if_getcapenable(ifp)) && 1991 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 1992 if_printf(ifp, "enable txcsum first.\n"); 1993 error = EAGAIN; 1994 goto fail; 1995 } 1996 if_togglecapenable(ifp, IFCAP_TSO4); 1997 } 1998 if (mask & IFCAP_TSO6) { 1999 if (!(IFCAP_TSO6 & if_getcapenable(ifp)) && 2000 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { 2001 if_printf(ifp, "enable txcsum6 first.\n"); 2002 error = EAGAIN; 2003 goto fail; 2004 } 2005 if_togglecapenable(ifp, IFCAP_TSO6); 2006 } 2007 if (mask & IFCAP_LRO) { 2008 if_togglecapenable(ifp, IFCAP_LRO); 2009 2010 /* Safe to do this even if cxgb_up not called yet */ 2011 cxgb_set_lro(p, if_getcapenable(ifp) & IFCAP_LRO); 2012 } 2013 #ifdef TCP_OFFLOAD 2014 if (mask & IFCAP_TOE4) { 2015 int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TOE4; 2016 2017 error = toe_capability(p, enable); 2018 if (error == 0) 2019 if_togglecapenable(ifp, mask); 2020 } 2021 #endif 2022 if (mask & IFCAP_VLAN_HWTAGGING) { 2023 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); 2024 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2025 PORT_LOCK(p); 2026 cxgb_update_mac_settings(p); 2027 PORT_UNLOCK(p); 2028 } 2029 } 2030 if (mask & IFCAP_VLAN_MTU) { 2031 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 2032 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2033 PORT_LOCK(p); 2034 cxgb_update_mac_settings(p); 2035 PORT_UNLOCK(p); 2036 } 2037 } 2038 if (mask & IFCAP_VLAN_HWTSO) 2039 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); 2040 if (mask & IFCAP_VLAN_HWCSUM) 2041 if_togglecapenable(ifp, IFCAP_VLAN_HWCSUM); 2042 2043 #ifdef VLAN_CAPABILITIES 2044 VLAN_CAPABILITIES(ifp); 2045 #endif 2046 ADAPTER_UNLOCK(sc); 2047 break; 2048 case SIOCSIFMEDIA: 2049 case SIOCGIFMEDIA: 2050 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 2051 break; 2052 default: 2053 error = ether_ioctl(ifp, command, data); 2054 } 2055 2056 return (error); 2057 } 2058 2059 static int 2060 cxgb_media_change(if_t ifp) 2061 { 2062 return (EOPNOTSUPP); 2063 } 2064 2065 /* 2066 * Translates phy->modtype to the correct Ethernet media subtype. 2067 */ 2068 static int 2069 cxgb_ifm_type(int mod) 2070 { 2071 switch (mod) { 2072 case phy_modtype_sr: 2073 return (IFM_10G_SR); 2074 case phy_modtype_lr: 2075 return (IFM_10G_LR); 2076 case phy_modtype_lrm: 2077 return (IFM_10G_LRM); 2078 case phy_modtype_twinax: 2079 return (IFM_10G_TWINAX); 2080 case phy_modtype_twinax_long: 2081 return (IFM_10G_TWINAX_LONG); 2082 case phy_modtype_none: 2083 return (IFM_NONE); 2084 case phy_modtype_unknown: 2085 return (IFM_UNKNOWN); 2086 } 2087 2088 KASSERT(0, ("%s: modtype %d unknown", __func__, mod)); 2089 return (IFM_UNKNOWN); 2090 } 2091 2092 /* 2093 * Rebuilds the ifmedia list for this port, and sets the current media. 2094 */ 2095 static void 2096 cxgb_build_medialist(struct port_info *p) 2097 { 2098 struct cphy *phy = &p->phy; 2099 struct ifmedia *media = &p->media; 2100 int mod = phy->modtype; 2101 int m = IFM_ETHER | IFM_FDX; 2102 2103 PORT_LOCK(p); 2104 2105 ifmedia_removeall(media); 2106 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) { 2107 /* Copper (RJ45) */ 2108 2109 if (phy->caps & SUPPORTED_10000baseT_Full) 2110 ifmedia_add(media, m | IFM_10G_T, mod, NULL); 2111 2112 if (phy->caps & SUPPORTED_1000baseT_Full) 2113 ifmedia_add(media, m | IFM_1000_T, mod, NULL); 2114 2115 if (phy->caps & SUPPORTED_100baseT_Full) 2116 ifmedia_add(media, m | IFM_100_TX, mod, NULL); 2117 2118 if (phy->caps & SUPPORTED_10baseT_Full) 2119 ifmedia_add(media, m | IFM_10_T, mod, NULL); 2120 2121 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL); 2122 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2123 2124 } else if (phy->caps & SUPPORTED_TP) { 2125 /* Copper (CX4) */ 2126 2127 KASSERT(phy->caps & SUPPORTED_10000baseT_Full, 2128 ("%s: unexpected cap 0x%x", __func__, phy->caps)); 2129 2130 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL); 2131 ifmedia_set(media, m | IFM_10G_CX4); 2132 2133 } else if (phy->caps & SUPPORTED_FIBRE && 2134 phy->caps & SUPPORTED_10000baseT_Full) { 2135 /* 10G optical (but includes SFP+ twinax) */ 2136 2137 m |= cxgb_ifm_type(mod); 2138 if (IFM_SUBTYPE(m) == IFM_NONE) 2139 m &= ~IFM_FDX; 2140 2141 ifmedia_add(media, m, mod, NULL); 2142 ifmedia_set(media, m); 2143 2144 } else if (phy->caps & SUPPORTED_FIBRE && 2145 phy->caps & SUPPORTED_1000baseT_Full) { 2146 /* 1G optical */ 2147 2148 /* XXX: Lie and claim to be SX, could actually be any 1G-X */ 2149 ifmedia_add(media, m | IFM_1000_SX, mod, NULL); 2150 ifmedia_set(media, m | IFM_1000_SX); 2151 2152 } else { 2153 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__, 2154 phy->caps)); 2155 } 2156 2157 PORT_UNLOCK(p); 2158 } 2159 2160 static void 2161 cxgb_media_status(if_t ifp, struct ifmediareq *ifmr) 2162 { 2163 struct port_info *p = if_getsoftc(ifp); 2164 struct ifmedia_entry *cur = p->media.ifm_cur; 2165 int speed = p->link_config.speed; 2166 2167 if (cur->ifm_data != p->phy.modtype) { 2168 cxgb_build_medialist(p); 2169 cur = p->media.ifm_cur; 2170 } 2171 2172 ifmr->ifm_status = IFM_AVALID; 2173 if (!p->link_config.link_ok) 2174 return; 2175 2176 ifmr->ifm_status |= IFM_ACTIVE; 2177 2178 /* 2179 * active and current will differ iff current media is autoselect. That 2180 * can happen only for copper RJ45. 2181 */ 2182 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 2183 return; 2184 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg, 2185 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps)); 2186 2187 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 2188 if (speed == SPEED_10000) 2189 ifmr->ifm_active |= IFM_10G_T; 2190 else if (speed == SPEED_1000) 2191 ifmr->ifm_active |= IFM_1000_T; 2192 else if (speed == SPEED_100) 2193 ifmr->ifm_active |= IFM_100_TX; 2194 else if (speed == SPEED_10) 2195 ifmr->ifm_active |= IFM_10_T; 2196 else 2197 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 2198 speed)); 2199 } 2200 2201 static uint64_t 2202 cxgb_get_counter(if_t ifp, ift_counter c) 2203 { 2204 struct port_info *pi = if_getsoftc(ifp); 2205 struct adapter *sc = pi->adapter; 2206 struct cmac *mac = &pi->mac; 2207 struct mac_stats *mstats = &mac->stats; 2208 2209 cxgb_refresh_stats(pi); 2210 2211 switch (c) { 2212 case IFCOUNTER_IPACKETS: 2213 return (mstats->rx_frames); 2214 2215 case IFCOUNTER_IERRORS: 2216 return (mstats->rx_jabber + mstats->rx_data_errs + 2217 mstats->rx_sequence_errs + mstats->rx_runt + 2218 mstats->rx_too_long + mstats->rx_mac_internal_errs + 2219 mstats->rx_short + mstats->rx_fcs_errs); 2220 2221 case IFCOUNTER_OPACKETS: 2222 return (mstats->tx_frames); 2223 2224 case IFCOUNTER_OERRORS: 2225 return (mstats->tx_excess_collisions + mstats->tx_underrun + 2226 mstats->tx_len_errs + mstats->tx_mac_internal_errs + 2227 mstats->tx_excess_deferral + mstats->tx_fcs_errs); 2228 2229 case IFCOUNTER_COLLISIONS: 2230 return (mstats->tx_total_collisions); 2231 2232 case IFCOUNTER_IBYTES: 2233 return (mstats->rx_octets); 2234 2235 case IFCOUNTER_OBYTES: 2236 return (mstats->tx_octets); 2237 2238 case IFCOUNTER_IMCASTS: 2239 return (mstats->rx_mcast_frames); 2240 2241 case IFCOUNTER_OMCASTS: 2242 return (mstats->tx_mcast_frames); 2243 2244 case IFCOUNTER_IQDROPS: 2245 return (mstats->rx_cong_drops); 2246 2247 case IFCOUNTER_OQDROPS: { 2248 int i; 2249 uint64_t drops; 2250 2251 drops = 0; 2252 if (sc->flags & FULL_INIT_DONE) { 2253 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) 2254 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops; 2255 } 2256 2257 return (drops); 2258 2259 } 2260 2261 default: 2262 return (if_get_counter_default(ifp, c)); 2263 } 2264 } 2265 2266 static void 2267 cxgb_async_intr(void *data) 2268 { 2269 adapter_t *sc = data; 2270 2271 t3_write_reg(sc, A_PL_INT_ENABLE0, 0); 2272 (void) t3_read_reg(sc, A_PL_INT_ENABLE0); 2273 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2274 } 2275 2276 static void 2277 link_check_callout(void *arg) 2278 { 2279 struct port_info *pi = arg; 2280 struct adapter *sc = pi->adapter; 2281 2282 if (!isset(&sc->open_device_map, pi->port_id)) 2283 return; 2284 2285 taskqueue_enqueue(sc->tq, &pi->link_check_task); 2286 } 2287 2288 static void 2289 check_link_status(void *arg, int pending) 2290 { 2291 struct port_info *pi = arg; 2292 struct adapter *sc = pi->adapter; 2293 2294 if (!isset(&sc->open_device_map, pi->port_id)) 2295 return; 2296 2297 t3_link_changed(sc, pi->port_id); 2298 2299 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) || 2300 pi->link_config.link_ok == 0) 2301 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi); 2302 } 2303 2304 void 2305 t3_os_link_intr(struct port_info *pi) 2306 { 2307 /* 2308 * Schedule a link check in the near future. If the link is flapping 2309 * rapidly we'll keep resetting the callout and delaying the check until 2310 * things stabilize a bit. 2311 */ 2312 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi); 2313 } 2314 2315 static void 2316 check_t3b2_mac(struct adapter *sc) 2317 { 2318 int i; 2319 2320 if (sc->flags & CXGB_SHUTDOWN) 2321 return; 2322 2323 for_each_port(sc, i) { 2324 struct port_info *p = &sc->port[i]; 2325 int status; 2326 #ifdef INVARIANTS 2327 if_t ifp = p->ifp; 2328 #endif 2329 2330 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault || 2331 !p->link_config.link_ok) 2332 continue; 2333 2334 KASSERT(if_getdrvflags(ifp) & IFF_DRV_RUNNING, 2335 ("%s: state mismatch (drv_flags %x, device_map %x)", 2336 __func__, if_getdrvflags(ifp), sc->open_device_map)); 2337 2338 PORT_LOCK(p); 2339 status = t3b2_mac_watchdog_task(&p->mac); 2340 if (status == 1) 2341 p->mac.stats.num_toggled++; 2342 else if (status == 2) { 2343 struct cmac *mac = &p->mac; 2344 2345 cxgb_update_mac_settings(p); 2346 t3_link_start(&p->phy, mac, &p->link_config); 2347 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2348 t3_port_intr_enable(sc, p->port_id); 2349 p->mac.stats.num_resets++; 2350 } 2351 PORT_UNLOCK(p); 2352 } 2353 } 2354 2355 static void 2356 cxgb_tick(void *arg) 2357 { 2358 adapter_t *sc = (adapter_t *)arg; 2359 2360 if (sc->flags & CXGB_SHUTDOWN) 2361 return; 2362 2363 taskqueue_enqueue(sc->tq, &sc->tick_task); 2364 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 2365 } 2366 2367 void 2368 cxgb_refresh_stats(struct port_info *pi) 2369 { 2370 struct timeval tv; 2371 const struct timeval interval = {0, 250000}; /* 250ms */ 2372 2373 getmicrotime(&tv); 2374 timevalsub(&tv, &interval); 2375 if (timevalcmp(&tv, &pi->last_refreshed, <)) 2376 return; 2377 2378 PORT_LOCK(pi); 2379 t3_mac_update_stats(&pi->mac); 2380 PORT_UNLOCK(pi); 2381 getmicrotime(&pi->last_refreshed); 2382 } 2383 2384 static void 2385 cxgb_tick_handler(void *arg, int count) 2386 { 2387 adapter_t *sc = (adapter_t *)arg; 2388 const struct adapter_params *p = &sc->params; 2389 int i; 2390 uint32_t cause, reset; 2391 2392 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE)) 2393 return; 2394 2395 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2396 check_t3b2_mac(sc); 2397 2398 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY); 2399 if (cause) { 2400 struct sge_qset *qs = &sc->sge.qs[0]; 2401 uint32_t mask, v; 2402 2403 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00; 2404 2405 mask = 1; 2406 for (i = 0; i < SGE_QSETS; i++) { 2407 if (v & mask) 2408 qs[i].rspq.starved++; 2409 mask <<= 1; 2410 } 2411 2412 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */ 2413 2414 for (i = 0; i < SGE_QSETS * 2; i++) { 2415 if (v & mask) { 2416 qs[i / 2].fl[i % 2].empty++; 2417 } 2418 mask <<= 1; 2419 } 2420 2421 /* clear */ 2422 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v); 2423 t3_write_reg(sc, A_SG_INT_CAUSE, cause); 2424 } 2425 2426 for (i = 0; i < sc->params.nports; i++) { 2427 struct port_info *pi = &sc->port[i]; 2428 struct cmac *mac = &pi->mac; 2429 2430 if (!isset(&sc->open_device_map, pi->port_id)) 2431 continue; 2432 2433 cxgb_refresh_stats(pi); 2434 2435 if (mac->multiport) 2436 continue; 2437 2438 /* Count rx fifo overflows, once per second */ 2439 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset); 2440 reset = 0; 2441 if (cause & F_RXFIFO_OVERFLOW) { 2442 mac->stats.rx_fifo_ovfl++; 2443 reset |= F_RXFIFO_OVERFLOW; 2444 } 2445 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset); 2446 } 2447 } 2448 2449 static void 2450 touch_bars(device_t dev) 2451 { 2452 /* 2453 * Don't enable yet 2454 */ 2455 #if !defined(__LP64__) && 0 2456 u32 v; 2457 2458 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2459 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2460 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2461 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2462 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2463 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2464 #endif 2465 } 2466 2467 static int 2468 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2469 { 2470 uint8_t *buf; 2471 int err = 0; 2472 u32 aligned_offset, aligned_len, *p; 2473 struct adapter *adapter = pi->adapter; 2474 2475 2476 aligned_offset = offset & ~3; 2477 aligned_len = (len + (offset & 3) + 3) & ~3; 2478 2479 if (aligned_offset != offset || aligned_len != len) { 2480 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2481 if (!buf) 2482 return (ENOMEM); 2483 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2484 if (!err && aligned_len > 4) 2485 err = t3_seeprom_read(adapter, 2486 aligned_offset + aligned_len - 4, 2487 (u32 *)&buf[aligned_len - 4]); 2488 if (err) 2489 goto out; 2490 memcpy(buf + (offset & 3), data, len); 2491 } else 2492 buf = (uint8_t *)(uintptr_t)data; 2493 2494 err = t3_seeprom_wp(adapter, 0); 2495 if (err) 2496 goto out; 2497 2498 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2499 err = t3_seeprom_write(adapter, aligned_offset, *p); 2500 aligned_offset += 4; 2501 } 2502 2503 if (!err) 2504 err = t3_seeprom_wp(adapter, 1); 2505 out: 2506 if (buf != data) 2507 free(buf, M_DEVBUF); 2508 return err; 2509 } 2510 2511 2512 static int 2513 in_range(int val, int lo, int hi) 2514 { 2515 return val < 0 || (val <= hi && val >= lo); 2516 } 2517 2518 static int 2519 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td) 2520 { 2521 return (0); 2522 } 2523 2524 static int 2525 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td) 2526 { 2527 return (0); 2528 } 2529 2530 static int 2531 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2532 int fflag, struct thread *td) 2533 { 2534 int mmd, error = 0; 2535 struct port_info *pi = dev->si_drv1; 2536 adapter_t *sc = pi->adapter; 2537 2538 #ifdef PRIV_SUPPORTED 2539 if (priv_check(td, PRIV_DRIVER)) { 2540 if (cxgb_debug) 2541 printf("user does not have access to privileged ioctls\n"); 2542 return (EPERM); 2543 } 2544 #else 2545 if (suser(td)) { 2546 if (cxgb_debug) 2547 printf("user does not have access to privileged ioctls\n"); 2548 return (EPERM); 2549 } 2550 #endif 2551 2552 switch (cmd) { 2553 case CHELSIO_GET_MIIREG: { 2554 uint32_t val; 2555 struct cphy *phy = &pi->phy; 2556 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2557 2558 if (!phy->mdio_read) 2559 return (EOPNOTSUPP); 2560 if (is_10G(sc)) { 2561 mmd = mid->phy_id >> 8; 2562 if (!mmd) 2563 mmd = MDIO_DEV_PCS; 2564 else if (mmd > MDIO_DEV_VEND2) 2565 return (EINVAL); 2566 2567 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2568 mid->reg_num, &val); 2569 } else 2570 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2571 mid->reg_num & 0x1f, &val); 2572 if (error == 0) 2573 mid->val_out = val; 2574 break; 2575 } 2576 case CHELSIO_SET_MIIREG: { 2577 struct cphy *phy = &pi->phy; 2578 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2579 2580 if (!phy->mdio_write) 2581 return (EOPNOTSUPP); 2582 if (is_10G(sc)) { 2583 mmd = mid->phy_id >> 8; 2584 if (!mmd) 2585 mmd = MDIO_DEV_PCS; 2586 else if (mmd > MDIO_DEV_VEND2) 2587 return (EINVAL); 2588 2589 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2590 mmd, mid->reg_num, mid->val_in); 2591 } else 2592 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2593 mid->reg_num & 0x1f, 2594 mid->val_in); 2595 break; 2596 } 2597 case CHELSIO_SETREG: { 2598 struct ch_reg *edata = (struct ch_reg *)data; 2599 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2600 return (EFAULT); 2601 t3_write_reg(sc, edata->addr, edata->val); 2602 break; 2603 } 2604 case CHELSIO_GETREG: { 2605 struct ch_reg *edata = (struct ch_reg *)data; 2606 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2607 return (EFAULT); 2608 edata->val = t3_read_reg(sc, edata->addr); 2609 break; 2610 } 2611 case CHELSIO_GET_SGE_CONTEXT: { 2612 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2613 mtx_lock_spin(&sc->sge.reg_lock); 2614 switch (ecntxt->cntxt_type) { 2615 case CNTXT_TYPE_EGRESS: 2616 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2617 ecntxt->data); 2618 break; 2619 case CNTXT_TYPE_FL: 2620 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2621 ecntxt->data); 2622 break; 2623 case CNTXT_TYPE_RSP: 2624 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2625 ecntxt->data); 2626 break; 2627 case CNTXT_TYPE_CQ: 2628 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2629 ecntxt->data); 2630 break; 2631 default: 2632 error = EINVAL; 2633 break; 2634 } 2635 mtx_unlock_spin(&sc->sge.reg_lock); 2636 break; 2637 } 2638 case CHELSIO_GET_SGE_DESC: { 2639 struct ch_desc *edesc = (struct ch_desc *)data; 2640 int ret; 2641 if (edesc->queue_num >= SGE_QSETS * 6) 2642 return (EINVAL); 2643 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2644 edesc->queue_num % 6, edesc->idx, edesc->data); 2645 if (ret < 0) 2646 return (EINVAL); 2647 edesc->size = ret; 2648 break; 2649 } 2650 case CHELSIO_GET_QSET_PARAMS: { 2651 struct qset_params *q; 2652 struct ch_qset_params *t = (struct ch_qset_params *)data; 2653 int q1 = pi->first_qset; 2654 int nqsets = pi->nqsets; 2655 int i; 2656 2657 if (t->qset_idx >= nqsets) 2658 return EINVAL; 2659 2660 i = q1 + t->qset_idx; 2661 q = &sc->params.sge.qset[i]; 2662 t->rspq_size = q->rspq_size; 2663 t->txq_size[0] = q->txq_size[0]; 2664 t->txq_size[1] = q->txq_size[1]; 2665 t->txq_size[2] = q->txq_size[2]; 2666 t->fl_size[0] = q->fl_size; 2667 t->fl_size[1] = q->jumbo_size; 2668 t->polling = q->polling; 2669 t->lro = q->lro; 2670 t->intr_lat = q->coalesce_usecs; 2671 t->cong_thres = q->cong_thres; 2672 t->qnum = i; 2673 2674 if ((sc->flags & FULL_INIT_DONE) == 0) 2675 t->vector = 0; 2676 else if (sc->flags & USING_MSIX) 2677 t->vector = rman_get_start(sc->msix_irq_res[i]); 2678 else 2679 t->vector = rman_get_start(sc->irq_res); 2680 2681 break; 2682 } 2683 case CHELSIO_GET_QSET_NUM: { 2684 struct ch_reg *edata = (struct ch_reg *)data; 2685 edata->val = pi->nqsets; 2686 break; 2687 } 2688 case CHELSIO_LOAD_FW: { 2689 uint8_t *fw_data; 2690 uint32_t vers; 2691 struct ch_mem_range *t = (struct ch_mem_range *)data; 2692 2693 /* 2694 * You're allowed to load a firmware only before FULL_INIT_DONE 2695 * 2696 * FW_UPTODATE is also set so the rest of the initialization 2697 * will not overwrite what was loaded here. This gives you the 2698 * flexibility to load any firmware (and maybe shoot yourself in 2699 * the foot). 2700 */ 2701 2702 ADAPTER_LOCK(sc); 2703 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2704 ADAPTER_UNLOCK(sc); 2705 return (EBUSY); 2706 } 2707 2708 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2709 if (!fw_data) 2710 error = ENOMEM; 2711 else 2712 error = copyin(t->buf, fw_data, t->len); 2713 2714 if (!error) 2715 error = -t3_load_fw(sc, fw_data, t->len); 2716 2717 if (t3_get_fw_version(sc, &vers) == 0) { 2718 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2719 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2720 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2721 } 2722 2723 if (!error) 2724 sc->flags |= FW_UPTODATE; 2725 2726 free(fw_data, M_DEVBUF); 2727 ADAPTER_UNLOCK(sc); 2728 break; 2729 } 2730 case CHELSIO_LOAD_BOOT: { 2731 uint8_t *boot_data; 2732 struct ch_mem_range *t = (struct ch_mem_range *)data; 2733 2734 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2735 if (!boot_data) 2736 return ENOMEM; 2737 2738 error = copyin(t->buf, boot_data, t->len); 2739 if (!error) 2740 error = -t3_load_boot(sc, boot_data, t->len); 2741 2742 free(boot_data, M_DEVBUF); 2743 break; 2744 } 2745 case CHELSIO_GET_PM: { 2746 struct ch_pm *m = (struct ch_pm *)data; 2747 struct tp_params *p = &sc->params.tp; 2748 2749 if (!is_offload(sc)) 2750 return (EOPNOTSUPP); 2751 2752 m->tx_pg_sz = p->tx_pg_size; 2753 m->tx_num_pg = p->tx_num_pgs; 2754 m->rx_pg_sz = p->rx_pg_size; 2755 m->rx_num_pg = p->rx_num_pgs; 2756 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2757 2758 break; 2759 } 2760 case CHELSIO_SET_PM: { 2761 struct ch_pm *m = (struct ch_pm *)data; 2762 struct tp_params *p = &sc->params.tp; 2763 2764 if (!is_offload(sc)) 2765 return (EOPNOTSUPP); 2766 if (sc->flags & FULL_INIT_DONE) 2767 return (EBUSY); 2768 2769 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2770 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2771 return (EINVAL); /* not power of 2 */ 2772 if (!(m->rx_pg_sz & 0x14000)) 2773 return (EINVAL); /* not 16KB or 64KB */ 2774 if (!(m->tx_pg_sz & 0x1554000)) 2775 return (EINVAL); 2776 if (m->tx_num_pg == -1) 2777 m->tx_num_pg = p->tx_num_pgs; 2778 if (m->rx_num_pg == -1) 2779 m->rx_num_pg = p->rx_num_pgs; 2780 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2781 return (EINVAL); 2782 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2783 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2784 return (EINVAL); 2785 2786 p->rx_pg_size = m->rx_pg_sz; 2787 p->tx_pg_size = m->tx_pg_sz; 2788 p->rx_num_pgs = m->rx_num_pg; 2789 p->tx_num_pgs = m->tx_num_pg; 2790 break; 2791 } 2792 case CHELSIO_SETMTUTAB: { 2793 struct ch_mtus *m = (struct ch_mtus *)data; 2794 int i; 2795 2796 if (!is_offload(sc)) 2797 return (EOPNOTSUPP); 2798 if (offload_running(sc)) 2799 return (EBUSY); 2800 if (m->nmtus != NMTUS) 2801 return (EINVAL); 2802 if (m->mtus[0] < 81) /* accommodate SACK */ 2803 return (EINVAL); 2804 2805 /* 2806 * MTUs must be in ascending order 2807 */ 2808 for (i = 1; i < NMTUS; ++i) 2809 if (m->mtus[i] < m->mtus[i - 1]) 2810 return (EINVAL); 2811 2812 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2813 break; 2814 } 2815 case CHELSIO_GETMTUTAB: { 2816 struct ch_mtus *m = (struct ch_mtus *)data; 2817 2818 if (!is_offload(sc)) 2819 return (EOPNOTSUPP); 2820 2821 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2822 m->nmtus = NMTUS; 2823 break; 2824 } 2825 case CHELSIO_GET_MEM: { 2826 struct ch_mem_range *t = (struct ch_mem_range *)data; 2827 struct mc7 *mem; 2828 uint8_t *useraddr; 2829 u64 buf[32]; 2830 2831 /* 2832 * Use these to avoid modifying len/addr in the return 2833 * struct 2834 */ 2835 uint32_t len = t->len, addr = t->addr; 2836 2837 if (!is_offload(sc)) 2838 return (EOPNOTSUPP); 2839 if (!(sc->flags & FULL_INIT_DONE)) 2840 return (EIO); /* need the memory controllers */ 2841 if ((addr & 0x7) || (len & 0x7)) 2842 return (EINVAL); 2843 if (t->mem_id == MEM_CM) 2844 mem = &sc->cm; 2845 else if (t->mem_id == MEM_PMRX) 2846 mem = &sc->pmrx; 2847 else if (t->mem_id == MEM_PMTX) 2848 mem = &sc->pmtx; 2849 else 2850 return (EINVAL); 2851 2852 /* 2853 * Version scheme: 2854 * bits 0..9: chip version 2855 * bits 10..15: chip revision 2856 */ 2857 t->version = 3 | (sc->params.rev << 10); 2858 2859 /* 2860 * Read 256 bytes at a time as len can be large and we don't 2861 * want to use huge intermediate buffers. 2862 */ 2863 useraddr = (uint8_t *)t->buf; 2864 while (len) { 2865 unsigned int chunk = min(len, sizeof(buf)); 2866 2867 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2868 if (error) 2869 return (-error); 2870 if (copyout(buf, useraddr, chunk)) 2871 return (EFAULT); 2872 useraddr += chunk; 2873 addr += chunk; 2874 len -= chunk; 2875 } 2876 break; 2877 } 2878 case CHELSIO_READ_TCAM_WORD: { 2879 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2880 2881 if (!is_offload(sc)) 2882 return (EOPNOTSUPP); 2883 if (!(sc->flags & FULL_INIT_DONE)) 2884 return (EIO); /* need MC5 */ 2885 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2886 break; 2887 } 2888 case CHELSIO_SET_TRACE_FILTER: { 2889 struct ch_trace *t = (struct ch_trace *)data; 2890 const struct trace_params *tp; 2891 2892 tp = (const struct trace_params *)&t->sip; 2893 if (t->config_tx) 2894 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2895 t->trace_tx); 2896 if (t->config_rx) 2897 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2898 t->trace_rx); 2899 break; 2900 } 2901 case CHELSIO_SET_PKTSCHED: { 2902 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2903 if (sc->open_device_map == 0) 2904 return (EAGAIN); 2905 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2906 p->binding); 2907 break; 2908 } 2909 case CHELSIO_IFCONF_GETREGS: { 2910 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2911 int reglen = cxgb_get_regs_len(); 2912 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2913 if (buf == NULL) { 2914 return (ENOMEM); 2915 } 2916 if (regs->len > reglen) 2917 regs->len = reglen; 2918 else if (regs->len < reglen) 2919 error = ENOBUFS; 2920 2921 if (!error) { 2922 cxgb_get_regs(sc, regs, buf); 2923 error = copyout(buf, regs->data, reglen); 2924 } 2925 free(buf, M_DEVBUF); 2926 2927 break; 2928 } 2929 case CHELSIO_SET_HW_SCHED: { 2930 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2931 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2932 2933 if ((sc->flags & FULL_INIT_DONE) == 0) 2934 return (EAGAIN); /* need TP to be initialized */ 2935 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2936 !in_range(t->channel, 0, 1) || 2937 !in_range(t->kbps, 0, 10000000) || 2938 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2939 !in_range(t->flow_ipg, 0, 2940 dack_ticks_to_usec(sc, 0x7ff))) 2941 return (EINVAL); 2942 2943 if (t->kbps >= 0) { 2944 error = t3_config_sched(sc, t->kbps, t->sched); 2945 if (error < 0) 2946 return (-error); 2947 } 2948 if (t->class_ipg >= 0) 2949 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2950 if (t->flow_ipg >= 0) { 2951 t->flow_ipg *= 1000; /* us -> ns */ 2952 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2953 } 2954 if (t->mode >= 0) { 2955 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2956 2957 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2958 bit, t->mode ? bit : 0); 2959 } 2960 if (t->channel >= 0) 2961 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2962 1 << t->sched, t->channel << t->sched); 2963 break; 2964 } 2965 case CHELSIO_GET_EEPROM: { 2966 int i; 2967 struct ch_eeprom *e = (struct ch_eeprom *)data; 2968 uint8_t *buf; 2969 2970 if (e->offset & 3 || e->offset >= EEPROMSIZE || 2971 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) { 2972 return (EINVAL); 2973 } 2974 2975 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2976 if (buf == NULL) { 2977 return (ENOMEM); 2978 } 2979 e->magic = EEPROM_MAGIC; 2980 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2981 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2982 2983 if (!error) 2984 error = copyout(buf + e->offset, e->data, e->len); 2985 2986 free(buf, M_DEVBUF); 2987 break; 2988 } 2989 case CHELSIO_CLEAR_STATS: { 2990 if (!(sc->flags & FULL_INIT_DONE)) 2991 return EAGAIN; 2992 2993 PORT_LOCK(pi); 2994 t3_mac_update_stats(&pi->mac); 2995 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 2996 PORT_UNLOCK(pi); 2997 break; 2998 } 2999 case CHELSIO_GET_UP_LA: { 3000 struct ch_up_la *la = (struct ch_up_la *)data; 3001 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT); 3002 if (buf == NULL) { 3003 return (ENOMEM); 3004 } 3005 if (la->bufsize < LA_BUFSIZE) 3006 error = ENOBUFS; 3007 3008 if (!error) 3009 error = -t3_get_up_la(sc, &la->stopped, &la->idx, 3010 &la->bufsize, buf); 3011 if (!error) 3012 error = copyout(buf, la->data, la->bufsize); 3013 3014 free(buf, M_DEVBUF); 3015 break; 3016 } 3017 case CHELSIO_GET_UP_IOQS: { 3018 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data; 3019 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT); 3020 uint32_t *v; 3021 3022 if (buf == NULL) { 3023 return (ENOMEM); 3024 } 3025 if (ioqs->bufsize < IOQS_BUFSIZE) 3026 error = ENOBUFS; 3027 3028 if (!error) 3029 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf); 3030 3031 if (!error) { 3032 v = (uint32_t *)buf; 3033 3034 ioqs->ioq_rx_enable = *v++; 3035 ioqs->ioq_tx_enable = *v++; 3036 ioqs->ioq_rx_status = *v++; 3037 ioqs->ioq_tx_status = *v++; 3038 3039 error = copyout(v, ioqs->data, ioqs->bufsize); 3040 } 3041 3042 free(buf, M_DEVBUF); 3043 break; 3044 } 3045 case CHELSIO_SET_FILTER: { 3046 struct ch_filter *f = (struct ch_filter *)data; 3047 struct filter_info *p; 3048 unsigned int nfilters = sc->params.mc5.nfilters; 3049 3050 if (!is_offload(sc)) 3051 return (EOPNOTSUPP); /* No TCAM */ 3052 if (!(sc->flags & FULL_INIT_DONE)) 3053 return (EAGAIN); /* mc5 not setup yet */ 3054 if (nfilters == 0) 3055 return (EBUSY); /* TOE will use TCAM */ 3056 3057 /* sanity checks */ 3058 if (f->filter_id >= nfilters || 3059 (f->val.dip && f->mask.dip != 0xffffffff) || 3060 (f->val.sport && f->mask.sport != 0xffff) || 3061 (f->val.dport && f->mask.dport != 0xffff) || 3062 (f->val.vlan && f->mask.vlan != 0xfff) || 3063 (f->val.vlan_prio && 3064 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) || 3065 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) || 3066 f->qset >= SGE_QSETS || 3067 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE) 3068 return (EINVAL); 3069 3070 /* Was allocated with M_WAITOK */ 3071 KASSERT(sc->filters, ("filter table NULL\n")); 3072 3073 p = &sc->filters[f->filter_id]; 3074 if (p->locked) 3075 return (EPERM); 3076 3077 bzero(p, sizeof(*p)); 3078 p->sip = f->val.sip; 3079 p->sip_mask = f->mask.sip; 3080 p->dip = f->val.dip; 3081 p->sport = f->val.sport; 3082 p->dport = f->val.dport; 3083 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff; 3084 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) : 3085 FILTER_NO_VLAN_PRI; 3086 p->mac_hit = f->mac_hit; 3087 p->mac_vld = f->mac_addr_idx != 0xffff; 3088 p->mac_idx = f->mac_addr_idx; 3089 p->pkt_type = f->proto; 3090 p->report_filter_id = f->want_filter_id; 3091 p->pass = f->pass; 3092 p->rss = f->rss; 3093 p->qset = f->qset; 3094 3095 error = set_filter(sc, f->filter_id, p); 3096 if (error == 0) 3097 p->valid = 1; 3098 break; 3099 } 3100 case CHELSIO_DEL_FILTER: { 3101 struct ch_filter *f = (struct ch_filter *)data; 3102 struct filter_info *p; 3103 unsigned int nfilters = sc->params.mc5.nfilters; 3104 3105 if (!is_offload(sc)) 3106 return (EOPNOTSUPP); 3107 if (!(sc->flags & FULL_INIT_DONE)) 3108 return (EAGAIN); 3109 if (nfilters == 0 || sc->filters == NULL) 3110 return (EINVAL); 3111 if (f->filter_id >= nfilters) 3112 return (EINVAL); 3113 3114 p = &sc->filters[f->filter_id]; 3115 if (p->locked) 3116 return (EPERM); 3117 if (!p->valid) 3118 return (EFAULT); /* Read "Bad address" as "Bad index" */ 3119 3120 bzero(p, sizeof(*p)); 3121 p->sip = p->sip_mask = 0xffffffff; 3122 p->vlan = 0xfff; 3123 p->vlan_prio = FILTER_NO_VLAN_PRI; 3124 p->pkt_type = 1; 3125 error = set_filter(sc, f->filter_id, p); 3126 break; 3127 } 3128 case CHELSIO_GET_FILTER: { 3129 struct ch_filter *f = (struct ch_filter *)data; 3130 struct filter_info *p; 3131 unsigned int i, nfilters = sc->params.mc5.nfilters; 3132 3133 if (!is_offload(sc)) 3134 return (EOPNOTSUPP); 3135 if (!(sc->flags & FULL_INIT_DONE)) 3136 return (EAGAIN); 3137 if (nfilters == 0 || sc->filters == NULL) 3138 return (EINVAL); 3139 3140 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1; 3141 for (; i < nfilters; i++) { 3142 p = &sc->filters[i]; 3143 if (!p->valid) 3144 continue; 3145 3146 bzero(f, sizeof(*f)); 3147 3148 f->filter_id = i; 3149 f->val.sip = p->sip; 3150 f->mask.sip = p->sip_mask; 3151 f->val.dip = p->dip; 3152 f->mask.dip = p->dip ? 0xffffffff : 0; 3153 f->val.sport = p->sport; 3154 f->mask.sport = p->sport ? 0xffff : 0; 3155 f->val.dport = p->dport; 3156 f->mask.dport = p->dport ? 0xffff : 0; 3157 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan; 3158 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff; 3159 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3160 0 : p->vlan_prio; 3161 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3162 0 : FILTER_NO_VLAN_PRI; 3163 f->mac_hit = p->mac_hit; 3164 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff; 3165 f->proto = p->pkt_type; 3166 f->want_filter_id = p->report_filter_id; 3167 f->pass = p->pass; 3168 f->rss = p->rss; 3169 f->qset = p->qset; 3170 3171 break; 3172 } 3173 3174 if (i == nfilters) 3175 f->filter_id = 0xffffffff; 3176 break; 3177 } 3178 default: 3179 return (EOPNOTSUPP); 3180 break; 3181 } 3182 3183 return (error); 3184 } 3185 3186 static __inline void 3187 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 3188 unsigned int end) 3189 { 3190 uint32_t *p = (uint32_t *)(buf + start); 3191 3192 for ( ; start <= end; start += sizeof(uint32_t)) 3193 *p++ = t3_read_reg(ap, start); 3194 } 3195 3196 #define T3_REGMAP_SIZE (3 * 1024) 3197 static int 3198 cxgb_get_regs_len(void) 3199 { 3200 return T3_REGMAP_SIZE; 3201 } 3202 3203 static void 3204 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 3205 { 3206 3207 /* 3208 * Version scheme: 3209 * bits 0..9: chip version 3210 * bits 10..15: chip revision 3211 * bit 31: set for PCIe cards 3212 */ 3213 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 3214 3215 /* 3216 * We skip the MAC statistics registers because they are clear-on-read. 3217 * Also reading multi-register stats would need to synchronize with the 3218 * periodic mac stats accumulation. Hard to justify the complexity. 3219 */ 3220 memset(buf, 0, cxgb_get_regs_len()); 3221 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 3222 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 3223 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 3224 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 3225 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 3226 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 3227 XGM_REG(A_XGM_SERDES_STAT3, 1)); 3228 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 3229 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 3230 } 3231 3232 static int 3233 alloc_filters(struct adapter *sc) 3234 { 3235 struct filter_info *p; 3236 unsigned int nfilters = sc->params.mc5.nfilters; 3237 3238 if (nfilters == 0) 3239 return (0); 3240 3241 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO); 3242 sc->filters = p; 3243 3244 p = &sc->filters[nfilters - 1]; 3245 p->vlan = 0xfff; 3246 p->vlan_prio = FILTER_NO_VLAN_PRI; 3247 p->pass = p->rss = p->valid = p->locked = 1; 3248 3249 return (0); 3250 } 3251 3252 static int 3253 setup_hw_filters(struct adapter *sc) 3254 { 3255 int i, rc; 3256 unsigned int nfilters = sc->params.mc5.nfilters; 3257 3258 if (!sc->filters) 3259 return (0); 3260 3261 t3_enable_filters(sc); 3262 3263 for (i = rc = 0; i < nfilters && !rc; i++) { 3264 if (sc->filters[i].locked) 3265 rc = set_filter(sc, i, &sc->filters[i]); 3266 } 3267 3268 return (rc); 3269 } 3270 3271 static int 3272 set_filter(struct adapter *sc, int id, const struct filter_info *f) 3273 { 3274 int len; 3275 struct mbuf *m; 3276 struct ulp_txpkt *txpkt; 3277 struct work_request_hdr *wr; 3278 struct cpl_pass_open_req *oreq; 3279 struct cpl_set_tcb_field *sreq; 3280 3281 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq); 3282 KASSERT(len <= MHLEN, ("filter request too big for an mbuf")); 3283 3284 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes - 3285 sc->params.mc5.nfilters; 3286 3287 m = m_gethdr(M_WAITOK, MT_DATA); 3288 m->m_len = m->m_pkthdr.len = len; 3289 bzero(mtod(m, char *), len); 3290 3291 wr = mtod(m, struct work_request_hdr *); 3292 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); 3293 3294 oreq = (struct cpl_pass_open_req *)(wr + 1); 3295 txpkt = (struct ulp_txpkt *)oreq; 3296 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3297 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8)); 3298 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id)); 3299 oreq->local_port = htons(f->dport); 3300 oreq->peer_port = htons(f->sport); 3301 oreq->local_ip = htonl(f->dip); 3302 oreq->peer_ip = htonl(f->sip); 3303 oreq->peer_netmask = htonl(f->sip_mask); 3304 oreq->opt0h = 0; 3305 oreq->opt0l = htonl(F_NO_OFFLOAD); 3306 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) | 3307 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) | 3308 V_VLAN_PRI(f->vlan_prio >> 1) | 3309 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) | 3310 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) | 3311 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4))); 3312 3313 sreq = (struct cpl_set_tcb_field *)(oreq + 1); 3314 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL, 3315 (f->report_filter_id << 15) | (1 << 23) | 3316 ((u64)f->pass << 35) | ((u64)!f->rss << 36)); 3317 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1); 3318 t3_mgmt_tx(sc, m); 3319 3320 if (f->pass && !f->rss) { 3321 len = sizeof(*sreq); 3322 m = m_gethdr(M_WAITOK, MT_DATA); 3323 m->m_len = m->m_pkthdr.len = len; 3324 bzero(mtod(m, char *), len); 3325 sreq = mtod(m, struct cpl_set_tcb_field *); 3326 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3327 mk_set_tcb_field(sreq, id, 25, 0x3f80000, 3328 (u64)sc->rrss_map[f->qset] << 19); 3329 t3_mgmt_tx(sc, m); 3330 } 3331 return 0; 3332 } 3333 3334 static inline void 3335 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid, 3336 unsigned int word, u64 mask, u64 val) 3337 { 3338 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 3339 req->reply = V_NO_REPLY(1); 3340 req->cpu_idx = 0; 3341 req->word = htons(word); 3342 req->mask = htobe64(mask); 3343 req->val = htobe64(val); 3344 } 3345 3346 static inline void 3347 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, 3348 unsigned int word, u64 mask, u64 val) 3349 { 3350 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 3351 3352 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3353 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 3354 mk_set_tcb_field(req, tid, word, mask, val); 3355 } 3356 3357 void 3358 t3_iterate(void (*func)(struct adapter *, void *), void *arg) 3359 { 3360 struct adapter *sc; 3361 3362 mtx_lock(&t3_list_lock); 3363 SLIST_FOREACH(sc, &t3_list, link) { 3364 /* 3365 * func should not make any assumptions about what state sc is 3366 * in - the only guarantee is that sc->sc_lock is a valid lock. 3367 */ 3368 func(sc, arg); 3369 } 3370 mtx_unlock(&t3_list_lock); 3371 } 3372 3373 #ifdef TCP_OFFLOAD 3374 static int 3375 toe_capability(struct port_info *pi, int enable) 3376 { 3377 int rc; 3378 struct adapter *sc = pi->adapter; 3379 3380 ADAPTER_LOCK_ASSERT_OWNED(sc); 3381 3382 if (!is_offload(sc)) 3383 return (ENODEV); 3384 3385 if (enable) { 3386 if (!(sc->flags & FULL_INIT_DONE)) { 3387 log(LOG_WARNING, 3388 "You must enable a cxgb interface first\n"); 3389 return (EAGAIN); 3390 } 3391 3392 if (isset(&sc->offload_map, pi->port_id)) 3393 return (0); 3394 3395 if (!(sc->flags & TOM_INIT_DONE)) { 3396 rc = t3_activate_uld(sc, ULD_TOM); 3397 if (rc == EAGAIN) { 3398 log(LOG_WARNING, 3399 "You must kldload t3_tom.ko before trying " 3400 "to enable TOE on a cxgb interface.\n"); 3401 } 3402 if (rc != 0) 3403 return (rc); 3404 KASSERT(sc->tom_softc != NULL, 3405 ("%s: TOM activated but softc NULL", __func__)); 3406 KASSERT(sc->flags & TOM_INIT_DONE, 3407 ("%s: TOM activated but flag not set", __func__)); 3408 } 3409 3410 setbit(&sc->offload_map, pi->port_id); 3411 3412 /* 3413 * XXX: Temporary code to allow iWARP to be enabled when TOE is 3414 * enabled on any port. Need to figure out how to enable, 3415 * disable, load, and unload iWARP cleanly. 3416 */ 3417 if (!isset(&sc->offload_map, MAX_NPORTS) && 3418 t3_activate_uld(sc, ULD_IWARP) == 0) 3419 setbit(&sc->offload_map, MAX_NPORTS); 3420 } else { 3421 if (!isset(&sc->offload_map, pi->port_id)) 3422 return (0); 3423 3424 KASSERT(sc->flags & TOM_INIT_DONE, 3425 ("%s: TOM never initialized?", __func__)); 3426 clrbit(&sc->offload_map, pi->port_id); 3427 } 3428 3429 return (0); 3430 } 3431 3432 /* 3433 * Add an upper layer driver to the global list. 3434 */ 3435 int 3436 t3_register_uld(struct uld_info *ui) 3437 { 3438 int rc = 0; 3439 struct uld_info *u; 3440 3441 mtx_lock(&t3_uld_list_lock); 3442 SLIST_FOREACH(u, &t3_uld_list, link) { 3443 if (u->uld_id == ui->uld_id) { 3444 rc = EEXIST; 3445 goto done; 3446 } 3447 } 3448 3449 SLIST_INSERT_HEAD(&t3_uld_list, ui, link); 3450 ui->refcount = 0; 3451 done: 3452 mtx_unlock(&t3_uld_list_lock); 3453 return (rc); 3454 } 3455 3456 int 3457 t3_unregister_uld(struct uld_info *ui) 3458 { 3459 int rc = EINVAL; 3460 struct uld_info *u; 3461 3462 mtx_lock(&t3_uld_list_lock); 3463 3464 SLIST_FOREACH(u, &t3_uld_list, link) { 3465 if (u == ui) { 3466 if (ui->refcount > 0) { 3467 rc = EBUSY; 3468 goto done; 3469 } 3470 3471 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link); 3472 rc = 0; 3473 goto done; 3474 } 3475 } 3476 done: 3477 mtx_unlock(&t3_uld_list_lock); 3478 return (rc); 3479 } 3480 3481 int 3482 t3_activate_uld(struct adapter *sc, int id) 3483 { 3484 int rc = EAGAIN; 3485 struct uld_info *ui; 3486 3487 mtx_lock(&t3_uld_list_lock); 3488 3489 SLIST_FOREACH(ui, &t3_uld_list, link) { 3490 if (ui->uld_id == id) { 3491 rc = ui->activate(sc); 3492 if (rc == 0) 3493 ui->refcount++; 3494 goto done; 3495 } 3496 } 3497 done: 3498 mtx_unlock(&t3_uld_list_lock); 3499 3500 return (rc); 3501 } 3502 3503 int 3504 t3_deactivate_uld(struct adapter *sc, int id) 3505 { 3506 int rc = EINVAL; 3507 struct uld_info *ui; 3508 3509 mtx_lock(&t3_uld_list_lock); 3510 3511 SLIST_FOREACH(ui, &t3_uld_list, link) { 3512 if (ui->uld_id == id) { 3513 rc = ui->deactivate(sc); 3514 if (rc == 0) 3515 ui->refcount--; 3516 goto done; 3517 } 3518 } 3519 done: 3520 mtx_unlock(&t3_uld_list_lock); 3521 3522 return (rc); 3523 } 3524 3525 static int 3526 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused, 3527 struct mbuf *m) 3528 { 3529 m_freem(m); 3530 return (EDOOFUS); 3531 } 3532 3533 int 3534 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3535 { 3536 uintptr_t *loc, new; 3537 3538 if (opcode >= NUM_CPL_HANDLERS) 3539 return (EINVAL); 3540 3541 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3542 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3543 atomic_store_rel_ptr(loc, new); 3544 3545 return (0); 3546 } 3547 #endif 3548 3549 static int 3550 cxgbc_mod_event(module_t mod, int cmd, void *arg) 3551 { 3552 int rc = 0; 3553 3554 switch (cmd) { 3555 case MOD_LOAD: 3556 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF); 3557 SLIST_INIT(&t3_list); 3558 #ifdef TCP_OFFLOAD 3559 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF); 3560 SLIST_INIT(&t3_uld_list); 3561 #endif 3562 break; 3563 3564 case MOD_UNLOAD: 3565 #ifdef TCP_OFFLOAD 3566 mtx_lock(&t3_uld_list_lock); 3567 if (!SLIST_EMPTY(&t3_uld_list)) { 3568 rc = EBUSY; 3569 mtx_unlock(&t3_uld_list_lock); 3570 break; 3571 } 3572 mtx_unlock(&t3_uld_list_lock); 3573 mtx_destroy(&t3_uld_list_lock); 3574 #endif 3575 mtx_lock(&t3_list_lock); 3576 if (!SLIST_EMPTY(&t3_list)) { 3577 rc = EBUSY; 3578 mtx_unlock(&t3_list_lock); 3579 break; 3580 } 3581 mtx_unlock(&t3_list_lock); 3582 mtx_destroy(&t3_list_lock); 3583 break; 3584 } 3585 3586 return (rc); 3587 } 3588 3589 #ifdef DEBUGNET 3590 static void 3591 cxgb_debugnet_init(if_t ifp, int *nrxr, int *ncl, int *clsize) 3592 { 3593 struct port_info *pi; 3594 adapter_t *adap; 3595 3596 pi = if_getsoftc(ifp); 3597 adap = pi->adapter; 3598 ADAPTER_LOCK(adap); 3599 *nrxr = adap->nqsets; 3600 *ncl = adap->sge.qs[0].fl[1].size; 3601 *clsize = adap->sge.qs[0].fl[1].buf_size; 3602 ADAPTER_UNLOCK(adap); 3603 } 3604 3605 static void 3606 cxgb_debugnet_event(if_t ifp, enum debugnet_ev event) 3607 { 3608 struct port_info *pi; 3609 struct sge_qset *qs; 3610 int i; 3611 3612 pi = if_getsoftc(ifp); 3613 if (event == DEBUGNET_START) 3614 for (i = 0; i < pi->adapter->nqsets; i++) { 3615 qs = &pi->adapter->sge.qs[i]; 3616 3617 /* Need to reinit after debugnet_mbuf_start(). */ 3618 qs->fl[0].zone = zone_pack; 3619 qs->fl[1].zone = zone_clust; 3620 qs->lro.enabled = 0; 3621 } 3622 } 3623 3624 static int 3625 cxgb_debugnet_transmit(if_t ifp, struct mbuf *m) 3626 { 3627 struct port_info *pi; 3628 struct sge_qset *qs; 3629 3630 pi = if_getsoftc(ifp); 3631 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 3632 IFF_DRV_RUNNING) 3633 return (ENOENT); 3634 3635 qs = &pi->adapter->sge.qs[pi->first_qset]; 3636 return (cxgb_debugnet_encap(qs, &m)); 3637 } 3638 3639 static int 3640 cxgb_debugnet_poll(if_t ifp, int count) 3641 { 3642 struct port_info *pi; 3643 adapter_t *adap; 3644 int i; 3645 3646 pi = if_getsoftc(ifp); 3647 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 3648 return (ENOENT); 3649 3650 adap = pi->adapter; 3651 for (i = 0; i < adap->nqsets; i++) 3652 (void)cxgb_debugnet_poll_rx(adap, &adap->sge.qs[i]); 3653 (void)cxgb_debugnet_poll_tx(&adap->sge.qs[pi->first_qset]); 3654 return (0); 3655 } 3656 #endif /* DEBUGNET */ 3657