1 /************************************************************************** 2 SPDX-License-Identifier: BSD-2-Clause 3 4 Copyright (c) 2007-2009, Chelsio Inc. 5 All rights reserved. 6 7 Redistribution and use in source and binary forms, with or without 8 modification, are permitted provided that the following conditions are met: 9 10 1. Redistributions of source code must retain the above copyright notice, 11 this list of conditions and the following disclaimer. 12 13 2. Neither the name of the Chelsio Corporation nor the names of its 14 contributors may be used to endorse or promote products derived from 15 this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 POSSIBILITY OF SUCH DAMAGE. 28 29 ***************************************************************************/ 30 31 #include <sys/cdefs.h> 32 #include "opt_inet.h" 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/bus.h> 38 #include <sys/module.h> 39 #include <sys/conf.h> 40 #include <machine/bus.h> 41 #include <machine/resource.h> 42 #include <sys/ktr.h> 43 #include <sys/rman.h> 44 #include <sys/ioccom.h> 45 #include <sys/mbuf.h> 46 #include <sys/linker.h> 47 #include <sys/firmware.h> 48 #include <sys/socket.h> 49 #include <sys/sockio.h> 50 #include <sys/smp.h> 51 #include <sys/sysctl.h> 52 #include <sys/syslog.h> 53 #include <sys/queue.h> 54 #include <sys/taskqueue.h> 55 #include <sys/proc.h> 56 57 #include <net/bpf.h> 58 #include <net/debugnet.h> 59 #include <net/ethernet.h> 60 #include <net/if.h> 61 #include <net/if_var.h> 62 #include <net/if_arp.h> 63 #include <net/if_dl.h> 64 #include <net/if_media.h> 65 #include <net/if_types.h> 66 #include <net/if_vlan_var.h> 67 68 #include <netinet/in_systm.h> 69 #include <netinet/in.h> 70 #include <netinet/if_ether.h> 71 #include <netinet/ip.h> 72 #include <netinet/ip.h> 73 #include <netinet/tcp.h> 74 #include <netinet/udp.h> 75 76 #include <dev/pci/pcireg.h> 77 #include <dev/pci/pcivar.h> 78 79 #include <cxgb_include.h> 80 81 #ifdef PRIV_SUPPORTED 82 #include <sys/priv.h> 83 #endif 84 85 static int cxgb_setup_interrupts(adapter_t *); 86 static void cxgb_teardown_interrupts(adapter_t *); 87 static void cxgb_init(void *); 88 static int cxgb_init_locked(struct port_info *); 89 static int cxgb_uninit_locked(struct port_info *); 90 static int cxgb_uninit_synchronized(struct port_info *); 91 static int cxgb_ioctl(if_t, unsigned long, caddr_t); 92 static int cxgb_media_change(if_t); 93 static int cxgb_ifm_type(int); 94 static void cxgb_build_medialist(struct port_info *); 95 static void cxgb_media_status(if_t, struct ifmediareq *); 96 static uint64_t cxgb_get_counter(if_t, ift_counter); 97 static int setup_sge_qsets(adapter_t *); 98 static void cxgb_async_intr(void *); 99 static void cxgb_tick_handler(void *, int); 100 static void cxgb_tick(void *); 101 static void link_check_callout(void *); 102 static void check_link_status(void *, int); 103 static void setup_rss(adapter_t *sc); 104 static int alloc_filters(struct adapter *); 105 static int setup_hw_filters(struct adapter *); 106 static int set_filter(struct adapter *, int, const struct filter_info *); 107 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int, 108 unsigned int, u64, u64); 109 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int, 110 unsigned int, u64, u64); 111 #ifdef TCP_OFFLOAD 112 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *); 113 #endif 114 115 /* Attachment glue for the PCI controller end of the device. Each port of 116 * the device is attached separately, as defined later. 117 */ 118 static int cxgb_controller_probe(device_t); 119 static int cxgb_controller_attach(device_t); 120 static int cxgb_controller_detach(device_t); 121 static void cxgb_free(struct adapter *); 122 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 123 unsigned int end); 124 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 125 static int cxgb_get_regs_len(void); 126 static void touch_bars(device_t dev); 127 static void cxgb_update_mac_settings(struct port_info *p); 128 #ifdef TCP_OFFLOAD 129 static int toe_capability(struct port_info *, int); 130 #endif 131 132 /* Table for probing the cards. The desc field isn't actually used */ 133 struct cxgb_ident { 134 uint16_t vendor; 135 uint16_t device; 136 int index; 137 char *desc; 138 } cxgb_identifiers[] = { 139 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 140 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 141 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 142 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 143 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 144 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 145 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 146 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 147 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 148 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 149 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 150 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"}, 151 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"}, 152 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"}, 153 {0, 0, 0, NULL} 154 }; 155 156 static device_method_t cxgb_controller_methods[] = { 157 DEVMETHOD(device_probe, cxgb_controller_probe), 158 DEVMETHOD(device_attach, cxgb_controller_attach), 159 DEVMETHOD(device_detach, cxgb_controller_detach), 160 161 DEVMETHOD_END 162 }; 163 164 static driver_t cxgb_controller_driver = { 165 "cxgbc", 166 cxgb_controller_methods, 167 sizeof(struct adapter) 168 }; 169 170 static int cxgbc_mod_event(module_t, int, void *); 171 172 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgbc_mod_event, NULL); 173 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers, 174 nitems(cxgb_identifiers) - 1); 175 MODULE_VERSION(cxgbc, 1); 176 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1); 177 178 /* 179 * Attachment glue for the ports. Attachment is done directly to the 180 * controller device. 181 */ 182 static int cxgb_port_probe(device_t); 183 static int cxgb_port_attach(device_t); 184 static int cxgb_port_detach(device_t); 185 186 static device_method_t cxgb_port_methods[] = { 187 DEVMETHOD(device_probe, cxgb_port_probe), 188 DEVMETHOD(device_attach, cxgb_port_attach), 189 DEVMETHOD(device_detach, cxgb_port_detach), 190 { 0, 0 } 191 }; 192 193 static driver_t cxgb_port_driver = { 194 "cxgb", 195 cxgb_port_methods, 196 0 197 }; 198 199 static d_ioctl_t cxgb_extension_ioctl; 200 static d_open_t cxgb_extension_open; 201 static d_close_t cxgb_extension_close; 202 203 static struct cdevsw cxgb_cdevsw = { 204 .d_version = D_VERSION, 205 .d_flags = 0, 206 .d_open = cxgb_extension_open, 207 .d_close = cxgb_extension_close, 208 .d_ioctl = cxgb_extension_ioctl, 209 .d_name = "cxgb", 210 }; 211 212 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, 0, 0); 213 MODULE_VERSION(cxgb, 1); 214 215 DEBUGNET_DEFINE(cxgb); 216 217 static struct mtx t3_list_lock; 218 static SLIST_HEAD(, adapter) t3_list; 219 #ifdef TCP_OFFLOAD 220 static struct mtx t3_uld_list_lock; 221 static SLIST_HEAD(, uld_info) t3_uld_list; 222 #endif 223 224 /* 225 * The driver uses the best interrupt scheme available on a platform in the 226 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 227 * of these schemes the driver may consider as follows: 228 * 229 * msi = 2: choose from among all three options 230 * msi = 1 : only consider MSI and pin interrupts 231 * msi = 0: force pin interrupts 232 */ 233 static int msi_allowed = 2; 234 235 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 236 "CXGB driver parameters"); 237 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 238 "MSI-X, MSI, INTx selector"); 239 240 /* 241 * The driver uses an auto-queue algorithm by default. 242 * To disable it and force a single queue-set per port, use multiq = 0 243 */ 244 static int multiq = 1; 245 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0, 246 "use min(ncpus/ports, 8) queue-sets per port"); 247 248 /* 249 * By default the driver will not update the firmware unless 250 * it was compiled against a newer version 251 * 252 */ 253 static int force_fw_update = 0; 254 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 255 "update firmware even if up to date"); 256 257 int cxgb_use_16k_clusters = -1; 258 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 259 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 260 261 static int nfilters = -1; 262 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN, 263 &nfilters, 0, "max number of entries in the filter table"); 264 265 enum { 266 MAX_TXQ_ENTRIES = 16384, 267 MAX_CTRL_TXQ_ENTRIES = 1024, 268 MAX_RSPQ_ENTRIES = 16384, 269 MAX_RX_BUFFERS = 16384, 270 MAX_RX_JUMBO_BUFFERS = 16384, 271 MIN_TXQ_ENTRIES = 4, 272 MIN_CTRL_TXQ_ENTRIES = 4, 273 MIN_RSPQ_ENTRIES = 32, 274 MIN_FL_ENTRIES = 32, 275 MIN_FL_JUMBO_ENTRIES = 32 276 }; 277 278 struct filter_info { 279 u32 sip; 280 u32 sip_mask; 281 u32 dip; 282 u16 sport; 283 u16 dport; 284 u32 vlan:12; 285 u32 vlan_prio:3; 286 u32 mac_hit:1; 287 u32 mac_idx:4; 288 u32 mac_vld:1; 289 u32 pkt_type:2; 290 u32 report_filter_id:1; 291 u32 pass:1; 292 u32 rss:1; 293 u32 qset:3; 294 u32 locked:1; 295 u32 valid:1; 296 }; 297 298 enum { FILTER_NO_VLAN_PRI = 7 }; 299 300 #define EEPROM_MAGIC 0x38E2F10C 301 302 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 303 304 305 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 306 307 308 static __inline char 309 t3rev2char(struct adapter *adapter) 310 { 311 char rev = 'z'; 312 313 switch(adapter->params.rev) { 314 case T3_REV_A: 315 rev = 'a'; 316 break; 317 case T3_REV_B: 318 case T3_REV_B2: 319 rev = 'b'; 320 break; 321 case T3_REV_C: 322 rev = 'c'; 323 break; 324 } 325 return rev; 326 } 327 328 static struct cxgb_ident * 329 cxgb_get_ident(device_t dev) 330 { 331 struct cxgb_ident *id; 332 333 for (id = cxgb_identifiers; id->desc != NULL; id++) { 334 if ((id->vendor == pci_get_vendor(dev)) && 335 (id->device == pci_get_device(dev))) { 336 return (id); 337 } 338 } 339 return (NULL); 340 } 341 342 static const struct adapter_info * 343 cxgb_get_adapter_info(device_t dev) 344 { 345 struct cxgb_ident *id; 346 const struct adapter_info *ai; 347 348 id = cxgb_get_ident(dev); 349 if (id == NULL) 350 return (NULL); 351 352 ai = t3_get_adapter_info(id->index); 353 354 return (ai); 355 } 356 357 static int 358 cxgb_controller_probe(device_t dev) 359 { 360 const struct adapter_info *ai; 361 const char *ports; 362 int nports; 363 364 ai = cxgb_get_adapter_info(dev); 365 if (ai == NULL) 366 return (ENXIO); 367 368 nports = ai->nports0 + ai->nports1; 369 if (nports == 1) 370 ports = "port"; 371 else 372 ports = "ports"; 373 374 device_set_descf(dev, "%s, %d %s", ai->desc, nports, ports); 375 return (BUS_PROBE_DEFAULT); 376 } 377 378 #define FW_FNAME "cxgb_t3fw" 379 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom" 380 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram" 381 382 static int 383 upgrade_fw(adapter_t *sc) 384 { 385 const struct firmware *fw; 386 int status; 387 u32 vers; 388 389 if ((fw = firmware_get(FW_FNAME)) == NULL) { 390 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 391 return (ENOENT); 392 } else 393 device_printf(sc->dev, "installing firmware on card\n"); 394 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 395 396 if (status != 0) { 397 device_printf(sc->dev, "failed to install firmware: %d\n", 398 status); 399 } else { 400 t3_get_fw_version(sc, &vers); 401 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 402 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 403 G_FW_VERSION_MICRO(vers)); 404 } 405 406 firmware_put(fw, FIRMWARE_UNLOAD); 407 408 return (status); 409 } 410 411 /* 412 * The cxgb_controller_attach function is responsible for the initial 413 * bringup of the device. Its responsibilities include: 414 * 415 * 1. Determine if the device supports MSI or MSI-X. 416 * 2. Allocate bus resources so that we can access the Base Address Register 417 * 3. Create and initialize mutexes for the controller and its control 418 * logic such as SGE and MDIO. 419 * 4. Call hardware specific setup routine for the adapter as a whole. 420 * 5. Allocate the BAR for doing MSI-X. 421 * 6. Setup the line interrupt iff MSI-X is not supported. 422 * 7. Create the driver's taskq. 423 * 8. Start one task queue service thread. 424 * 9. Check if the firmware and SRAM are up-to-date. They will be 425 * auto-updated later (before FULL_INIT_DONE), if required. 426 * 10. Create a child device for each MAC (port) 427 * 11. Initialize T3 private state. 428 * 12. Trigger the LED 429 * 13. Setup offload iff supported. 430 * 14. Reset/restart the tick callout. 431 * 15. Attach sysctls 432 * 433 * NOTE: Any modification or deviation from this list MUST be reflected in 434 * the above comment. Failure to do so will result in problems on various 435 * error conditions including link flapping. 436 */ 437 static int 438 cxgb_controller_attach(device_t dev) 439 { 440 device_t child; 441 const struct adapter_info *ai; 442 struct adapter *sc; 443 int i, error = 0; 444 uint32_t vers; 445 int port_qsets = 1; 446 int msi_needed, reg; 447 448 sc = device_get_softc(dev); 449 sc->dev = dev; 450 sc->msi_count = 0; 451 ai = cxgb_get_adapter_info(dev); 452 453 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 454 device_get_unit(dev)); 455 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 456 457 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 458 device_get_unit(dev)); 459 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 460 device_get_unit(dev)); 461 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 462 device_get_unit(dev)); 463 464 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 465 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 466 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 467 468 mtx_lock(&t3_list_lock); 469 SLIST_INSERT_HEAD(&t3_list, sc, link); 470 mtx_unlock(&t3_list_lock); 471 472 /* find the PCIe link width and set max read request to 4KB*/ 473 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 474 uint16_t lnk; 475 476 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2); 477 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4; 478 if (sc->link_width < 8 && 479 (ai->caps & SUPPORTED_10000baseT_Full)) { 480 device_printf(sc->dev, 481 "PCIe x%d Link, expect reduced performance\n", 482 sc->link_width); 483 } 484 485 pci_set_max_read_req(dev, 4096); 486 } 487 488 touch_bars(dev); 489 pci_enable_busmaster(dev); 490 /* 491 * Allocate the registers and make them available to the driver. 492 * The registers that we care about for NIC mode are in BAR 0 493 */ 494 sc->regs_rid = PCIR_BAR(0); 495 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 496 &sc->regs_rid, RF_ACTIVE)) == NULL) { 497 device_printf(dev, "Cannot allocate BAR region 0\n"); 498 error = ENXIO; 499 goto out; 500 } 501 502 sc->bt = rman_get_bustag(sc->regs_res); 503 sc->bh = rman_get_bushandle(sc->regs_res); 504 sc->mmio_len = rman_get_size(sc->regs_res); 505 506 for (i = 0; i < MAX_NPORTS; i++) 507 sc->port[i].adapter = sc; 508 509 if (t3_prep_adapter(sc, ai, 1) < 0) { 510 printf("prep adapter failed\n"); 511 error = ENODEV; 512 goto out; 513 } 514 515 sc->udbs_rid = PCIR_BAR(2); 516 sc->udbs_res = NULL; 517 if (is_offload(sc) && 518 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 519 &sc->udbs_rid, RF_ACTIVE)) == NULL)) { 520 device_printf(dev, "Cannot allocate BAR region 1\n"); 521 error = ENXIO; 522 goto out; 523 } 524 525 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 526 * enough messages for the queue sets. If that fails, try falling 527 * back to MSI. If that fails, then try falling back to the legacy 528 * interrupt pin model. 529 */ 530 sc->msix_regs_rid = 0x20; 531 if ((msi_allowed >= 2) && 532 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 533 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 534 535 if (multiq) 536 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus); 537 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1; 538 539 if (pci_msix_count(dev) == 0 || 540 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 || 541 sc->msi_count != msi_needed) { 542 device_printf(dev, "alloc msix failed - " 543 "msi_count=%d, msi_needed=%d, err=%d; " 544 "will try MSI\n", sc->msi_count, 545 msi_needed, error); 546 sc->msi_count = 0; 547 port_qsets = 1; 548 pci_release_msi(dev); 549 bus_release_resource(dev, SYS_RES_MEMORY, 550 sc->msix_regs_rid, sc->msix_regs_res); 551 sc->msix_regs_res = NULL; 552 } else { 553 sc->flags |= USING_MSIX; 554 sc->cxgb_intr = cxgb_async_intr; 555 device_printf(dev, 556 "using MSI-X interrupts (%u vectors)\n", 557 sc->msi_count); 558 } 559 } 560 561 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 562 sc->msi_count = 1; 563 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) { 564 device_printf(dev, "alloc msi failed - " 565 "err=%d; will try INTx\n", error); 566 sc->msi_count = 0; 567 port_qsets = 1; 568 pci_release_msi(dev); 569 } else { 570 sc->flags |= USING_MSI; 571 sc->cxgb_intr = t3_intr_msi; 572 device_printf(dev, "using MSI interrupts\n"); 573 } 574 } 575 if (sc->msi_count == 0) { 576 device_printf(dev, "using line interrupts\n"); 577 sc->cxgb_intr = t3b_intr; 578 } 579 580 /* Create a private taskqueue thread for handling driver events */ 581 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 582 taskqueue_thread_enqueue, &sc->tq); 583 if (sc->tq == NULL) { 584 device_printf(dev, "failed to allocate controller task queue\n"); 585 goto out; 586 } 587 588 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 589 device_get_nameunit(dev)); 590 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 591 592 593 /* Create a periodic callout for checking adapter status */ 594 callout_init(&sc->cxgb_tick_ch, 1); 595 596 if (t3_check_fw_version(sc) < 0 || force_fw_update) { 597 /* 598 * Warn user that a firmware update will be attempted in init. 599 */ 600 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 601 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 602 sc->flags &= ~FW_UPTODATE; 603 } else { 604 sc->flags |= FW_UPTODATE; 605 } 606 607 if (t3_check_tpsram_version(sc) < 0) { 608 /* 609 * Warn user that a firmware update will be attempted in init. 610 */ 611 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 612 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 613 sc->flags &= ~TPS_UPTODATE; 614 } else { 615 sc->flags |= TPS_UPTODATE; 616 } 617 618 /* 619 * Create a child device for each MAC. The ethernet attachment 620 * will be done in these children. 621 */ 622 for (i = 0; i < (sc)->params.nports; i++) { 623 struct port_info *pi; 624 625 if ((child = device_add_child(dev, "cxgb", 626 DEVICE_UNIT_ANY)) == NULL) { 627 device_printf(dev, "failed to add child port\n"); 628 error = EINVAL; 629 goto out; 630 } 631 pi = &sc->port[i]; 632 pi->adapter = sc; 633 pi->nqsets = port_qsets; 634 pi->first_qset = i*port_qsets; 635 pi->port_id = i; 636 pi->tx_chan = i >= ai->nports0; 637 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 638 sc->rxpkt_map[pi->txpkt_intf] = i; 639 sc->port[i].tx_chan = i >= ai->nports0; 640 sc->portdev[i] = child; 641 device_set_softc(child, pi); 642 } 643 bus_attach_children(dev); 644 645 /* initialize sge private state */ 646 t3_sge_init_adapter(sc); 647 648 t3_led_ready(sc); 649 650 error = t3_get_fw_version(sc, &vers); 651 if (error) 652 goto out; 653 654 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 655 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 656 G_FW_VERSION_MICRO(vers)); 657 658 device_set_descf(dev, "%s %sNIC\t E/C: %s S/N: %s", 659 ai->desc, is_offload(sc) ? "R" : "", 660 sc->params.vpd.ec, sc->params.vpd.sn); 661 662 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x", 663 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1], 664 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]); 665 666 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 667 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 668 t3_add_attach_sysctls(sc); 669 670 #ifdef TCP_OFFLOAD 671 for (i = 0; i < NUM_CPL_HANDLERS; i++) 672 sc->cpl_handler[i] = cpl_not_handled; 673 #endif 674 675 t3_intr_clear(sc); 676 error = cxgb_setup_interrupts(sc); 677 out: 678 if (error) 679 cxgb_free(sc); 680 681 return (error); 682 } 683 684 /* 685 * The cxgb_controller_detach routine is called with the device is 686 * unloaded from the system. 687 */ 688 689 static int 690 cxgb_controller_detach(device_t dev) 691 { 692 struct adapter *sc; 693 694 sc = device_get_softc(dev); 695 696 cxgb_free(sc); 697 698 return (0); 699 } 700 701 /* 702 * The cxgb_free() is called by the cxgb_controller_detach() routine 703 * to tear down the structures that were built up in 704 * cxgb_controller_attach(), and should be the final piece of work 705 * done when fully unloading the driver. 706 * 707 * 708 * 1. Shutting down the threads started by the cxgb_controller_attach() 709 * routine. 710 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()). 711 * 3. Detaching all of the port devices created during the 712 * cxgb_controller_attach() routine. 713 * 4. Removing the device children created via cxgb_controller_attach(). 714 * 5. Releasing PCI resources associated with the device. 715 * 6. Turning off the offload support, iff it was turned on. 716 * 7. Destroying the mutexes created in cxgb_controller_attach(). 717 * 718 */ 719 static void 720 cxgb_free(struct adapter *sc) 721 { 722 int i, nqsets = 0; 723 724 ADAPTER_LOCK(sc); 725 sc->flags |= CXGB_SHUTDOWN; 726 ADAPTER_UNLOCK(sc); 727 728 /* 729 * Make sure all child devices are gone. 730 */ 731 bus_detach_children(sc->dev); 732 for (i = 0; i < (sc)->params.nports; i++) { 733 if (sc->portdev[i] && 734 device_delete_child(sc->dev, sc->portdev[i]) != 0) 735 device_printf(sc->dev, "failed to delete child port\n"); 736 nqsets += sc->port[i].nqsets; 737 } 738 739 /* 740 * At this point, it is as if cxgb_port_detach has run on all ports, and 741 * cxgb_down has run on the adapter. All interrupts have been silenced, 742 * all open devices have been closed. 743 */ 744 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)", 745 __func__, sc->open_device_map)); 746 for (i = 0; i < sc->params.nports; i++) { 747 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!", 748 __func__, i)); 749 } 750 751 /* 752 * Finish off the adapter's callouts. 753 */ 754 callout_drain(&sc->cxgb_tick_ch); 755 callout_drain(&sc->sge_timer_ch); 756 757 /* 758 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The 759 * sysctls are cleaned up by the kernel linker. 760 */ 761 if (sc->flags & FULL_INIT_DONE) { 762 t3_free_sge_resources(sc, nqsets); 763 sc->flags &= ~FULL_INIT_DONE; 764 } 765 766 /* 767 * Release all interrupt resources. 768 */ 769 cxgb_teardown_interrupts(sc); 770 if (sc->flags & (USING_MSI | USING_MSIX)) { 771 device_printf(sc->dev, "releasing msi message(s)\n"); 772 pci_release_msi(sc->dev); 773 } else { 774 device_printf(sc->dev, "no msi message to release\n"); 775 } 776 777 if (sc->msix_regs_res != NULL) { 778 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 779 sc->msix_regs_res); 780 } 781 782 /* 783 * Free the adapter's taskqueue. 784 */ 785 if (sc->tq != NULL) { 786 taskqueue_free(sc->tq); 787 sc->tq = NULL; 788 } 789 790 free(sc->filters, M_DEVBUF); 791 t3_sge_free(sc); 792 793 if (sc->udbs_res != NULL) 794 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 795 sc->udbs_res); 796 797 if (sc->regs_res != NULL) 798 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 799 sc->regs_res); 800 801 MTX_DESTROY(&sc->mdio_lock); 802 MTX_DESTROY(&sc->sge.reg_lock); 803 MTX_DESTROY(&sc->elmer_lock); 804 mtx_lock(&t3_list_lock); 805 SLIST_REMOVE(&t3_list, sc, adapter, link); 806 mtx_unlock(&t3_list_lock); 807 ADAPTER_LOCK_DEINIT(sc); 808 } 809 810 /** 811 * setup_sge_qsets - configure SGE Tx/Rx/response queues 812 * @sc: the controller softc 813 * 814 * Determines how many sets of SGE queues to use and initializes them. 815 * We support multiple queue sets per port if we have MSI-X, otherwise 816 * just one queue set per port. 817 */ 818 static int 819 setup_sge_qsets(adapter_t *sc) 820 { 821 int i, j, err, irq_idx = 0, qset_idx = 0; 822 u_int ntxq = SGE_TXQ_PER_SET; 823 824 if ((err = t3_sge_alloc(sc)) != 0) { 825 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 826 return (err); 827 } 828 829 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 830 irq_idx = -1; 831 832 for (i = 0; i < (sc)->params.nports; i++) { 833 struct port_info *pi = &sc->port[i]; 834 835 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 836 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 837 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 838 &sc->params.sge.qset[qset_idx], ntxq, pi); 839 if (err) { 840 t3_free_sge_resources(sc, qset_idx); 841 device_printf(sc->dev, 842 "t3_sge_alloc_qset failed with %d\n", err); 843 return (err); 844 } 845 } 846 } 847 848 sc->nqsets = qset_idx; 849 850 return (0); 851 } 852 853 static void 854 cxgb_teardown_interrupts(adapter_t *sc) 855 { 856 int i; 857 858 for (i = 0; i < SGE_QSETS; i++) { 859 if (sc->msix_intr_tag[i] == NULL) { 860 861 /* Should have been setup fully or not at all */ 862 KASSERT(sc->msix_irq_res[i] == NULL && 863 sc->msix_irq_rid[i] == 0, 864 ("%s: half-done interrupt (%d).", __func__, i)); 865 866 continue; 867 } 868 869 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 870 sc->msix_intr_tag[i]); 871 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i], 872 sc->msix_irq_res[i]); 873 874 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL; 875 sc->msix_irq_rid[i] = 0; 876 } 877 878 if (sc->intr_tag) { 879 KASSERT(sc->irq_res != NULL, 880 ("%s: half-done interrupt.", __func__)); 881 882 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 883 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 884 sc->irq_res); 885 886 sc->irq_res = sc->intr_tag = NULL; 887 sc->irq_rid = 0; 888 } 889 } 890 891 static int 892 cxgb_setup_interrupts(adapter_t *sc) 893 { 894 struct resource *res; 895 void *tag; 896 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX); 897 898 sc->irq_rid = intr_flag ? 1 : 0; 899 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, 900 RF_SHAREABLE | RF_ACTIVE); 901 if (sc->irq_res == NULL) { 902 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n", 903 intr_flag, sc->irq_rid); 904 err = EINVAL; 905 sc->irq_rid = 0; 906 } else { 907 err = bus_setup_intr(sc->dev, sc->irq_res, 908 INTR_MPSAFE | INTR_TYPE_NET, NULL, 909 sc->cxgb_intr, sc, &sc->intr_tag); 910 911 if (err) { 912 device_printf(sc->dev, 913 "Cannot set up interrupt (%x, %u, %d)\n", 914 intr_flag, sc->irq_rid, err); 915 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 916 sc->irq_res); 917 sc->irq_res = sc->intr_tag = NULL; 918 sc->irq_rid = 0; 919 } 920 } 921 922 /* That's all for INTx or MSI */ 923 if (!(intr_flag & USING_MSIX) || err) 924 return (err); 925 926 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err"); 927 for (i = 0; i < sc->msi_count - 1; i++) { 928 rid = i + 2; 929 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, 930 RF_SHAREABLE | RF_ACTIVE); 931 if (res == NULL) { 932 device_printf(sc->dev, "Cannot allocate interrupt " 933 "for message %d\n", rid); 934 err = EINVAL; 935 break; 936 } 937 938 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET, 939 NULL, t3_intr_msix, &sc->sge.qs[i], &tag); 940 if (err) { 941 device_printf(sc->dev, "Cannot set up interrupt " 942 "for message %d (%d)\n", rid, err); 943 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res); 944 break; 945 } 946 947 sc->msix_irq_rid[i] = rid; 948 sc->msix_irq_res[i] = res; 949 sc->msix_intr_tag[i] = tag; 950 bus_describe_intr(sc->dev, res, tag, "qs%d", i); 951 } 952 953 if (err) 954 cxgb_teardown_interrupts(sc); 955 956 return (err); 957 } 958 959 960 static int 961 cxgb_port_probe(device_t dev) 962 { 963 struct port_info *p; 964 const char *desc; 965 966 p = device_get_softc(dev); 967 desc = p->phy.desc; 968 device_set_descf(dev, "Port %d %s", p->port_id, desc); 969 return (0); 970 } 971 972 973 static int 974 cxgb_makedev(struct port_info *pi) 975 { 976 977 pi->port_cdev = make_dev(&cxgb_cdevsw, if_getdunit(pi->ifp), 978 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp)); 979 980 if (pi->port_cdev == NULL) 981 return (ENOMEM); 982 983 pi->port_cdev->si_drv1 = (void *)pi; 984 985 return (0); 986 } 987 988 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 989 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 990 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 991 #define CXGB_CAP_ENABLE CXGB_CAP 992 993 static int 994 cxgb_port_attach(device_t dev) 995 { 996 struct port_info *p; 997 if_t ifp; 998 int err; 999 struct adapter *sc; 1000 1001 p = device_get_softc(dev); 1002 sc = p->adapter; 1003 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 1004 device_get_unit(device_get_parent(dev)), p->port_id); 1005 PORT_LOCK_INIT(p, p->lockbuf); 1006 1007 callout_init(&p->link_check_ch, 1); 1008 TASK_INIT(&p->link_check_task, 0, check_link_status, p); 1009 1010 /* Allocate an ifnet object and set it up */ 1011 ifp = p->ifp = if_alloc(IFT_ETHER); 1012 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1013 if_setinitfn(ifp, cxgb_init); 1014 if_setsoftc(ifp, p); 1015 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 1016 if_setioctlfn(ifp, cxgb_ioctl); 1017 if_settransmitfn(ifp, cxgb_transmit); 1018 if_setqflushfn(ifp, cxgb_qflush); 1019 if_setgetcounterfn(ifp, cxgb_get_counter); 1020 1021 if_setcapabilities(ifp, CXGB_CAP); 1022 #ifdef TCP_OFFLOAD 1023 if (is_offload(sc)) 1024 if_setcapabilitiesbit(ifp, IFCAP_TOE4, 0); 1025 #endif 1026 if_setcapenable(ifp, CXGB_CAP_ENABLE); 1027 if_sethwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1028 CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1029 if_sethwtsomax(ifp, IP_MAXPACKET); 1030 if_sethwtsomaxsegcount(ifp, 36); 1031 if_sethwtsomaxsegsize(ifp, 65536); 1032 1033 /* 1034 * Disable TSO on 4-port - it isn't supported by the firmware. 1035 */ 1036 if (sc->params.nports > 2) { 1037 if_setcapabilitiesbit(ifp, 0, IFCAP_TSO | IFCAP_VLAN_HWTSO); 1038 if_setcapenablebit(ifp, 0, IFCAP_TSO | IFCAP_VLAN_HWTSO); 1039 if_sethwassistbits(ifp, 0, CSUM_TSO); 1040 } 1041 1042 /* Create a list of media supported by this port */ 1043 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 1044 cxgb_media_status); 1045 cxgb_build_medialist(p); 1046 1047 ether_ifattach(ifp, p->hw_addr); 1048 1049 /* Attach driver debugnet methods. */ 1050 DEBUGNET_SET(ifp, cxgb); 1051 1052 #ifdef DEFAULT_JUMBO 1053 if (sc->params.nports <= 2) 1054 if_setmtu(ifp, ETHERMTU_JUMBO); 1055 #endif 1056 if ((err = cxgb_makedev(p)) != 0) { 1057 printf("makedev failed %d\n", err); 1058 return (err); 1059 } 1060 1061 t3_sge_init_port(p); 1062 1063 return (err); 1064 } 1065 1066 /* 1067 * cxgb_port_detach() is called via the device_detach methods when 1068 * cxgb_free() calls the bus_detach_children. It is responsible for 1069 * removing the device from the view of the kernel, i.e. from all 1070 * interfaces lists etc. This routine is only called when the driver is 1071 * being unloaded, not when the link goes down. 1072 */ 1073 static int 1074 cxgb_port_detach(device_t dev) 1075 { 1076 struct port_info *p; 1077 struct adapter *sc; 1078 int i; 1079 1080 p = device_get_softc(dev); 1081 sc = p->adapter; 1082 1083 /* Tell cxgb_ioctl and if_init that the port is going away */ 1084 ADAPTER_LOCK(sc); 1085 SET_DOOMED(p); 1086 wakeup(&sc->flags); 1087 while (IS_BUSY(sc)) 1088 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0); 1089 SET_BUSY(sc); 1090 ADAPTER_UNLOCK(sc); 1091 1092 if (p->port_cdev != NULL) 1093 destroy_dev(p->port_cdev); 1094 1095 cxgb_uninit_synchronized(p); 1096 ether_ifdetach(p->ifp); 1097 1098 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1099 struct sge_qset *qs = &sc->sge.qs[i]; 1100 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1101 1102 callout_drain(&txq->txq_watchdog); 1103 callout_drain(&txq->txq_timer); 1104 } 1105 1106 PORT_LOCK_DEINIT(p); 1107 if_free(p->ifp); 1108 p->ifp = NULL; 1109 1110 ADAPTER_LOCK(sc); 1111 CLR_BUSY(sc); 1112 wakeup_one(&sc->flags); 1113 ADAPTER_UNLOCK(sc); 1114 return (0); 1115 } 1116 1117 void 1118 t3_fatal_err(struct adapter *sc) 1119 { 1120 u_int fw_status[4]; 1121 1122 if (sc->flags & FULL_INIT_DONE) { 1123 t3_sge_stop(sc); 1124 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1125 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1126 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1127 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1128 t3_intr_disable(sc); 1129 } 1130 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1131 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1132 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1133 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1134 } 1135 1136 int 1137 t3_os_find_pci_capability(adapter_t *sc, int cap) 1138 { 1139 int rc, reg = 0; 1140 1141 rc = pci_find_cap(sc->dev, cap, ®); 1142 return (rc == 0 ? reg : 0); 1143 } 1144 1145 int 1146 t3_os_pci_save_state(struct adapter *sc) 1147 { 1148 pci_save_state(sc->dev); 1149 return (0); 1150 } 1151 1152 int 1153 t3_os_pci_restore_state(struct adapter *sc) 1154 { 1155 pci_restore_state(sc->dev); 1156 return (0); 1157 } 1158 1159 /** 1160 * t3_os_link_changed - handle link status changes 1161 * @sc: the adapter associated with the link change 1162 * @port_id: the port index whose link status has changed 1163 * @link_status: the new status of the link 1164 * @speed: the new speed setting 1165 * @duplex: the new duplex setting 1166 * @fc: the new flow-control setting 1167 * 1168 * This is the OS-dependent handler for link status changes. The OS 1169 * neutral handler takes care of most of the processing for these events, 1170 * then calls this handler for any OS-specific processing. 1171 */ 1172 void 1173 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1174 int duplex, int fc, int mac_was_reset) 1175 { 1176 struct port_info *pi = &adapter->port[port_id]; 1177 if_t ifp = pi->ifp; 1178 1179 /* no race with detach, so ifp should always be good */ 1180 KASSERT(ifp, ("%s: if detached.", __func__)); 1181 1182 /* Reapply mac settings if they were lost due to a reset */ 1183 if (mac_was_reset) { 1184 PORT_LOCK(pi); 1185 cxgb_update_mac_settings(pi); 1186 PORT_UNLOCK(pi); 1187 } 1188 1189 if (link_status) { 1190 if_setbaudrate(ifp, IF_Mbps(speed)); 1191 if_link_state_change(ifp, LINK_STATE_UP); 1192 } else 1193 if_link_state_change(ifp, LINK_STATE_DOWN); 1194 } 1195 1196 /** 1197 * t3_os_phymod_changed - handle PHY module changes 1198 * @phy: the PHY reporting the module change 1199 * @mod_type: new module type 1200 * 1201 * This is the OS-dependent handler for PHY module changes. It is 1202 * invoked when a PHY module is removed or inserted for any OS-specific 1203 * processing. 1204 */ 1205 void t3_os_phymod_changed(struct adapter *adap, int port_id) 1206 { 1207 static const char *mod_str[] = { 1208 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown" 1209 }; 1210 struct port_info *pi = &adap->port[port_id]; 1211 int mod = pi->phy.modtype; 1212 1213 if (mod != pi->media.ifm_cur->ifm_data) 1214 cxgb_build_medialist(pi); 1215 1216 if (mod == phy_modtype_none) 1217 if_printf(pi->ifp, "PHY module unplugged\n"); 1218 else { 1219 KASSERT(mod < ARRAY_SIZE(mod_str), 1220 ("invalid PHY module type %d", mod)); 1221 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]); 1222 } 1223 } 1224 1225 void 1226 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1227 { 1228 1229 /* 1230 * The ifnet might not be allocated before this gets called, 1231 * as this is called early on in attach by t3_prep_adapter 1232 * save the address off in the port structure 1233 */ 1234 if (cxgb_debug) 1235 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1236 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1237 } 1238 1239 /* 1240 * Programs the XGMAC based on the settings in the ifnet. These settings 1241 * include MTU, MAC address, mcast addresses, etc. 1242 */ 1243 static void 1244 cxgb_update_mac_settings(struct port_info *p) 1245 { 1246 if_t ifp = p->ifp; 1247 struct t3_rx_mode rm; 1248 struct cmac *mac = &p->mac; 1249 int mtu, hwtagging; 1250 1251 PORT_LOCK_ASSERT_OWNED(p); 1252 1253 bcopy(if_getlladdr(ifp), p->hw_addr, ETHER_ADDR_LEN); 1254 1255 mtu = if_getmtu(ifp); 1256 if (if_getcapenable(ifp) & IFCAP_VLAN_MTU) 1257 mtu += ETHER_VLAN_ENCAP_LEN; 1258 1259 hwtagging = (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) != 0; 1260 1261 t3_mac_set_mtu(mac, mtu); 1262 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1263 t3_mac_set_address(mac, 0, p->hw_addr); 1264 t3_init_rx_mode(&rm, p); 1265 t3_mac_set_rx_mode(mac, &rm); 1266 } 1267 1268 1269 static int 1270 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1271 unsigned long n) 1272 { 1273 int attempts = 5; 1274 1275 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1276 if (!--attempts) 1277 return (ETIMEDOUT); 1278 t3_os_sleep(10); 1279 } 1280 return 0; 1281 } 1282 1283 static int 1284 init_tp_parity(struct adapter *adap) 1285 { 1286 int i; 1287 struct mbuf *m; 1288 struct cpl_set_tcb_field *greq; 1289 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1290 1291 t3_tp_set_offload_mode(adap, 1); 1292 1293 for (i = 0; i < 16; i++) { 1294 struct cpl_smt_write_req *req; 1295 1296 m = m_gethdr(M_WAITOK, MT_DATA); 1297 req = mtod(m, struct cpl_smt_write_req *); 1298 m->m_len = m->m_pkthdr.len = sizeof(*req); 1299 memset(req, 0, sizeof(*req)); 1300 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1301 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1302 req->iff = i; 1303 t3_mgmt_tx(adap, m); 1304 } 1305 1306 for (i = 0; i < 2048; i++) { 1307 struct cpl_l2t_write_req *req; 1308 1309 m = m_gethdr(M_WAITOK, MT_DATA); 1310 req = mtod(m, struct cpl_l2t_write_req *); 1311 m->m_len = m->m_pkthdr.len = sizeof(*req); 1312 memset(req, 0, sizeof(*req)); 1313 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1314 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1315 req->params = htonl(V_L2T_W_IDX(i)); 1316 t3_mgmt_tx(adap, m); 1317 } 1318 1319 for (i = 0; i < 2048; i++) { 1320 struct cpl_rte_write_req *req; 1321 1322 m = m_gethdr(M_WAITOK, MT_DATA); 1323 req = mtod(m, struct cpl_rte_write_req *); 1324 m->m_len = m->m_pkthdr.len = sizeof(*req); 1325 memset(req, 0, sizeof(*req)); 1326 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1327 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1328 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1329 t3_mgmt_tx(adap, m); 1330 } 1331 1332 m = m_gethdr(M_WAITOK, MT_DATA); 1333 greq = mtod(m, struct cpl_set_tcb_field *); 1334 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1335 memset(greq, 0, sizeof(*greq)); 1336 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1337 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1338 greq->mask = htobe64(1); 1339 t3_mgmt_tx(adap, m); 1340 1341 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1342 t3_tp_set_offload_mode(adap, 0); 1343 return (i); 1344 } 1345 1346 /** 1347 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1348 * @adap: the adapter 1349 * 1350 * Sets up RSS to distribute packets to multiple receive queues. We 1351 * configure the RSS CPU lookup table to distribute to the number of HW 1352 * receive queues, and the response queue lookup table to narrow that 1353 * down to the response queues actually configured for each port. 1354 * We always configure the RSS mapping for two ports since the mapping 1355 * table has plenty of entries. 1356 */ 1357 static void 1358 setup_rss(adapter_t *adap) 1359 { 1360 int i; 1361 u_int nq[2]; 1362 uint8_t cpus[SGE_QSETS + 1]; 1363 uint16_t rspq_map[RSS_TABLE_SIZE]; 1364 1365 for (i = 0; i < SGE_QSETS; ++i) 1366 cpus[i] = i; 1367 cpus[SGE_QSETS] = 0xff; 1368 1369 nq[0] = nq[1] = 0; 1370 for_each_port(adap, i) { 1371 const struct port_info *pi = adap2pinfo(adap, i); 1372 1373 nq[pi->tx_chan] += pi->nqsets; 1374 } 1375 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1376 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1377 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1378 } 1379 1380 /* Calculate the reverse RSS map table */ 1381 for (i = 0; i < SGE_QSETS; ++i) 1382 adap->rrss_map[i] = 0xff; 1383 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1384 if (adap->rrss_map[rspq_map[i]] == 0xff) 1385 adap->rrss_map[rspq_map[i]] = i; 1386 1387 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1388 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1389 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1390 cpus, rspq_map); 1391 1392 } 1393 static void 1394 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1395 int hi, int port) 1396 { 1397 struct mbuf *m; 1398 struct mngt_pktsched_wr *req; 1399 1400 m = m_gethdr(M_NOWAIT, MT_DATA); 1401 if (m) { 1402 req = mtod(m, struct mngt_pktsched_wr *); 1403 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1404 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1405 req->sched = sched; 1406 req->idx = qidx; 1407 req->min = lo; 1408 req->max = hi; 1409 req->binding = port; 1410 m->m_len = m->m_pkthdr.len = sizeof(*req); 1411 t3_mgmt_tx(adap, m); 1412 } 1413 } 1414 1415 static void 1416 bind_qsets(adapter_t *sc) 1417 { 1418 int i, j; 1419 1420 for (i = 0; i < (sc)->params.nports; ++i) { 1421 const struct port_info *pi = adap2pinfo(sc, i); 1422 1423 for (j = 0; j < pi->nqsets; ++j) { 1424 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1425 -1, pi->tx_chan); 1426 1427 } 1428 } 1429 } 1430 1431 static void 1432 update_tpeeprom(struct adapter *adap) 1433 { 1434 const struct firmware *tpeeprom; 1435 1436 uint32_t version; 1437 unsigned int major, minor; 1438 int ret, len; 1439 char rev, name[32]; 1440 1441 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1442 1443 major = G_TP_VERSION_MAJOR(version); 1444 minor = G_TP_VERSION_MINOR(version); 1445 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1446 return; 1447 1448 rev = t3rev2char(adap); 1449 snprintf(name, sizeof(name), TPEEPROM_NAME, rev); 1450 1451 tpeeprom = firmware_get(name); 1452 if (tpeeprom == NULL) { 1453 device_printf(adap->dev, 1454 "could not load TP EEPROM: unable to load %s\n", 1455 name); 1456 return; 1457 } 1458 1459 len = tpeeprom->datasize - 4; 1460 1461 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1462 if (ret) 1463 goto release_tpeeprom; 1464 1465 if (len != TP_SRAM_LEN) { 1466 device_printf(adap->dev, 1467 "%s length is wrong len=%d expected=%d\n", name, 1468 len, TP_SRAM_LEN); 1469 return; 1470 } 1471 1472 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1473 TP_SRAM_OFFSET); 1474 1475 if (!ret) { 1476 device_printf(adap->dev, 1477 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1478 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1479 } else 1480 device_printf(adap->dev, 1481 "Protocol SRAM image update in EEPROM failed\n"); 1482 1483 release_tpeeprom: 1484 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1485 1486 return; 1487 } 1488 1489 static int 1490 update_tpsram(struct adapter *adap) 1491 { 1492 const struct firmware *tpsram; 1493 int ret; 1494 char rev, name[32]; 1495 1496 rev = t3rev2char(adap); 1497 snprintf(name, sizeof(name), TPSRAM_NAME, rev); 1498 1499 update_tpeeprom(adap); 1500 1501 tpsram = firmware_get(name); 1502 if (tpsram == NULL){ 1503 device_printf(adap->dev, "could not load TP SRAM\n"); 1504 return (EINVAL); 1505 } else 1506 device_printf(adap->dev, "updating TP SRAM\n"); 1507 1508 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1509 if (ret) 1510 goto release_tpsram; 1511 1512 ret = t3_set_proto_sram(adap, tpsram->data); 1513 if (ret) 1514 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1515 1516 release_tpsram: 1517 firmware_put(tpsram, FIRMWARE_UNLOAD); 1518 1519 return ret; 1520 } 1521 1522 /** 1523 * cxgb_up - enable the adapter 1524 * @adap: adapter being enabled 1525 * 1526 * Called when the first port is enabled, this function performs the 1527 * actions necessary to make an adapter operational, such as completing 1528 * the initialization of HW modules, and enabling interrupts. 1529 */ 1530 static int 1531 cxgb_up(struct adapter *sc) 1532 { 1533 int err = 0; 1534 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS; 1535 1536 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)", 1537 __func__, sc->open_device_map)); 1538 1539 if ((sc->flags & FULL_INIT_DONE) == 0) { 1540 1541 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1542 1543 if ((sc->flags & FW_UPTODATE) == 0) 1544 if ((err = upgrade_fw(sc))) 1545 goto out; 1546 1547 if ((sc->flags & TPS_UPTODATE) == 0) 1548 if ((err = update_tpsram(sc))) 1549 goto out; 1550 1551 if (is_offload(sc) && nfilters != 0) { 1552 sc->params.mc5.nservers = 0; 1553 1554 if (nfilters < 0) 1555 sc->params.mc5.nfilters = mxf; 1556 else 1557 sc->params.mc5.nfilters = min(nfilters, mxf); 1558 } 1559 1560 err = t3_init_hw(sc, 0); 1561 if (err) 1562 goto out; 1563 1564 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1565 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1566 1567 err = setup_sge_qsets(sc); 1568 if (err) 1569 goto out; 1570 1571 alloc_filters(sc); 1572 setup_rss(sc); 1573 1574 t3_add_configured_sysctls(sc); 1575 sc->flags |= FULL_INIT_DONE; 1576 } 1577 1578 t3_intr_clear(sc); 1579 t3_sge_start(sc); 1580 t3_intr_enable(sc); 1581 1582 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1583 is_offload(sc) && init_tp_parity(sc) == 0) 1584 sc->flags |= TP_PARITY_INIT; 1585 1586 if (sc->flags & TP_PARITY_INIT) { 1587 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR); 1588 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1589 } 1590 1591 if (!(sc->flags & QUEUES_BOUND)) { 1592 bind_qsets(sc); 1593 setup_hw_filters(sc); 1594 sc->flags |= QUEUES_BOUND; 1595 } 1596 1597 t3_sge_reset_adapter(sc); 1598 out: 1599 return (err); 1600 } 1601 1602 /* 1603 * Called when the last open device is closed. Does NOT undo all of cxgb_up's 1604 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released 1605 * during controller_detach, not here. 1606 */ 1607 static void 1608 cxgb_down(struct adapter *sc) 1609 { 1610 t3_sge_stop(sc); 1611 t3_intr_disable(sc); 1612 } 1613 1614 /* 1615 * if_init for cxgb ports. 1616 */ 1617 static void 1618 cxgb_init(void *arg) 1619 { 1620 struct port_info *p = arg; 1621 struct adapter *sc = p->adapter; 1622 1623 ADAPTER_LOCK(sc); 1624 cxgb_init_locked(p); /* releases adapter lock */ 1625 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1626 } 1627 1628 static int 1629 cxgb_init_locked(struct port_info *p) 1630 { 1631 struct adapter *sc = p->adapter; 1632 if_t ifp = p->ifp; 1633 struct cmac *mac = &p->mac; 1634 int i, rc = 0, may_sleep = 0, gave_up_lock = 0; 1635 1636 ADAPTER_LOCK_ASSERT_OWNED(sc); 1637 1638 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1639 gave_up_lock = 1; 1640 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) { 1641 rc = EINTR; 1642 goto done; 1643 } 1644 } 1645 if (IS_DOOMED(p)) { 1646 rc = ENXIO; 1647 goto done; 1648 } 1649 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1650 1651 /* 1652 * The code that runs during one-time adapter initialization can sleep 1653 * so it's important not to hold any locks across it. 1654 */ 1655 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1; 1656 1657 if (may_sleep) { 1658 SET_BUSY(sc); 1659 gave_up_lock = 1; 1660 ADAPTER_UNLOCK(sc); 1661 } 1662 1663 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0)) 1664 goto done; 1665 1666 PORT_LOCK(p); 1667 if (isset(&sc->open_device_map, p->port_id) && 1668 (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { 1669 PORT_UNLOCK(p); 1670 goto done; 1671 } 1672 t3_port_intr_enable(sc, p->port_id); 1673 if (!mac->multiport) 1674 t3_mac_init(mac); 1675 cxgb_update_mac_settings(p); 1676 t3_link_start(&p->phy, mac, &p->link_config); 1677 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1678 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 1679 PORT_UNLOCK(p); 1680 1681 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1682 struct sge_qset *qs = &sc->sge.qs[i]; 1683 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1684 1685 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs, 1686 txq->txq_watchdog.c_cpu); 1687 } 1688 1689 /* all ok */ 1690 setbit(&sc->open_device_map, p->port_id); 1691 callout_reset(&p->link_check_ch, 1692 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4, 1693 link_check_callout, p); 1694 1695 done: 1696 if (may_sleep) { 1697 ADAPTER_LOCK(sc); 1698 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1699 CLR_BUSY(sc); 1700 } 1701 if (gave_up_lock) 1702 wakeup_one(&sc->flags); 1703 ADAPTER_UNLOCK(sc); 1704 return (rc); 1705 } 1706 1707 static int 1708 cxgb_uninit_locked(struct port_info *p) 1709 { 1710 struct adapter *sc = p->adapter; 1711 int rc; 1712 1713 ADAPTER_LOCK_ASSERT_OWNED(sc); 1714 1715 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1716 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) { 1717 rc = EINTR; 1718 goto done; 1719 } 1720 } 1721 if (IS_DOOMED(p)) { 1722 rc = ENXIO; 1723 goto done; 1724 } 1725 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1726 SET_BUSY(sc); 1727 ADAPTER_UNLOCK(sc); 1728 1729 rc = cxgb_uninit_synchronized(p); 1730 1731 ADAPTER_LOCK(sc); 1732 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1733 CLR_BUSY(sc); 1734 wakeup_one(&sc->flags); 1735 done: 1736 ADAPTER_UNLOCK(sc); 1737 return (rc); 1738 } 1739 1740 /* 1741 * Called on "ifconfig down", and from port_detach 1742 */ 1743 static int 1744 cxgb_uninit_synchronized(struct port_info *pi) 1745 { 1746 struct adapter *sc = pi->adapter; 1747 if_t ifp = pi->ifp; 1748 1749 /* 1750 * taskqueue_drain may cause a deadlock if the adapter lock is held. 1751 */ 1752 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1753 1754 /* 1755 * Clear this port's bit from the open device map, and then drain all 1756 * the tasks that can access/manipulate this port's port_info or ifp. 1757 * We disable this port's interrupts here and so the slow/ext 1758 * interrupt tasks won't be enqueued. The tick task will continue to 1759 * be enqueued every second but the runs after this drain will not see 1760 * this port in the open device map. 1761 * 1762 * A well behaved task must take open_device_map into account and ignore 1763 * ports that are not open. 1764 */ 1765 clrbit(&sc->open_device_map, pi->port_id); 1766 t3_port_intr_disable(sc, pi->port_id); 1767 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1768 taskqueue_drain(sc->tq, &sc->tick_task); 1769 1770 callout_drain(&pi->link_check_ch); 1771 taskqueue_drain(sc->tq, &pi->link_check_task); 1772 1773 PORT_LOCK(pi); 1774 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1775 1776 /* disable pause frames */ 1777 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0); 1778 1779 /* Reset RX FIFO HWM */ 1780 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset, 1781 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1782 1783 DELAY(100 * 1000); 1784 1785 /* Wait for TXFIFO empty */ 1786 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset, 1787 F_TXFIFO_EMPTY, 1, 20, 5); 1788 1789 DELAY(100 * 1000); 1790 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX); 1791 1792 pi->phy.ops->power_down(&pi->phy, 1); 1793 1794 PORT_UNLOCK(pi); 1795 1796 pi->link_config.link_ok = 0; 1797 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0); 1798 1799 if (sc->open_device_map == 0) 1800 cxgb_down(pi->adapter); 1801 1802 return (0); 1803 } 1804 1805 /* 1806 * Mark lro enabled or disabled in all qsets for this port 1807 */ 1808 static int 1809 cxgb_set_lro(struct port_info *p, int enabled) 1810 { 1811 int i; 1812 struct adapter *adp = p->adapter; 1813 struct sge_qset *q; 1814 1815 for (i = 0; i < p->nqsets; i++) { 1816 q = &adp->sge.qs[p->first_qset + i]; 1817 q->lro.enabled = (enabled != 0); 1818 } 1819 return (0); 1820 } 1821 1822 static int 1823 cxgb_ioctl(if_t ifp, unsigned long command, caddr_t data) 1824 { 1825 struct port_info *p = if_getsoftc(ifp); 1826 struct adapter *sc = p->adapter; 1827 struct ifreq *ifr = (struct ifreq *)data; 1828 int flags, error = 0, mtu; 1829 uint32_t mask; 1830 1831 switch (command) { 1832 case SIOCSIFMTU: 1833 ADAPTER_LOCK(sc); 1834 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1835 if (error) { 1836 fail: 1837 ADAPTER_UNLOCK(sc); 1838 return (error); 1839 } 1840 1841 mtu = ifr->ifr_mtu; 1842 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 1843 error = EINVAL; 1844 } else { 1845 if_setmtu(ifp, mtu); 1846 PORT_LOCK(p); 1847 cxgb_update_mac_settings(p); 1848 PORT_UNLOCK(p); 1849 } 1850 ADAPTER_UNLOCK(sc); 1851 break; 1852 case SIOCSIFFLAGS: 1853 ADAPTER_LOCK(sc); 1854 if (IS_DOOMED(p)) { 1855 error = ENXIO; 1856 goto fail; 1857 } 1858 if (if_getflags(ifp) & IFF_UP) { 1859 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1860 flags = p->if_flags; 1861 if (((if_getflags(ifp) ^ flags) & IFF_PROMISC) || 1862 ((if_getflags(ifp) ^ flags) & IFF_ALLMULTI)) { 1863 if (IS_BUSY(sc)) { 1864 error = EBUSY; 1865 goto fail; 1866 } 1867 PORT_LOCK(p); 1868 cxgb_update_mac_settings(p); 1869 PORT_UNLOCK(p); 1870 } 1871 ADAPTER_UNLOCK(sc); 1872 } else 1873 error = cxgb_init_locked(p); 1874 p->if_flags = if_getflags(ifp); 1875 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 1876 error = cxgb_uninit_locked(p); 1877 else 1878 ADAPTER_UNLOCK(sc); 1879 1880 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1881 break; 1882 case SIOCADDMULTI: 1883 case SIOCDELMULTI: 1884 ADAPTER_LOCK(sc); 1885 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1886 if (error) 1887 goto fail; 1888 1889 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1890 PORT_LOCK(p); 1891 cxgb_update_mac_settings(p); 1892 PORT_UNLOCK(p); 1893 } 1894 ADAPTER_UNLOCK(sc); 1895 1896 break; 1897 case SIOCSIFCAP: 1898 ADAPTER_LOCK(sc); 1899 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1900 if (error) 1901 goto fail; 1902 1903 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); 1904 if (mask & IFCAP_TXCSUM) { 1905 if_togglecapenable(ifp, IFCAP_TXCSUM); 1906 if_togglehwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP); 1907 1908 if (IFCAP_TSO4 & if_getcapenable(ifp) && 1909 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 1910 mask &= ~IFCAP_TSO4; 1911 if_setcapenablebit(ifp, 0, IFCAP_TSO4); 1912 if_printf(ifp, 1913 "tso4 disabled due to -txcsum.\n"); 1914 } 1915 } 1916 if (mask & IFCAP_TXCSUM_IPV6) { 1917 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); 1918 if_togglehwassist(ifp, CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1919 1920 if (IFCAP_TSO6 & if_getcapenable(ifp) && 1921 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { 1922 mask &= ~IFCAP_TSO6; 1923 if_setcapenablebit(ifp, 0, IFCAP_TSO6); 1924 if_printf(ifp, 1925 "tso6 disabled due to -txcsum6.\n"); 1926 } 1927 } 1928 if (mask & IFCAP_RXCSUM) 1929 if_togglecapenable(ifp, IFCAP_RXCSUM); 1930 if (mask & IFCAP_RXCSUM_IPV6) 1931 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); 1932 1933 /* 1934 * Note that we leave CSUM_TSO alone (it is always set). The 1935 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1936 * sending a TSO request our way, so it's sufficient to toggle 1937 * IFCAP_TSOx only. 1938 */ 1939 if (mask & IFCAP_TSO4) { 1940 if (!(IFCAP_TSO4 & if_getcapenable(ifp)) && 1941 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 1942 if_printf(ifp, "enable txcsum first.\n"); 1943 error = EAGAIN; 1944 goto fail; 1945 } 1946 if_togglecapenable(ifp, IFCAP_TSO4); 1947 } 1948 if (mask & IFCAP_TSO6) { 1949 if (!(IFCAP_TSO6 & if_getcapenable(ifp)) && 1950 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { 1951 if_printf(ifp, "enable txcsum6 first.\n"); 1952 error = EAGAIN; 1953 goto fail; 1954 } 1955 if_togglecapenable(ifp, IFCAP_TSO6); 1956 } 1957 if (mask & IFCAP_LRO) { 1958 if_togglecapenable(ifp, IFCAP_LRO); 1959 1960 /* Safe to do this even if cxgb_up not called yet */ 1961 cxgb_set_lro(p, if_getcapenable(ifp) & IFCAP_LRO); 1962 } 1963 #ifdef TCP_OFFLOAD 1964 if (mask & IFCAP_TOE4) { 1965 int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TOE4; 1966 1967 error = toe_capability(p, enable); 1968 if (error == 0) 1969 if_togglecapenable(ifp, mask); 1970 } 1971 #endif 1972 if (mask & IFCAP_VLAN_HWTAGGING) { 1973 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); 1974 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1975 PORT_LOCK(p); 1976 cxgb_update_mac_settings(p); 1977 PORT_UNLOCK(p); 1978 } 1979 } 1980 if (mask & IFCAP_VLAN_MTU) { 1981 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 1982 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1983 PORT_LOCK(p); 1984 cxgb_update_mac_settings(p); 1985 PORT_UNLOCK(p); 1986 } 1987 } 1988 if (mask & IFCAP_VLAN_HWTSO) 1989 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); 1990 if (mask & IFCAP_VLAN_HWCSUM) 1991 if_togglecapenable(ifp, IFCAP_VLAN_HWCSUM); 1992 1993 #ifdef VLAN_CAPABILITIES 1994 VLAN_CAPABILITIES(ifp); 1995 #endif 1996 ADAPTER_UNLOCK(sc); 1997 break; 1998 case SIOCSIFMEDIA: 1999 case SIOCGIFMEDIA: 2000 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 2001 break; 2002 default: 2003 error = ether_ioctl(ifp, command, data); 2004 } 2005 2006 return (error); 2007 } 2008 2009 static int 2010 cxgb_media_change(if_t ifp) 2011 { 2012 return (EOPNOTSUPP); 2013 } 2014 2015 /* 2016 * Translates phy->modtype to the correct Ethernet media subtype. 2017 */ 2018 static int 2019 cxgb_ifm_type(int mod) 2020 { 2021 switch (mod) { 2022 case phy_modtype_sr: 2023 return (IFM_10G_SR); 2024 case phy_modtype_lr: 2025 return (IFM_10G_LR); 2026 case phy_modtype_lrm: 2027 return (IFM_10G_LRM); 2028 case phy_modtype_twinax: 2029 return (IFM_10G_TWINAX); 2030 case phy_modtype_twinax_long: 2031 return (IFM_10G_TWINAX_LONG); 2032 case phy_modtype_none: 2033 return (IFM_NONE); 2034 case phy_modtype_unknown: 2035 return (IFM_UNKNOWN); 2036 } 2037 2038 KASSERT(0, ("%s: modtype %d unknown", __func__, mod)); 2039 return (IFM_UNKNOWN); 2040 } 2041 2042 /* 2043 * Rebuilds the ifmedia list for this port, and sets the current media. 2044 */ 2045 static void 2046 cxgb_build_medialist(struct port_info *p) 2047 { 2048 struct cphy *phy = &p->phy; 2049 struct ifmedia *media = &p->media; 2050 int mod = phy->modtype; 2051 int m = IFM_ETHER | IFM_FDX; 2052 2053 PORT_LOCK(p); 2054 2055 ifmedia_removeall(media); 2056 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) { 2057 /* Copper (RJ45) */ 2058 2059 if (phy->caps & SUPPORTED_10000baseT_Full) 2060 ifmedia_add(media, m | IFM_10G_T, mod, NULL); 2061 2062 if (phy->caps & SUPPORTED_1000baseT_Full) 2063 ifmedia_add(media, m | IFM_1000_T, mod, NULL); 2064 2065 if (phy->caps & SUPPORTED_100baseT_Full) 2066 ifmedia_add(media, m | IFM_100_TX, mod, NULL); 2067 2068 if (phy->caps & SUPPORTED_10baseT_Full) 2069 ifmedia_add(media, m | IFM_10_T, mod, NULL); 2070 2071 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL); 2072 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2073 2074 } else if (phy->caps & SUPPORTED_TP) { 2075 /* Copper (CX4) */ 2076 2077 KASSERT(phy->caps & SUPPORTED_10000baseT_Full, 2078 ("%s: unexpected cap 0x%x", __func__, phy->caps)); 2079 2080 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL); 2081 ifmedia_set(media, m | IFM_10G_CX4); 2082 2083 } else if (phy->caps & SUPPORTED_FIBRE && 2084 phy->caps & SUPPORTED_10000baseT_Full) { 2085 /* 10G optical (but includes SFP+ twinax) */ 2086 2087 m |= cxgb_ifm_type(mod); 2088 if (IFM_SUBTYPE(m) == IFM_NONE) 2089 m &= ~IFM_FDX; 2090 2091 ifmedia_add(media, m, mod, NULL); 2092 ifmedia_set(media, m); 2093 2094 } else if (phy->caps & SUPPORTED_FIBRE && 2095 phy->caps & SUPPORTED_1000baseT_Full) { 2096 /* 1G optical */ 2097 2098 /* XXX: Lie and claim to be SX, could actually be any 1G-X */ 2099 ifmedia_add(media, m | IFM_1000_SX, mod, NULL); 2100 ifmedia_set(media, m | IFM_1000_SX); 2101 2102 } else { 2103 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__, 2104 phy->caps)); 2105 } 2106 2107 PORT_UNLOCK(p); 2108 } 2109 2110 static void 2111 cxgb_media_status(if_t ifp, struct ifmediareq *ifmr) 2112 { 2113 struct port_info *p = if_getsoftc(ifp); 2114 struct ifmedia_entry *cur = p->media.ifm_cur; 2115 int speed = p->link_config.speed; 2116 2117 if (cur->ifm_data != p->phy.modtype) { 2118 cxgb_build_medialist(p); 2119 cur = p->media.ifm_cur; 2120 } 2121 2122 ifmr->ifm_status = IFM_AVALID; 2123 if (!p->link_config.link_ok) 2124 return; 2125 2126 ifmr->ifm_status |= IFM_ACTIVE; 2127 2128 /* 2129 * active and current will differ iff current media is autoselect. That 2130 * can happen only for copper RJ45. 2131 */ 2132 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 2133 return; 2134 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg, 2135 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps)); 2136 2137 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 2138 if (speed == SPEED_10000) 2139 ifmr->ifm_active |= IFM_10G_T; 2140 else if (speed == SPEED_1000) 2141 ifmr->ifm_active |= IFM_1000_T; 2142 else if (speed == SPEED_100) 2143 ifmr->ifm_active |= IFM_100_TX; 2144 else if (speed == SPEED_10) 2145 ifmr->ifm_active |= IFM_10_T; 2146 else 2147 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 2148 speed)); 2149 } 2150 2151 static uint64_t 2152 cxgb_get_counter(if_t ifp, ift_counter c) 2153 { 2154 struct port_info *pi = if_getsoftc(ifp); 2155 struct adapter *sc = pi->adapter; 2156 struct cmac *mac = &pi->mac; 2157 struct mac_stats *mstats = &mac->stats; 2158 2159 cxgb_refresh_stats(pi); 2160 2161 switch (c) { 2162 case IFCOUNTER_IPACKETS: 2163 return (mstats->rx_frames); 2164 2165 case IFCOUNTER_IERRORS: 2166 return (mstats->rx_jabber + mstats->rx_data_errs + 2167 mstats->rx_sequence_errs + mstats->rx_runt + 2168 mstats->rx_too_long + mstats->rx_mac_internal_errs + 2169 mstats->rx_short + mstats->rx_fcs_errs); 2170 2171 case IFCOUNTER_OPACKETS: 2172 return (mstats->tx_frames); 2173 2174 case IFCOUNTER_OERRORS: 2175 return (mstats->tx_excess_collisions + mstats->tx_underrun + 2176 mstats->tx_len_errs + mstats->tx_mac_internal_errs + 2177 mstats->tx_excess_deferral + mstats->tx_fcs_errs); 2178 2179 case IFCOUNTER_COLLISIONS: 2180 return (mstats->tx_total_collisions); 2181 2182 case IFCOUNTER_IBYTES: 2183 return (mstats->rx_octets); 2184 2185 case IFCOUNTER_OBYTES: 2186 return (mstats->tx_octets); 2187 2188 case IFCOUNTER_IMCASTS: 2189 return (mstats->rx_mcast_frames); 2190 2191 case IFCOUNTER_OMCASTS: 2192 return (mstats->tx_mcast_frames); 2193 2194 case IFCOUNTER_IQDROPS: 2195 return (mstats->rx_cong_drops); 2196 2197 case IFCOUNTER_OQDROPS: { 2198 int i; 2199 uint64_t drops; 2200 2201 drops = 0; 2202 if (sc->flags & FULL_INIT_DONE) { 2203 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) 2204 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops; 2205 } 2206 2207 return (drops); 2208 2209 } 2210 2211 default: 2212 return (if_get_counter_default(ifp, c)); 2213 } 2214 } 2215 2216 static void 2217 cxgb_async_intr(void *data) 2218 { 2219 adapter_t *sc = data; 2220 2221 t3_write_reg(sc, A_PL_INT_ENABLE0, 0); 2222 (void) t3_read_reg(sc, A_PL_INT_ENABLE0); 2223 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2224 } 2225 2226 static void 2227 link_check_callout(void *arg) 2228 { 2229 struct port_info *pi = arg; 2230 struct adapter *sc = pi->adapter; 2231 2232 if (!isset(&sc->open_device_map, pi->port_id)) 2233 return; 2234 2235 taskqueue_enqueue(sc->tq, &pi->link_check_task); 2236 } 2237 2238 static void 2239 check_link_status(void *arg, int pending) 2240 { 2241 struct port_info *pi = arg; 2242 struct adapter *sc = pi->adapter; 2243 2244 if (!isset(&sc->open_device_map, pi->port_id)) 2245 return; 2246 2247 t3_link_changed(sc, pi->port_id); 2248 2249 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) || 2250 pi->link_config.link_ok == 0) 2251 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi); 2252 } 2253 2254 void 2255 t3_os_link_intr(struct port_info *pi) 2256 { 2257 /* 2258 * Schedule a link check in the near future. If the link is flapping 2259 * rapidly we'll keep resetting the callout and delaying the check until 2260 * things stabilize a bit. 2261 */ 2262 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi); 2263 } 2264 2265 static void 2266 check_t3b2_mac(struct adapter *sc) 2267 { 2268 int i; 2269 2270 if (sc->flags & CXGB_SHUTDOWN) 2271 return; 2272 2273 for_each_port(sc, i) { 2274 struct port_info *p = &sc->port[i]; 2275 int status; 2276 #ifdef INVARIANTS 2277 if_t ifp = p->ifp; 2278 #endif 2279 2280 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault || 2281 !p->link_config.link_ok) 2282 continue; 2283 2284 KASSERT(if_getdrvflags(ifp) & IFF_DRV_RUNNING, 2285 ("%s: state mismatch (drv_flags %x, device_map %x)", 2286 __func__, if_getdrvflags(ifp), sc->open_device_map)); 2287 2288 PORT_LOCK(p); 2289 status = t3b2_mac_watchdog_task(&p->mac); 2290 if (status == 1) 2291 p->mac.stats.num_toggled++; 2292 else if (status == 2) { 2293 struct cmac *mac = &p->mac; 2294 2295 cxgb_update_mac_settings(p); 2296 t3_link_start(&p->phy, mac, &p->link_config); 2297 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2298 t3_port_intr_enable(sc, p->port_id); 2299 p->mac.stats.num_resets++; 2300 } 2301 PORT_UNLOCK(p); 2302 } 2303 } 2304 2305 static void 2306 cxgb_tick(void *arg) 2307 { 2308 adapter_t *sc = (adapter_t *)arg; 2309 2310 if (sc->flags & CXGB_SHUTDOWN) 2311 return; 2312 2313 taskqueue_enqueue(sc->tq, &sc->tick_task); 2314 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 2315 } 2316 2317 void 2318 cxgb_refresh_stats(struct port_info *pi) 2319 { 2320 struct timeval tv; 2321 const struct timeval interval = {0, 250000}; /* 250ms */ 2322 2323 getmicrotime(&tv); 2324 timevalsub(&tv, &interval); 2325 if (timevalcmp(&tv, &pi->last_refreshed, <)) 2326 return; 2327 2328 PORT_LOCK(pi); 2329 t3_mac_update_stats(&pi->mac); 2330 PORT_UNLOCK(pi); 2331 getmicrotime(&pi->last_refreshed); 2332 } 2333 2334 static void 2335 cxgb_tick_handler(void *arg, int count) 2336 { 2337 adapter_t *sc = (adapter_t *)arg; 2338 const struct adapter_params *p = &sc->params; 2339 int i; 2340 uint32_t cause, reset; 2341 2342 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE)) 2343 return; 2344 2345 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2346 check_t3b2_mac(sc); 2347 2348 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY); 2349 if (cause) { 2350 struct sge_qset *qs = &sc->sge.qs[0]; 2351 uint32_t mask, v; 2352 2353 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00; 2354 2355 mask = 1; 2356 for (i = 0; i < SGE_QSETS; i++) { 2357 if (v & mask) 2358 qs[i].rspq.starved++; 2359 mask <<= 1; 2360 } 2361 2362 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */ 2363 2364 for (i = 0; i < SGE_QSETS * 2; i++) { 2365 if (v & mask) { 2366 qs[i / 2].fl[i % 2].empty++; 2367 } 2368 mask <<= 1; 2369 } 2370 2371 /* clear */ 2372 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v); 2373 t3_write_reg(sc, A_SG_INT_CAUSE, cause); 2374 } 2375 2376 for (i = 0; i < sc->params.nports; i++) { 2377 struct port_info *pi = &sc->port[i]; 2378 struct cmac *mac = &pi->mac; 2379 2380 if (!isset(&sc->open_device_map, pi->port_id)) 2381 continue; 2382 2383 cxgb_refresh_stats(pi); 2384 2385 if (mac->multiport) 2386 continue; 2387 2388 /* Count rx fifo overflows, once per second */ 2389 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset); 2390 reset = 0; 2391 if (cause & F_RXFIFO_OVERFLOW) { 2392 mac->stats.rx_fifo_ovfl++; 2393 reset |= F_RXFIFO_OVERFLOW; 2394 } 2395 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset); 2396 } 2397 } 2398 2399 static void 2400 touch_bars(device_t dev) 2401 { 2402 /* 2403 * Don't enable yet 2404 */ 2405 #if !defined(__LP64__) && 0 2406 u32 v; 2407 2408 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2409 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2410 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2411 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2412 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2413 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2414 #endif 2415 } 2416 2417 static int 2418 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2419 { 2420 uint8_t *buf; 2421 int err = 0; 2422 u32 aligned_offset, aligned_len, *p; 2423 struct adapter *adapter = pi->adapter; 2424 2425 2426 aligned_offset = offset & ~3; 2427 aligned_len = (len + (offset & 3) + 3) & ~3; 2428 2429 if (aligned_offset != offset || aligned_len != len) { 2430 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK | M_ZERO); 2431 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2432 if (!err && aligned_len > 4) 2433 err = t3_seeprom_read(adapter, 2434 aligned_offset + aligned_len - 4, 2435 (u32 *)&buf[aligned_len - 4]); 2436 if (err) 2437 goto out; 2438 memcpy(buf + (offset & 3), data, len); 2439 } else 2440 buf = (uint8_t *)(uintptr_t)data; 2441 2442 err = t3_seeprom_wp(adapter, 0); 2443 if (err) 2444 goto out; 2445 2446 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2447 err = t3_seeprom_write(adapter, aligned_offset, *p); 2448 aligned_offset += 4; 2449 } 2450 2451 if (!err) 2452 err = t3_seeprom_wp(adapter, 1); 2453 out: 2454 if (buf != data) 2455 free(buf, M_DEVBUF); 2456 return err; 2457 } 2458 2459 2460 static int 2461 in_range(int val, int lo, int hi) 2462 { 2463 return val < 0 || (val <= hi && val >= lo); 2464 } 2465 2466 static int 2467 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td) 2468 { 2469 return (0); 2470 } 2471 2472 static int 2473 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td) 2474 { 2475 return (0); 2476 } 2477 2478 static int 2479 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2480 int fflag, struct thread *td) 2481 { 2482 int mmd, error = 0; 2483 struct port_info *pi = dev->si_drv1; 2484 adapter_t *sc = pi->adapter; 2485 2486 #ifdef PRIV_SUPPORTED 2487 if (priv_check(td, PRIV_DRIVER)) { 2488 if (cxgb_debug) 2489 printf("user does not have access to privileged ioctls\n"); 2490 return (EPERM); 2491 } 2492 #else 2493 if (suser(td)) { 2494 if (cxgb_debug) 2495 printf("user does not have access to privileged ioctls\n"); 2496 return (EPERM); 2497 } 2498 #endif 2499 2500 switch (cmd) { 2501 case CHELSIO_GET_MIIREG: { 2502 uint32_t val; 2503 struct cphy *phy = &pi->phy; 2504 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2505 2506 if (!phy->mdio_read) 2507 return (EOPNOTSUPP); 2508 if (is_10G(sc)) { 2509 mmd = mid->phy_id >> 8; 2510 if (!mmd) 2511 mmd = MDIO_DEV_PCS; 2512 else if (mmd > MDIO_DEV_VEND2) 2513 return (EINVAL); 2514 2515 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2516 mid->reg_num, &val); 2517 } else 2518 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2519 mid->reg_num & 0x1f, &val); 2520 if (error == 0) 2521 mid->val_out = val; 2522 break; 2523 } 2524 case CHELSIO_SET_MIIREG: { 2525 struct cphy *phy = &pi->phy; 2526 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2527 2528 if (!phy->mdio_write) 2529 return (EOPNOTSUPP); 2530 if (is_10G(sc)) { 2531 mmd = mid->phy_id >> 8; 2532 if (!mmd) 2533 mmd = MDIO_DEV_PCS; 2534 else if (mmd > MDIO_DEV_VEND2) 2535 return (EINVAL); 2536 2537 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2538 mmd, mid->reg_num, mid->val_in); 2539 } else 2540 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2541 mid->reg_num & 0x1f, 2542 mid->val_in); 2543 break; 2544 } 2545 case CHELSIO_SETREG: { 2546 struct ch_reg *edata = (struct ch_reg *)data; 2547 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2548 return (EFAULT); 2549 t3_write_reg(sc, edata->addr, edata->val); 2550 break; 2551 } 2552 case CHELSIO_GETREG: { 2553 struct ch_reg *edata = (struct ch_reg *)data; 2554 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2555 return (EFAULT); 2556 edata->val = t3_read_reg(sc, edata->addr); 2557 break; 2558 } 2559 case CHELSIO_GET_SGE_CONTEXT: { 2560 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2561 mtx_lock_spin(&sc->sge.reg_lock); 2562 switch (ecntxt->cntxt_type) { 2563 case CNTXT_TYPE_EGRESS: 2564 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2565 ecntxt->data); 2566 break; 2567 case CNTXT_TYPE_FL: 2568 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2569 ecntxt->data); 2570 break; 2571 case CNTXT_TYPE_RSP: 2572 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2573 ecntxt->data); 2574 break; 2575 case CNTXT_TYPE_CQ: 2576 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2577 ecntxt->data); 2578 break; 2579 default: 2580 error = EINVAL; 2581 break; 2582 } 2583 mtx_unlock_spin(&sc->sge.reg_lock); 2584 break; 2585 } 2586 case CHELSIO_GET_SGE_DESC: { 2587 struct ch_desc *edesc = (struct ch_desc *)data; 2588 int ret; 2589 if (edesc->queue_num >= SGE_QSETS * 6) 2590 return (EINVAL); 2591 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2592 edesc->queue_num % 6, edesc->idx, edesc->data); 2593 if (ret < 0) 2594 return (EINVAL); 2595 edesc->size = ret; 2596 break; 2597 } 2598 case CHELSIO_GET_QSET_PARAMS: { 2599 struct qset_params *q; 2600 struct ch_qset_params *t = (struct ch_qset_params *)data; 2601 int q1 = pi->first_qset; 2602 int nqsets = pi->nqsets; 2603 int i; 2604 2605 if (t->qset_idx >= nqsets) 2606 return EINVAL; 2607 2608 i = q1 + t->qset_idx; 2609 q = &sc->params.sge.qset[i]; 2610 t->rspq_size = q->rspq_size; 2611 t->txq_size[0] = q->txq_size[0]; 2612 t->txq_size[1] = q->txq_size[1]; 2613 t->txq_size[2] = q->txq_size[2]; 2614 t->fl_size[0] = q->fl_size; 2615 t->fl_size[1] = q->jumbo_size; 2616 t->polling = q->polling; 2617 t->lro = q->lro; 2618 t->intr_lat = q->coalesce_usecs; 2619 t->cong_thres = q->cong_thres; 2620 t->qnum = i; 2621 2622 if ((sc->flags & FULL_INIT_DONE) == 0) 2623 t->vector = 0; 2624 else if (sc->flags & USING_MSIX) 2625 t->vector = rman_get_start(sc->msix_irq_res[i]); 2626 else 2627 t->vector = rman_get_start(sc->irq_res); 2628 2629 break; 2630 } 2631 case CHELSIO_GET_QSET_NUM: { 2632 struct ch_reg *edata = (struct ch_reg *)data; 2633 edata->val = pi->nqsets; 2634 break; 2635 } 2636 case CHELSIO_LOAD_FW: { 2637 uint8_t *fw_data; 2638 uint32_t vers; 2639 struct ch_mem_range *t = (struct ch_mem_range *)data; 2640 2641 /* 2642 * You're allowed to load a firmware only before FULL_INIT_DONE 2643 * 2644 * FW_UPTODATE is also set so the rest of the initialization 2645 * will not overwrite what was loaded here. This gives you the 2646 * flexibility to load any firmware (and maybe shoot yourself in 2647 * the foot). 2648 */ 2649 2650 ADAPTER_LOCK(sc); 2651 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2652 ADAPTER_UNLOCK(sc); 2653 return (EBUSY); 2654 } 2655 2656 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2657 if (!fw_data) 2658 error = ENOMEM; 2659 else 2660 error = copyin(t->buf, fw_data, t->len); 2661 2662 if (!error) 2663 error = -t3_load_fw(sc, fw_data, t->len); 2664 2665 if (t3_get_fw_version(sc, &vers) == 0) { 2666 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2667 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2668 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2669 } 2670 2671 if (!error) 2672 sc->flags |= FW_UPTODATE; 2673 2674 free(fw_data, M_DEVBUF); 2675 ADAPTER_UNLOCK(sc); 2676 break; 2677 } 2678 case CHELSIO_LOAD_BOOT: { 2679 uint8_t *boot_data; 2680 struct ch_mem_range *t = (struct ch_mem_range *)data; 2681 2682 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2683 if (!boot_data) 2684 return ENOMEM; 2685 2686 error = copyin(t->buf, boot_data, t->len); 2687 if (!error) 2688 error = -t3_load_boot(sc, boot_data, t->len); 2689 2690 free(boot_data, M_DEVBUF); 2691 break; 2692 } 2693 case CHELSIO_GET_PM: { 2694 struct ch_pm *m = (struct ch_pm *)data; 2695 struct tp_params *p = &sc->params.tp; 2696 2697 if (!is_offload(sc)) 2698 return (EOPNOTSUPP); 2699 2700 m->tx_pg_sz = p->tx_pg_size; 2701 m->tx_num_pg = p->tx_num_pgs; 2702 m->rx_pg_sz = p->rx_pg_size; 2703 m->rx_num_pg = p->rx_num_pgs; 2704 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2705 2706 break; 2707 } 2708 case CHELSIO_SET_PM: { 2709 struct ch_pm *m = (struct ch_pm *)data; 2710 struct tp_params *p = &sc->params.tp; 2711 2712 if (!is_offload(sc)) 2713 return (EOPNOTSUPP); 2714 if (sc->flags & FULL_INIT_DONE) 2715 return (EBUSY); 2716 2717 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2718 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2719 return (EINVAL); /* not power of 2 */ 2720 if (!(m->rx_pg_sz & 0x14000)) 2721 return (EINVAL); /* not 16KB or 64KB */ 2722 if (!(m->tx_pg_sz & 0x1554000)) 2723 return (EINVAL); 2724 if (m->tx_num_pg == -1) 2725 m->tx_num_pg = p->tx_num_pgs; 2726 if (m->rx_num_pg == -1) 2727 m->rx_num_pg = p->rx_num_pgs; 2728 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2729 return (EINVAL); 2730 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2731 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2732 return (EINVAL); 2733 2734 p->rx_pg_size = m->rx_pg_sz; 2735 p->tx_pg_size = m->tx_pg_sz; 2736 p->rx_num_pgs = m->rx_num_pg; 2737 p->tx_num_pgs = m->tx_num_pg; 2738 break; 2739 } 2740 case CHELSIO_SETMTUTAB: { 2741 struct ch_mtus *m = (struct ch_mtus *)data; 2742 int i; 2743 2744 if (!is_offload(sc)) 2745 return (EOPNOTSUPP); 2746 if (offload_running(sc)) 2747 return (EBUSY); 2748 if (m->nmtus != NMTUS) 2749 return (EINVAL); 2750 if (m->mtus[0] < 81) /* accommodate SACK */ 2751 return (EINVAL); 2752 2753 /* 2754 * MTUs must be in ascending order 2755 */ 2756 for (i = 1; i < NMTUS; ++i) 2757 if (m->mtus[i] < m->mtus[i - 1]) 2758 return (EINVAL); 2759 2760 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2761 break; 2762 } 2763 case CHELSIO_GETMTUTAB: { 2764 struct ch_mtus *m = (struct ch_mtus *)data; 2765 2766 if (!is_offload(sc)) 2767 return (EOPNOTSUPP); 2768 2769 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2770 m->nmtus = NMTUS; 2771 break; 2772 } 2773 case CHELSIO_GET_MEM: { 2774 struct ch_mem_range *t = (struct ch_mem_range *)data; 2775 struct mc7 *mem; 2776 uint8_t *useraddr; 2777 u64 buf[32]; 2778 2779 /* 2780 * Use these to avoid modifying len/addr in the return 2781 * struct 2782 */ 2783 uint32_t len = t->len, addr = t->addr; 2784 2785 if (!is_offload(sc)) 2786 return (EOPNOTSUPP); 2787 if (!(sc->flags & FULL_INIT_DONE)) 2788 return (EIO); /* need the memory controllers */ 2789 if ((addr & 0x7) || (len & 0x7)) 2790 return (EINVAL); 2791 if (t->mem_id == MEM_CM) 2792 mem = &sc->cm; 2793 else if (t->mem_id == MEM_PMRX) 2794 mem = &sc->pmrx; 2795 else if (t->mem_id == MEM_PMTX) 2796 mem = &sc->pmtx; 2797 else 2798 return (EINVAL); 2799 2800 /* 2801 * Version scheme: 2802 * bits 0..9: chip version 2803 * bits 10..15: chip revision 2804 */ 2805 t->version = 3 | (sc->params.rev << 10); 2806 2807 /* 2808 * Read 256 bytes at a time as len can be large and we don't 2809 * want to use huge intermediate buffers. 2810 */ 2811 useraddr = (uint8_t *)t->buf; 2812 while (len) { 2813 unsigned int chunk = min(len, sizeof(buf)); 2814 2815 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2816 if (error) 2817 return (-error); 2818 if (copyout(buf, useraddr, chunk)) 2819 return (EFAULT); 2820 useraddr += chunk; 2821 addr += chunk; 2822 len -= chunk; 2823 } 2824 break; 2825 } 2826 case CHELSIO_READ_TCAM_WORD: { 2827 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2828 2829 if (!is_offload(sc)) 2830 return (EOPNOTSUPP); 2831 if (!(sc->flags & FULL_INIT_DONE)) 2832 return (EIO); /* need MC5 */ 2833 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2834 break; 2835 } 2836 case CHELSIO_SET_TRACE_FILTER: { 2837 struct ch_trace *t = (struct ch_trace *)data; 2838 const struct trace_params *tp; 2839 2840 tp = (const struct trace_params *)&t->sip; 2841 if (t->config_tx) 2842 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2843 t->trace_tx); 2844 if (t->config_rx) 2845 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2846 t->trace_rx); 2847 break; 2848 } 2849 case CHELSIO_SET_PKTSCHED: { 2850 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2851 if (sc->open_device_map == 0) 2852 return (EAGAIN); 2853 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2854 p->binding); 2855 break; 2856 } 2857 case CHELSIO_IFCONF_GETREGS: { 2858 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2859 int reglen = cxgb_get_regs_len(); 2860 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2861 if (buf == NULL) { 2862 return (ENOMEM); 2863 } 2864 if (regs->len > reglen) 2865 regs->len = reglen; 2866 else if (regs->len < reglen) 2867 error = ENOBUFS; 2868 2869 if (!error) { 2870 cxgb_get_regs(sc, regs, buf); 2871 error = copyout(buf, regs->data, reglen); 2872 } 2873 free(buf, M_DEVBUF); 2874 2875 break; 2876 } 2877 case CHELSIO_SET_HW_SCHED: { 2878 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2879 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2880 2881 if ((sc->flags & FULL_INIT_DONE) == 0) 2882 return (EAGAIN); /* need TP to be initialized */ 2883 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2884 !in_range(t->channel, 0, 1) || 2885 !in_range(t->kbps, 0, 10000000) || 2886 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2887 !in_range(t->flow_ipg, 0, 2888 dack_ticks_to_usec(sc, 0x7ff))) 2889 return (EINVAL); 2890 2891 if (t->kbps >= 0) { 2892 error = t3_config_sched(sc, t->kbps, t->sched); 2893 if (error < 0) 2894 return (-error); 2895 } 2896 if (t->class_ipg >= 0) 2897 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2898 if (t->flow_ipg >= 0) { 2899 t->flow_ipg *= 1000; /* us -> ns */ 2900 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2901 } 2902 if (t->mode >= 0) { 2903 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2904 2905 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2906 bit, t->mode ? bit : 0); 2907 } 2908 if (t->channel >= 0) 2909 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2910 1 << t->sched, t->channel << t->sched); 2911 break; 2912 } 2913 case CHELSIO_GET_EEPROM: { 2914 int i; 2915 struct ch_eeprom *e = (struct ch_eeprom *)data; 2916 uint8_t *buf; 2917 2918 if (e->offset & 3 || e->offset >= EEPROMSIZE || 2919 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) { 2920 return (EINVAL); 2921 } 2922 2923 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2924 if (buf == NULL) { 2925 return (ENOMEM); 2926 } 2927 e->magic = EEPROM_MAGIC; 2928 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2929 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2930 2931 if (!error) 2932 error = copyout(buf + e->offset, e->data, e->len); 2933 2934 free(buf, M_DEVBUF); 2935 break; 2936 } 2937 case CHELSIO_CLEAR_STATS: { 2938 if (!(sc->flags & FULL_INIT_DONE)) 2939 return EAGAIN; 2940 2941 PORT_LOCK(pi); 2942 t3_mac_update_stats(&pi->mac); 2943 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 2944 PORT_UNLOCK(pi); 2945 break; 2946 } 2947 case CHELSIO_GET_UP_LA: { 2948 struct ch_up_la *la = (struct ch_up_la *)data; 2949 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT); 2950 if (buf == NULL) { 2951 return (ENOMEM); 2952 } 2953 if (la->bufsize < LA_BUFSIZE) 2954 error = ENOBUFS; 2955 2956 if (!error) 2957 error = -t3_get_up_la(sc, &la->stopped, &la->idx, 2958 &la->bufsize, buf); 2959 if (!error) 2960 error = copyout(buf, la->data, la->bufsize); 2961 2962 free(buf, M_DEVBUF); 2963 break; 2964 } 2965 case CHELSIO_GET_UP_IOQS: { 2966 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data; 2967 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT); 2968 uint32_t *v; 2969 2970 if (buf == NULL) { 2971 return (ENOMEM); 2972 } 2973 if (ioqs->bufsize < IOQS_BUFSIZE) 2974 error = ENOBUFS; 2975 2976 if (!error) 2977 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf); 2978 2979 if (!error) { 2980 v = (uint32_t *)buf; 2981 2982 ioqs->ioq_rx_enable = *v++; 2983 ioqs->ioq_tx_enable = *v++; 2984 ioqs->ioq_rx_status = *v++; 2985 ioqs->ioq_tx_status = *v++; 2986 2987 error = copyout(v, ioqs->data, ioqs->bufsize); 2988 } 2989 2990 free(buf, M_DEVBUF); 2991 break; 2992 } 2993 case CHELSIO_SET_FILTER: { 2994 struct ch_filter *f = (struct ch_filter *)data; 2995 struct filter_info *p; 2996 unsigned int nfilters = sc->params.mc5.nfilters; 2997 2998 if (!is_offload(sc)) 2999 return (EOPNOTSUPP); /* No TCAM */ 3000 if (!(sc->flags & FULL_INIT_DONE)) 3001 return (EAGAIN); /* mc5 not setup yet */ 3002 if (nfilters == 0) 3003 return (EBUSY); /* TOE will use TCAM */ 3004 3005 /* sanity checks */ 3006 if (f->filter_id >= nfilters || 3007 (f->val.dip && f->mask.dip != 0xffffffff) || 3008 (f->val.sport && f->mask.sport != 0xffff) || 3009 (f->val.dport && f->mask.dport != 0xffff) || 3010 (f->val.vlan && f->mask.vlan != 0xfff) || 3011 (f->val.vlan_prio && 3012 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) || 3013 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) || 3014 f->qset >= SGE_QSETS || 3015 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE) 3016 return (EINVAL); 3017 3018 /* Was allocated with M_WAITOK */ 3019 KASSERT(sc->filters, ("filter table NULL\n")); 3020 3021 p = &sc->filters[f->filter_id]; 3022 if (p->locked) 3023 return (EPERM); 3024 3025 bzero(p, sizeof(*p)); 3026 p->sip = f->val.sip; 3027 p->sip_mask = f->mask.sip; 3028 p->dip = f->val.dip; 3029 p->sport = f->val.sport; 3030 p->dport = f->val.dport; 3031 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff; 3032 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) : 3033 FILTER_NO_VLAN_PRI; 3034 p->mac_hit = f->mac_hit; 3035 p->mac_vld = f->mac_addr_idx != 0xffff; 3036 p->mac_idx = f->mac_addr_idx; 3037 p->pkt_type = f->proto; 3038 p->report_filter_id = f->want_filter_id; 3039 p->pass = f->pass; 3040 p->rss = f->rss; 3041 p->qset = f->qset; 3042 3043 error = set_filter(sc, f->filter_id, p); 3044 if (error == 0) 3045 p->valid = 1; 3046 break; 3047 } 3048 case CHELSIO_DEL_FILTER: { 3049 struct ch_filter *f = (struct ch_filter *)data; 3050 struct filter_info *p; 3051 unsigned int nfilters = sc->params.mc5.nfilters; 3052 3053 if (!is_offload(sc)) 3054 return (EOPNOTSUPP); 3055 if (!(sc->flags & FULL_INIT_DONE)) 3056 return (EAGAIN); 3057 if (nfilters == 0 || sc->filters == NULL) 3058 return (EINVAL); 3059 if (f->filter_id >= nfilters) 3060 return (EINVAL); 3061 3062 p = &sc->filters[f->filter_id]; 3063 if (p->locked) 3064 return (EPERM); 3065 if (!p->valid) 3066 return (EFAULT); /* Read "Bad address" as "Bad index" */ 3067 3068 bzero(p, sizeof(*p)); 3069 p->sip = p->sip_mask = 0xffffffff; 3070 p->vlan = 0xfff; 3071 p->vlan_prio = FILTER_NO_VLAN_PRI; 3072 p->pkt_type = 1; 3073 error = set_filter(sc, f->filter_id, p); 3074 break; 3075 } 3076 case CHELSIO_GET_FILTER: { 3077 struct ch_filter *f = (struct ch_filter *)data; 3078 struct filter_info *p; 3079 unsigned int i, nfilters = sc->params.mc5.nfilters; 3080 3081 if (!is_offload(sc)) 3082 return (EOPNOTSUPP); 3083 if (!(sc->flags & FULL_INIT_DONE)) 3084 return (EAGAIN); 3085 if (nfilters == 0 || sc->filters == NULL) 3086 return (EINVAL); 3087 3088 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1; 3089 for (; i < nfilters; i++) { 3090 p = &sc->filters[i]; 3091 if (!p->valid) 3092 continue; 3093 3094 bzero(f, sizeof(*f)); 3095 3096 f->filter_id = i; 3097 f->val.sip = p->sip; 3098 f->mask.sip = p->sip_mask; 3099 f->val.dip = p->dip; 3100 f->mask.dip = p->dip ? 0xffffffff : 0; 3101 f->val.sport = p->sport; 3102 f->mask.sport = p->sport ? 0xffff : 0; 3103 f->val.dport = p->dport; 3104 f->mask.dport = p->dport ? 0xffff : 0; 3105 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan; 3106 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff; 3107 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3108 0 : p->vlan_prio; 3109 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3110 0 : FILTER_NO_VLAN_PRI; 3111 f->mac_hit = p->mac_hit; 3112 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff; 3113 f->proto = p->pkt_type; 3114 f->want_filter_id = p->report_filter_id; 3115 f->pass = p->pass; 3116 f->rss = p->rss; 3117 f->qset = p->qset; 3118 3119 break; 3120 } 3121 3122 if (i == nfilters) 3123 f->filter_id = 0xffffffff; 3124 break; 3125 } 3126 default: 3127 return (EOPNOTSUPP); 3128 break; 3129 } 3130 3131 return (error); 3132 } 3133 3134 static __inline void 3135 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 3136 unsigned int end) 3137 { 3138 uint32_t *p = (uint32_t *)(buf + start); 3139 3140 for ( ; start <= end; start += sizeof(uint32_t)) 3141 *p++ = t3_read_reg(ap, start); 3142 } 3143 3144 #define T3_REGMAP_SIZE (3 * 1024) 3145 static int 3146 cxgb_get_regs_len(void) 3147 { 3148 return T3_REGMAP_SIZE; 3149 } 3150 3151 static void 3152 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 3153 { 3154 3155 /* 3156 * Version scheme: 3157 * bits 0..9: chip version 3158 * bits 10..15: chip revision 3159 * bit 31: set for PCIe cards 3160 */ 3161 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 3162 3163 /* 3164 * We skip the MAC statistics registers because they are clear-on-read. 3165 * Also reading multi-register stats would need to synchronize with the 3166 * periodic mac stats accumulation. Hard to justify the complexity. 3167 */ 3168 memset(buf, 0, cxgb_get_regs_len()); 3169 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 3170 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 3171 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 3172 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 3173 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 3174 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 3175 XGM_REG(A_XGM_SERDES_STAT3, 1)); 3176 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 3177 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 3178 } 3179 3180 static int 3181 alloc_filters(struct adapter *sc) 3182 { 3183 struct filter_info *p; 3184 unsigned int nfilters = sc->params.mc5.nfilters; 3185 3186 if (nfilters == 0) 3187 return (0); 3188 3189 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO); 3190 sc->filters = p; 3191 3192 p = &sc->filters[nfilters - 1]; 3193 p->vlan = 0xfff; 3194 p->vlan_prio = FILTER_NO_VLAN_PRI; 3195 p->pass = p->rss = p->valid = p->locked = 1; 3196 3197 return (0); 3198 } 3199 3200 static int 3201 setup_hw_filters(struct adapter *sc) 3202 { 3203 int i, rc; 3204 unsigned int nfilters = sc->params.mc5.nfilters; 3205 3206 if (!sc->filters) 3207 return (0); 3208 3209 t3_enable_filters(sc); 3210 3211 for (i = rc = 0; i < nfilters && !rc; i++) { 3212 if (sc->filters[i].locked) 3213 rc = set_filter(sc, i, &sc->filters[i]); 3214 } 3215 3216 return (rc); 3217 } 3218 3219 static int 3220 set_filter(struct adapter *sc, int id, const struct filter_info *f) 3221 { 3222 int len; 3223 struct mbuf *m; 3224 struct ulp_txpkt *txpkt; 3225 struct work_request_hdr *wr; 3226 struct cpl_pass_open_req *oreq; 3227 struct cpl_set_tcb_field *sreq; 3228 3229 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq); 3230 KASSERT(len <= MHLEN, ("filter request too big for an mbuf")); 3231 3232 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes - 3233 sc->params.mc5.nfilters; 3234 3235 m = m_gethdr(M_WAITOK, MT_DATA); 3236 m->m_len = m->m_pkthdr.len = len; 3237 bzero(mtod(m, char *), len); 3238 3239 wr = mtod(m, struct work_request_hdr *); 3240 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); 3241 3242 oreq = (struct cpl_pass_open_req *)(wr + 1); 3243 txpkt = (struct ulp_txpkt *)oreq; 3244 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3245 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8)); 3246 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id)); 3247 oreq->local_port = htons(f->dport); 3248 oreq->peer_port = htons(f->sport); 3249 oreq->local_ip = htonl(f->dip); 3250 oreq->peer_ip = htonl(f->sip); 3251 oreq->peer_netmask = htonl(f->sip_mask); 3252 oreq->opt0h = 0; 3253 oreq->opt0l = htonl(F_NO_OFFLOAD); 3254 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) | 3255 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) | 3256 V_VLAN_PRI(f->vlan_prio >> 1) | 3257 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) | 3258 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) | 3259 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4))); 3260 3261 sreq = (struct cpl_set_tcb_field *)(oreq + 1); 3262 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL, 3263 (f->report_filter_id << 15) | (1 << 23) | 3264 ((u64)f->pass << 35) | ((u64)!f->rss << 36)); 3265 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1); 3266 t3_mgmt_tx(sc, m); 3267 3268 if (f->pass && !f->rss) { 3269 len = sizeof(*sreq); 3270 m = m_gethdr(M_WAITOK, MT_DATA); 3271 m->m_len = m->m_pkthdr.len = len; 3272 bzero(mtod(m, char *), len); 3273 sreq = mtod(m, struct cpl_set_tcb_field *); 3274 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3275 mk_set_tcb_field(sreq, id, 25, 0x3f80000, 3276 (u64)sc->rrss_map[f->qset] << 19); 3277 t3_mgmt_tx(sc, m); 3278 } 3279 return 0; 3280 } 3281 3282 static inline void 3283 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid, 3284 unsigned int word, u64 mask, u64 val) 3285 { 3286 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 3287 req->reply = V_NO_REPLY(1); 3288 req->cpu_idx = 0; 3289 req->word = htons(word); 3290 req->mask = htobe64(mask); 3291 req->val = htobe64(val); 3292 } 3293 3294 static inline void 3295 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, 3296 unsigned int word, u64 mask, u64 val) 3297 { 3298 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 3299 3300 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3301 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 3302 mk_set_tcb_field(req, tid, word, mask, val); 3303 } 3304 3305 void 3306 t3_iterate(void (*func)(struct adapter *, void *), void *arg) 3307 { 3308 struct adapter *sc; 3309 3310 mtx_lock(&t3_list_lock); 3311 SLIST_FOREACH(sc, &t3_list, link) { 3312 /* 3313 * func should not make any assumptions about what state sc is 3314 * in - the only guarantee is that sc->sc_lock is a valid lock. 3315 */ 3316 func(sc, arg); 3317 } 3318 mtx_unlock(&t3_list_lock); 3319 } 3320 3321 #ifdef TCP_OFFLOAD 3322 static int 3323 toe_capability(struct port_info *pi, int enable) 3324 { 3325 int rc; 3326 struct adapter *sc = pi->adapter; 3327 3328 ADAPTER_LOCK_ASSERT_OWNED(sc); 3329 3330 if (!is_offload(sc)) 3331 return (ENODEV); 3332 3333 if (enable) { 3334 if (!(sc->flags & FULL_INIT_DONE)) { 3335 log(LOG_WARNING, 3336 "You must enable a cxgb interface first\n"); 3337 return (EAGAIN); 3338 } 3339 3340 if (isset(&sc->offload_map, pi->port_id)) 3341 return (0); 3342 3343 if (!(sc->flags & TOM_INIT_DONE)) { 3344 rc = t3_activate_uld(sc, ULD_TOM); 3345 if (rc == EAGAIN) { 3346 log(LOG_WARNING, 3347 "You must kldload t3_tom.ko before trying " 3348 "to enable TOE on a cxgb interface.\n"); 3349 } 3350 if (rc != 0) 3351 return (rc); 3352 KASSERT(sc->tom_softc != NULL, 3353 ("%s: TOM activated but softc NULL", __func__)); 3354 KASSERT(sc->flags & TOM_INIT_DONE, 3355 ("%s: TOM activated but flag not set", __func__)); 3356 } 3357 3358 setbit(&sc->offload_map, pi->port_id); 3359 3360 /* 3361 * XXX: Temporary code to allow iWARP to be enabled when TOE is 3362 * enabled on any port. Need to figure out how to enable, 3363 * disable, load, and unload iWARP cleanly. 3364 */ 3365 if (!isset(&sc->offload_map, MAX_NPORTS) && 3366 t3_activate_uld(sc, ULD_IWARP) == 0) 3367 setbit(&sc->offload_map, MAX_NPORTS); 3368 } else { 3369 if (!isset(&sc->offload_map, pi->port_id)) 3370 return (0); 3371 3372 KASSERT(sc->flags & TOM_INIT_DONE, 3373 ("%s: TOM never initialized?", __func__)); 3374 clrbit(&sc->offload_map, pi->port_id); 3375 } 3376 3377 return (0); 3378 } 3379 3380 /* 3381 * Add an upper layer driver to the global list. 3382 */ 3383 int 3384 t3_register_uld(struct uld_info *ui) 3385 { 3386 int rc = 0; 3387 struct uld_info *u; 3388 3389 mtx_lock(&t3_uld_list_lock); 3390 SLIST_FOREACH(u, &t3_uld_list, link) { 3391 if (u->uld_id == ui->uld_id) { 3392 rc = EEXIST; 3393 goto done; 3394 } 3395 } 3396 3397 SLIST_INSERT_HEAD(&t3_uld_list, ui, link); 3398 ui->refcount = 0; 3399 done: 3400 mtx_unlock(&t3_uld_list_lock); 3401 return (rc); 3402 } 3403 3404 int 3405 t3_unregister_uld(struct uld_info *ui) 3406 { 3407 int rc = EINVAL; 3408 struct uld_info *u; 3409 3410 mtx_lock(&t3_uld_list_lock); 3411 3412 SLIST_FOREACH(u, &t3_uld_list, link) { 3413 if (u == ui) { 3414 if (ui->refcount > 0) { 3415 rc = EBUSY; 3416 goto done; 3417 } 3418 3419 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link); 3420 rc = 0; 3421 goto done; 3422 } 3423 } 3424 done: 3425 mtx_unlock(&t3_uld_list_lock); 3426 return (rc); 3427 } 3428 3429 int 3430 t3_activate_uld(struct adapter *sc, int id) 3431 { 3432 int rc = EAGAIN; 3433 struct uld_info *ui; 3434 3435 mtx_lock(&t3_uld_list_lock); 3436 3437 SLIST_FOREACH(ui, &t3_uld_list, link) { 3438 if (ui->uld_id == id) { 3439 rc = ui->activate(sc); 3440 if (rc == 0) 3441 ui->refcount++; 3442 goto done; 3443 } 3444 } 3445 done: 3446 mtx_unlock(&t3_uld_list_lock); 3447 3448 return (rc); 3449 } 3450 3451 int 3452 t3_deactivate_uld(struct adapter *sc, int id) 3453 { 3454 int rc = EINVAL; 3455 struct uld_info *ui; 3456 3457 mtx_lock(&t3_uld_list_lock); 3458 3459 SLIST_FOREACH(ui, &t3_uld_list, link) { 3460 if (ui->uld_id == id) { 3461 rc = ui->deactivate(sc); 3462 if (rc == 0) 3463 ui->refcount--; 3464 goto done; 3465 } 3466 } 3467 done: 3468 mtx_unlock(&t3_uld_list_lock); 3469 3470 return (rc); 3471 } 3472 3473 static int 3474 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused, 3475 struct mbuf *m) 3476 { 3477 m_freem(m); 3478 return (EDOOFUS); 3479 } 3480 3481 int 3482 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3483 { 3484 uintptr_t *loc, new; 3485 3486 if (opcode >= NUM_CPL_HANDLERS) 3487 return (EINVAL); 3488 3489 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3490 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3491 atomic_store_rel_ptr(loc, new); 3492 3493 return (0); 3494 } 3495 #endif 3496 3497 static int 3498 cxgbc_mod_event(module_t mod, int cmd, void *arg) 3499 { 3500 int rc = 0; 3501 3502 switch (cmd) { 3503 case MOD_LOAD: 3504 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF); 3505 SLIST_INIT(&t3_list); 3506 #ifdef TCP_OFFLOAD 3507 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF); 3508 SLIST_INIT(&t3_uld_list); 3509 #endif 3510 break; 3511 3512 case MOD_UNLOAD: 3513 #ifdef TCP_OFFLOAD 3514 mtx_lock(&t3_uld_list_lock); 3515 if (!SLIST_EMPTY(&t3_uld_list)) { 3516 rc = EBUSY; 3517 mtx_unlock(&t3_uld_list_lock); 3518 break; 3519 } 3520 mtx_unlock(&t3_uld_list_lock); 3521 mtx_destroy(&t3_uld_list_lock); 3522 #endif 3523 mtx_lock(&t3_list_lock); 3524 if (!SLIST_EMPTY(&t3_list)) { 3525 rc = EBUSY; 3526 mtx_unlock(&t3_list_lock); 3527 break; 3528 } 3529 mtx_unlock(&t3_list_lock); 3530 mtx_destroy(&t3_list_lock); 3531 break; 3532 } 3533 3534 return (rc); 3535 } 3536 3537 #ifdef DEBUGNET 3538 static void 3539 cxgb_debugnet_init(if_t ifp, int *nrxr, int *ncl, int *clsize) 3540 { 3541 struct port_info *pi; 3542 adapter_t *adap; 3543 3544 pi = if_getsoftc(ifp); 3545 adap = pi->adapter; 3546 ADAPTER_LOCK(adap); 3547 *nrxr = adap->nqsets; 3548 *ncl = adap->sge.qs[0].fl[1].size; 3549 *clsize = adap->sge.qs[0].fl[1].buf_size; 3550 ADAPTER_UNLOCK(adap); 3551 } 3552 3553 static void 3554 cxgb_debugnet_event(if_t ifp, enum debugnet_ev event) 3555 { 3556 struct port_info *pi; 3557 struct sge_qset *qs; 3558 int i; 3559 3560 pi = if_getsoftc(ifp); 3561 if (event == DEBUGNET_START) 3562 for (i = 0; i < pi->adapter->nqsets; i++) { 3563 qs = &pi->adapter->sge.qs[i]; 3564 3565 /* Need to reinit after debugnet_mbuf_start(). */ 3566 qs->fl[0].zone = zone_pack; 3567 qs->fl[1].zone = zone_clust; 3568 qs->lro.enabled = 0; 3569 } 3570 } 3571 3572 static int 3573 cxgb_debugnet_transmit(if_t ifp, struct mbuf *m) 3574 { 3575 struct port_info *pi; 3576 struct sge_qset *qs; 3577 3578 pi = if_getsoftc(ifp); 3579 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 3580 IFF_DRV_RUNNING) 3581 return (ENOENT); 3582 3583 qs = &pi->adapter->sge.qs[pi->first_qset]; 3584 return (cxgb_debugnet_encap(qs, &m)); 3585 } 3586 3587 static int 3588 cxgb_debugnet_poll(if_t ifp, int count) 3589 { 3590 struct port_info *pi; 3591 adapter_t *adap; 3592 int i; 3593 3594 pi = if_getsoftc(ifp); 3595 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 3596 return (ENOENT); 3597 3598 adap = pi->adapter; 3599 for (i = 0; i < adap->nqsets; i++) 3600 (void)cxgb_debugnet_poll_rx(adap, &adap->sge.qs[i]); 3601 (void)cxgb_debugnet_poll_tx(&adap->sge.qs[pi->first_qset]); 3602 return (0); 3603 } 3604 #endif /* DEBUGNET */ 3605