1 /************************************************************************** 2 SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 4 Copyright (c) 2007-2009, Chelsio Inc. 5 All rights reserved. 6 7 Redistribution and use in source and binary forms, with or without 8 modification, are permitted provided that the following conditions are met: 9 10 1. Redistributions of source code must retain the above copyright notice, 11 this list of conditions and the following disclaimer. 12 13 2. Neither the name of the Chelsio Corporation nor the names of its 14 contributors may be used to endorse or promote products derived from 15 this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 POSSIBILITY OF SUCH DAMAGE. 28 29 ***************************************************************************/ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_inet.h" 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/pciio.h> 42 #include <sys/conf.h> 43 #include <machine/bus.h> 44 #include <machine/resource.h> 45 #include <sys/ktr.h> 46 #include <sys/rman.h> 47 #include <sys/ioccom.h> 48 #include <sys/mbuf.h> 49 #include <sys/linker.h> 50 #include <sys/firmware.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/smp.h> 54 #include <sys/sysctl.h> 55 #include <sys/syslog.h> 56 #include <sys/queue.h> 57 #include <sys/taskqueue.h> 58 #include <sys/proc.h> 59 60 #include <net/bpf.h> 61 #include <net/ethernet.h> 62 #include <net/if.h> 63 #include <net/if_var.h> 64 #include <net/if_arp.h> 65 #include <net/if_dl.h> 66 #include <net/if_media.h> 67 #include <net/if_types.h> 68 #include <net/if_vlan_var.h> 69 70 #include <netinet/in_systm.h> 71 #include <netinet/in.h> 72 #include <netinet/if_ether.h> 73 #include <netinet/ip.h> 74 #include <netinet/ip.h> 75 #include <netinet/tcp.h> 76 #include <netinet/udp.h> 77 #include <netinet/netdump/netdump.h> 78 79 #include <dev/pci/pcireg.h> 80 #include <dev/pci/pcivar.h> 81 #include <dev/pci/pci_private.h> 82 83 #include <cxgb_include.h> 84 85 #ifdef PRIV_SUPPORTED 86 #include <sys/priv.h> 87 #endif 88 89 static int cxgb_setup_interrupts(adapter_t *); 90 static void cxgb_teardown_interrupts(adapter_t *); 91 static void cxgb_init(void *); 92 static int cxgb_init_locked(struct port_info *); 93 static int cxgb_uninit_locked(struct port_info *); 94 static int cxgb_uninit_synchronized(struct port_info *); 95 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 96 static int cxgb_media_change(struct ifnet *); 97 static int cxgb_ifm_type(int); 98 static void cxgb_build_medialist(struct port_info *); 99 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 100 static uint64_t cxgb_get_counter(struct ifnet *, ift_counter); 101 static int setup_sge_qsets(adapter_t *); 102 static void cxgb_async_intr(void *); 103 static void cxgb_tick_handler(void *, int); 104 static void cxgb_tick(void *); 105 static void link_check_callout(void *); 106 static void check_link_status(void *, int); 107 static void setup_rss(adapter_t *sc); 108 static int alloc_filters(struct adapter *); 109 static int setup_hw_filters(struct adapter *); 110 static int set_filter(struct adapter *, int, const struct filter_info *); 111 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int, 112 unsigned int, u64, u64); 113 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int, 114 unsigned int, u64, u64); 115 #ifdef TCP_OFFLOAD 116 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *); 117 #endif 118 119 /* Attachment glue for the PCI controller end of the device. Each port of 120 * the device is attached separately, as defined later. 121 */ 122 static int cxgb_controller_probe(device_t); 123 static int cxgb_controller_attach(device_t); 124 static int cxgb_controller_detach(device_t); 125 static void cxgb_free(struct adapter *); 126 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 127 unsigned int end); 128 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 129 static int cxgb_get_regs_len(void); 130 static void touch_bars(device_t dev); 131 static void cxgb_update_mac_settings(struct port_info *p); 132 #ifdef TCP_OFFLOAD 133 static int toe_capability(struct port_info *, int); 134 #endif 135 136 static device_method_t cxgb_controller_methods[] = { 137 DEVMETHOD(device_probe, cxgb_controller_probe), 138 DEVMETHOD(device_attach, cxgb_controller_attach), 139 DEVMETHOD(device_detach, cxgb_controller_detach), 140 141 DEVMETHOD_END 142 }; 143 144 static driver_t cxgb_controller_driver = { 145 "cxgbc", 146 cxgb_controller_methods, 147 sizeof(struct adapter) 148 }; 149 150 static int cxgbc_mod_event(module_t, int, void *); 151 static devclass_t cxgb_controller_devclass; 152 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 153 cxgbc_mod_event, 0); 154 MODULE_VERSION(cxgbc, 1); 155 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1); 156 157 /* 158 * Attachment glue for the ports. Attachment is done directly to the 159 * controller device. 160 */ 161 static int cxgb_port_probe(device_t); 162 static int cxgb_port_attach(device_t); 163 static int cxgb_port_detach(device_t); 164 165 static device_method_t cxgb_port_methods[] = { 166 DEVMETHOD(device_probe, cxgb_port_probe), 167 DEVMETHOD(device_attach, cxgb_port_attach), 168 DEVMETHOD(device_detach, cxgb_port_detach), 169 { 0, 0 } 170 }; 171 172 static driver_t cxgb_port_driver = { 173 "cxgb", 174 cxgb_port_methods, 175 0 176 }; 177 178 static d_ioctl_t cxgb_extension_ioctl; 179 static d_open_t cxgb_extension_open; 180 static d_close_t cxgb_extension_close; 181 182 static struct cdevsw cxgb_cdevsw = { 183 .d_version = D_VERSION, 184 .d_flags = 0, 185 .d_open = cxgb_extension_open, 186 .d_close = cxgb_extension_close, 187 .d_ioctl = cxgb_extension_ioctl, 188 .d_name = "cxgb", 189 }; 190 191 static devclass_t cxgb_port_devclass; 192 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 193 MODULE_VERSION(cxgb, 1); 194 195 NETDUMP_DEFINE(cxgb); 196 197 static struct mtx t3_list_lock; 198 static SLIST_HEAD(, adapter) t3_list; 199 #ifdef TCP_OFFLOAD 200 static struct mtx t3_uld_list_lock; 201 static SLIST_HEAD(, uld_info) t3_uld_list; 202 #endif 203 204 /* 205 * The driver uses the best interrupt scheme available on a platform in the 206 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 207 * of these schemes the driver may consider as follows: 208 * 209 * msi = 2: choose from among all three options 210 * msi = 1 : only consider MSI and pin interrupts 211 * msi = 0: force pin interrupts 212 */ 213 static int msi_allowed = 2; 214 215 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 216 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 217 "MSI-X, MSI, INTx selector"); 218 219 /* 220 * The driver uses an auto-queue algorithm by default. 221 * To disable it and force a single queue-set per port, use multiq = 0 222 */ 223 static int multiq = 1; 224 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0, 225 "use min(ncpus/ports, 8) queue-sets per port"); 226 227 /* 228 * By default the driver will not update the firmware unless 229 * it was compiled against a newer version 230 * 231 */ 232 static int force_fw_update = 0; 233 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 234 "update firmware even if up to date"); 235 236 int cxgb_use_16k_clusters = -1; 237 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 238 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 239 240 static int nfilters = -1; 241 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN, 242 &nfilters, 0, "max number of entries in the filter table"); 243 244 enum { 245 MAX_TXQ_ENTRIES = 16384, 246 MAX_CTRL_TXQ_ENTRIES = 1024, 247 MAX_RSPQ_ENTRIES = 16384, 248 MAX_RX_BUFFERS = 16384, 249 MAX_RX_JUMBO_BUFFERS = 16384, 250 MIN_TXQ_ENTRIES = 4, 251 MIN_CTRL_TXQ_ENTRIES = 4, 252 MIN_RSPQ_ENTRIES = 32, 253 MIN_FL_ENTRIES = 32, 254 MIN_FL_JUMBO_ENTRIES = 32 255 }; 256 257 struct filter_info { 258 u32 sip; 259 u32 sip_mask; 260 u32 dip; 261 u16 sport; 262 u16 dport; 263 u32 vlan:12; 264 u32 vlan_prio:3; 265 u32 mac_hit:1; 266 u32 mac_idx:4; 267 u32 mac_vld:1; 268 u32 pkt_type:2; 269 u32 report_filter_id:1; 270 u32 pass:1; 271 u32 rss:1; 272 u32 qset:3; 273 u32 locked:1; 274 u32 valid:1; 275 }; 276 277 enum { FILTER_NO_VLAN_PRI = 7 }; 278 279 #define EEPROM_MAGIC 0x38E2F10C 280 281 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 282 283 /* Table for probing the cards. The desc field isn't actually used */ 284 struct cxgb_ident { 285 uint16_t vendor; 286 uint16_t device; 287 int index; 288 char *desc; 289 } cxgb_identifiers[] = { 290 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 291 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 292 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 293 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 294 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 295 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 296 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 297 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 298 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 299 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 300 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 301 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"}, 302 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"}, 303 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"}, 304 {0, 0, 0, NULL} 305 }; 306 307 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 308 309 310 static __inline char 311 t3rev2char(struct adapter *adapter) 312 { 313 char rev = 'z'; 314 315 switch(adapter->params.rev) { 316 case T3_REV_A: 317 rev = 'a'; 318 break; 319 case T3_REV_B: 320 case T3_REV_B2: 321 rev = 'b'; 322 break; 323 case T3_REV_C: 324 rev = 'c'; 325 break; 326 } 327 return rev; 328 } 329 330 static struct cxgb_ident * 331 cxgb_get_ident(device_t dev) 332 { 333 struct cxgb_ident *id; 334 335 for (id = cxgb_identifiers; id->desc != NULL; id++) { 336 if ((id->vendor == pci_get_vendor(dev)) && 337 (id->device == pci_get_device(dev))) { 338 return (id); 339 } 340 } 341 return (NULL); 342 } 343 344 static const struct adapter_info * 345 cxgb_get_adapter_info(device_t dev) 346 { 347 struct cxgb_ident *id; 348 const struct adapter_info *ai; 349 350 id = cxgb_get_ident(dev); 351 if (id == NULL) 352 return (NULL); 353 354 ai = t3_get_adapter_info(id->index); 355 356 return (ai); 357 } 358 359 static int 360 cxgb_controller_probe(device_t dev) 361 { 362 const struct adapter_info *ai; 363 char *ports, buf[80]; 364 int nports; 365 366 ai = cxgb_get_adapter_info(dev); 367 if (ai == NULL) 368 return (ENXIO); 369 370 nports = ai->nports0 + ai->nports1; 371 if (nports == 1) 372 ports = "port"; 373 else 374 ports = "ports"; 375 376 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports); 377 device_set_desc_copy(dev, buf); 378 return (BUS_PROBE_DEFAULT); 379 } 380 381 #define FW_FNAME "cxgb_t3fw" 382 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom" 383 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram" 384 385 static int 386 upgrade_fw(adapter_t *sc) 387 { 388 const struct firmware *fw; 389 int status; 390 u32 vers; 391 392 if ((fw = firmware_get(FW_FNAME)) == NULL) { 393 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 394 return (ENOENT); 395 } else 396 device_printf(sc->dev, "installing firmware on card\n"); 397 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 398 399 if (status != 0) { 400 device_printf(sc->dev, "failed to install firmware: %d\n", 401 status); 402 } else { 403 t3_get_fw_version(sc, &vers); 404 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 405 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 406 G_FW_VERSION_MICRO(vers)); 407 } 408 409 firmware_put(fw, FIRMWARE_UNLOAD); 410 411 return (status); 412 } 413 414 /* 415 * The cxgb_controller_attach function is responsible for the initial 416 * bringup of the device. Its responsibilities include: 417 * 418 * 1. Determine if the device supports MSI or MSI-X. 419 * 2. Allocate bus resources so that we can access the Base Address Register 420 * 3. Create and initialize mutexes for the controller and its control 421 * logic such as SGE and MDIO. 422 * 4. Call hardware specific setup routine for the adapter as a whole. 423 * 5. Allocate the BAR for doing MSI-X. 424 * 6. Setup the line interrupt iff MSI-X is not supported. 425 * 7. Create the driver's taskq. 426 * 8. Start one task queue service thread. 427 * 9. Check if the firmware and SRAM are up-to-date. They will be 428 * auto-updated later (before FULL_INIT_DONE), if required. 429 * 10. Create a child device for each MAC (port) 430 * 11. Initialize T3 private state. 431 * 12. Trigger the LED 432 * 13. Setup offload iff supported. 433 * 14. Reset/restart the tick callout. 434 * 15. Attach sysctls 435 * 436 * NOTE: Any modification or deviation from this list MUST be reflected in 437 * the above comment. Failure to do so will result in problems on various 438 * error conditions including link flapping. 439 */ 440 static int 441 cxgb_controller_attach(device_t dev) 442 { 443 device_t child; 444 const struct adapter_info *ai; 445 struct adapter *sc; 446 int i, error = 0; 447 uint32_t vers; 448 int port_qsets = 1; 449 int msi_needed, reg; 450 char buf[80]; 451 452 sc = device_get_softc(dev); 453 sc->dev = dev; 454 sc->msi_count = 0; 455 ai = cxgb_get_adapter_info(dev); 456 457 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 458 device_get_unit(dev)); 459 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 460 461 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 462 device_get_unit(dev)); 463 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 464 device_get_unit(dev)); 465 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 466 device_get_unit(dev)); 467 468 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 469 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 470 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 471 472 mtx_lock(&t3_list_lock); 473 SLIST_INSERT_HEAD(&t3_list, sc, link); 474 mtx_unlock(&t3_list_lock); 475 476 /* find the PCIe link width and set max read request to 4KB*/ 477 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 478 uint16_t lnk; 479 480 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2); 481 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4; 482 if (sc->link_width < 8 && 483 (ai->caps & SUPPORTED_10000baseT_Full)) { 484 device_printf(sc->dev, 485 "PCIe x%d Link, expect reduced performance\n", 486 sc->link_width); 487 } 488 489 pci_set_max_read_req(dev, 4096); 490 } 491 492 touch_bars(dev); 493 pci_enable_busmaster(dev); 494 /* 495 * Allocate the registers and make them available to the driver. 496 * The registers that we care about for NIC mode are in BAR 0 497 */ 498 sc->regs_rid = PCIR_BAR(0); 499 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 500 &sc->regs_rid, RF_ACTIVE)) == NULL) { 501 device_printf(dev, "Cannot allocate BAR region 0\n"); 502 error = ENXIO; 503 goto out; 504 } 505 506 sc->bt = rman_get_bustag(sc->regs_res); 507 sc->bh = rman_get_bushandle(sc->regs_res); 508 sc->mmio_len = rman_get_size(sc->regs_res); 509 510 for (i = 0; i < MAX_NPORTS; i++) 511 sc->port[i].adapter = sc; 512 513 if (t3_prep_adapter(sc, ai, 1) < 0) { 514 printf("prep adapter failed\n"); 515 error = ENODEV; 516 goto out; 517 } 518 519 sc->udbs_rid = PCIR_BAR(2); 520 sc->udbs_res = NULL; 521 if (is_offload(sc) && 522 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 523 &sc->udbs_rid, RF_ACTIVE)) == NULL)) { 524 device_printf(dev, "Cannot allocate BAR region 1\n"); 525 error = ENXIO; 526 goto out; 527 } 528 529 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 530 * enough messages for the queue sets. If that fails, try falling 531 * back to MSI. If that fails, then try falling back to the legacy 532 * interrupt pin model. 533 */ 534 sc->msix_regs_rid = 0x20; 535 if ((msi_allowed >= 2) && 536 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 537 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 538 539 if (multiq) 540 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus); 541 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1; 542 543 if (pci_msix_count(dev) == 0 || 544 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 || 545 sc->msi_count != msi_needed) { 546 device_printf(dev, "alloc msix failed - " 547 "msi_count=%d, msi_needed=%d, err=%d; " 548 "will try MSI\n", sc->msi_count, 549 msi_needed, error); 550 sc->msi_count = 0; 551 port_qsets = 1; 552 pci_release_msi(dev); 553 bus_release_resource(dev, SYS_RES_MEMORY, 554 sc->msix_regs_rid, sc->msix_regs_res); 555 sc->msix_regs_res = NULL; 556 } else { 557 sc->flags |= USING_MSIX; 558 sc->cxgb_intr = cxgb_async_intr; 559 device_printf(dev, 560 "using MSI-X interrupts (%u vectors)\n", 561 sc->msi_count); 562 } 563 } 564 565 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 566 sc->msi_count = 1; 567 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) { 568 device_printf(dev, "alloc msi failed - " 569 "err=%d; will try INTx\n", error); 570 sc->msi_count = 0; 571 port_qsets = 1; 572 pci_release_msi(dev); 573 } else { 574 sc->flags |= USING_MSI; 575 sc->cxgb_intr = t3_intr_msi; 576 device_printf(dev, "using MSI interrupts\n"); 577 } 578 } 579 if (sc->msi_count == 0) { 580 device_printf(dev, "using line interrupts\n"); 581 sc->cxgb_intr = t3b_intr; 582 } 583 584 /* Create a private taskqueue thread for handling driver events */ 585 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 586 taskqueue_thread_enqueue, &sc->tq); 587 if (sc->tq == NULL) { 588 device_printf(dev, "failed to allocate controller task queue\n"); 589 goto out; 590 } 591 592 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 593 device_get_nameunit(dev)); 594 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 595 596 597 /* Create a periodic callout for checking adapter status */ 598 callout_init(&sc->cxgb_tick_ch, 1); 599 600 if (t3_check_fw_version(sc) < 0 || force_fw_update) { 601 /* 602 * Warn user that a firmware update will be attempted in init. 603 */ 604 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 605 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 606 sc->flags &= ~FW_UPTODATE; 607 } else { 608 sc->flags |= FW_UPTODATE; 609 } 610 611 if (t3_check_tpsram_version(sc) < 0) { 612 /* 613 * Warn user that a firmware update will be attempted in init. 614 */ 615 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 616 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 617 sc->flags &= ~TPS_UPTODATE; 618 } else { 619 sc->flags |= TPS_UPTODATE; 620 } 621 622 /* 623 * Create a child device for each MAC. The ethernet attachment 624 * will be done in these children. 625 */ 626 for (i = 0; i < (sc)->params.nports; i++) { 627 struct port_info *pi; 628 629 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 630 device_printf(dev, "failed to add child port\n"); 631 error = EINVAL; 632 goto out; 633 } 634 pi = &sc->port[i]; 635 pi->adapter = sc; 636 pi->nqsets = port_qsets; 637 pi->first_qset = i*port_qsets; 638 pi->port_id = i; 639 pi->tx_chan = i >= ai->nports0; 640 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 641 sc->rxpkt_map[pi->txpkt_intf] = i; 642 sc->port[i].tx_chan = i >= ai->nports0; 643 sc->portdev[i] = child; 644 device_set_softc(child, pi); 645 } 646 if ((error = bus_generic_attach(dev)) != 0) 647 goto out; 648 649 /* initialize sge private state */ 650 t3_sge_init_adapter(sc); 651 652 t3_led_ready(sc); 653 654 error = t3_get_fw_version(sc, &vers); 655 if (error) 656 goto out; 657 658 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 659 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 660 G_FW_VERSION_MICRO(vers)); 661 662 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s", 663 ai->desc, is_offload(sc) ? "R" : "", 664 sc->params.vpd.ec, sc->params.vpd.sn); 665 device_set_desc_copy(dev, buf); 666 667 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x", 668 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1], 669 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]); 670 671 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 672 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 673 t3_add_attach_sysctls(sc); 674 675 #ifdef TCP_OFFLOAD 676 for (i = 0; i < NUM_CPL_HANDLERS; i++) 677 sc->cpl_handler[i] = cpl_not_handled; 678 #endif 679 680 t3_intr_clear(sc); 681 error = cxgb_setup_interrupts(sc); 682 out: 683 if (error) 684 cxgb_free(sc); 685 686 return (error); 687 } 688 689 /* 690 * The cxgb_controller_detach routine is called with the device is 691 * unloaded from the system. 692 */ 693 694 static int 695 cxgb_controller_detach(device_t dev) 696 { 697 struct adapter *sc; 698 699 sc = device_get_softc(dev); 700 701 cxgb_free(sc); 702 703 return (0); 704 } 705 706 /* 707 * The cxgb_free() is called by the cxgb_controller_detach() routine 708 * to tear down the structures that were built up in 709 * cxgb_controller_attach(), and should be the final piece of work 710 * done when fully unloading the driver. 711 * 712 * 713 * 1. Shutting down the threads started by the cxgb_controller_attach() 714 * routine. 715 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()). 716 * 3. Detaching all of the port devices created during the 717 * cxgb_controller_attach() routine. 718 * 4. Removing the device children created via cxgb_controller_attach(). 719 * 5. Releasing PCI resources associated with the device. 720 * 6. Turning off the offload support, iff it was turned on. 721 * 7. Destroying the mutexes created in cxgb_controller_attach(). 722 * 723 */ 724 static void 725 cxgb_free(struct adapter *sc) 726 { 727 int i, nqsets = 0; 728 729 ADAPTER_LOCK(sc); 730 sc->flags |= CXGB_SHUTDOWN; 731 ADAPTER_UNLOCK(sc); 732 733 /* 734 * Make sure all child devices are gone. 735 */ 736 bus_generic_detach(sc->dev); 737 for (i = 0; i < (sc)->params.nports; i++) { 738 if (sc->portdev[i] && 739 device_delete_child(sc->dev, sc->portdev[i]) != 0) 740 device_printf(sc->dev, "failed to delete child port\n"); 741 nqsets += sc->port[i].nqsets; 742 } 743 744 /* 745 * At this point, it is as if cxgb_port_detach has run on all ports, and 746 * cxgb_down has run on the adapter. All interrupts have been silenced, 747 * all open devices have been closed. 748 */ 749 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)", 750 __func__, sc->open_device_map)); 751 for (i = 0; i < sc->params.nports; i++) { 752 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!", 753 __func__, i)); 754 } 755 756 /* 757 * Finish off the adapter's callouts. 758 */ 759 callout_drain(&sc->cxgb_tick_ch); 760 callout_drain(&sc->sge_timer_ch); 761 762 /* 763 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The 764 * sysctls are cleaned up by the kernel linker. 765 */ 766 if (sc->flags & FULL_INIT_DONE) { 767 t3_free_sge_resources(sc, nqsets); 768 sc->flags &= ~FULL_INIT_DONE; 769 } 770 771 /* 772 * Release all interrupt resources. 773 */ 774 cxgb_teardown_interrupts(sc); 775 if (sc->flags & (USING_MSI | USING_MSIX)) { 776 device_printf(sc->dev, "releasing msi message(s)\n"); 777 pci_release_msi(sc->dev); 778 } else { 779 device_printf(sc->dev, "no msi message to release\n"); 780 } 781 782 if (sc->msix_regs_res != NULL) { 783 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 784 sc->msix_regs_res); 785 } 786 787 /* 788 * Free the adapter's taskqueue. 789 */ 790 if (sc->tq != NULL) { 791 taskqueue_free(sc->tq); 792 sc->tq = NULL; 793 } 794 795 free(sc->filters, M_DEVBUF); 796 t3_sge_free(sc); 797 798 if (sc->udbs_res != NULL) 799 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 800 sc->udbs_res); 801 802 if (sc->regs_res != NULL) 803 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 804 sc->regs_res); 805 806 MTX_DESTROY(&sc->mdio_lock); 807 MTX_DESTROY(&sc->sge.reg_lock); 808 MTX_DESTROY(&sc->elmer_lock); 809 mtx_lock(&t3_list_lock); 810 SLIST_REMOVE(&t3_list, sc, adapter, link); 811 mtx_unlock(&t3_list_lock); 812 ADAPTER_LOCK_DEINIT(sc); 813 } 814 815 /** 816 * setup_sge_qsets - configure SGE Tx/Rx/response queues 817 * @sc: the controller softc 818 * 819 * Determines how many sets of SGE queues to use and initializes them. 820 * We support multiple queue sets per port if we have MSI-X, otherwise 821 * just one queue set per port. 822 */ 823 static int 824 setup_sge_qsets(adapter_t *sc) 825 { 826 int i, j, err, irq_idx = 0, qset_idx = 0; 827 u_int ntxq = SGE_TXQ_PER_SET; 828 829 if ((err = t3_sge_alloc(sc)) != 0) { 830 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 831 return (err); 832 } 833 834 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 835 irq_idx = -1; 836 837 for (i = 0; i < (sc)->params.nports; i++) { 838 struct port_info *pi = &sc->port[i]; 839 840 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 841 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 842 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 843 &sc->params.sge.qset[qset_idx], ntxq, pi); 844 if (err) { 845 t3_free_sge_resources(sc, qset_idx); 846 device_printf(sc->dev, 847 "t3_sge_alloc_qset failed with %d\n", err); 848 return (err); 849 } 850 } 851 } 852 853 return (0); 854 } 855 856 static void 857 cxgb_teardown_interrupts(adapter_t *sc) 858 { 859 int i; 860 861 for (i = 0; i < SGE_QSETS; i++) { 862 if (sc->msix_intr_tag[i] == NULL) { 863 864 /* Should have been setup fully or not at all */ 865 KASSERT(sc->msix_irq_res[i] == NULL && 866 sc->msix_irq_rid[i] == 0, 867 ("%s: half-done interrupt (%d).", __func__, i)); 868 869 continue; 870 } 871 872 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 873 sc->msix_intr_tag[i]); 874 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i], 875 sc->msix_irq_res[i]); 876 877 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL; 878 sc->msix_irq_rid[i] = 0; 879 } 880 881 if (sc->intr_tag) { 882 KASSERT(sc->irq_res != NULL, 883 ("%s: half-done interrupt.", __func__)); 884 885 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 886 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 887 sc->irq_res); 888 889 sc->irq_res = sc->intr_tag = NULL; 890 sc->irq_rid = 0; 891 } 892 } 893 894 static int 895 cxgb_setup_interrupts(adapter_t *sc) 896 { 897 struct resource *res; 898 void *tag; 899 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX); 900 901 sc->irq_rid = intr_flag ? 1 : 0; 902 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, 903 RF_SHAREABLE | RF_ACTIVE); 904 if (sc->irq_res == NULL) { 905 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n", 906 intr_flag, sc->irq_rid); 907 err = EINVAL; 908 sc->irq_rid = 0; 909 } else { 910 err = bus_setup_intr(sc->dev, sc->irq_res, 911 INTR_MPSAFE | INTR_TYPE_NET, NULL, 912 sc->cxgb_intr, sc, &sc->intr_tag); 913 914 if (err) { 915 device_printf(sc->dev, 916 "Cannot set up interrupt (%x, %u, %d)\n", 917 intr_flag, sc->irq_rid, err); 918 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 919 sc->irq_res); 920 sc->irq_res = sc->intr_tag = NULL; 921 sc->irq_rid = 0; 922 } 923 } 924 925 /* That's all for INTx or MSI */ 926 if (!(intr_flag & USING_MSIX) || err) 927 return (err); 928 929 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err"); 930 for (i = 0; i < sc->msi_count - 1; i++) { 931 rid = i + 2; 932 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, 933 RF_SHAREABLE | RF_ACTIVE); 934 if (res == NULL) { 935 device_printf(sc->dev, "Cannot allocate interrupt " 936 "for message %d\n", rid); 937 err = EINVAL; 938 break; 939 } 940 941 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET, 942 NULL, t3_intr_msix, &sc->sge.qs[i], &tag); 943 if (err) { 944 device_printf(sc->dev, "Cannot set up interrupt " 945 "for message %d (%d)\n", rid, err); 946 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res); 947 break; 948 } 949 950 sc->msix_irq_rid[i] = rid; 951 sc->msix_irq_res[i] = res; 952 sc->msix_intr_tag[i] = tag; 953 bus_describe_intr(sc->dev, res, tag, "qs%d", i); 954 } 955 956 if (err) 957 cxgb_teardown_interrupts(sc); 958 959 return (err); 960 } 961 962 963 static int 964 cxgb_port_probe(device_t dev) 965 { 966 struct port_info *p; 967 char buf[80]; 968 const char *desc; 969 970 p = device_get_softc(dev); 971 desc = p->phy.desc; 972 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc); 973 device_set_desc_copy(dev, buf); 974 return (0); 975 } 976 977 978 static int 979 cxgb_makedev(struct port_info *pi) 980 { 981 982 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 983 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp)); 984 985 if (pi->port_cdev == NULL) 986 return (ENOMEM); 987 988 pi->port_cdev->si_drv1 = (void *)pi; 989 990 return (0); 991 } 992 993 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 994 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 995 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 996 #define CXGB_CAP_ENABLE CXGB_CAP 997 998 static int 999 cxgb_port_attach(device_t dev) 1000 { 1001 struct port_info *p; 1002 struct ifnet *ifp; 1003 int err; 1004 struct adapter *sc; 1005 1006 p = device_get_softc(dev); 1007 sc = p->adapter; 1008 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 1009 device_get_unit(device_get_parent(dev)), p->port_id); 1010 PORT_LOCK_INIT(p, p->lockbuf); 1011 1012 callout_init(&p->link_check_ch, 1); 1013 TASK_INIT(&p->link_check_task, 0, check_link_status, p); 1014 1015 /* Allocate an ifnet object and set it up */ 1016 ifp = p->ifp = if_alloc(IFT_ETHER); 1017 if (ifp == NULL) { 1018 device_printf(dev, "Cannot allocate ifnet\n"); 1019 return (ENOMEM); 1020 } 1021 1022 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1023 ifp->if_init = cxgb_init; 1024 ifp->if_softc = p; 1025 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1026 ifp->if_ioctl = cxgb_ioctl; 1027 ifp->if_transmit = cxgb_transmit; 1028 ifp->if_qflush = cxgb_qflush; 1029 ifp->if_get_counter = cxgb_get_counter; 1030 1031 ifp->if_capabilities = CXGB_CAP; 1032 #ifdef TCP_OFFLOAD 1033 if (is_offload(sc)) 1034 ifp->if_capabilities |= IFCAP_TOE4; 1035 #endif 1036 ifp->if_capenable = CXGB_CAP_ENABLE; 1037 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1038 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1039 1040 /* 1041 * Disable TSO on 4-port - it isn't supported by the firmware. 1042 */ 1043 if (sc->params.nports > 2) { 1044 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO); 1045 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO); 1046 ifp->if_hwassist &= ~CSUM_TSO; 1047 } 1048 1049 ether_ifattach(ifp, p->hw_addr); 1050 1051 /* Attach driver netdump methods. */ 1052 NETDUMP_SET(ifp, cxgb); 1053 1054 #ifdef DEFAULT_JUMBO 1055 if (sc->params.nports <= 2) 1056 ifp->if_mtu = ETHERMTU_JUMBO; 1057 #endif 1058 if ((err = cxgb_makedev(p)) != 0) { 1059 printf("makedev failed %d\n", err); 1060 return (err); 1061 } 1062 1063 /* Create a list of media supported by this port */ 1064 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 1065 cxgb_media_status); 1066 cxgb_build_medialist(p); 1067 1068 t3_sge_init_port(p); 1069 1070 return (err); 1071 } 1072 1073 /* 1074 * cxgb_port_detach() is called via the device_detach methods when 1075 * cxgb_free() calls the bus_generic_detach. It is responsible for 1076 * removing the device from the view of the kernel, i.e. from all 1077 * interfaces lists etc. This routine is only called when the driver is 1078 * being unloaded, not when the link goes down. 1079 */ 1080 static int 1081 cxgb_port_detach(device_t dev) 1082 { 1083 struct port_info *p; 1084 struct adapter *sc; 1085 int i; 1086 1087 p = device_get_softc(dev); 1088 sc = p->adapter; 1089 1090 /* Tell cxgb_ioctl and if_init that the port is going away */ 1091 ADAPTER_LOCK(sc); 1092 SET_DOOMED(p); 1093 wakeup(&sc->flags); 1094 while (IS_BUSY(sc)) 1095 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0); 1096 SET_BUSY(sc); 1097 ADAPTER_UNLOCK(sc); 1098 1099 if (p->port_cdev != NULL) 1100 destroy_dev(p->port_cdev); 1101 1102 cxgb_uninit_synchronized(p); 1103 ether_ifdetach(p->ifp); 1104 1105 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1106 struct sge_qset *qs = &sc->sge.qs[i]; 1107 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1108 1109 callout_drain(&txq->txq_watchdog); 1110 callout_drain(&txq->txq_timer); 1111 } 1112 1113 PORT_LOCK_DEINIT(p); 1114 if_free(p->ifp); 1115 p->ifp = NULL; 1116 1117 ADAPTER_LOCK(sc); 1118 CLR_BUSY(sc); 1119 wakeup_one(&sc->flags); 1120 ADAPTER_UNLOCK(sc); 1121 return (0); 1122 } 1123 1124 void 1125 t3_fatal_err(struct adapter *sc) 1126 { 1127 u_int fw_status[4]; 1128 1129 if (sc->flags & FULL_INIT_DONE) { 1130 t3_sge_stop(sc); 1131 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1132 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1133 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1134 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1135 t3_intr_disable(sc); 1136 } 1137 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1138 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1139 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1140 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1141 } 1142 1143 int 1144 t3_os_find_pci_capability(adapter_t *sc, int cap) 1145 { 1146 device_t dev; 1147 struct pci_devinfo *dinfo; 1148 pcicfgregs *cfg; 1149 uint32_t status; 1150 uint8_t ptr; 1151 1152 dev = sc->dev; 1153 dinfo = device_get_ivars(dev); 1154 cfg = &dinfo->cfg; 1155 1156 status = pci_read_config(dev, PCIR_STATUS, 2); 1157 if (!(status & PCIM_STATUS_CAPPRESENT)) 1158 return (0); 1159 1160 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1161 case 0: 1162 case 1: 1163 ptr = PCIR_CAP_PTR; 1164 break; 1165 case 2: 1166 ptr = PCIR_CAP_PTR_2; 1167 break; 1168 default: 1169 return (0); 1170 break; 1171 } 1172 ptr = pci_read_config(dev, ptr, 1); 1173 1174 while (ptr != 0) { 1175 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1176 return (ptr); 1177 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1178 } 1179 1180 return (0); 1181 } 1182 1183 int 1184 t3_os_pci_save_state(struct adapter *sc) 1185 { 1186 device_t dev; 1187 struct pci_devinfo *dinfo; 1188 1189 dev = sc->dev; 1190 dinfo = device_get_ivars(dev); 1191 1192 pci_cfg_save(dev, dinfo, 0); 1193 return (0); 1194 } 1195 1196 int 1197 t3_os_pci_restore_state(struct adapter *sc) 1198 { 1199 device_t dev; 1200 struct pci_devinfo *dinfo; 1201 1202 dev = sc->dev; 1203 dinfo = device_get_ivars(dev); 1204 1205 pci_cfg_restore(dev, dinfo); 1206 return (0); 1207 } 1208 1209 /** 1210 * t3_os_link_changed - handle link status changes 1211 * @sc: the adapter associated with the link change 1212 * @port_id: the port index whose link status has changed 1213 * @link_status: the new status of the link 1214 * @speed: the new speed setting 1215 * @duplex: the new duplex setting 1216 * @fc: the new flow-control setting 1217 * 1218 * This is the OS-dependent handler for link status changes. The OS 1219 * neutral handler takes care of most of the processing for these events, 1220 * then calls this handler for any OS-specific processing. 1221 */ 1222 void 1223 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1224 int duplex, int fc, int mac_was_reset) 1225 { 1226 struct port_info *pi = &adapter->port[port_id]; 1227 struct ifnet *ifp = pi->ifp; 1228 1229 /* no race with detach, so ifp should always be good */ 1230 KASSERT(ifp, ("%s: if detached.", __func__)); 1231 1232 /* Reapply mac settings if they were lost due to a reset */ 1233 if (mac_was_reset) { 1234 PORT_LOCK(pi); 1235 cxgb_update_mac_settings(pi); 1236 PORT_UNLOCK(pi); 1237 } 1238 1239 if (link_status) { 1240 ifp->if_baudrate = IF_Mbps(speed); 1241 if_link_state_change(ifp, LINK_STATE_UP); 1242 } else 1243 if_link_state_change(ifp, LINK_STATE_DOWN); 1244 } 1245 1246 /** 1247 * t3_os_phymod_changed - handle PHY module changes 1248 * @phy: the PHY reporting the module change 1249 * @mod_type: new module type 1250 * 1251 * This is the OS-dependent handler for PHY module changes. It is 1252 * invoked when a PHY module is removed or inserted for any OS-specific 1253 * processing. 1254 */ 1255 void t3_os_phymod_changed(struct adapter *adap, int port_id) 1256 { 1257 static const char *mod_str[] = { 1258 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown" 1259 }; 1260 struct port_info *pi = &adap->port[port_id]; 1261 int mod = pi->phy.modtype; 1262 1263 if (mod != pi->media.ifm_cur->ifm_data) 1264 cxgb_build_medialist(pi); 1265 1266 if (mod == phy_modtype_none) 1267 if_printf(pi->ifp, "PHY module unplugged\n"); 1268 else { 1269 KASSERT(mod < ARRAY_SIZE(mod_str), 1270 ("invalid PHY module type %d", mod)); 1271 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]); 1272 } 1273 } 1274 1275 void 1276 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1277 { 1278 1279 /* 1280 * The ifnet might not be allocated before this gets called, 1281 * as this is called early on in attach by t3_prep_adapter 1282 * save the address off in the port structure 1283 */ 1284 if (cxgb_debug) 1285 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1286 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1287 } 1288 1289 /* 1290 * Programs the XGMAC based on the settings in the ifnet. These settings 1291 * include MTU, MAC address, mcast addresses, etc. 1292 */ 1293 static void 1294 cxgb_update_mac_settings(struct port_info *p) 1295 { 1296 struct ifnet *ifp = p->ifp; 1297 struct t3_rx_mode rm; 1298 struct cmac *mac = &p->mac; 1299 int mtu, hwtagging; 1300 1301 PORT_LOCK_ASSERT_OWNED(p); 1302 1303 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); 1304 1305 mtu = ifp->if_mtu; 1306 if (ifp->if_capenable & IFCAP_VLAN_MTU) 1307 mtu += ETHER_VLAN_ENCAP_LEN; 1308 1309 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; 1310 1311 t3_mac_set_mtu(mac, mtu); 1312 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1313 t3_mac_set_address(mac, 0, p->hw_addr); 1314 t3_init_rx_mode(&rm, p); 1315 t3_mac_set_rx_mode(mac, &rm); 1316 } 1317 1318 1319 static int 1320 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1321 unsigned long n) 1322 { 1323 int attempts = 5; 1324 1325 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1326 if (!--attempts) 1327 return (ETIMEDOUT); 1328 t3_os_sleep(10); 1329 } 1330 return 0; 1331 } 1332 1333 static int 1334 init_tp_parity(struct adapter *adap) 1335 { 1336 int i; 1337 struct mbuf *m; 1338 struct cpl_set_tcb_field *greq; 1339 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1340 1341 t3_tp_set_offload_mode(adap, 1); 1342 1343 for (i = 0; i < 16; i++) { 1344 struct cpl_smt_write_req *req; 1345 1346 m = m_gethdr(M_WAITOK, MT_DATA); 1347 req = mtod(m, struct cpl_smt_write_req *); 1348 m->m_len = m->m_pkthdr.len = sizeof(*req); 1349 memset(req, 0, sizeof(*req)); 1350 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1351 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1352 req->iff = i; 1353 t3_mgmt_tx(adap, m); 1354 } 1355 1356 for (i = 0; i < 2048; i++) { 1357 struct cpl_l2t_write_req *req; 1358 1359 m = m_gethdr(M_WAITOK, MT_DATA); 1360 req = mtod(m, struct cpl_l2t_write_req *); 1361 m->m_len = m->m_pkthdr.len = sizeof(*req); 1362 memset(req, 0, sizeof(*req)); 1363 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1364 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1365 req->params = htonl(V_L2T_W_IDX(i)); 1366 t3_mgmt_tx(adap, m); 1367 } 1368 1369 for (i = 0; i < 2048; i++) { 1370 struct cpl_rte_write_req *req; 1371 1372 m = m_gethdr(M_WAITOK, MT_DATA); 1373 req = mtod(m, struct cpl_rte_write_req *); 1374 m->m_len = m->m_pkthdr.len = sizeof(*req); 1375 memset(req, 0, sizeof(*req)); 1376 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1377 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1378 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1379 t3_mgmt_tx(adap, m); 1380 } 1381 1382 m = m_gethdr(M_WAITOK, MT_DATA); 1383 greq = mtod(m, struct cpl_set_tcb_field *); 1384 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1385 memset(greq, 0, sizeof(*greq)); 1386 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1387 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1388 greq->mask = htobe64(1); 1389 t3_mgmt_tx(adap, m); 1390 1391 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1392 t3_tp_set_offload_mode(adap, 0); 1393 return (i); 1394 } 1395 1396 /** 1397 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1398 * @adap: the adapter 1399 * 1400 * Sets up RSS to distribute packets to multiple receive queues. We 1401 * configure the RSS CPU lookup table to distribute to the number of HW 1402 * receive queues, and the response queue lookup table to narrow that 1403 * down to the response queues actually configured for each port. 1404 * We always configure the RSS mapping for two ports since the mapping 1405 * table has plenty of entries. 1406 */ 1407 static void 1408 setup_rss(adapter_t *adap) 1409 { 1410 int i; 1411 u_int nq[2]; 1412 uint8_t cpus[SGE_QSETS + 1]; 1413 uint16_t rspq_map[RSS_TABLE_SIZE]; 1414 1415 for (i = 0; i < SGE_QSETS; ++i) 1416 cpus[i] = i; 1417 cpus[SGE_QSETS] = 0xff; 1418 1419 nq[0] = nq[1] = 0; 1420 for_each_port(adap, i) { 1421 const struct port_info *pi = adap2pinfo(adap, i); 1422 1423 nq[pi->tx_chan] += pi->nqsets; 1424 } 1425 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1426 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1427 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1428 } 1429 1430 /* Calculate the reverse RSS map table */ 1431 for (i = 0; i < SGE_QSETS; ++i) 1432 adap->rrss_map[i] = 0xff; 1433 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1434 if (adap->rrss_map[rspq_map[i]] == 0xff) 1435 adap->rrss_map[rspq_map[i]] = i; 1436 1437 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1438 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1439 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1440 cpus, rspq_map); 1441 1442 } 1443 static void 1444 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1445 int hi, int port) 1446 { 1447 struct mbuf *m; 1448 struct mngt_pktsched_wr *req; 1449 1450 m = m_gethdr(M_NOWAIT, MT_DATA); 1451 if (m) { 1452 req = mtod(m, struct mngt_pktsched_wr *); 1453 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1454 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1455 req->sched = sched; 1456 req->idx = qidx; 1457 req->min = lo; 1458 req->max = hi; 1459 req->binding = port; 1460 m->m_len = m->m_pkthdr.len = sizeof(*req); 1461 t3_mgmt_tx(adap, m); 1462 } 1463 } 1464 1465 static void 1466 bind_qsets(adapter_t *sc) 1467 { 1468 int i, j; 1469 1470 for (i = 0; i < (sc)->params.nports; ++i) { 1471 const struct port_info *pi = adap2pinfo(sc, i); 1472 1473 for (j = 0; j < pi->nqsets; ++j) { 1474 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1475 -1, pi->tx_chan); 1476 1477 } 1478 } 1479 } 1480 1481 static void 1482 update_tpeeprom(struct adapter *adap) 1483 { 1484 const struct firmware *tpeeprom; 1485 1486 uint32_t version; 1487 unsigned int major, minor; 1488 int ret, len; 1489 char rev, name[32]; 1490 1491 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1492 1493 major = G_TP_VERSION_MAJOR(version); 1494 minor = G_TP_VERSION_MINOR(version); 1495 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1496 return; 1497 1498 rev = t3rev2char(adap); 1499 snprintf(name, sizeof(name), TPEEPROM_NAME, rev); 1500 1501 tpeeprom = firmware_get(name); 1502 if (tpeeprom == NULL) { 1503 device_printf(adap->dev, 1504 "could not load TP EEPROM: unable to load %s\n", 1505 name); 1506 return; 1507 } 1508 1509 len = tpeeprom->datasize - 4; 1510 1511 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1512 if (ret) 1513 goto release_tpeeprom; 1514 1515 if (len != TP_SRAM_LEN) { 1516 device_printf(adap->dev, 1517 "%s length is wrong len=%d expected=%d\n", name, 1518 len, TP_SRAM_LEN); 1519 return; 1520 } 1521 1522 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1523 TP_SRAM_OFFSET); 1524 1525 if (!ret) { 1526 device_printf(adap->dev, 1527 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1528 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1529 } else 1530 device_printf(adap->dev, 1531 "Protocol SRAM image update in EEPROM failed\n"); 1532 1533 release_tpeeprom: 1534 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1535 1536 return; 1537 } 1538 1539 static int 1540 update_tpsram(struct adapter *adap) 1541 { 1542 const struct firmware *tpsram; 1543 int ret; 1544 char rev, name[32]; 1545 1546 rev = t3rev2char(adap); 1547 snprintf(name, sizeof(name), TPSRAM_NAME, rev); 1548 1549 update_tpeeprom(adap); 1550 1551 tpsram = firmware_get(name); 1552 if (tpsram == NULL){ 1553 device_printf(adap->dev, "could not load TP SRAM\n"); 1554 return (EINVAL); 1555 } else 1556 device_printf(adap->dev, "updating TP SRAM\n"); 1557 1558 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1559 if (ret) 1560 goto release_tpsram; 1561 1562 ret = t3_set_proto_sram(adap, tpsram->data); 1563 if (ret) 1564 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1565 1566 release_tpsram: 1567 firmware_put(tpsram, FIRMWARE_UNLOAD); 1568 1569 return ret; 1570 } 1571 1572 /** 1573 * cxgb_up - enable the adapter 1574 * @adap: adapter being enabled 1575 * 1576 * Called when the first port is enabled, this function performs the 1577 * actions necessary to make an adapter operational, such as completing 1578 * the initialization of HW modules, and enabling interrupts. 1579 */ 1580 static int 1581 cxgb_up(struct adapter *sc) 1582 { 1583 int err = 0; 1584 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS; 1585 1586 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)", 1587 __func__, sc->open_device_map)); 1588 1589 if ((sc->flags & FULL_INIT_DONE) == 0) { 1590 1591 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1592 1593 if ((sc->flags & FW_UPTODATE) == 0) 1594 if ((err = upgrade_fw(sc))) 1595 goto out; 1596 1597 if ((sc->flags & TPS_UPTODATE) == 0) 1598 if ((err = update_tpsram(sc))) 1599 goto out; 1600 1601 if (is_offload(sc) && nfilters != 0) { 1602 sc->params.mc5.nservers = 0; 1603 1604 if (nfilters < 0) 1605 sc->params.mc5.nfilters = mxf; 1606 else 1607 sc->params.mc5.nfilters = min(nfilters, mxf); 1608 } 1609 1610 err = t3_init_hw(sc, 0); 1611 if (err) 1612 goto out; 1613 1614 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1615 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1616 1617 err = setup_sge_qsets(sc); 1618 if (err) 1619 goto out; 1620 1621 alloc_filters(sc); 1622 setup_rss(sc); 1623 1624 t3_add_configured_sysctls(sc); 1625 sc->flags |= FULL_INIT_DONE; 1626 } 1627 1628 t3_intr_clear(sc); 1629 t3_sge_start(sc); 1630 t3_intr_enable(sc); 1631 1632 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1633 is_offload(sc) && init_tp_parity(sc) == 0) 1634 sc->flags |= TP_PARITY_INIT; 1635 1636 if (sc->flags & TP_PARITY_INIT) { 1637 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR); 1638 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1639 } 1640 1641 if (!(sc->flags & QUEUES_BOUND)) { 1642 bind_qsets(sc); 1643 setup_hw_filters(sc); 1644 sc->flags |= QUEUES_BOUND; 1645 } 1646 1647 t3_sge_reset_adapter(sc); 1648 out: 1649 return (err); 1650 } 1651 1652 /* 1653 * Called when the last open device is closed. Does NOT undo all of cxgb_up's 1654 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released 1655 * during controller_detach, not here. 1656 */ 1657 static void 1658 cxgb_down(struct adapter *sc) 1659 { 1660 t3_sge_stop(sc); 1661 t3_intr_disable(sc); 1662 } 1663 1664 /* 1665 * if_init for cxgb ports. 1666 */ 1667 static void 1668 cxgb_init(void *arg) 1669 { 1670 struct port_info *p = arg; 1671 struct adapter *sc = p->adapter; 1672 1673 ADAPTER_LOCK(sc); 1674 cxgb_init_locked(p); /* releases adapter lock */ 1675 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1676 } 1677 1678 static int 1679 cxgb_init_locked(struct port_info *p) 1680 { 1681 struct adapter *sc = p->adapter; 1682 struct ifnet *ifp = p->ifp; 1683 struct cmac *mac = &p->mac; 1684 int i, rc = 0, may_sleep = 0, gave_up_lock = 0; 1685 1686 ADAPTER_LOCK_ASSERT_OWNED(sc); 1687 1688 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1689 gave_up_lock = 1; 1690 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) { 1691 rc = EINTR; 1692 goto done; 1693 } 1694 } 1695 if (IS_DOOMED(p)) { 1696 rc = ENXIO; 1697 goto done; 1698 } 1699 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1700 1701 /* 1702 * The code that runs during one-time adapter initialization can sleep 1703 * so it's important not to hold any locks across it. 1704 */ 1705 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1; 1706 1707 if (may_sleep) { 1708 SET_BUSY(sc); 1709 gave_up_lock = 1; 1710 ADAPTER_UNLOCK(sc); 1711 } 1712 1713 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0)) 1714 goto done; 1715 1716 PORT_LOCK(p); 1717 if (isset(&sc->open_device_map, p->port_id) && 1718 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1719 PORT_UNLOCK(p); 1720 goto done; 1721 } 1722 t3_port_intr_enable(sc, p->port_id); 1723 if (!mac->multiport) 1724 t3_mac_init(mac); 1725 cxgb_update_mac_settings(p); 1726 t3_link_start(&p->phy, mac, &p->link_config); 1727 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1728 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1729 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1730 PORT_UNLOCK(p); 1731 1732 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1733 struct sge_qset *qs = &sc->sge.qs[i]; 1734 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1735 1736 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs, 1737 txq->txq_watchdog.c_cpu); 1738 } 1739 1740 /* all ok */ 1741 setbit(&sc->open_device_map, p->port_id); 1742 callout_reset(&p->link_check_ch, 1743 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4, 1744 link_check_callout, p); 1745 1746 done: 1747 if (may_sleep) { 1748 ADAPTER_LOCK(sc); 1749 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1750 CLR_BUSY(sc); 1751 } 1752 if (gave_up_lock) 1753 wakeup_one(&sc->flags); 1754 ADAPTER_UNLOCK(sc); 1755 return (rc); 1756 } 1757 1758 static int 1759 cxgb_uninit_locked(struct port_info *p) 1760 { 1761 struct adapter *sc = p->adapter; 1762 int rc; 1763 1764 ADAPTER_LOCK_ASSERT_OWNED(sc); 1765 1766 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1767 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) { 1768 rc = EINTR; 1769 goto done; 1770 } 1771 } 1772 if (IS_DOOMED(p)) { 1773 rc = ENXIO; 1774 goto done; 1775 } 1776 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1777 SET_BUSY(sc); 1778 ADAPTER_UNLOCK(sc); 1779 1780 rc = cxgb_uninit_synchronized(p); 1781 1782 ADAPTER_LOCK(sc); 1783 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1784 CLR_BUSY(sc); 1785 wakeup_one(&sc->flags); 1786 done: 1787 ADAPTER_UNLOCK(sc); 1788 return (rc); 1789 } 1790 1791 /* 1792 * Called on "ifconfig down", and from port_detach 1793 */ 1794 static int 1795 cxgb_uninit_synchronized(struct port_info *pi) 1796 { 1797 struct adapter *sc = pi->adapter; 1798 struct ifnet *ifp = pi->ifp; 1799 1800 /* 1801 * taskqueue_drain may cause a deadlock if the adapter lock is held. 1802 */ 1803 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1804 1805 /* 1806 * Clear this port's bit from the open device map, and then drain all 1807 * the tasks that can access/manipulate this port's port_info or ifp. 1808 * We disable this port's interrupts here and so the slow/ext 1809 * interrupt tasks won't be enqueued. The tick task will continue to 1810 * be enqueued every second but the runs after this drain will not see 1811 * this port in the open device map. 1812 * 1813 * A well behaved task must take open_device_map into account and ignore 1814 * ports that are not open. 1815 */ 1816 clrbit(&sc->open_device_map, pi->port_id); 1817 t3_port_intr_disable(sc, pi->port_id); 1818 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1819 taskqueue_drain(sc->tq, &sc->tick_task); 1820 1821 callout_drain(&pi->link_check_ch); 1822 taskqueue_drain(sc->tq, &pi->link_check_task); 1823 1824 PORT_LOCK(pi); 1825 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1826 1827 /* disable pause frames */ 1828 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0); 1829 1830 /* Reset RX FIFO HWM */ 1831 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset, 1832 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1833 1834 DELAY(100 * 1000); 1835 1836 /* Wait for TXFIFO empty */ 1837 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset, 1838 F_TXFIFO_EMPTY, 1, 20, 5); 1839 1840 DELAY(100 * 1000); 1841 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX); 1842 1843 pi->phy.ops->power_down(&pi->phy, 1); 1844 1845 PORT_UNLOCK(pi); 1846 1847 pi->link_config.link_ok = 0; 1848 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0); 1849 1850 if (sc->open_device_map == 0) 1851 cxgb_down(pi->adapter); 1852 1853 return (0); 1854 } 1855 1856 /* 1857 * Mark lro enabled or disabled in all qsets for this port 1858 */ 1859 static int 1860 cxgb_set_lro(struct port_info *p, int enabled) 1861 { 1862 int i; 1863 struct adapter *adp = p->adapter; 1864 struct sge_qset *q; 1865 1866 for (i = 0; i < p->nqsets; i++) { 1867 q = &adp->sge.qs[p->first_qset + i]; 1868 q->lro.enabled = (enabled != 0); 1869 } 1870 return (0); 1871 } 1872 1873 static int 1874 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1875 { 1876 struct port_info *p = ifp->if_softc; 1877 struct adapter *sc = p->adapter; 1878 struct ifreq *ifr = (struct ifreq *)data; 1879 int flags, error = 0, mtu; 1880 uint32_t mask; 1881 1882 switch (command) { 1883 case SIOCSIFMTU: 1884 ADAPTER_LOCK(sc); 1885 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1886 if (error) { 1887 fail: 1888 ADAPTER_UNLOCK(sc); 1889 return (error); 1890 } 1891 1892 mtu = ifr->ifr_mtu; 1893 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 1894 error = EINVAL; 1895 } else { 1896 ifp->if_mtu = mtu; 1897 PORT_LOCK(p); 1898 cxgb_update_mac_settings(p); 1899 PORT_UNLOCK(p); 1900 } 1901 ADAPTER_UNLOCK(sc); 1902 break; 1903 case SIOCSIFFLAGS: 1904 ADAPTER_LOCK(sc); 1905 if (IS_DOOMED(p)) { 1906 error = ENXIO; 1907 goto fail; 1908 } 1909 if (ifp->if_flags & IFF_UP) { 1910 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1911 flags = p->if_flags; 1912 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1913 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) { 1914 if (IS_BUSY(sc)) { 1915 error = EBUSY; 1916 goto fail; 1917 } 1918 PORT_LOCK(p); 1919 cxgb_update_mac_settings(p); 1920 PORT_UNLOCK(p); 1921 } 1922 ADAPTER_UNLOCK(sc); 1923 } else 1924 error = cxgb_init_locked(p); 1925 p->if_flags = ifp->if_flags; 1926 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1927 error = cxgb_uninit_locked(p); 1928 else 1929 ADAPTER_UNLOCK(sc); 1930 1931 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1932 break; 1933 case SIOCADDMULTI: 1934 case SIOCDELMULTI: 1935 ADAPTER_LOCK(sc); 1936 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1937 if (error) 1938 goto fail; 1939 1940 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1941 PORT_LOCK(p); 1942 cxgb_update_mac_settings(p); 1943 PORT_UNLOCK(p); 1944 } 1945 ADAPTER_UNLOCK(sc); 1946 1947 break; 1948 case SIOCSIFCAP: 1949 ADAPTER_LOCK(sc); 1950 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1951 if (error) 1952 goto fail; 1953 1954 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1955 if (mask & IFCAP_TXCSUM) { 1956 ifp->if_capenable ^= IFCAP_TXCSUM; 1957 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1958 1959 if (IFCAP_TSO4 & ifp->if_capenable && 1960 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1961 ifp->if_capenable &= ~IFCAP_TSO4; 1962 if_printf(ifp, 1963 "tso4 disabled due to -txcsum.\n"); 1964 } 1965 } 1966 if (mask & IFCAP_TXCSUM_IPV6) { 1967 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1968 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1969 1970 if (IFCAP_TSO6 & ifp->if_capenable && 1971 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1972 ifp->if_capenable &= ~IFCAP_TSO6; 1973 if_printf(ifp, 1974 "tso6 disabled due to -txcsum6.\n"); 1975 } 1976 } 1977 if (mask & IFCAP_RXCSUM) 1978 ifp->if_capenable ^= IFCAP_RXCSUM; 1979 if (mask & IFCAP_RXCSUM_IPV6) 1980 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1981 1982 /* 1983 * Note that we leave CSUM_TSO alone (it is always set). The 1984 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1985 * sending a TSO request our way, so it's sufficient to toggle 1986 * IFCAP_TSOx only. 1987 */ 1988 if (mask & IFCAP_TSO4) { 1989 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1990 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1991 if_printf(ifp, "enable txcsum first.\n"); 1992 error = EAGAIN; 1993 goto fail; 1994 } 1995 ifp->if_capenable ^= IFCAP_TSO4; 1996 } 1997 if (mask & IFCAP_TSO6) { 1998 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1999 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 2000 if_printf(ifp, "enable txcsum6 first.\n"); 2001 error = EAGAIN; 2002 goto fail; 2003 } 2004 ifp->if_capenable ^= IFCAP_TSO6; 2005 } 2006 if (mask & IFCAP_LRO) { 2007 ifp->if_capenable ^= IFCAP_LRO; 2008 2009 /* Safe to do this even if cxgb_up not called yet */ 2010 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO); 2011 } 2012 #ifdef TCP_OFFLOAD 2013 if (mask & IFCAP_TOE4) { 2014 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4; 2015 2016 error = toe_capability(p, enable); 2017 if (error == 0) 2018 ifp->if_capenable ^= mask; 2019 } 2020 #endif 2021 if (mask & IFCAP_VLAN_HWTAGGING) { 2022 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2023 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2024 PORT_LOCK(p); 2025 cxgb_update_mac_settings(p); 2026 PORT_UNLOCK(p); 2027 } 2028 } 2029 if (mask & IFCAP_VLAN_MTU) { 2030 ifp->if_capenable ^= IFCAP_VLAN_MTU; 2031 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2032 PORT_LOCK(p); 2033 cxgb_update_mac_settings(p); 2034 PORT_UNLOCK(p); 2035 } 2036 } 2037 if (mask & IFCAP_VLAN_HWTSO) 2038 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 2039 if (mask & IFCAP_VLAN_HWCSUM) 2040 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 2041 2042 #ifdef VLAN_CAPABILITIES 2043 VLAN_CAPABILITIES(ifp); 2044 #endif 2045 ADAPTER_UNLOCK(sc); 2046 break; 2047 case SIOCSIFMEDIA: 2048 case SIOCGIFMEDIA: 2049 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 2050 break; 2051 default: 2052 error = ether_ioctl(ifp, command, data); 2053 } 2054 2055 return (error); 2056 } 2057 2058 static int 2059 cxgb_media_change(struct ifnet *ifp) 2060 { 2061 return (EOPNOTSUPP); 2062 } 2063 2064 /* 2065 * Translates phy->modtype to the correct Ethernet media subtype. 2066 */ 2067 static int 2068 cxgb_ifm_type(int mod) 2069 { 2070 switch (mod) { 2071 case phy_modtype_sr: 2072 return (IFM_10G_SR); 2073 case phy_modtype_lr: 2074 return (IFM_10G_LR); 2075 case phy_modtype_lrm: 2076 return (IFM_10G_LRM); 2077 case phy_modtype_twinax: 2078 return (IFM_10G_TWINAX); 2079 case phy_modtype_twinax_long: 2080 return (IFM_10G_TWINAX_LONG); 2081 case phy_modtype_none: 2082 return (IFM_NONE); 2083 case phy_modtype_unknown: 2084 return (IFM_UNKNOWN); 2085 } 2086 2087 KASSERT(0, ("%s: modtype %d unknown", __func__, mod)); 2088 return (IFM_UNKNOWN); 2089 } 2090 2091 /* 2092 * Rebuilds the ifmedia list for this port, and sets the current media. 2093 */ 2094 static void 2095 cxgb_build_medialist(struct port_info *p) 2096 { 2097 struct cphy *phy = &p->phy; 2098 struct ifmedia *media = &p->media; 2099 int mod = phy->modtype; 2100 int m = IFM_ETHER | IFM_FDX; 2101 2102 PORT_LOCK(p); 2103 2104 ifmedia_removeall(media); 2105 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) { 2106 /* Copper (RJ45) */ 2107 2108 if (phy->caps & SUPPORTED_10000baseT_Full) 2109 ifmedia_add(media, m | IFM_10G_T, mod, NULL); 2110 2111 if (phy->caps & SUPPORTED_1000baseT_Full) 2112 ifmedia_add(media, m | IFM_1000_T, mod, NULL); 2113 2114 if (phy->caps & SUPPORTED_100baseT_Full) 2115 ifmedia_add(media, m | IFM_100_TX, mod, NULL); 2116 2117 if (phy->caps & SUPPORTED_10baseT_Full) 2118 ifmedia_add(media, m | IFM_10_T, mod, NULL); 2119 2120 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL); 2121 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2122 2123 } else if (phy->caps & SUPPORTED_TP) { 2124 /* Copper (CX4) */ 2125 2126 KASSERT(phy->caps & SUPPORTED_10000baseT_Full, 2127 ("%s: unexpected cap 0x%x", __func__, phy->caps)); 2128 2129 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL); 2130 ifmedia_set(media, m | IFM_10G_CX4); 2131 2132 } else if (phy->caps & SUPPORTED_FIBRE && 2133 phy->caps & SUPPORTED_10000baseT_Full) { 2134 /* 10G optical (but includes SFP+ twinax) */ 2135 2136 m |= cxgb_ifm_type(mod); 2137 if (IFM_SUBTYPE(m) == IFM_NONE) 2138 m &= ~IFM_FDX; 2139 2140 ifmedia_add(media, m, mod, NULL); 2141 ifmedia_set(media, m); 2142 2143 } else if (phy->caps & SUPPORTED_FIBRE && 2144 phy->caps & SUPPORTED_1000baseT_Full) { 2145 /* 1G optical */ 2146 2147 /* XXX: Lie and claim to be SX, could actually be any 1G-X */ 2148 ifmedia_add(media, m | IFM_1000_SX, mod, NULL); 2149 ifmedia_set(media, m | IFM_1000_SX); 2150 2151 } else { 2152 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__, 2153 phy->caps)); 2154 } 2155 2156 PORT_UNLOCK(p); 2157 } 2158 2159 static void 2160 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2161 { 2162 struct port_info *p = ifp->if_softc; 2163 struct ifmedia_entry *cur = p->media.ifm_cur; 2164 int speed = p->link_config.speed; 2165 2166 if (cur->ifm_data != p->phy.modtype) { 2167 cxgb_build_medialist(p); 2168 cur = p->media.ifm_cur; 2169 } 2170 2171 ifmr->ifm_status = IFM_AVALID; 2172 if (!p->link_config.link_ok) 2173 return; 2174 2175 ifmr->ifm_status |= IFM_ACTIVE; 2176 2177 /* 2178 * active and current will differ iff current media is autoselect. That 2179 * can happen only for copper RJ45. 2180 */ 2181 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 2182 return; 2183 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg, 2184 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps)); 2185 2186 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 2187 if (speed == SPEED_10000) 2188 ifmr->ifm_active |= IFM_10G_T; 2189 else if (speed == SPEED_1000) 2190 ifmr->ifm_active |= IFM_1000_T; 2191 else if (speed == SPEED_100) 2192 ifmr->ifm_active |= IFM_100_TX; 2193 else if (speed == SPEED_10) 2194 ifmr->ifm_active |= IFM_10_T; 2195 else 2196 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 2197 speed)); 2198 } 2199 2200 static uint64_t 2201 cxgb_get_counter(struct ifnet *ifp, ift_counter c) 2202 { 2203 struct port_info *pi = ifp->if_softc; 2204 struct adapter *sc = pi->adapter; 2205 struct cmac *mac = &pi->mac; 2206 struct mac_stats *mstats = &mac->stats; 2207 2208 cxgb_refresh_stats(pi); 2209 2210 switch (c) { 2211 case IFCOUNTER_IPACKETS: 2212 return (mstats->rx_frames); 2213 2214 case IFCOUNTER_IERRORS: 2215 return (mstats->rx_jabber + mstats->rx_data_errs + 2216 mstats->rx_sequence_errs + mstats->rx_runt + 2217 mstats->rx_too_long + mstats->rx_mac_internal_errs + 2218 mstats->rx_short + mstats->rx_fcs_errs); 2219 2220 case IFCOUNTER_OPACKETS: 2221 return (mstats->tx_frames); 2222 2223 case IFCOUNTER_OERRORS: 2224 return (mstats->tx_excess_collisions + mstats->tx_underrun + 2225 mstats->tx_len_errs + mstats->tx_mac_internal_errs + 2226 mstats->tx_excess_deferral + mstats->tx_fcs_errs); 2227 2228 case IFCOUNTER_COLLISIONS: 2229 return (mstats->tx_total_collisions); 2230 2231 case IFCOUNTER_IBYTES: 2232 return (mstats->rx_octets); 2233 2234 case IFCOUNTER_OBYTES: 2235 return (mstats->tx_octets); 2236 2237 case IFCOUNTER_IMCASTS: 2238 return (mstats->rx_mcast_frames); 2239 2240 case IFCOUNTER_OMCASTS: 2241 return (mstats->tx_mcast_frames); 2242 2243 case IFCOUNTER_IQDROPS: 2244 return (mstats->rx_cong_drops); 2245 2246 case IFCOUNTER_OQDROPS: { 2247 int i; 2248 uint64_t drops; 2249 2250 drops = 0; 2251 if (sc->flags & FULL_INIT_DONE) { 2252 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) 2253 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops; 2254 } 2255 2256 return (drops); 2257 2258 } 2259 2260 default: 2261 return (if_get_counter_default(ifp, c)); 2262 } 2263 } 2264 2265 static void 2266 cxgb_async_intr(void *data) 2267 { 2268 adapter_t *sc = data; 2269 2270 t3_write_reg(sc, A_PL_INT_ENABLE0, 0); 2271 (void) t3_read_reg(sc, A_PL_INT_ENABLE0); 2272 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2273 } 2274 2275 static void 2276 link_check_callout(void *arg) 2277 { 2278 struct port_info *pi = arg; 2279 struct adapter *sc = pi->adapter; 2280 2281 if (!isset(&sc->open_device_map, pi->port_id)) 2282 return; 2283 2284 taskqueue_enqueue(sc->tq, &pi->link_check_task); 2285 } 2286 2287 static void 2288 check_link_status(void *arg, int pending) 2289 { 2290 struct port_info *pi = arg; 2291 struct adapter *sc = pi->adapter; 2292 2293 if (!isset(&sc->open_device_map, pi->port_id)) 2294 return; 2295 2296 t3_link_changed(sc, pi->port_id); 2297 2298 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) || 2299 pi->link_config.link_ok == 0) 2300 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi); 2301 } 2302 2303 void 2304 t3_os_link_intr(struct port_info *pi) 2305 { 2306 /* 2307 * Schedule a link check in the near future. If the link is flapping 2308 * rapidly we'll keep resetting the callout and delaying the check until 2309 * things stabilize a bit. 2310 */ 2311 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi); 2312 } 2313 2314 static void 2315 check_t3b2_mac(struct adapter *sc) 2316 { 2317 int i; 2318 2319 if (sc->flags & CXGB_SHUTDOWN) 2320 return; 2321 2322 for_each_port(sc, i) { 2323 struct port_info *p = &sc->port[i]; 2324 int status; 2325 #ifdef INVARIANTS 2326 struct ifnet *ifp = p->ifp; 2327 #endif 2328 2329 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault || 2330 !p->link_config.link_ok) 2331 continue; 2332 2333 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2334 ("%s: state mismatch (drv_flags %x, device_map %x)", 2335 __func__, ifp->if_drv_flags, sc->open_device_map)); 2336 2337 PORT_LOCK(p); 2338 status = t3b2_mac_watchdog_task(&p->mac); 2339 if (status == 1) 2340 p->mac.stats.num_toggled++; 2341 else if (status == 2) { 2342 struct cmac *mac = &p->mac; 2343 2344 cxgb_update_mac_settings(p); 2345 t3_link_start(&p->phy, mac, &p->link_config); 2346 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2347 t3_port_intr_enable(sc, p->port_id); 2348 p->mac.stats.num_resets++; 2349 } 2350 PORT_UNLOCK(p); 2351 } 2352 } 2353 2354 static void 2355 cxgb_tick(void *arg) 2356 { 2357 adapter_t *sc = (adapter_t *)arg; 2358 2359 if (sc->flags & CXGB_SHUTDOWN) 2360 return; 2361 2362 taskqueue_enqueue(sc->tq, &sc->tick_task); 2363 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 2364 } 2365 2366 void 2367 cxgb_refresh_stats(struct port_info *pi) 2368 { 2369 struct timeval tv; 2370 const struct timeval interval = {0, 250000}; /* 250ms */ 2371 2372 getmicrotime(&tv); 2373 timevalsub(&tv, &interval); 2374 if (timevalcmp(&tv, &pi->last_refreshed, <)) 2375 return; 2376 2377 PORT_LOCK(pi); 2378 t3_mac_update_stats(&pi->mac); 2379 PORT_UNLOCK(pi); 2380 getmicrotime(&pi->last_refreshed); 2381 } 2382 2383 static void 2384 cxgb_tick_handler(void *arg, int count) 2385 { 2386 adapter_t *sc = (adapter_t *)arg; 2387 const struct adapter_params *p = &sc->params; 2388 int i; 2389 uint32_t cause, reset; 2390 2391 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE)) 2392 return; 2393 2394 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2395 check_t3b2_mac(sc); 2396 2397 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY); 2398 if (cause) { 2399 struct sge_qset *qs = &sc->sge.qs[0]; 2400 uint32_t mask, v; 2401 2402 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00; 2403 2404 mask = 1; 2405 for (i = 0; i < SGE_QSETS; i++) { 2406 if (v & mask) 2407 qs[i].rspq.starved++; 2408 mask <<= 1; 2409 } 2410 2411 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */ 2412 2413 for (i = 0; i < SGE_QSETS * 2; i++) { 2414 if (v & mask) { 2415 qs[i / 2].fl[i % 2].empty++; 2416 } 2417 mask <<= 1; 2418 } 2419 2420 /* clear */ 2421 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v); 2422 t3_write_reg(sc, A_SG_INT_CAUSE, cause); 2423 } 2424 2425 for (i = 0; i < sc->params.nports; i++) { 2426 struct port_info *pi = &sc->port[i]; 2427 struct cmac *mac = &pi->mac; 2428 2429 if (!isset(&sc->open_device_map, pi->port_id)) 2430 continue; 2431 2432 cxgb_refresh_stats(pi); 2433 2434 if (mac->multiport) 2435 continue; 2436 2437 /* Count rx fifo overflows, once per second */ 2438 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset); 2439 reset = 0; 2440 if (cause & F_RXFIFO_OVERFLOW) { 2441 mac->stats.rx_fifo_ovfl++; 2442 reset |= F_RXFIFO_OVERFLOW; 2443 } 2444 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset); 2445 } 2446 } 2447 2448 static void 2449 touch_bars(device_t dev) 2450 { 2451 /* 2452 * Don't enable yet 2453 */ 2454 #if !defined(__LP64__) && 0 2455 u32 v; 2456 2457 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2458 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2459 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2460 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2461 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2462 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2463 #endif 2464 } 2465 2466 static int 2467 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2468 { 2469 uint8_t *buf; 2470 int err = 0; 2471 u32 aligned_offset, aligned_len, *p; 2472 struct adapter *adapter = pi->adapter; 2473 2474 2475 aligned_offset = offset & ~3; 2476 aligned_len = (len + (offset & 3) + 3) & ~3; 2477 2478 if (aligned_offset != offset || aligned_len != len) { 2479 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2480 if (!buf) 2481 return (ENOMEM); 2482 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2483 if (!err && aligned_len > 4) 2484 err = t3_seeprom_read(adapter, 2485 aligned_offset + aligned_len - 4, 2486 (u32 *)&buf[aligned_len - 4]); 2487 if (err) 2488 goto out; 2489 memcpy(buf + (offset & 3), data, len); 2490 } else 2491 buf = (uint8_t *)(uintptr_t)data; 2492 2493 err = t3_seeprom_wp(adapter, 0); 2494 if (err) 2495 goto out; 2496 2497 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2498 err = t3_seeprom_write(adapter, aligned_offset, *p); 2499 aligned_offset += 4; 2500 } 2501 2502 if (!err) 2503 err = t3_seeprom_wp(adapter, 1); 2504 out: 2505 if (buf != data) 2506 free(buf, M_DEVBUF); 2507 return err; 2508 } 2509 2510 2511 static int 2512 in_range(int val, int lo, int hi) 2513 { 2514 return val < 0 || (val <= hi && val >= lo); 2515 } 2516 2517 static int 2518 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td) 2519 { 2520 return (0); 2521 } 2522 2523 static int 2524 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td) 2525 { 2526 return (0); 2527 } 2528 2529 static int 2530 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2531 int fflag, struct thread *td) 2532 { 2533 int mmd, error = 0; 2534 struct port_info *pi = dev->si_drv1; 2535 adapter_t *sc = pi->adapter; 2536 2537 #ifdef PRIV_SUPPORTED 2538 if (priv_check(td, PRIV_DRIVER)) { 2539 if (cxgb_debug) 2540 printf("user does not have access to privileged ioctls\n"); 2541 return (EPERM); 2542 } 2543 #else 2544 if (suser(td)) { 2545 if (cxgb_debug) 2546 printf("user does not have access to privileged ioctls\n"); 2547 return (EPERM); 2548 } 2549 #endif 2550 2551 switch (cmd) { 2552 case CHELSIO_GET_MIIREG: { 2553 uint32_t val; 2554 struct cphy *phy = &pi->phy; 2555 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2556 2557 if (!phy->mdio_read) 2558 return (EOPNOTSUPP); 2559 if (is_10G(sc)) { 2560 mmd = mid->phy_id >> 8; 2561 if (!mmd) 2562 mmd = MDIO_DEV_PCS; 2563 else if (mmd > MDIO_DEV_VEND2) 2564 return (EINVAL); 2565 2566 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2567 mid->reg_num, &val); 2568 } else 2569 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2570 mid->reg_num & 0x1f, &val); 2571 if (error == 0) 2572 mid->val_out = val; 2573 break; 2574 } 2575 case CHELSIO_SET_MIIREG: { 2576 struct cphy *phy = &pi->phy; 2577 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2578 2579 if (!phy->mdio_write) 2580 return (EOPNOTSUPP); 2581 if (is_10G(sc)) { 2582 mmd = mid->phy_id >> 8; 2583 if (!mmd) 2584 mmd = MDIO_DEV_PCS; 2585 else if (mmd > MDIO_DEV_VEND2) 2586 return (EINVAL); 2587 2588 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2589 mmd, mid->reg_num, mid->val_in); 2590 } else 2591 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2592 mid->reg_num & 0x1f, 2593 mid->val_in); 2594 break; 2595 } 2596 case CHELSIO_SETREG: { 2597 struct ch_reg *edata = (struct ch_reg *)data; 2598 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2599 return (EFAULT); 2600 t3_write_reg(sc, edata->addr, edata->val); 2601 break; 2602 } 2603 case CHELSIO_GETREG: { 2604 struct ch_reg *edata = (struct ch_reg *)data; 2605 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2606 return (EFAULT); 2607 edata->val = t3_read_reg(sc, edata->addr); 2608 break; 2609 } 2610 case CHELSIO_GET_SGE_CONTEXT: { 2611 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2612 mtx_lock_spin(&sc->sge.reg_lock); 2613 switch (ecntxt->cntxt_type) { 2614 case CNTXT_TYPE_EGRESS: 2615 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2616 ecntxt->data); 2617 break; 2618 case CNTXT_TYPE_FL: 2619 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2620 ecntxt->data); 2621 break; 2622 case CNTXT_TYPE_RSP: 2623 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2624 ecntxt->data); 2625 break; 2626 case CNTXT_TYPE_CQ: 2627 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2628 ecntxt->data); 2629 break; 2630 default: 2631 error = EINVAL; 2632 break; 2633 } 2634 mtx_unlock_spin(&sc->sge.reg_lock); 2635 break; 2636 } 2637 case CHELSIO_GET_SGE_DESC: { 2638 struct ch_desc *edesc = (struct ch_desc *)data; 2639 int ret; 2640 if (edesc->queue_num >= SGE_QSETS * 6) 2641 return (EINVAL); 2642 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2643 edesc->queue_num % 6, edesc->idx, edesc->data); 2644 if (ret < 0) 2645 return (EINVAL); 2646 edesc->size = ret; 2647 break; 2648 } 2649 case CHELSIO_GET_QSET_PARAMS: { 2650 struct qset_params *q; 2651 struct ch_qset_params *t = (struct ch_qset_params *)data; 2652 int q1 = pi->first_qset; 2653 int nqsets = pi->nqsets; 2654 int i; 2655 2656 if (t->qset_idx >= nqsets) 2657 return EINVAL; 2658 2659 i = q1 + t->qset_idx; 2660 q = &sc->params.sge.qset[i]; 2661 t->rspq_size = q->rspq_size; 2662 t->txq_size[0] = q->txq_size[0]; 2663 t->txq_size[1] = q->txq_size[1]; 2664 t->txq_size[2] = q->txq_size[2]; 2665 t->fl_size[0] = q->fl_size; 2666 t->fl_size[1] = q->jumbo_size; 2667 t->polling = q->polling; 2668 t->lro = q->lro; 2669 t->intr_lat = q->coalesce_usecs; 2670 t->cong_thres = q->cong_thres; 2671 t->qnum = i; 2672 2673 if ((sc->flags & FULL_INIT_DONE) == 0) 2674 t->vector = 0; 2675 else if (sc->flags & USING_MSIX) 2676 t->vector = rman_get_start(sc->msix_irq_res[i]); 2677 else 2678 t->vector = rman_get_start(sc->irq_res); 2679 2680 break; 2681 } 2682 case CHELSIO_GET_QSET_NUM: { 2683 struct ch_reg *edata = (struct ch_reg *)data; 2684 edata->val = pi->nqsets; 2685 break; 2686 } 2687 case CHELSIO_LOAD_FW: { 2688 uint8_t *fw_data; 2689 uint32_t vers; 2690 struct ch_mem_range *t = (struct ch_mem_range *)data; 2691 2692 /* 2693 * You're allowed to load a firmware only before FULL_INIT_DONE 2694 * 2695 * FW_UPTODATE is also set so the rest of the initialization 2696 * will not overwrite what was loaded here. This gives you the 2697 * flexibility to load any firmware (and maybe shoot yourself in 2698 * the foot). 2699 */ 2700 2701 ADAPTER_LOCK(sc); 2702 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2703 ADAPTER_UNLOCK(sc); 2704 return (EBUSY); 2705 } 2706 2707 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2708 if (!fw_data) 2709 error = ENOMEM; 2710 else 2711 error = copyin(t->buf, fw_data, t->len); 2712 2713 if (!error) 2714 error = -t3_load_fw(sc, fw_data, t->len); 2715 2716 if (t3_get_fw_version(sc, &vers) == 0) { 2717 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2718 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2719 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2720 } 2721 2722 if (!error) 2723 sc->flags |= FW_UPTODATE; 2724 2725 free(fw_data, M_DEVBUF); 2726 ADAPTER_UNLOCK(sc); 2727 break; 2728 } 2729 case CHELSIO_LOAD_BOOT: { 2730 uint8_t *boot_data; 2731 struct ch_mem_range *t = (struct ch_mem_range *)data; 2732 2733 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2734 if (!boot_data) 2735 return ENOMEM; 2736 2737 error = copyin(t->buf, boot_data, t->len); 2738 if (!error) 2739 error = -t3_load_boot(sc, boot_data, t->len); 2740 2741 free(boot_data, M_DEVBUF); 2742 break; 2743 } 2744 case CHELSIO_GET_PM: { 2745 struct ch_pm *m = (struct ch_pm *)data; 2746 struct tp_params *p = &sc->params.tp; 2747 2748 if (!is_offload(sc)) 2749 return (EOPNOTSUPP); 2750 2751 m->tx_pg_sz = p->tx_pg_size; 2752 m->tx_num_pg = p->tx_num_pgs; 2753 m->rx_pg_sz = p->rx_pg_size; 2754 m->rx_num_pg = p->rx_num_pgs; 2755 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2756 2757 break; 2758 } 2759 case CHELSIO_SET_PM: { 2760 struct ch_pm *m = (struct ch_pm *)data; 2761 struct tp_params *p = &sc->params.tp; 2762 2763 if (!is_offload(sc)) 2764 return (EOPNOTSUPP); 2765 if (sc->flags & FULL_INIT_DONE) 2766 return (EBUSY); 2767 2768 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2769 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2770 return (EINVAL); /* not power of 2 */ 2771 if (!(m->rx_pg_sz & 0x14000)) 2772 return (EINVAL); /* not 16KB or 64KB */ 2773 if (!(m->tx_pg_sz & 0x1554000)) 2774 return (EINVAL); 2775 if (m->tx_num_pg == -1) 2776 m->tx_num_pg = p->tx_num_pgs; 2777 if (m->rx_num_pg == -1) 2778 m->rx_num_pg = p->rx_num_pgs; 2779 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2780 return (EINVAL); 2781 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2782 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2783 return (EINVAL); 2784 2785 p->rx_pg_size = m->rx_pg_sz; 2786 p->tx_pg_size = m->tx_pg_sz; 2787 p->rx_num_pgs = m->rx_num_pg; 2788 p->tx_num_pgs = m->tx_num_pg; 2789 break; 2790 } 2791 case CHELSIO_SETMTUTAB: { 2792 struct ch_mtus *m = (struct ch_mtus *)data; 2793 int i; 2794 2795 if (!is_offload(sc)) 2796 return (EOPNOTSUPP); 2797 if (offload_running(sc)) 2798 return (EBUSY); 2799 if (m->nmtus != NMTUS) 2800 return (EINVAL); 2801 if (m->mtus[0] < 81) /* accommodate SACK */ 2802 return (EINVAL); 2803 2804 /* 2805 * MTUs must be in ascending order 2806 */ 2807 for (i = 1; i < NMTUS; ++i) 2808 if (m->mtus[i] < m->mtus[i - 1]) 2809 return (EINVAL); 2810 2811 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2812 break; 2813 } 2814 case CHELSIO_GETMTUTAB: { 2815 struct ch_mtus *m = (struct ch_mtus *)data; 2816 2817 if (!is_offload(sc)) 2818 return (EOPNOTSUPP); 2819 2820 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2821 m->nmtus = NMTUS; 2822 break; 2823 } 2824 case CHELSIO_GET_MEM: { 2825 struct ch_mem_range *t = (struct ch_mem_range *)data; 2826 struct mc7 *mem; 2827 uint8_t *useraddr; 2828 u64 buf[32]; 2829 2830 /* 2831 * Use these to avoid modifying len/addr in the return 2832 * struct 2833 */ 2834 uint32_t len = t->len, addr = t->addr; 2835 2836 if (!is_offload(sc)) 2837 return (EOPNOTSUPP); 2838 if (!(sc->flags & FULL_INIT_DONE)) 2839 return (EIO); /* need the memory controllers */ 2840 if ((addr & 0x7) || (len & 0x7)) 2841 return (EINVAL); 2842 if (t->mem_id == MEM_CM) 2843 mem = &sc->cm; 2844 else if (t->mem_id == MEM_PMRX) 2845 mem = &sc->pmrx; 2846 else if (t->mem_id == MEM_PMTX) 2847 mem = &sc->pmtx; 2848 else 2849 return (EINVAL); 2850 2851 /* 2852 * Version scheme: 2853 * bits 0..9: chip version 2854 * bits 10..15: chip revision 2855 */ 2856 t->version = 3 | (sc->params.rev << 10); 2857 2858 /* 2859 * Read 256 bytes at a time as len can be large and we don't 2860 * want to use huge intermediate buffers. 2861 */ 2862 useraddr = (uint8_t *)t->buf; 2863 while (len) { 2864 unsigned int chunk = min(len, sizeof(buf)); 2865 2866 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2867 if (error) 2868 return (-error); 2869 if (copyout(buf, useraddr, chunk)) 2870 return (EFAULT); 2871 useraddr += chunk; 2872 addr += chunk; 2873 len -= chunk; 2874 } 2875 break; 2876 } 2877 case CHELSIO_READ_TCAM_WORD: { 2878 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2879 2880 if (!is_offload(sc)) 2881 return (EOPNOTSUPP); 2882 if (!(sc->flags & FULL_INIT_DONE)) 2883 return (EIO); /* need MC5 */ 2884 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2885 break; 2886 } 2887 case CHELSIO_SET_TRACE_FILTER: { 2888 struct ch_trace *t = (struct ch_trace *)data; 2889 const struct trace_params *tp; 2890 2891 tp = (const struct trace_params *)&t->sip; 2892 if (t->config_tx) 2893 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2894 t->trace_tx); 2895 if (t->config_rx) 2896 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2897 t->trace_rx); 2898 break; 2899 } 2900 case CHELSIO_SET_PKTSCHED: { 2901 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2902 if (sc->open_device_map == 0) 2903 return (EAGAIN); 2904 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2905 p->binding); 2906 break; 2907 } 2908 case CHELSIO_IFCONF_GETREGS: { 2909 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2910 int reglen = cxgb_get_regs_len(); 2911 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2912 if (buf == NULL) { 2913 return (ENOMEM); 2914 } 2915 if (regs->len > reglen) 2916 regs->len = reglen; 2917 else if (regs->len < reglen) 2918 error = ENOBUFS; 2919 2920 if (!error) { 2921 cxgb_get_regs(sc, regs, buf); 2922 error = copyout(buf, regs->data, reglen); 2923 } 2924 free(buf, M_DEVBUF); 2925 2926 break; 2927 } 2928 case CHELSIO_SET_HW_SCHED: { 2929 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2930 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2931 2932 if ((sc->flags & FULL_INIT_DONE) == 0) 2933 return (EAGAIN); /* need TP to be initialized */ 2934 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2935 !in_range(t->channel, 0, 1) || 2936 !in_range(t->kbps, 0, 10000000) || 2937 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2938 !in_range(t->flow_ipg, 0, 2939 dack_ticks_to_usec(sc, 0x7ff))) 2940 return (EINVAL); 2941 2942 if (t->kbps >= 0) { 2943 error = t3_config_sched(sc, t->kbps, t->sched); 2944 if (error < 0) 2945 return (-error); 2946 } 2947 if (t->class_ipg >= 0) 2948 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2949 if (t->flow_ipg >= 0) { 2950 t->flow_ipg *= 1000; /* us -> ns */ 2951 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2952 } 2953 if (t->mode >= 0) { 2954 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2955 2956 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2957 bit, t->mode ? bit : 0); 2958 } 2959 if (t->channel >= 0) 2960 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2961 1 << t->sched, t->channel << t->sched); 2962 break; 2963 } 2964 case CHELSIO_GET_EEPROM: { 2965 int i; 2966 struct ch_eeprom *e = (struct ch_eeprom *)data; 2967 uint8_t *buf; 2968 2969 if (e->offset & 3 || e->offset >= EEPROMSIZE || 2970 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) { 2971 return (EINVAL); 2972 } 2973 2974 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2975 if (buf == NULL) { 2976 return (ENOMEM); 2977 } 2978 e->magic = EEPROM_MAGIC; 2979 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2980 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2981 2982 if (!error) 2983 error = copyout(buf + e->offset, e->data, e->len); 2984 2985 free(buf, M_DEVBUF); 2986 break; 2987 } 2988 case CHELSIO_CLEAR_STATS: { 2989 if (!(sc->flags & FULL_INIT_DONE)) 2990 return EAGAIN; 2991 2992 PORT_LOCK(pi); 2993 t3_mac_update_stats(&pi->mac); 2994 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 2995 PORT_UNLOCK(pi); 2996 break; 2997 } 2998 case CHELSIO_GET_UP_LA: { 2999 struct ch_up_la *la = (struct ch_up_la *)data; 3000 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT); 3001 if (buf == NULL) { 3002 return (ENOMEM); 3003 } 3004 if (la->bufsize < LA_BUFSIZE) 3005 error = ENOBUFS; 3006 3007 if (!error) 3008 error = -t3_get_up_la(sc, &la->stopped, &la->idx, 3009 &la->bufsize, buf); 3010 if (!error) 3011 error = copyout(buf, la->data, la->bufsize); 3012 3013 free(buf, M_DEVBUF); 3014 break; 3015 } 3016 case CHELSIO_GET_UP_IOQS: { 3017 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data; 3018 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT); 3019 uint32_t *v; 3020 3021 if (buf == NULL) { 3022 return (ENOMEM); 3023 } 3024 if (ioqs->bufsize < IOQS_BUFSIZE) 3025 error = ENOBUFS; 3026 3027 if (!error) 3028 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf); 3029 3030 if (!error) { 3031 v = (uint32_t *)buf; 3032 3033 ioqs->ioq_rx_enable = *v++; 3034 ioqs->ioq_tx_enable = *v++; 3035 ioqs->ioq_rx_status = *v++; 3036 ioqs->ioq_tx_status = *v++; 3037 3038 error = copyout(v, ioqs->data, ioqs->bufsize); 3039 } 3040 3041 free(buf, M_DEVBUF); 3042 break; 3043 } 3044 case CHELSIO_SET_FILTER: { 3045 struct ch_filter *f = (struct ch_filter *)data; 3046 struct filter_info *p; 3047 unsigned int nfilters = sc->params.mc5.nfilters; 3048 3049 if (!is_offload(sc)) 3050 return (EOPNOTSUPP); /* No TCAM */ 3051 if (!(sc->flags & FULL_INIT_DONE)) 3052 return (EAGAIN); /* mc5 not setup yet */ 3053 if (nfilters == 0) 3054 return (EBUSY); /* TOE will use TCAM */ 3055 3056 /* sanity checks */ 3057 if (f->filter_id >= nfilters || 3058 (f->val.dip && f->mask.dip != 0xffffffff) || 3059 (f->val.sport && f->mask.sport != 0xffff) || 3060 (f->val.dport && f->mask.dport != 0xffff) || 3061 (f->val.vlan && f->mask.vlan != 0xfff) || 3062 (f->val.vlan_prio && 3063 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) || 3064 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) || 3065 f->qset >= SGE_QSETS || 3066 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE) 3067 return (EINVAL); 3068 3069 /* Was allocated with M_WAITOK */ 3070 KASSERT(sc->filters, ("filter table NULL\n")); 3071 3072 p = &sc->filters[f->filter_id]; 3073 if (p->locked) 3074 return (EPERM); 3075 3076 bzero(p, sizeof(*p)); 3077 p->sip = f->val.sip; 3078 p->sip_mask = f->mask.sip; 3079 p->dip = f->val.dip; 3080 p->sport = f->val.sport; 3081 p->dport = f->val.dport; 3082 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff; 3083 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) : 3084 FILTER_NO_VLAN_PRI; 3085 p->mac_hit = f->mac_hit; 3086 p->mac_vld = f->mac_addr_idx != 0xffff; 3087 p->mac_idx = f->mac_addr_idx; 3088 p->pkt_type = f->proto; 3089 p->report_filter_id = f->want_filter_id; 3090 p->pass = f->pass; 3091 p->rss = f->rss; 3092 p->qset = f->qset; 3093 3094 error = set_filter(sc, f->filter_id, p); 3095 if (error == 0) 3096 p->valid = 1; 3097 break; 3098 } 3099 case CHELSIO_DEL_FILTER: { 3100 struct ch_filter *f = (struct ch_filter *)data; 3101 struct filter_info *p; 3102 unsigned int nfilters = sc->params.mc5.nfilters; 3103 3104 if (!is_offload(sc)) 3105 return (EOPNOTSUPP); 3106 if (!(sc->flags & FULL_INIT_DONE)) 3107 return (EAGAIN); 3108 if (nfilters == 0 || sc->filters == NULL) 3109 return (EINVAL); 3110 if (f->filter_id >= nfilters) 3111 return (EINVAL); 3112 3113 p = &sc->filters[f->filter_id]; 3114 if (p->locked) 3115 return (EPERM); 3116 if (!p->valid) 3117 return (EFAULT); /* Read "Bad address" as "Bad index" */ 3118 3119 bzero(p, sizeof(*p)); 3120 p->sip = p->sip_mask = 0xffffffff; 3121 p->vlan = 0xfff; 3122 p->vlan_prio = FILTER_NO_VLAN_PRI; 3123 p->pkt_type = 1; 3124 error = set_filter(sc, f->filter_id, p); 3125 break; 3126 } 3127 case CHELSIO_GET_FILTER: { 3128 struct ch_filter *f = (struct ch_filter *)data; 3129 struct filter_info *p; 3130 unsigned int i, nfilters = sc->params.mc5.nfilters; 3131 3132 if (!is_offload(sc)) 3133 return (EOPNOTSUPP); 3134 if (!(sc->flags & FULL_INIT_DONE)) 3135 return (EAGAIN); 3136 if (nfilters == 0 || sc->filters == NULL) 3137 return (EINVAL); 3138 3139 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1; 3140 for (; i < nfilters; i++) { 3141 p = &sc->filters[i]; 3142 if (!p->valid) 3143 continue; 3144 3145 bzero(f, sizeof(*f)); 3146 3147 f->filter_id = i; 3148 f->val.sip = p->sip; 3149 f->mask.sip = p->sip_mask; 3150 f->val.dip = p->dip; 3151 f->mask.dip = p->dip ? 0xffffffff : 0; 3152 f->val.sport = p->sport; 3153 f->mask.sport = p->sport ? 0xffff : 0; 3154 f->val.dport = p->dport; 3155 f->mask.dport = p->dport ? 0xffff : 0; 3156 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan; 3157 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff; 3158 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3159 0 : p->vlan_prio; 3160 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3161 0 : FILTER_NO_VLAN_PRI; 3162 f->mac_hit = p->mac_hit; 3163 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff; 3164 f->proto = p->pkt_type; 3165 f->want_filter_id = p->report_filter_id; 3166 f->pass = p->pass; 3167 f->rss = p->rss; 3168 f->qset = p->qset; 3169 3170 break; 3171 } 3172 3173 if (i == nfilters) 3174 f->filter_id = 0xffffffff; 3175 break; 3176 } 3177 default: 3178 return (EOPNOTSUPP); 3179 break; 3180 } 3181 3182 return (error); 3183 } 3184 3185 static __inline void 3186 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 3187 unsigned int end) 3188 { 3189 uint32_t *p = (uint32_t *)(buf + start); 3190 3191 for ( ; start <= end; start += sizeof(uint32_t)) 3192 *p++ = t3_read_reg(ap, start); 3193 } 3194 3195 #define T3_REGMAP_SIZE (3 * 1024) 3196 static int 3197 cxgb_get_regs_len(void) 3198 { 3199 return T3_REGMAP_SIZE; 3200 } 3201 3202 static void 3203 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 3204 { 3205 3206 /* 3207 * Version scheme: 3208 * bits 0..9: chip version 3209 * bits 10..15: chip revision 3210 * bit 31: set for PCIe cards 3211 */ 3212 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 3213 3214 /* 3215 * We skip the MAC statistics registers because they are clear-on-read. 3216 * Also reading multi-register stats would need to synchronize with the 3217 * periodic mac stats accumulation. Hard to justify the complexity. 3218 */ 3219 memset(buf, 0, cxgb_get_regs_len()); 3220 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 3221 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 3222 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 3223 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 3224 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 3225 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 3226 XGM_REG(A_XGM_SERDES_STAT3, 1)); 3227 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 3228 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 3229 } 3230 3231 static int 3232 alloc_filters(struct adapter *sc) 3233 { 3234 struct filter_info *p; 3235 unsigned int nfilters = sc->params.mc5.nfilters; 3236 3237 if (nfilters == 0) 3238 return (0); 3239 3240 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO); 3241 sc->filters = p; 3242 3243 p = &sc->filters[nfilters - 1]; 3244 p->vlan = 0xfff; 3245 p->vlan_prio = FILTER_NO_VLAN_PRI; 3246 p->pass = p->rss = p->valid = p->locked = 1; 3247 3248 return (0); 3249 } 3250 3251 static int 3252 setup_hw_filters(struct adapter *sc) 3253 { 3254 int i, rc; 3255 unsigned int nfilters = sc->params.mc5.nfilters; 3256 3257 if (!sc->filters) 3258 return (0); 3259 3260 t3_enable_filters(sc); 3261 3262 for (i = rc = 0; i < nfilters && !rc; i++) { 3263 if (sc->filters[i].locked) 3264 rc = set_filter(sc, i, &sc->filters[i]); 3265 } 3266 3267 return (rc); 3268 } 3269 3270 static int 3271 set_filter(struct adapter *sc, int id, const struct filter_info *f) 3272 { 3273 int len; 3274 struct mbuf *m; 3275 struct ulp_txpkt *txpkt; 3276 struct work_request_hdr *wr; 3277 struct cpl_pass_open_req *oreq; 3278 struct cpl_set_tcb_field *sreq; 3279 3280 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq); 3281 KASSERT(len <= MHLEN, ("filter request too big for an mbuf")); 3282 3283 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes - 3284 sc->params.mc5.nfilters; 3285 3286 m = m_gethdr(M_WAITOK, MT_DATA); 3287 m->m_len = m->m_pkthdr.len = len; 3288 bzero(mtod(m, char *), len); 3289 3290 wr = mtod(m, struct work_request_hdr *); 3291 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); 3292 3293 oreq = (struct cpl_pass_open_req *)(wr + 1); 3294 txpkt = (struct ulp_txpkt *)oreq; 3295 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3296 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8)); 3297 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id)); 3298 oreq->local_port = htons(f->dport); 3299 oreq->peer_port = htons(f->sport); 3300 oreq->local_ip = htonl(f->dip); 3301 oreq->peer_ip = htonl(f->sip); 3302 oreq->peer_netmask = htonl(f->sip_mask); 3303 oreq->opt0h = 0; 3304 oreq->opt0l = htonl(F_NO_OFFLOAD); 3305 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) | 3306 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) | 3307 V_VLAN_PRI(f->vlan_prio >> 1) | 3308 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) | 3309 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) | 3310 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4))); 3311 3312 sreq = (struct cpl_set_tcb_field *)(oreq + 1); 3313 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL, 3314 (f->report_filter_id << 15) | (1 << 23) | 3315 ((u64)f->pass << 35) | ((u64)!f->rss << 36)); 3316 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1); 3317 t3_mgmt_tx(sc, m); 3318 3319 if (f->pass && !f->rss) { 3320 len = sizeof(*sreq); 3321 m = m_gethdr(M_WAITOK, MT_DATA); 3322 m->m_len = m->m_pkthdr.len = len; 3323 bzero(mtod(m, char *), len); 3324 sreq = mtod(m, struct cpl_set_tcb_field *); 3325 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3326 mk_set_tcb_field(sreq, id, 25, 0x3f80000, 3327 (u64)sc->rrss_map[f->qset] << 19); 3328 t3_mgmt_tx(sc, m); 3329 } 3330 return 0; 3331 } 3332 3333 static inline void 3334 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid, 3335 unsigned int word, u64 mask, u64 val) 3336 { 3337 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 3338 req->reply = V_NO_REPLY(1); 3339 req->cpu_idx = 0; 3340 req->word = htons(word); 3341 req->mask = htobe64(mask); 3342 req->val = htobe64(val); 3343 } 3344 3345 static inline void 3346 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, 3347 unsigned int word, u64 mask, u64 val) 3348 { 3349 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 3350 3351 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3352 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 3353 mk_set_tcb_field(req, tid, word, mask, val); 3354 } 3355 3356 void 3357 t3_iterate(void (*func)(struct adapter *, void *), void *arg) 3358 { 3359 struct adapter *sc; 3360 3361 mtx_lock(&t3_list_lock); 3362 SLIST_FOREACH(sc, &t3_list, link) { 3363 /* 3364 * func should not make any assumptions about what state sc is 3365 * in - the only guarantee is that sc->sc_lock is a valid lock. 3366 */ 3367 func(sc, arg); 3368 } 3369 mtx_unlock(&t3_list_lock); 3370 } 3371 3372 #ifdef TCP_OFFLOAD 3373 static int 3374 toe_capability(struct port_info *pi, int enable) 3375 { 3376 int rc; 3377 struct adapter *sc = pi->adapter; 3378 3379 ADAPTER_LOCK_ASSERT_OWNED(sc); 3380 3381 if (!is_offload(sc)) 3382 return (ENODEV); 3383 3384 if (enable) { 3385 if (!(sc->flags & FULL_INIT_DONE)) { 3386 log(LOG_WARNING, 3387 "You must enable a cxgb interface first\n"); 3388 return (EAGAIN); 3389 } 3390 3391 if (isset(&sc->offload_map, pi->port_id)) 3392 return (0); 3393 3394 if (!(sc->flags & TOM_INIT_DONE)) { 3395 rc = t3_activate_uld(sc, ULD_TOM); 3396 if (rc == EAGAIN) { 3397 log(LOG_WARNING, 3398 "You must kldload t3_tom.ko before trying " 3399 "to enable TOE on a cxgb interface.\n"); 3400 } 3401 if (rc != 0) 3402 return (rc); 3403 KASSERT(sc->tom_softc != NULL, 3404 ("%s: TOM activated but softc NULL", __func__)); 3405 KASSERT(sc->flags & TOM_INIT_DONE, 3406 ("%s: TOM activated but flag not set", __func__)); 3407 } 3408 3409 setbit(&sc->offload_map, pi->port_id); 3410 3411 /* 3412 * XXX: Temporary code to allow iWARP to be enabled when TOE is 3413 * enabled on any port. Need to figure out how to enable, 3414 * disable, load, and unload iWARP cleanly. 3415 */ 3416 if (!isset(&sc->offload_map, MAX_NPORTS) && 3417 t3_activate_uld(sc, ULD_IWARP) == 0) 3418 setbit(&sc->offload_map, MAX_NPORTS); 3419 } else { 3420 if (!isset(&sc->offload_map, pi->port_id)) 3421 return (0); 3422 3423 KASSERT(sc->flags & TOM_INIT_DONE, 3424 ("%s: TOM never initialized?", __func__)); 3425 clrbit(&sc->offload_map, pi->port_id); 3426 } 3427 3428 return (0); 3429 } 3430 3431 /* 3432 * Add an upper layer driver to the global list. 3433 */ 3434 int 3435 t3_register_uld(struct uld_info *ui) 3436 { 3437 int rc = 0; 3438 struct uld_info *u; 3439 3440 mtx_lock(&t3_uld_list_lock); 3441 SLIST_FOREACH(u, &t3_uld_list, link) { 3442 if (u->uld_id == ui->uld_id) { 3443 rc = EEXIST; 3444 goto done; 3445 } 3446 } 3447 3448 SLIST_INSERT_HEAD(&t3_uld_list, ui, link); 3449 ui->refcount = 0; 3450 done: 3451 mtx_unlock(&t3_uld_list_lock); 3452 return (rc); 3453 } 3454 3455 int 3456 t3_unregister_uld(struct uld_info *ui) 3457 { 3458 int rc = EINVAL; 3459 struct uld_info *u; 3460 3461 mtx_lock(&t3_uld_list_lock); 3462 3463 SLIST_FOREACH(u, &t3_uld_list, link) { 3464 if (u == ui) { 3465 if (ui->refcount > 0) { 3466 rc = EBUSY; 3467 goto done; 3468 } 3469 3470 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link); 3471 rc = 0; 3472 goto done; 3473 } 3474 } 3475 done: 3476 mtx_unlock(&t3_uld_list_lock); 3477 return (rc); 3478 } 3479 3480 int 3481 t3_activate_uld(struct adapter *sc, int id) 3482 { 3483 int rc = EAGAIN; 3484 struct uld_info *ui; 3485 3486 mtx_lock(&t3_uld_list_lock); 3487 3488 SLIST_FOREACH(ui, &t3_uld_list, link) { 3489 if (ui->uld_id == id) { 3490 rc = ui->activate(sc); 3491 if (rc == 0) 3492 ui->refcount++; 3493 goto done; 3494 } 3495 } 3496 done: 3497 mtx_unlock(&t3_uld_list_lock); 3498 3499 return (rc); 3500 } 3501 3502 int 3503 t3_deactivate_uld(struct adapter *sc, int id) 3504 { 3505 int rc = EINVAL; 3506 struct uld_info *ui; 3507 3508 mtx_lock(&t3_uld_list_lock); 3509 3510 SLIST_FOREACH(ui, &t3_uld_list, link) { 3511 if (ui->uld_id == id) { 3512 rc = ui->deactivate(sc); 3513 if (rc == 0) 3514 ui->refcount--; 3515 goto done; 3516 } 3517 } 3518 done: 3519 mtx_unlock(&t3_uld_list_lock); 3520 3521 return (rc); 3522 } 3523 3524 static int 3525 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused, 3526 struct mbuf *m) 3527 { 3528 m_freem(m); 3529 return (EDOOFUS); 3530 } 3531 3532 int 3533 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3534 { 3535 uintptr_t *loc, new; 3536 3537 if (opcode >= NUM_CPL_HANDLERS) 3538 return (EINVAL); 3539 3540 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3541 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3542 atomic_store_rel_ptr(loc, new); 3543 3544 return (0); 3545 } 3546 #endif 3547 3548 static int 3549 cxgbc_mod_event(module_t mod, int cmd, void *arg) 3550 { 3551 int rc = 0; 3552 3553 switch (cmd) { 3554 case MOD_LOAD: 3555 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF); 3556 SLIST_INIT(&t3_list); 3557 #ifdef TCP_OFFLOAD 3558 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF); 3559 SLIST_INIT(&t3_uld_list); 3560 #endif 3561 break; 3562 3563 case MOD_UNLOAD: 3564 #ifdef TCP_OFFLOAD 3565 mtx_lock(&t3_uld_list_lock); 3566 if (!SLIST_EMPTY(&t3_uld_list)) { 3567 rc = EBUSY; 3568 mtx_unlock(&t3_uld_list_lock); 3569 break; 3570 } 3571 mtx_unlock(&t3_uld_list_lock); 3572 mtx_destroy(&t3_uld_list_lock); 3573 #endif 3574 mtx_lock(&t3_list_lock); 3575 if (!SLIST_EMPTY(&t3_list)) { 3576 rc = EBUSY; 3577 mtx_unlock(&t3_list_lock); 3578 break; 3579 } 3580 mtx_unlock(&t3_list_lock); 3581 mtx_destroy(&t3_list_lock); 3582 break; 3583 } 3584 3585 return (rc); 3586 } 3587 3588 #ifdef NETDUMP 3589 static void 3590 cxgb_netdump_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize) 3591 { 3592 struct port_info *pi; 3593 adapter_t *adap; 3594 3595 pi = if_getsoftc(ifp); 3596 adap = pi->adapter; 3597 ADAPTER_LOCK(adap); 3598 *nrxr = SGE_QSETS; 3599 *ncl = adap->sge.qs[0].fl[1].size; 3600 *clsize = adap->sge.qs[0].fl[1].buf_size; 3601 ADAPTER_UNLOCK(adap); 3602 } 3603 3604 static void 3605 cxgb_netdump_event(struct ifnet *ifp, enum netdump_ev event) 3606 { 3607 struct port_info *pi; 3608 struct sge_qset *qs; 3609 int i; 3610 3611 pi = if_getsoftc(ifp); 3612 if (event == NETDUMP_START) 3613 for (i = 0; i < SGE_QSETS; i++) { 3614 qs = &pi->adapter->sge.qs[i]; 3615 3616 /* Need to reinit after netdump_mbuf_dump(). */ 3617 qs->fl[0].zone = zone_pack; 3618 qs->fl[1].zone = zone_clust; 3619 qs->lro.enabled = 0; 3620 } 3621 } 3622 3623 static int 3624 cxgb_netdump_transmit(struct ifnet *ifp, struct mbuf *m) 3625 { 3626 struct port_info *pi; 3627 struct sge_qset *qs; 3628 3629 pi = if_getsoftc(ifp); 3630 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 3631 IFF_DRV_RUNNING) 3632 return (ENOENT); 3633 3634 qs = &pi->adapter->sge.qs[pi->first_qset]; 3635 return (cxgb_netdump_encap(qs, &m)); 3636 } 3637 3638 static int 3639 cxgb_netdump_poll(struct ifnet *ifp, int count) 3640 { 3641 struct port_info *pi; 3642 adapter_t *adap; 3643 int i; 3644 3645 pi = if_getsoftc(ifp); 3646 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 3647 return (ENOENT); 3648 3649 adap = pi->adapter; 3650 for (i = 0; i < SGE_QSETS; i++) 3651 (void)cxgb_netdump_poll_rx(adap, &adap->sge.qs[i]); 3652 (void)cxgb_netdump_poll_tx(&adap->sge.qs[pi->first_qset]); 3653 return (0); 3654 } 3655 #endif /* NETDUMP */ 3656