1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 33 #include <sys/param.h> 34 #include <sys/conf.h> 35 #include <sys/priv.h> 36 #include <sys/kernel.h> 37 #include <sys/bus.h> 38 #include <sys/module.h> 39 #include <sys/malloc.h> 40 #include <sys/queue.h> 41 #include <sys/taskqueue.h> 42 #include <sys/pciio.h> 43 #include <dev/pci/pcireg.h> 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pci_private.h> 46 #include <sys/firmware.h> 47 #include <sys/sbuf.h> 48 #include <sys/smp.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/sysctl.h> 52 #include <net/ethernet.h> 53 #include <net/if.h> 54 #include <net/if_types.h> 55 #include <net/if_dl.h> 56 #include <net/if_vlan_var.h> 57 58 #include "common/common.h" 59 #include "common/t4_msg.h" 60 #include "common/t4_regs.h" 61 #include "common/t4_regs_values.h" 62 #include "t4_ioctl.h" 63 #include "t4_l2t.h" 64 65 /* T4 bus driver interface */ 66 static int t4_probe(device_t); 67 static int t4_attach(device_t); 68 static int t4_detach(device_t); 69 static device_method_t t4_methods[] = { 70 DEVMETHOD(device_probe, t4_probe), 71 DEVMETHOD(device_attach, t4_attach), 72 DEVMETHOD(device_detach, t4_detach), 73 74 DEVMETHOD_END 75 }; 76 static driver_t t4_driver = { 77 "t4nex", 78 t4_methods, 79 sizeof(struct adapter) 80 }; 81 82 83 /* T4 port (cxgbe) interface */ 84 static int cxgbe_probe(device_t); 85 static int cxgbe_attach(device_t); 86 static int cxgbe_detach(device_t); 87 static device_method_t cxgbe_methods[] = { 88 DEVMETHOD(device_probe, cxgbe_probe), 89 DEVMETHOD(device_attach, cxgbe_attach), 90 DEVMETHOD(device_detach, cxgbe_detach), 91 { 0, 0 } 92 }; 93 static driver_t cxgbe_driver = { 94 "cxgbe", 95 cxgbe_methods, 96 sizeof(struct port_info) 97 }; 98 99 static d_ioctl_t t4_ioctl; 100 static d_open_t t4_open; 101 static d_close_t t4_close; 102 103 static struct cdevsw t4_cdevsw = { 104 .d_version = D_VERSION, 105 .d_flags = 0, 106 .d_open = t4_open, 107 .d_close = t4_close, 108 .d_ioctl = t4_ioctl, 109 .d_name = "t4nex", 110 }; 111 112 /* ifnet + media interface */ 113 static void cxgbe_init(void *); 114 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 115 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 116 static void cxgbe_qflush(struct ifnet *); 117 static int cxgbe_media_change(struct ifnet *); 118 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 119 120 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services"); 121 122 static struct mtx t4_list_lock; 123 static SLIST_HEAD(, adapter) t4_list; 124 #ifndef TCP_OFFLOAD_DISABLE 125 static struct mtx t4_uld_list_lock; 126 static SLIST_HEAD(, uld_info) t4_uld_list; 127 #endif 128 129 /* 130 * Tunables. See tweak_tunables() too. 131 */ 132 133 /* 134 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 135 */ 136 #define NTXQ_10G 16 137 static int t4_ntxq10g = -1; 138 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 139 140 #define NRXQ_10G 8 141 static int t4_nrxq10g = -1; 142 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 143 144 #define NTXQ_1G 4 145 static int t4_ntxq1g = -1; 146 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 147 148 #define NRXQ_1G 2 149 static int t4_nrxq1g = -1; 150 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 151 152 #ifndef TCP_OFFLOAD_DISABLE 153 #define NOFLDTXQ_10G 8 154 static int t4_nofldtxq10g = -1; 155 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 156 157 #define NOFLDRXQ_10G 2 158 static int t4_nofldrxq10g = -1; 159 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 160 161 #define NOFLDTXQ_1G 2 162 static int t4_nofldtxq1g = -1; 163 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 164 165 #define NOFLDRXQ_1G 1 166 static int t4_nofldrxq1g = -1; 167 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 168 #endif 169 170 /* 171 * Holdoff parameters for 10G and 1G ports. 172 */ 173 #define TMR_IDX_10G 1 174 static int t4_tmr_idx_10g = TMR_IDX_10G; 175 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 176 177 #define PKTC_IDX_10G (-1) 178 static int t4_pktc_idx_10g = PKTC_IDX_10G; 179 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 180 181 #define TMR_IDX_1G 1 182 static int t4_tmr_idx_1g = TMR_IDX_1G; 183 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 184 185 #define PKTC_IDX_1G (-1) 186 static int t4_pktc_idx_1g = PKTC_IDX_1G; 187 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 188 189 /* 190 * Size (# of entries) of each tx and rx queue. 191 */ 192 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 193 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 194 195 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 196 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 197 198 /* 199 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 200 */ 201 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 202 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 203 204 /* 205 * Configuration file. 206 */ 207 static char t4_cfg_file[32] = "default"; 208 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 209 210 /* 211 * ASIC features that will be used. Disable the ones you don't want so that the 212 * chip resources aren't wasted on features that will not be used. 213 */ 214 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 215 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 216 217 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 218 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 219 220 static int t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 221 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 222 223 static int t4_rdmacaps_allowed = 0; 224 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 225 226 static int t4_iscsicaps_allowed = 0; 227 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 228 229 static int t4_fcoecaps_allowed = 0; 230 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 231 232 struct intrs_and_queues { 233 int intr_type; /* INTx, MSI, or MSI-X */ 234 int nirq; /* Number of vectors */ 235 int intr_flags; 236 int ntxq10g; /* # of NIC txq's for each 10G port */ 237 int nrxq10g; /* # of NIC rxq's for each 10G port */ 238 int ntxq1g; /* # of NIC txq's for each 1G port */ 239 int nrxq1g; /* # of NIC rxq's for each 1G port */ 240 #ifndef TCP_OFFLOAD_DISABLE 241 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 242 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 243 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 244 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 245 #endif 246 }; 247 248 struct filter_entry { 249 uint32_t valid:1; /* filter allocated and valid */ 250 uint32_t locked:1; /* filter is administratively locked */ 251 uint32_t pending:1; /* filter action is pending firmware reply */ 252 uint32_t smtidx:8; /* Source MAC Table index for smac */ 253 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 254 255 struct t4_filter_specification fs; 256 }; 257 258 enum { 259 XGMAC_MTU = (1 << 0), 260 XGMAC_PROMISC = (1 << 1), 261 XGMAC_ALLMULTI = (1 << 2), 262 XGMAC_VLANEX = (1 << 3), 263 XGMAC_UCADDR = (1 << 4), 264 XGMAC_MCADDRS = (1 << 5), 265 266 XGMAC_ALL = 0xffff 267 }; 268 269 static int map_bars(struct adapter *); 270 static void setup_memwin(struct adapter *); 271 static int cfg_itype_and_nqueues(struct adapter *, int, int, 272 struct intrs_and_queues *); 273 static int prep_firmware(struct adapter *); 274 static int upload_config_file(struct adapter *, const struct firmware *, 275 uint32_t *, uint32_t *); 276 static int partition_resources(struct adapter *, const struct firmware *); 277 static int get_params__pre_init(struct adapter *); 278 static int get_params__post_init(struct adapter *); 279 static void t4_set_desc(struct adapter *); 280 static void build_medialist(struct port_info *); 281 static int update_mac_settings(struct port_info *, int); 282 static int cxgbe_init_locked(struct port_info *); 283 static int cxgbe_init_synchronized(struct port_info *); 284 static int cxgbe_uninit_locked(struct port_info *); 285 static int cxgbe_uninit_synchronized(struct port_info *); 286 static int adapter_full_init(struct adapter *); 287 static int adapter_full_uninit(struct adapter *); 288 static int port_full_init(struct port_info *); 289 static int port_full_uninit(struct port_info *); 290 static void quiesce_eq(struct adapter *, struct sge_eq *); 291 static void quiesce_iq(struct adapter *, struct sge_iq *); 292 static void quiesce_fl(struct adapter *, struct sge_fl *); 293 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 294 driver_intr_t *, void *, char *); 295 static int t4_free_irq(struct adapter *, struct irq *); 296 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 297 unsigned int); 298 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 299 static void cxgbe_tick(void *); 300 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 301 struct mbuf *); 302 static int t4_sysctls(struct adapter *); 303 static int cxgbe_sysctls(struct port_info *); 304 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 305 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 306 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 307 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 308 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 309 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 310 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 311 #ifdef SBUF_DRAIN 312 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 313 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 314 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 315 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 316 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 317 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 318 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 319 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 320 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 321 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 322 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 323 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 324 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 325 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 326 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 327 #endif 328 static inline void txq_start(struct ifnet *, struct sge_txq *); 329 static uint32_t fconf_to_mode(uint32_t); 330 static uint32_t mode_to_fconf(uint32_t); 331 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 332 static int get_filter_mode(struct adapter *, uint32_t *); 333 static int set_filter_mode(struct adapter *, uint32_t); 334 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 335 static int get_filter(struct adapter *, struct t4_filter *); 336 static int set_filter(struct adapter *, struct t4_filter *); 337 static int del_filter(struct adapter *, struct t4_filter *); 338 static void clear_filter(struct filter_entry *); 339 static int set_filter_wr(struct adapter *, int); 340 static int del_filter_wr(struct adapter *, int); 341 static int filter_rpl(struct sge_iq *, const struct rss_header *, 342 struct mbuf *); 343 static int get_sge_context(struct adapter *, struct t4_sge_context *); 344 static int read_card_mem(struct adapter *, struct t4_mem_range *); 345 #ifndef TCP_OFFLOAD_DISABLE 346 static int toe_capability(struct port_info *, int); 347 static int activate_uld(struct adapter *, int, struct uld_softc *); 348 static int deactivate_uld(struct uld_softc *); 349 #endif 350 static int t4_mod_event(module_t, int, void *); 351 352 struct t4_pciids { 353 uint16_t device; 354 uint8_t mpf; 355 char *desc; 356 } t4_pciids[] = { 357 {0xa000, 0, "Chelsio Terminator 4 FPGA"}, 358 {0x4400, 4, "Chelsio T440-dbg"}, 359 {0x4401, 4, "Chelsio T420-CR"}, 360 {0x4402, 4, "Chelsio T422-CR"}, 361 {0x4403, 4, "Chelsio T440-CR"}, 362 {0x4404, 4, "Chelsio T420-BCH"}, 363 {0x4405, 4, "Chelsio T440-BCH"}, 364 {0x4406, 4, "Chelsio T440-CH"}, 365 {0x4407, 4, "Chelsio T420-SO"}, 366 {0x4408, 4, "Chelsio T420-CX"}, 367 {0x4409, 4, "Chelsio T420-BT"}, 368 {0x440a, 4, "Chelsio T404-BT"}, 369 }; 370 371 #ifndef TCP_OFFLOAD_DISABLE 372 /* This is used in service_iq() to get to the fl associated with an iq. */ 373 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 374 #endif 375 376 static int 377 t4_probe(device_t dev) 378 { 379 int i; 380 uint16_t v = pci_get_vendor(dev); 381 uint16_t d = pci_get_device(dev); 382 383 if (v != PCI_VENDOR_ID_CHELSIO) 384 return (ENXIO); 385 386 for (i = 0; i < ARRAY_SIZE(t4_pciids); i++) { 387 if (d == t4_pciids[i].device && 388 pci_get_function(dev) == t4_pciids[i].mpf) { 389 device_set_desc(dev, t4_pciids[i].desc); 390 return (BUS_PROBE_DEFAULT); 391 } 392 } 393 394 return (ENXIO); 395 } 396 397 static int 398 t4_attach(device_t dev) 399 { 400 struct adapter *sc; 401 int rc = 0, i, n10g, n1g, rqidx, tqidx; 402 struct intrs_and_queues iaq; 403 struct sge *s; 404 #ifndef TCP_OFFLOAD_DISABLE 405 int ofld_rqidx, ofld_tqidx; 406 #endif 407 408 sc = device_get_softc(dev); 409 sc->dev = dev; 410 sc->pf = pci_get_function(dev); 411 sc->mbox = sc->pf; 412 413 pci_enable_busmaster(dev); 414 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 415 uint32_t v; 416 417 pci_set_max_read_req(dev, 4096); 418 v = pci_read_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, 2); 419 v |= PCIM_EXP_CTL_RELAXED_ORD_ENABLE; 420 pci_write_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, v, 2); 421 } 422 423 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 424 device_get_nameunit(dev)); 425 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 426 mtx_lock(&t4_list_lock); 427 SLIST_INSERT_HEAD(&t4_list, sc, link); 428 mtx_unlock(&t4_list_lock); 429 430 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 431 TAILQ_INIT(&sc->sfl); 432 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 433 434 rc = map_bars(sc); 435 if (rc != 0) 436 goto done; /* error message displayed already */ 437 438 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 439 for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) 440 sc->cpl_handler[i] = cpl_not_handled; 441 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, filter_rpl); 442 443 /* Prepare the adapter for operation */ 444 rc = -t4_prep_adapter(sc); 445 if (rc != 0) { 446 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 447 goto done; 448 } 449 450 /* 451 * Do this really early, with the memory windows set up even before the 452 * character device. The userland tool's register i/o and mem read 453 * will work even in "recovery mode". 454 */ 455 setup_memwin(sc); 456 sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT, 457 GID_WHEEL, 0600, "%s", device_get_nameunit(dev)); 458 sc->cdev->si_drv1 = sc; 459 460 /* Go no further if recovery mode has been requested. */ 461 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 462 device_printf(dev, "recovery mode.\n"); 463 goto done; 464 } 465 466 /* Prepare the firmware for operation */ 467 rc = prep_firmware(sc); 468 if (rc != 0) 469 goto done; /* error message displayed already */ 470 471 rc = get_params__pre_init(sc); 472 if (rc != 0) 473 goto done; /* error message displayed already */ 474 475 rc = t4_sge_init(sc); 476 if (rc != 0) 477 goto done; /* error message displayed already */ 478 479 if (sc->flags & MASTER_PF) { 480 /* get basic stuff going */ 481 rc = -t4_fw_initialize(sc, sc->mbox); 482 if (rc != 0) { 483 device_printf(dev, "early init failed: %d.\n", rc); 484 goto done; 485 } 486 } 487 488 rc = get_params__post_init(sc); 489 if (rc != 0) 490 goto done; /* error message displayed already */ 491 492 if (sc->flags & MASTER_PF) { 493 494 /* final tweaks to some settings */ 495 496 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, 497 sc->params.b_wnd); 498 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 499 t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 | 500 F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3, 0); 501 t4_set_reg_field(sc, A_TP_PARA_REG5, 502 V_INDICATESIZE(M_INDICATESIZE) | 503 F_REARMDDPOFFSET | F_RESETDDPOFFSET, 504 V_INDICATESIZE(M_INDICATESIZE) | 505 F_REARMDDPOFFSET | F_RESETDDPOFFSET); 506 } else { 507 /* 508 * XXX: Verify that we can live with whatever the master driver 509 * has done so far, and hope that it doesn't change any global 510 * setting from underneath us in the future. 511 */ 512 } 513 514 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 515 A_TP_VLAN_PRI_MAP); 516 517 for (i = 0; i < NCHAN; i++) 518 sc->params.tp.tx_modq[i] = i; 519 520 rc = t4_create_dma_tag(sc); 521 if (rc != 0) 522 goto done; /* error message displayed already */ 523 524 /* 525 * First pass over all the ports - allocate VIs and initialize some 526 * basic parameters like mac address, port type, etc. We also figure 527 * out whether a port is 10G or 1G and use that information when 528 * calculating how many interrupts to attempt to allocate. 529 */ 530 n10g = n1g = 0; 531 for_each_port(sc, i) { 532 struct port_info *pi; 533 534 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 535 sc->port[i] = pi; 536 537 /* These must be set before t4_port_init */ 538 pi->adapter = sc; 539 pi->port_id = i; 540 541 /* Allocate the vi and initialize parameters like mac addr */ 542 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 543 if (rc != 0) { 544 device_printf(dev, "unable to initialize port %d: %d\n", 545 i, rc); 546 free(pi, M_CXGBE); 547 sc->port[i] = NULL; 548 goto done; 549 } 550 551 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 552 device_get_nameunit(dev), i); 553 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 554 555 if (is_10G_port(pi)) { 556 n10g++; 557 pi->tmr_idx = t4_tmr_idx_10g; 558 pi->pktc_idx = t4_pktc_idx_10g; 559 } else { 560 n1g++; 561 pi->tmr_idx = t4_tmr_idx_1g; 562 pi->pktc_idx = t4_pktc_idx_1g; 563 } 564 565 pi->xact_addr_filt = -1; 566 567 pi->qsize_rxq = t4_qsize_rxq; 568 pi->qsize_txq = t4_qsize_txq; 569 570 pi->dev = device_add_child(dev, "cxgbe", -1); 571 if (pi->dev == NULL) { 572 device_printf(dev, 573 "failed to add device for port %d.\n", i); 574 rc = ENXIO; 575 goto done; 576 } 577 device_set_softc(pi->dev, pi); 578 } 579 580 /* 581 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 582 */ 583 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 584 if (rc != 0) 585 goto done; /* error message displayed already */ 586 587 sc->intr_type = iaq.intr_type; 588 sc->intr_count = iaq.nirq; 589 sc->flags |= iaq.intr_flags; 590 591 s = &sc->sge; 592 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 593 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 594 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 595 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 596 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 597 598 #ifndef TCP_OFFLOAD_DISABLE 599 if (is_offload(sc)) { 600 601 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 602 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 603 s->neq += s->nofldtxq + s->nofldrxq; 604 s->niq += s->nofldrxq; 605 606 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 607 M_CXGBE, M_ZERO | M_WAITOK); 608 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 609 M_CXGBE, M_ZERO | M_WAITOK); 610 } 611 #endif 612 613 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 614 M_ZERO | M_WAITOK); 615 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 616 M_ZERO | M_WAITOK); 617 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 618 M_ZERO | M_WAITOK); 619 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 620 M_ZERO | M_WAITOK); 621 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 622 M_ZERO | M_WAITOK); 623 624 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 625 M_ZERO | M_WAITOK); 626 627 t4_init_l2t(sc, M_WAITOK); 628 629 /* 630 * Second pass over the ports. This time we know the number of rx and 631 * tx queues that each port should get. 632 */ 633 rqidx = tqidx = 0; 634 #ifndef TCP_OFFLOAD_DISABLE 635 ofld_rqidx = ofld_tqidx = 0; 636 #endif 637 for_each_port(sc, i) { 638 struct port_info *pi = sc->port[i]; 639 640 if (pi == NULL) 641 continue; 642 643 pi->first_rxq = rqidx; 644 pi->first_txq = tqidx; 645 if (is_10G_port(pi)) { 646 pi->nrxq = iaq.nrxq10g; 647 pi->ntxq = iaq.ntxq10g; 648 } else { 649 pi->nrxq = iaq.nrxq1g; 650 pi->ntxq = iaq.ntxq1g; 651 } 652 653 rqidx += pi->nrxq; 654 tqidx += pi->ntxq; 655 656 #ifndef TCP_OFFLOAD_DISABLE 657 if (is_offload(sc)) { 658 pi->first_ofld_rxq = ofld_rqidx; 659 pi->first_ofld_txq = ofld_tqidx; 660 if (is_10G_port(pi)) { 661 pi->nofldrxq = iaq.nofldrxq10g; 662 pi->nofldtxq = iaq.nofldtxq10g; 663 } else { 664 pi->nofldrxq = iaq.nofldrxq1g; 665 pi->nofldtxq = iaq.nofldtxq1g; 666 } 667 ofld_rqidx += pi->nofldrxq; 668 ofld_tqidx += pi->nofldtxq; 669 } 670 #endif 671 } 672 673 rc = bus_generic_attach(dev); 674 if (rc != 0) { 675 device_printf(dev, 676 "failed to attach all child ports: %d\n", rc); 677 goto done; 678 } 679 680 device_printf(dev, 681 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 682 sc->params.pci.width, sc->params.nports, sc->intr_count, 683 sc->intr_type == INTR_MSIX ? "MSI-X" : 684 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 685 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 686 687 t4_set_desc(sc); 688 689 done: 690 if (rc != 0 && sc->cdev) { 691 /* cdev was created and so cxgbetool works; recover that way. */ 692 device_printf(dev, 693 "error during attach, adapter is now in recovery mode.\n"); 694 rc = 0; 695 } 696 697 if (rc != 0) 698 t4_detach(dev); 699 else 700 t4_sysctls(sc); 701 702 return (rc); 703 } 704 705 /* 706 * Idempotent 707 */ 708 static int 709 t4_detach(device_t dev) 710 { 711 struct adapter *sc; 712 struct port_info *pi; 713 int i, rc; 714 715 sc = device_get_softc(dev); 716 717 if (sc->flags & FULL_INIT_DONE) 718 t4_intr_disable(sc); 719 720 if (sc->cdev) { 721 destroy_dev(sc->cdev); 722 sc->cdev = NULL; 723 } 724 725 rc = bus_generic_detach(dev); 726 if (rc) { 727 device_printf(dev, 728 "failed to detach child devices: %d\n", rc); 729 return (rc); 730 } 731 732 for (i = 0; i < MAX_NPORTS; i++) { 733 pi = sc->port[i]; 734 if (pi) { 735 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 736 if (pi->dev) 737 device_delete_child(dev, pi->dev); 738 739 mtx_destroy(&pi->pi_lock); 740 free(pi, M_CXGBE); 741 } 742 } 743 744 if (sc->flags & FULL_INIT_DONE) 745 adapter_full_uninit(sc); 746 747 if (sc->flags & FW_OK) 748 t4_fw_bye(sc, sc->mbox); 749 750 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 751 pci_release_msi(dev); 752 753 if (sc->regs_res) 754 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 755 sc->regs_res); 756 757 if (sc->msix_res) 758 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 759 sc->msix_res); 760 761 if (sc->l2t) 762 t4_free_l2t(sc->l2t); 763 764 #ifndef TCP_OFFLOAD_DISABLE 765 free(sc->sge.ofld_rxq, M_CXGBE); 766 free(sc->sge.ofld_txq, M_CXGBE); 767 #endif 768 free(sc->irq, M_CXGBE); 769 free(sc->sge.rxq, M_CXGBE); 770 free(sc->sge.txq, M_CXGBE); 771 free(sc->sge.ctrlq, M_CXGBE); 772 free(sc->sge.iqmap, M_CXGBE); 773 free(sc->sge.eqmap, M_CXGBE); 774 free(sc->tids.ftid_tab, M_CXGBE); 775 t4_destroy_dma_tag(sc); 776 if (mtx_initialized(&sc->sc_lock)) { 777 mtx_lock(&t4_list_lock); 778 SLIST_REMOVE(&t4_list, sc, adapter, link); 779 mtx_unlock(&t4_list_lock); 780 mtx_destroy(&sc->sc_lock); 781 } 782 783 if (mtx_initialized(&sc->sfl_lock)) 784 mtx_destroy(&sc->sfl_lock); 785 786 bzero(sc, sizeof(*sc)); 787 788 return (0); 789 } 790 791 792 static int 793 cxgbe_probe(device_t dev) 794 { 795 char buf[128]; 796 struct port_info *pi = device_get_softc(dev); 797 798 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 799 device_set_desc_copy(dev, buf); 800 801 return (BUS_PROBE_DEFAULT); 802 } 803 804 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 805 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 806 IFCAP_VLAN_HWTSO) 807 #define T4_CAP_ENABLE (T4_CAP & ~IFCAP_TSO6) 808 809 static int 810 cxgbe_attach(device_t dev) 811 { 812 struct port_info *pi = device_get_softc(dev); 813 struct ifnet *ifp; 814 815 /* Allocate an ifnet and set it up */ 816 ifp = if_alloc(IFT_ETHER); 817 if (ifp == NULL) { 818 device_printf(dev, "Cannot allocate ifnet\n"); 819 return (ENOMEM); 820 } 821 pi->ifp = ifp; 822 ifp->if_softc = pi; 823 824 callout_init(&pi->tick, CALLOUT_MPSAFE); 825 826 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 827 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 828 829 ifp->if_init = cxgbe_init; 830 ifp->if_ioctl = cxgbe_ioctl; 831 ifp->if_transmit = cxgbe_transmit; 832 ifp->if_qflush = cxgbe_qflush; 833 834 ifp->if_capabilities = T4_CAP; 835 #ifndef TCP_OFFLOAD_DISABLE 836 if (is_offload(pi->adapter)) 837 ifp->if_capabilities |= IFCAP_TOE4; 838 #endif 839 ifp->if_capenable = T4_CAP_ENABLE; 840 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; 841 842 /* Initialize ifmedia for this port */ 843 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 844 cxgbe_media_status); 845 build_medialist(pi); 846 847 ether_ifattach(ifp, pi->hw_addr); 848 849 #ifndef TCP_OFFLOAD_DISABLE 850 if (is_offload(pi->adapter)) { 851 device_printf(dev, 852 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 853 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 854 } else 855 #endif 856 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 857 858 cxgbe_sysctls(pi); 859 860 return (0); 861 } 862 863 static int 864 cxgbe_detach(device_t dev) 865 { 866 struct port_info *pi = device_get_softc(dev); 867 struct adapter *sc = pi->adapter; 868 struct ifnet *ifp = pi->ifp; 869 870 /* Tell if_ioctl and if_init that the port is going away */ 871 ADAPTER_LOCK(sc); 872 SET_DOOMED(pi); 873 wakeup(&sc->flags); 874 while (IS_BUSY(sc)) 875 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 876 SET_BUSY(sc); 877 ADAPTER_UNLOCK(sc); 878 879 PORT_LOCK(pi); 880 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 881 callout_stop(&pi->tick); 882 PORT_UNLOCK(pi); 883 callout_drain(&pi->tick); 884 885 /* Let detach proceed even if these fail. */ 886 cxgbe_uninit_synchronized(pi); 887 port_full_uninit(pi); 888 889 ifmedia_removeall(&pi->media); 890 ether_ifdetach(pi->ifp); 891 if_free(pi->ifp); 892 893 ADAPTER_LOCK(sc); 894 CLR_BUSY(sc); 895 wakeup_one(&sc->flags); 896 ADAPTER_UNLOCK(sc); 897 898 return (0); 899 } 900 901 static void 902 cxgbe_init(void *arg) 903 { 904 struct port_info *pi = arg; 905 struct adapter *sc = pi->adapter; 906 907 ADAPTER_LOCK(sc); 908 cxgbe_init_locked(pi); /* releases adapter lock */ 909 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 910 } 911 912 static int 913 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 914 { 915 int rc = 0, mtu, flags; 916 struct port_info *pi = ifp->if_softc; 917 struct adapter *sc = pi->adapter; 918 struct ifreq *ifr = (struct ifreq *)data; 919 uint32_t mask; 920 921 switch (cmd) { 922 case SIOCSIFMTU: 923 ADAPTER_LOCK(sc); 924 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 925 if (rc) { 926 fail: 927 ADAPTER_UNLOCK(sc); 928 return (rc); 929 } 930 931 mtu = ifr->ifr_mtu; 932 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 933 rc = EINVAL; 934 } else { 935 ifp->if_mtu = mtu; 936 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 937 t4_update_fl_bufsize(ifp); 938 PORT_LOCK(pi); 939 rc = update_mac_settings(pi, XGMAC_MTU); 940 PORT_UNLOCK(pi); 941 } 942 } 943 ADAPTER_UNLOCK(sc); 944 break; 945 946 case SIOCSIFFLAGS: 947 ADAPTER_LOCK(sc); 948 if (IS_DOOMED(pi)) { 949 rc = ENXIO; 950 goto fail; 951 } 952 if (ifp->if_flags & IFF_UP) { 953 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 954 flags = pi->if_flags; 955 if ((ifp->if_flags ^ flags) & 956 (IFF_PROMISC | IFF_ALLMULTI)) { 957 if (IS_BUSY(sc)) { 958 rc = EBUSY; 959 goto fail; 960 } 961 PORT_LOCK(pi); 962 rc = update_mac_settings(pi, 963 XGMAC_PROMISC | XGMAC_ALLMULTI); 964 PORT_UNLOCK(pi); 965 } 966 ADAPTER_UNLOCK(sc); 967 } else 968 rc = cxgbe_init_locked(pi); 969 pi->if_flags = ifp->if_flags; 970 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 971 rc = cxgbe_uninit_locked(pi); 972 else 973 ADAPTER_UNLOCK(sc); 974 975 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 976 break; 977 978 case SIOCADDMULTI: 979 case SIOCDELMULTI: /* these two can be called with a mutex held :-( */ 980 ADAPTER_LOCK(sc); 981 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 982 if (rc) 983 goto fail; 984 985 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 986 PORT_LOCK(pi); 987 rc = update_mac_settings(pi, XGMAC_MCADDRS); 988 PORT_UNLOCK(pi); 989 } 990 ADAPTER_UNLOCK(sc); 991 break; 992 993 case SIOCSIFCAP: 994 ADAPTER_LOCK(sc); 995 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 996 if (rc) 997 goto fail; 998 999 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1000 if (mask & IFCAP_TXCSUM) { 1001 ifp->if_capenable ^= IFCAP_TXCSUM; 1002 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1003 1004 if (IFCAP_TSO & ifp->if_capenable && 1005 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1006 ifp->if_capenable &= ~IFCAP_TSO; 1007 ifp->if_hwassist &= ~CSUM_TSO; 1008 if_printf(ifp, 1009 "tso disabled due to -txcsum.\n"); 1010 } 1011 } 1012 if (mask & IFCAP_RXCSUM) 1013 ifp->if_capenable ^= IFCAP_RXCSUM; 1014 if (mask & IFCAP_TSO4) { 1015 ifp->if_capenable ^= IFCAP_TSO4; 1016 1017 if (IFCAP_TSO & ifp->if_capenable) { 1018 if (IFCAP_TXCSUM & ifp->if_capenable) 1019 ifp->if_hwassist |= CSUM_TSO; 1020 else { 1021 ifp->if_capenable &= ~IFCAP_TSO; 1022 ifp->if_hwassist &= ~CSUM_TSO; 1023 if_printf(ifp, 1024 "enable txcsum first.\n"); 1025 rc = EAGAIN; 1026 goto fail; 1027 } 1028 } else 1029 ifp->if_hwassist &= ~CSUM_TSO; 1030 } 1031 if (mask & IFCAP_LRO) { 1032 #ifdef INET 1033 int i; 1034 struct sge_rxq *rxq; 1035 1036 ifp->if_capenable ^= IFCAP_LRO; 1037 for_each_rxq(pi, i, rxq) { 1038 if (ifp->if_capenable & IFCAP_LRO) 1039 rxq->iq.flags |= IQ_LRO_ENABLED; 1040 else 1041 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1042 } 1043 #endif 1044 } 1045 #ifndef TCP_OFFLOAD_DISABLE 1046 if (mask & IFCAP_TOE) { 1047 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1048 1049 rc = toe_capability(pi, enable); 1050 if (rc != 0) 1051 goto fail; 1052 1053 ifp->if_capenable ^= mask; 1054 } 1055 #endif 1056 if (mask & IFCAP_VLAN_HWTAGGING) { 1057 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1058 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1059 PORT_LOCK(pi); 1060 rc = update_mac_settings(pi, XGMAC_VLANEX); 1061 PORT_UNLOCK(pi); 1062 } 1063 } 1064 if (mask & IFCAP_VLAN_MTU) { 1065 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1066 1067 /* Need to find out how to disable auto-mtu-inflation */ 1068 } 1069 if (mask & IFCAP_VLAN_HWTSO) 1070 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1071 if (mask & IFCAP_VLAN_HWCSUM) 1072 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1073 1074 #ifdef VLAN_CAPABILITIES 1075 VLAN_CAPABILITIES(ifp); 1076 #endif 1077 ADAPTER_UNLOCK(sc); 1078 break; 1079 1080 case SIOCSIFMEDIA: 1081 case SIOCGIFMEDIA: 1082 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1083 break; 1084 1085 default: 1086 rc = ether_ioctl(ifp, cmd, data); 1087 } 1088 1089 return (rc); 1090 } 1091 1092 static int 1093 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1094 { 1095 struct port_info *pi = ifp->if_softc; 1096 struct adapter *sc = pi->adapter; 1097 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1098 struct buf_ring *br; 1099 int rc; 1100 1101 M_ASSERTPKTHDR(m); 1102 1103 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1104 m_freem(m); 1105 return (ENETDOWN); 1106 } 1107 1108 if (m->m_flags & M_FLOWID) 1109 txq += (m->m_pkthdr.flowid % pi->ntxq); 1110 br = txq->br; 1111 1112 if (TXQ_TRYLOCK(txq) == 0) { 1113 struct sge_eq *eq = &txq->eq; 1114 1115 /* 1116 * It is possible that t4_eth_tx finishes up and releases the 1117 * lock between the TRYLOCK above and the drbr_enqueue here. We 1118 * need to make sure that this mbuf doesn't just sit there in 1119 * the drbr. 1120 */ 1121 1122 rc = drbr_enqueue(ifp, br, m); 1123 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1124 !(eq->flags & EQ_DOOMED)) 1125 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1126 return (rc); 1127 } 1128 1129 /* 1130 * txq->m is the mbuf that is held up due to a temporary shortage of 1131 * resources and it should be put on the wire first. Then what's in 1132 * drbr and finally the mbuf that was just passed in to us. 1133 * 1134 * Return code should indicate the fate of the mbuf that was passed in 1135 * this time. 1136 */ 1137 1138 TXQ_LOCK_ASSERT_OWNED(txq); 1139 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1140 1141 /* Queued for transmission. */ 1142 1143 rc = drbr_enqueue(ifp, br, m); 1144 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1145 (void) t4_eth_tx(ifp, txq, m); 1146 TXQ_UNLOCK(txq); 1147 return (rc); 1148 } 1149 1150 /* Direct transmission. */ 1151 rc = t4_eth_tx(ifp, txq, m); 1152 if (rc != 0 && txq->m) 1153 rc = 0; /* held, will be transmitted soon (hopefully) */ 1154 1155 TXQ_UNLOCK(txq); 1156 return (rc); 1157 } 1158 1159 static void 1160 cxgbe_qflush(struct ifnet *ifp) 1161 { 1162 struct port_info *pi = ifp->if_softc; 1163 struct sge_txq *txq; 1164 int i; 1165 struct mbuf *m; 1166 1167 /* queues do not exist if !PORT_INIT_DONE. */ 1168 if (pi->flags & PORT_INIT_DONE) { 1169 for_each_txq(pi, i, txq) { 1170 TXQ_LOCK(txq); 1171 m_freem(txq->m); 1172 txq->m = NULL; 1173 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1174 m_freem(m); 1175 TXQ_UNLOCK(txq); 1176 } 1177 } 1178 if_qflush(ifp); 1179 } 1180 1181 static int 1182 cxgbe_media_change(struct ifnet *ifp) 1183 { 1184 struct port_info *pi = ifp->if_softc; 1185 1186 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1187 1188 return (EOPNOTSUPP); 1189 } 1190 1191 static void 1192 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1193 { 1194 struct port_info *pi = ifp->if_softc; 1195 struct ifmedia_entry *cur = pi->media.ifm_cur; 1196 int speed = pi->link_cfg.speed; 1197 int data = (pi->port_type << 8) | pi->mod_type; 1198 1199 if (cur->ifm_data != data) { 1200 build_medialist(pi); 1201 cur = pi->media.ifm_cur; 1202 } 1203 1204 ifmr->ifm_status = IFM_AVALID; 1205 if (!pi->link_cfg.link_ok) 1206 return; 1207 1208 ifmr->ifm_status |= IFM_ACTIVE; 1209 1210 /* active and current will differ iff current media is autoselect. */ 1211 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1212 return; 1213 1214 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1215 if (speed == SPEED_10000) 1216 ifmr->ifm_active |= IFM_10G_T; 1217 else if (speed == SPEED_1000) 1218 ifmr->ifm_active |= IFM_1000_T; 1219 else if (speed == SPEED_100) 1220 ifmr->ifm_active |= IFM_100_TX; 1221 else if (speed == SPEED_10) 1222 ifmr->ifm_active |= IFM_10_T; 1223 else 1224 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1225 speed)); 1226 } 1227 1228 void 1229 t4_fatal_err(struct adapter *sc) 1230 { 1231 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1232 t4_intr_disable(sc); 1233 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1234 device_get_nameunit(sc->dev)); 1235 } 1236 1237 static int 1238 map_bars(struct adapter *sc) 1239 { 1240 sc->regs_rid = PCIR_BAR(0); 1241 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1242 &sc->regs_rid, RF_ACTIVE); 1243 if (sc->regs_res == NULL) { 1244 device_printf(sc->dev, "cannot map registers.\n"); 1245 return (ENXIO); 1246 } 1247 sc->bt = rman_get_bustag(sc->regs_res); 1248 sc->bh = rman_get_bushandle(sc->regs_res); 1249 sc->mmio_len = rman_get_size(sc->regs_res); 1250 1251 sc->msix_rid = PCIR_BAR(4); 1252 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1253 &sc->msix_rid, RF_ACTIVE); 1254 if (sc->msix_res == NULL) { 1255 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1256 return (ENXIO); 1257 } 1258 1259 return (0); 1260 } 1261 1262 static void 1263 setup_memwin(struct adapter *sc) 1264 { 1265 u_long bar0; 1266 1267 bar0 = rman_get_start(sc->regs_res); 1268 1269 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1270 (bar0 + MEMWIN0_BASE) | V_BIR(0) | 1271 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1272 1273 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1274 (bar0 + MEMWIN1_BASE) | V_BIR(0) | 1275 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1276 1277 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1278 (bar0 + MEMWIN2_BASE) | V_BIR(0) | 1279 V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10)); 1280 } 1281 1282 static int 1283 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1284 struct intrs_and_queues *iaq) 1285 { 1286 int rc, itype, navail, nrxq10g, nrxq1g, n; 1287 int nofldrxq10g = 0, nofldrxq1g = 0; 1288 1289 bzero(iaq, sizeof(*iaq)); 1290 1291 iaq->ntxq10g = t4_ntxq10g; 1292 iaq->ntxq1g = t4_ntxq1g; 1293 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1294 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1295 #ifndef TCP_OFFLOAD_DISABLE 1296 iaq->nofldtxq10g = t4_nofldtxq10g; 1297 iaq->nofldtxq1g = t4_nofldtxq1g; 1298 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1299 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1300 #endif 1301 1302 for (itype = INTR_MSIX; itype; itype >>= 1) { 1303 1304 if ((itype & t4_intr_types) == 0) 1305 continue; /* not allowed */ 1306 1307 if (itype == INTR_MSIX) 1308 navail = pci_msix_count(sc->dev); 1309 else if (itype == INTR_MSI) 1310 navail = pci_msi_count(sc->dev); 1311 else 1312 navail = 1; 1313 restart: 1314 if (navail == 0) 1315 continue; 1316 1317 iaq->intr_type = itype; 1318 iaq->intr_flags = 0; 1319 1320 /* 1321 * Best option: an interrupt vector for errors, one for the 1322 * firmware event queue, and one each for each rxq (NIC as well 1323 * as offload). 1324 */ 1325 iaq->nirq = T4_EXTRA_INTR; 1326 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1327 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1328 if (iaq->nirq <= navail && 1329 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1330 iaq->intr_flags |= INTR_DIRECT; 1331 goto allocate; 1332 } 1333 1334 /* 1335 * Second best option: an interrupt vector for errors, one for 1336 * the firmware event queue, and one each for either NIC or 1337 * offload rxq's. 1338 */ 1339 iaq->nirq = T4_EXTRA_INTR; 1340 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1341 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1342 if (iaq->nirq <= navail && 1343 (itype != INTR_MSI || powerof2(iaq->nirq))) 1344 goto allocate; 1345 1346 /* 1347 * Next best option: an interrupt vector for errors, one for the 1348 * firmware event queue, and at least one per port. At this 1349 * point we know we'll have to downsize nrxq or nofldrxq to fit 1350 * what's available to us. 1351 */ 1352 iaq->nirq = T4_EXTRA_INTR; 1353 iaq->nirq += n10g + n1g; 1354 if (iaq->nirq <= navail) { 1355 int leftover = navail - iaq->nirq; 1356 1357 if (n10g > 0) { 1358 int target = max(nrxq10g, nofldrxq10g); 1359 1360 n = 1; 1361 while (n < target && leftover >= n10g) { 1362 leftover -= n10g; 1363 iaq->nirq += n10g; 1364 n++; 1365 } 1366 iaq->nrxq10g = min(n, nrxq10g); 1367 #ifndef TCP_OFFLOAD_DISABLE 1368 iaq->nofldrxq10g = min(n, nofldrxq10g); 1369 #endif 1370 } 1371 1372 if (n1g > 0) { 1373 int target = max(nrxq1g, nofldrxq1g); 1374 1375 n = 1; 1376 while (n < target && leftover >= n1g) { 1377 leftover -= n1g; 1378 iaq->nirq += n1g; 1379 n++; 1380 } 1381 iaq->nrxq1g = min(n, nrxq1g); 1382 #ifndef TCP_OFFLOAD_DISABLE 1383 iaq->nofldrxq1g = min(n, nofldrxq1g); 1384 #endif 1385 } 1386 1387 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1388 goto allocate; 1389 } 1390 1391 /* 1392 * Least desirable option: one interrupt vector for everything. 1393 */ 1394 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1395 #ifndef TCP_OFFLOAD_DISABLE 1396 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1397 #endif 1398 1399 allocate: 1400 navail = iaq->nirq; 1401 rc = 0; 1402 if (itype == INTR_MSIX) 1403 rc = pci_alloc_msix(sc->dev, &navail); 1404 else if (itype == INTR_MSI) 1405 rc = pci_alloc_msi(sc->dev, &navail); 1406 1407 if (rc == 0) { 1408 if (navail == iaq->nirq) 1409 return (0); 1410 1411 /* 1412 * Didn't get the number requested. Use whatever number 1413 * the kernel is willing to allocate (it's in navail). 1414 */ 1415 device_printf(sc->dev, "fewer vectors than requested, " 1416 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1417 itype, iaq->nirq, navail); 1418 pci_release_msi(sc->dev); 1419 goto restart; 1420 } 1421 1422 device_printf(sc->dev, 1423 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1424 itype, rc, iaq->nirq, navail); 1425 } 1426 1427 device_printf(sc->dev, 1428 "failed to find a usable interrupt type. " 1429 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1430 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1431 1432 return (ENXIO); 1433 } 1434 1435 /* 1436 * Install a compatible firmware (if required), establish contact with it (by 1437 * saying hello), and reset the device. If we end up as the master driver, 1438 * partition adapter resources by providing a configuration file to the 1439 * firmware. 1440 */ 1441 static int 1442 prep_firmware(struct adapter *sc) 1443 { 1444 const struct firmware *fw = NULL, *cfg = NULL, *default_cfg; 1445 int rc; 1446 enum dev_state state; 1447 1448 default_cfg = firmware_get(T4_CFGNAME); 1449 1450 /* Check firmware version and install a different one if necessary */ 1451 rc = t4_check_fw_version(sc); 1452 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 1453 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1454 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1455 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1456 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1457 if (rc != 0) { 1458 uint32_t v = 0; 1459 1460 fw = firmware_get(T4_FWNAME); 1461 if (fw != NULL) { 1462 const struct fw_hdr *hdr = (const void *)fw->data; 1463 1464 v = ntohl(hdr->fw_ver); 1465 1466 /* 1467 * The firmware module will not be used if it isn't the 1468 * same major version as what the driver was compiled 1469 * with. 1470 */ 1471 if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) { 1472 device_printf(sc->dev, 1473 "Found firmware image but version %d " 1474 "can not be used with this driver (%d)\n", 1475 G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR); 1476 1477 firmware_put(fw, FIRMWARE_UNLOAD); 1478 fw = NULL; 1479 } 1480 } 1481 1482 if (fw == NULL && rc < 0) { 1483 device_printf(sc->dev, "No usable firmware. " 1484 "card has %d.%d.%d, driver compiled with %d.%d.%d", 1485 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1486 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1487 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1488 FW_VERSION_MAJOR, FW_VERSION_MINOR, 1489 FW_VERSION_MICRO); 1490 rc = EAGAIN; 1491 goto done; 1492 } 1493 1494 /* 1495 * Always upgrade, even for minor/micro/build mismatches. 1496 * Downgrade only for a major version mismatch or if 1497 * force_firmware_install was specified. 1498 */ 1499 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) { 1500 device_printf(sc->dev, 1501 "installing firmware %d.%d.%d.%d on card.\n", 1502 G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v), 1503 G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v)); 1504 1505 rc = -t4_load_fw(sc, fw->data, fw->datasize); 1506 if (rc != 0) { 1507 device_printf(sc->dev, 1508 "failed to install firmware: %d\n", rc); 1509 goto done; 1510 } else { 1511 /* refresh */ 1512 (void) t4_check_fw_version(sc); 1513 snprintf(sc->fw_version, 1514 sizeof(sc->fw_version), "%u.%u.%u.%u", 1515 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1516 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1517 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1518 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1519 } 1520 } 1521 } 1522 1523 /* Contact firmware. */ 1524 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1525 if (rc < 0) { 1526 rc = -rc; 1527 device_printf(sc->dev, 1528 "failed to connect to the firmware: %d.\n", rc); 1529 goto done; 1530 } 1531 if (rc == sc->mbox) 1532 sc->flags |= MASTER_PF; 1533 1534 /* Reset device */ 1535 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1536 if (rc != 0) { 1537 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 1538 if (rc != ETIMEDOUT && rc != EIO) 1539 t4_fw_bye(sc, sc->mbox); 1540 goto done; 1541 } 1542 1543 /* Partition adapter resources as specified in the config file. */ 1544 if (sc->flags & MASTER_PF) { 1545 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) { 1546 char s[32]; 1547 1548 snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file); 1549 cfg = firmware_get(s); 1550 if (cfg == NULL) { 1551 device_printf(sc->dev, 1552 "unable to locate %s module, " 1553 "will use default config file.\n", s); 1554 } 1555 } 1556 1557 rc = partition_resources(sc, cfg ? cfg : default_cfg); 1558 if (rc != 0) 1559 goto done; /* error message displayed already */ 1560 } 1561 1562 sc->flags |= FW_OK; 1563 1564 done: 1565 if (fw != NULL) 1566 firmware_put(fw, FIRMWARE_UNLOAD); 1567 if (cfg != NULL) 1568 firmware_put(cfg, FIRMWARE_UNLOAD); 1569 if (default_cfg != NULL) 1570 firmware_put(default_cfg, FIRMWARE_UNLOAD); 1571 1572 return (rc); 1573 } 1574 1575 #define FW_PARAM_DEV(param) \ 1576 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1577 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1578 #define FW_PARAM_PFVF(param) \ 1579 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1580 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1581 1582 /* 1583 * Upload configuration file to card's memory. 1584 */ 1585 static int 1586 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt, 1587 uint32_t *ma) 1588 { 1589 int rc, i; 1590 uint32_t param, val, mtype, maddr, bar, off, win, remaining; 1591 const uint32_t *b; 1592 1593 /* Figure out where the firmware wants us to upload it. */ 1594 param = FW_PARAM_DEV(CF); 1595 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1596 if (rc != 0) { 1597 /* Firmwares without config file support will fail this way */ 1598 device_printf(sc->dev, 1599 "failed to query config file location: %d.\n", rc); 1600 return (rc); 1601 } 1602 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1603 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1604 1605 if (maddr & 3) { 1606 device_printf(sc->dev, 1607 "cannot upload config file (type %u, addr %x).\n", 1608 mtype, maddr); 1609 return (EFAULT); 1610 } 1611 1612 /* Translate mtype/maddr to an address suitable for the PCIe window */ 1613 val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1614 val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE; 1615 switch (mtype) { 1616 case FW_MEMTYPE_CF_EDC0: 1617 if (!(val & F_EDRAM0_ENABLE)) 1618 goto err; 1619 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1620 maddr += G_EDRAM0_BASE(bar) << 20; 1621 break; 1622 1623 case FW_MEMTYPE_CF_EDC1: 1624 if (!(val & F_EDRAM1_ENABLE)) 1625 goto err; 1626 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1627 maddr += G_EDRAM1_BASE(bar) << 20; 1628 break; 1629 1630 case FW_MEMTYPE_CF_EXTMEM: 1631 if (!(val & F_EXT_MEM_ENABLE)) 1632 goto err; 1633 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1634 maddr += G_EXT_MEM_BASE(bar) << 20; 1635 break; 1636 1637 default: 1638 err: 1639 device_printf(sc->dev, 1640 "cannot upload config file (type %u, enabled %u).\n", 1641 mtype, val); 1642 return (EFAULT); 1643 } 1644 1645 /* 1646 * Position the PCIe window (we use memwin2) to the 16B aligned area 1647 * just at/before the upload location. 1648 */ 1649 win = maddr & ~0xf; 1650 off = maddr - win; /* offset from the start of the window. */ 1651 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 1652 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 1653 1654 remaining = fw->datasize; 1655 if (remaining > FLASH_CFG_MAX_SIZE || 1656 remaining > MEMWIN2_APERTURE - off) { 1657 device_printf(sc->dev, "cannot upload config file all at once " 1658 "(size %u, max %u, room %u).\n", 1659 remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off); 1660 return (EFBIG); 1661 } 1662 1663 /* 1664 * XXX: sheer laziness. We deliberately added 4 bytes of useless 1665 * stuffing/comments at the end of the config file so it's ok to simply 1666 * throw away the last remaining bytes when the config file is not an 1667 * exact multiple of 4. 1668 */ 1669 b = fw->data; 1670 for (i = 0; remaining >= 4; i += 4, remaining -= 4) 1671 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++); 1672 1673 return (rc); 1674 } 1675 1676 /* 1677 * Partition chip resources for use between various PFs, VFs, etc. This is done 1678 * by uploading the firmware configuration file to the adapter and instructing 1679 * the firmware to process it. 1680 */ 1681 static int 1682 partition_resources(struct adapter *sc, const struct firmware *cfg) 1683 { 1684 int rc; 1685 struct fw_caps_config_cmd caps; 1686 uint32_t mtype, maddr, finicsum, cfcsum; 1687 1688 rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT; 1689 if (rc != 0) { 1690 mtype = FW_MEMTYPE_CF_FLASH; 1691 maddr = t4_flash_cfg_addr(sc); 1692 } 1693 1694 bzero(&caps, sizeof(caps)); 1695 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1696 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1697 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1698 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1699 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1700 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1701 if (rc != 0) { 1702 device_printf(sc->dev, 1703 "failed to pre-process config file: %d.\n", rc); 1704 return (rc); 1705 } 1706 1707 finicsum = be32toh(caps.finicsum); 1708 cfcsum = be32toh(caps.cfcsum); 1709 if (finicsum != cfcsum) { 1710 device_printf(sc->dev, 1711 "WARNING: config file checksum mismatch: %08x %08x\n", 1712 finicsum, cfcsum); 1713 } 1714 sc->cfcsum = cfcsum; 1715 1716 #define LIMIT_CAPS(x) do { \ 1717 caps.x &= htobe16(t4_##x##_allowed); \ 1718 sc->x = htobe16(caps.x); \ 1719 } while (0) 1720 1721 /* 1722 * Let the firmware know what features will (not) be used so it can tune 1723 * things accordingly. 1724 */ 1725 LIMIT_CAPS(linkcaps); 1726 LIMIT_CAPS(niccaps); 1727 LIMIT_CAPS(toecaps); 1728 LIMIT_CAPS(rdmacaps); 1729 LIMIT_CAPS(iscsicaps); 1730 LIMIT_CAPS(fcoecaps); 1731 #undef LIMIT_CAPS 1732 1733 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1734 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1735 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1736 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 1737 if (rc != 0) { 1738 device_printf(sc->dev, 1739 "failed to process config file: %d.\n", rc); 1740 return (rc); 1741 } 1742 1743 return (0); 1744 } 1745 1746 /* 1747 * Retrieve parameters that are needed (or nice to have) prior to calling 1748 * t4_sge_init and t4_fw_initialize. 1749 */ 1750 static int 1751 get_params__pre_init(struct adapter *sc) 1752 { 1753 int rc; 1754 uint32_t param[2], val[2]; 1755 struct fw_devlog_cmd cmd; 1756 struct devlog_params *dlog = &sc->params.devlog; 1757 1758 param[0] = FW_PARAM_DEV(PORTVEC); 1759 param[1] = FW_PARAM_DEV(CCLK); 1760 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1761 if (rc != 0) { 1762 device_printf(sc->dev, 1763 "failed to query parameters (pre_init): %d.\n", rc); 1764 return (rc); 1765 } 1766 1767 sc->params.portvec = val[0]; 1768 sc->params.nports = 0; 1769 while (val[0]) { 1770 sc->params.nports++; 1771 val[0] &= val[0] - 1; 1772 } 1773 1774 sc->params.vpd.cclk = val[1]; 1775 1776 /* Read device log parameters. */ 1777 bzero(&cmd, sizeof(cmd)); 1778 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1779 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1780 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 1781 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 1782 if (rc != 0) { 1783 device_printf(sc->dev, 1784 "failed to get devlog parameters: %d.\n", rc); 1785 bzero(dlog, sizeof (*dlog)); 1786 rc = 0; /* devlog isn't critical for device operation */ 1787 } else { 1788 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 1789 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1790 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1791 dlog->size = be32toh(cmd.memsize_devlog); 1792 } 1793 1794 return (rc); 1795 } 1796 1797 /* 1798 * Retrieve various parameters that are of interest to the driver. The device 1799 * has been initialized by the firmware at this point. 1800 */ 1801 static int 1802 get_params__post_init(struct adapter *sc) 1803 { 1804 int rc; 1805 uint32_t param[7], val[7]; 1806 struct fw_caps_config_cmd caps; 1807 1808 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1809 param[1] = FW_PARAM_PFVF(EQ_START); 1810 param[2] = FW_PARAM_PFVF(FILTER_START); 1811 param[3] = FW_PARAM_PFVF(FILTER_END); 1812 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); 1813 if (rc != 0) { 1814 device_printf(sc->dev, 1815 "failed to query parameters (post_init): %d.\n", rc); 1816 return (rc); 1817 } 1818 1819 sc->sge.iq_start = val[0]; 1820 sc->sge.eq_start = val[1]; 1821 sc->tids.ftid_base = val[2]; 1822 sc->tids.nftids = val[3] - val[2] + 1; 1823 1824 /* get capabilites */ 1825 bzero(&caps, sizeof(caps)); 1826 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1827 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1828 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1829 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1830 if (rc != 0) { 1831 device_printf(sc->dev, 1832 "failed to get card capabilities: %d.\n", rc); 1833 return (rc); 1834 } 1835 1836 if (caps.toecaps) { 1837 /* query offload-related parameters */ 1838 param[0] = FW_PARAM_DEV(NTID); 1839 param[1] = FW_PARAM_PFVF(SERVER_START); 1840 param[2] = FW_PARAM_PFVF(SERVER_END); 1841 param[3] = FW_PARAM_PFVF(TDDP_START); 1842 param[4] = FW_PARAM_PFVF(TDDP_END); 1843 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1844 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1845 if (rc != 0) { 1846 device_printf(sc->dev, 1847 "failed to query TOE parameters: %d.\n", rc); 1848 return (rc); 1849 } 1850 sc->tids.ntids = val[0]; 1851 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1852 sc->tids.stid_base = val[1]; 1853 sc->tids.nstids = val[2] - val[1] + 1; 1854 sc->vres.ddp.start = val[3]; 1855 sc->vres.ddp.size = val[4] - val[3] + 1; 1856 sc->params.ofldq_wr_cred = val[5]; 1857 sc->params.offload = 1; 1858 } 1859 if (caps.rdmacaps) { 1860 param[0] = FW_PARAM_PFVF(STAG_START); 1861 param[1] = FW_PARAM_PFVF(STAG_END); 1862 param[2] = FW_PARAM_PFVF(RQ_START); 1863 param[3] = FW_PARAM_PFVF(RQ_END); 1864 param[4] = FW_PARAM_PFVF(PBL_START); 1865 param[5] = FW_PARAM_PFVF(PBL_END); 1866 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1867 if (rc != 0) { 1868 device_printf(sc->dev, 1869 "failed to query RDMA parameters(1): %d.\n", rc); 1870 return (rc); 1871 } 1872 sc->vres.stag.start = val[0]; 1873 sc->vres.stag.size = val[1] - val[0] + 1; 1874 sc->vres.rq.start = val[2]; 1875 sc->vres.rq.size = val[3] - val[2] + 1; 1876 sc->vres.pbl.start = val[4]; 1877 sc->vres.pbl.size = val[5] - val[4] + 1; 1878 1879 param[0] = FW_PARAM_PFVF(SQRQ_START); 1880 param[1] = FW_PARAM_PFVF(SQRQ_END); 1881 param[2] = FW_PARAM_PFVF(CQ_START); 1882 param[3] = FW_PARAM_PFVF(CQ_END); 1883 param[4] = FW_PARAM_PFVF(OCQ_START); 1884 param[5] = FW_PARAM_PFVF(OCQ_END); 1885 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 1886 if (rc != 0) { 1887 device_printf(sc->dev, 1888 "failed to query RDMA parameters(2): %d.\n", rc); 1889 return (rc); 1890 } 1891 sc->vres.qp.start = val[0]; 1892 sc->vres.qp.size = val[1] - val[0] + 1; 1893 sc->vres.cq.start = val[2]; 1894 sc->vres.cq.size = val[3] - val[2] + 1; 1895 sc->vres.ocq.start = val[4]; 1896 sc->vres.ocq.size = val[5] - val[4] + 1; 1897 } 1898 if (caps.iscsicaps) { 1899 param[0] = FW_PARAM_PFVF(ISCSI_START); 1900 param[1] = FW_PARAM_PFVF(ISCSI_END); 1901 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1902 if (rc != 0) { 1903 device_printf(sc->dev, 1904 "failed to query iSCSI parameters: %d.\n", rc); 1905 return (rc); 1906 } 1907 sc->vres.iscsi.start = val[0]; 1908 sc->vres.iscsi.size = val[1] - val[0] + 1; 1909 } 1910 1911 /* These are finalized by FW initialization, load their values now */ 1912 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1913 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1914 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1915 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1916 1917 return (rc); 1918 } 1919 1920 #undef FW_PARAM_PFVF 1921 #undef FW_PARAM_DEV 1922 1923 static void 1924 t4_set_desc(struct adapter *sc) 1925 { 1926 char buf[128]; 1927 struct adapter_params *p = &sc->params; 1928 1929 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 1930 p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec); 1931 1932 device_set_desc_copy(sc->dev, buf); 1933 } 1934 1935 static void 1936 build_medialist(struct port_info *pi) 1937 { 1938 struct ifmedia *media = &pi->media; 1939 int data, m; 1940 1941 PORT_LOCK(pi); 1942 1943 ifmedia_removeall(media); 1944 1945 m = IFM_ETHER | IFM_FDX; 1946 data = (pi->port_type << 8) | pi->mod_type; 1947 1948 switch(pi->port_type) { 1949 case FW_PORT_TYPE_BT_XFI: 1950 ifmedia_add(media, m | IFM_10G_T, data, NULL); 1951 break; 1952 1953 case FW_PORT_TYPE_BT_XAUI: 1954 ifmedia_add(media, m | IFM_10G_T, data, NULL); 1955 /* fall through */ 1956 1957 case FW_PORT_TYPE_BT_SGMII: 1958 ifmedia_add(media, m | IFM_1000_T, data, NULL); 1959 ifmedia_add(media, m | IFM_100_TX, data, NULL); 1960 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 1961 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 1962 break; 1963 1964 case FW_PORT_TYPE_CX4: 1965 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 1966 ifmedia_set(media, m | IFM_10G_CX4); 1967 break; 1968 1969 case FW_PORT_TYPE_SFP: 1970 case FW_PORT_TYPE_FIBER_XFI: 1971 case FW_PORT_TYPE_FIBER_XAUI: 1972 switch (pi->mod_type) { 1973 1974 case FW_PORT_MOD_TYPE_LR: 1975 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 1976 ifmedia_set(media, m | IFM_10G_LR); 1977 break; 1978 1979 case FW_PORT_MOD_TYPE_SR: 1980 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 1981 ifmedia_set(media, m | IFM_10G_SR); 1982 break; 1983 1984 case FW_PORT_MOD_TYPE_LRM: 1985 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 1986 ifmedia_set(media, m | IFM_10G_LRM); 1987 break; 1988 1989 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 1990 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 1991 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 1992 ifmedia_set(media, m | IFM_10G_TWINAX); 1993 break; 1994 1995 case FW_PORT_MOD_TYPE_NONE: 1996 m &= ~IFM_FDX; 1997 ifmedia_add(media, m | IFM_NONE, data, NULL); 1998 ifmedia_set(media, m | IFM_NONE); 1999 break; 2000 2001 case FW_PORT_MOD_TYPE_NA: 2002 case FW_PORT_MOD_TYPE_ER: 2003 default: 2004 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2005 ifmedia_set(media, m | IFM_UNKNOWN); 2006 break; 2007 } 2008 break; 2009 2010 case FW_PORT_TYPE_KX4: 2011 case FW_PORT_TYPE_KX: 2012 case FW_PORT_TYPE_KR: 2013 default: 2014 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2015 ifmedia_set(media, m | IFM_UNKNOWN); 2016 break; 2017 } 2018 2019 PORT_UNLOCK(pi); 2020 } 2021 2022 #define FW_MAC_EXACT_CHUNK 7 2023 2024 /* 2025 * Program the port's XGMAC based on parameters in ifnet. The caller also 2026 * indicates which parameters should be programmed (the rest are left alone). 2027 */ 2028 static int 2029 update_mac_settings(struct port_info *pi, int flags) 2030 { 2031 int rc; 2032 struct ifnet *ifp = pi->ifp; 2033 struct adapter *sc = pi->adapter; 2034 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2035 2036 PORT_LOCK_ASSERT_OWNED(pi); 2037 KASSERT(flags, ("%s: not told what to update.", __func__)); 2038 2039 if (flags & XGMAC_MTU) 2040 mtu = ifp->if_mtu; 2041 2042 if (flags & XGMAC_PROMISC) 2043 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2044 2045 if (flags & XGMAC_ALLMULTI) 2046 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2047 2048 if (flags & XGMAC_VLANEX) 2049 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2050 2051 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2052 vlanex, false); 2053 if (rc) { 2054 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2055 return (rc); 2056 } 2057 2058 if (flags & XGMAC_UCADDR) { 2059 uint8_t ucaddr[ETHER_ADDR_LEN]; 2060 2061 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2062 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2063 ucaddr, true, true); 2064 if (rc < 0) { 2065 rc = -rc; 2066 if_printf(ifp, "change_mac failed: %d\n", rc); 2067 return (rc); 2068 } else { 2069 pi->xact_addr_filt = rc; 2070 rc = 0; 2071 } 2072 } 2073 2074 if (flags & XGMAC_MCADDRS) { 2075 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2076 int del = 1; 2077 uint64_t hash = 0; 2078 struct ifmultiaddr *ifma; 2079 int i = 0, j; 2080 2081 if_maddr_rlock(ifp); 2082 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2083 if (ifma->ifma_addr->sa_family == AF_LINK) 2084 continue; 2085 mcaddr[i++] = 2086 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2087 2088 if (i == FW_MAC_EXACT_CHUNK) { 2089 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2090 del, i, mcaddr, NULL, &hash, 0); 2091 if (rc < 0) { 2092 rc = -rc; 2093 for (j = 0; j < i; j++) { 2094 if_printf(ifp, 2095 "failed to add mc address" 2096 " %02x:%02x:%02x:" 2097 "%02x:%02x:%02x rc=%d\n", 2098 mcaddr[j][0], mcaddr[j][1], 2099 mcaddr[j][2], mcaddr[j][3], 2100 mcaddr[j][4], mcaddr[j][5], 2101 rc); 2102 } 2103 goto mcfail; 2104 } 2105 del = 0; 2106 i = 0; 2107 } 2108 } 2109 if (i > 0) { 2110 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2111 del, i, mcaddr, NULL, &hash, 0); 2112 if (rc < 0) { 2113 rc = -rc; 2114 for (j = 0; j < i; j++) { 2115 if_printf(ifp, 2116 "failed to add mc address" 2117 " %02x:%02x:%02x:" 2118 "%02x:%02x:%02x rc=%d\n", 2119 mcaddr[j][0], mcaddr[j][1], 2120 mcaddr[j][2], mcaddr[j][3], 2121 mcaddr[j][4], mcaddr[j][5], 2122 rc); 2123 } 2124 goto mcfail; 2125 } 2126 } 2127 2128 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2129 if (rc != 0) 2130 if_printf(ifp, "failed to set mc address hash: %d", rc); 2131 mcfail: 2132 if_maddr_runlock(ifp); 2133 } 2134 2135 return (rc); 2136 } 2137 2138 static int 2139 cxgbe_init_locked(struct port_info *pi) 2140 { 2141 struct adapter *sc = pi->adapter; 2142 int rc = 0; 2143 2144 ADAPTER_LOCK_ASSERT_OWNED(sc); 2145 2146 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2147 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) { 2148 rc = EINTR; 2149 goto done; 2150 } 2151 } 2152 if (IS_DOOMED(pi)) { 2153 rc = ENXIO; 2154 goto done; 2155 } 2156 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2157 2158 /* Give up the adapter lock, port init code can sleep. */ 2159 SET_BUSY(sc); 2160 ADAPTER_UNLOCK(sc); 2161 2162 rc = cxgbe_init_synchronized(pi); 2163 2164 done: 2165 ADAPTER_LOCK(sc); 2166 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2167 CLR_BUSY(sc); 2168 wakeup_one(&sc->flags); 2169 ADAPTER_UNLOCK(sc); 2170 return (rc); 2171 } 2172 2173 static int 2174 cxgbe_init_synchronized(struct port_info *pi) 2175 { 2176 struct adapter *sc = pi->adapter; 2177 struct ifnet *ifp = pi->ifp; 2178 int rc = 0; 2179 2180 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2181 2182 if (isset(&sc->open_device_map, pi->port_id)) { 2183 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2184 ("mismatch between open_device_map and if_drv_flags")); 2185 return (0); /* already running */ 2186 } 2187 2188 if (!(sc->flags & FULL_INIT_DONE) && 2189 ((rc = adapter_full_init(sc)) != 0)) 2190 return (rc); /* error message displayed already */ 2191 2192 if (!(pi->flags & PORT_INIT_DONE) && 2193 ((rc = port_full_init(pi)) != 0)) 2194 return (rc); /* error message displayed already */ 2195 2196 PORT_LOCK(pi); 2197 rc = update_mac_settings(pi, XGMAC_ALL); 2198 PORT_UNLOCK(pi); 2199 if (rc) 2200 goto done; /* error message displayed already */ 2201 2202 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2203 if (rc != 0) { 2204 if_printf(ifp, "start_link failed: %d\n", rc); 2205 goto done; 2206 } 2207 2208 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2209 if (rc != 0) { 2210 if_printf(ifp, "enable_vi failed: %d\n", rc); 2211 goto done; 2212 } 2213 2214 /* all ok */ 2215 setbit(&sc->open_device_map, pi->port_id); 2216 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2217 2218 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2219 done: 2220 if (rc != 0) 2221 cxgbe_uninit_synchronized(pi); 2222 2223 return (rc); 2224 } 2225 2226 static int 2227 cxgbe_uninit_locked(struct port_info *pi) 2228 { 2229 struct adapter *sc = pi->adapter; 2230 int rc; 2231 2232 ADAPTER_LOCK_ASSERT_OWNED(sc); 2233 2234 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2235 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) { 2236 rc = EINTR; 2237 goto done; 2238 } 2239 } 2240 if (IS_DOOMED(pi)) { 2241 rc = ENXIO; 2242 goto done; 2243 } 2244 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2245 SET_BUSY(sc); 2246 ADAPTER_UNLOCK(sc); 2247 2248 rc = cxgbe_uninit_synchronized(pi); 2249 2250 ADAPTER_LOCK(sc); 2251 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2252 CLR_BUSY(sc); 2253 wakeup_one(&sc->flags); 2254 done: 2255 ADAPTER_UNLOCK(sc); 2256 return (rc); 2257 } 2258 2259 /* 2260 * Idempotent. 2261 */ 2262 static int 2263 cxgbe_uninit_synchronized(struct port_info *pi) 2264 { 2265 struct adapter *sc = pi->adapter; 2266 struct ifnet *ifp = pi->ifp; 2267 int rc; 2268 2269 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2270 2271 /* 2272 * Disable the VI so that all its data in either direction is discarded 2273 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2274 * tick) intact as the TP can deliver negative advice or data that it's 2275 * holding in its RAM (for an offloaded connection) even after the VI is 2276 * disabled. 2277 */ 2278 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2279 if (rc) { 2280 if_printf(ifp, "disable_vi failed: %d\n", rc); 2281 return (rc); 2282 } 2283 2284 clrbit(&sc->open_device_map, pi->port_id); 2285 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2286 2287 pi->link_cfg.link_ok = 0; 2288 pi->link_cfg.speed = 0; 2289 t4_os_link_changed(sc, pi->port_id, 0); 2290 2291 return (0); 2292 } 2293 2294 #define T4_ALLOC_IRQ(sc, irq, rid, handler, arg, name) do { \ 2295 rc = t4_alloc_irq(sc, irq, rid, handler, arg, name); \ 2296 if (rc != 0) \ 2297 goto done; \ 2298 } while (0) 2299 2300 static int 2301 adapter_full_init(struct adapter *sc) 2302 { 2303 int rc, i, rid, p, q; 2304 char s[8]; 2305 struct irq *irq; 2306 struct port_info *pi; 2307 struct sge_rxq *rxq; 2308 #ifndef TCP_OFFLOAD_DISABLE 2309 struct sge_ofld_rxq *ofld_rxq; 2310 #endif 2311 2312 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2313 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2314 ("%s: FULL_INIT_DONE already", __func__)); 2315 2316 /* 2317 * queues that belong to the adapter (not any particular port). 2318 */ 2319 rc = t4_setup_adapter_queues(sc); 2320 if (rc != 0) 2321 goto done; 2322 2323 for (i = 0; i < ARRAY_SIZE(sc->tq); i++) { 2324 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 2325 taskqueue_thread_enqueue, &sc->tq[i]); 2326 if (sc->tq[i] == NULL) { 2327 device_printf(sc->dev, 2328 "failed to allocate task queue %d\n", i); 2329 rc = ENOMEM; 2330 goto done; 2331 } 2332 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 2333 device_get_nameunit(sc->dev), i); 2334 } 2335 2336 /* 2337 * Setup interrupts. 2338 */ 2339 irq = &sc->irq[0]; 2340 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2341 if (sc->intr_count == 1) { 2342 KASSERT(!(sc->flags & INTR_DIRECT), 2343 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2344 2345 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_all, sc, "all"); 2346 } else { 2347 /* Multiple interrupts. */ 2348 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2349 ("%s: too few intr.", __func__)); 2350 2351 /* The first one is always error intr */ 2352 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_err, sc, "err"); 2353 irq++; 2354 rid++; 2355 2356 /* The second one is always the firmware event queue */ 2357 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 2358 irq++; 2359 rid++; 2360 2361 /* 2362 * Note that if INTR_DIRECT is not set then either the NIC rx 2363 * queues or (exclusive or) the TOE rx queueus will be taking 2364 * direct interrupts. 2365 * 2366 * There is no need to check for is_offload(sc) as nofldrxq 2367 * will be 0 if offload is disabled. 2368 */ 2369 for_each_port(sc, p) { 2370 pi = sc->port[p]; 2371 2372 #ifndef TCP_OFFLOAD_DISABLE 2373 /* 2374 * Skip over the NIC queues if they aren't taking direct 2375 * interrupts. 2376 */ 2377 if (!(sc->flags & INTR_DIRECT) && 2378 pi->nofldrxq > pi->nrxq) 2379 goto ofld_queues; 2380 #endif 2381 rxq = &sc->sge.rxq[pi->first_rxq]; 2382 for (q = 0; q < pi->nrxq; q++, rxq++) { 2383 snprintf(s, sizeof(s), "%d.%d", p, q); 2384 T4_ALLOC_IRQ(sc, irq, rid, t4_intr, rxq, s); 2385 irq++; 2386 rid++; 2387 } 2388 2389 #ifndef TCP_OFFLOAD_DISABLE 2390 /* 2391 * Skip over the offload queues if they aren't taking 2392 * direct interrupts. 2393 */ 2394 if (!(sc->flags & INTR_DIRECT)) 2395 continue; 2396 ofld_queues: 2397 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2398 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2399 snprintf(s, sizeof(s), "%d,%d", p, q); 2400 T4_ALLOC_IRQ(sc, irq, rid, t4_intr, ofld_rxq, s); 2401 irq++; 2402 rid++; 2403 } 2404 #endif 2405 } 2406 } 2407 2408 t4_intr_enable(sc); 2409 sc->flags |= FULL_INIT_DONE; 2410 done: 2411 if (rc != 0) 2412 adapter_full_uninit(sc); 2413 2414 return (rc); 2415 } 2416 #undef T4_ALLOC_IRQ 2417 2418 static int 2419 adapter_full_uninit(struct adapter *sc) 2420 { 2421 int i; 2422 2423 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2424 2425 t4_teardown_adapter_queues(sc); 2426 2427 for (i = 0; i < sc->intr_count; i++) 2428 t4_free_irq(sc, &sc->irq[i]); 2429 2430 for (i = 0; i < ARRAY_SIZE(sc->tq) && sc->tq[i]; i++) { 2431 taskqueue_free(sc->tq[i]); 2432 sc->tq[i] = NULL; 2433 } 2434 2435 sc->flags &= ~FULL_INIT_DONE; 2436 2437 return (0); 2438 } 2439 2440 static int 2441 port_full_init(struct port_info *pi) 2442 { 2443 struct adapter *sc = pi->adapter; 2444 struct ifnet *ifp = pi->ifp; 2445 uint16_t *rss; 2446 struct sge_rxq *rxq; 2447 int rc, i; 2448 2449 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2450 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 2451 ("%s: PORT_INIT_DONE already", __func__)); 2452 2453 sysctl_ctx_init(&pi->ctx); 2454 pi->flags |= PORT_SYSCTL_CTX; 2455 2456 /* 2457 * Allocate tx/rx/fl queues for this port. 2458 */ 2459 rc = t4_setup_port_queues(pi); 2460 if (rc != 0) 2461 goto done; /* error message displayed already */ 2462 2463 /* 2464 * Setup RSS for this port. 2465 */ 2466 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 2467 M_ZERO | M_WAITOK); 2468 for_each_rxq(pi, i, rxq) { 2469 rss[i] = rxq->iq.abs_id; 2470 } 2471 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2472 pi->rss_size, rss, pi->nrxq); 2473 free(rss, M_CXGBE); 2474 if (rc != 0) { 2475 if_printf(ifp, "rss_config failed: %d\n", rc); 2476 goto done; 2477 } 2478 2479 pi->flags |= PORT_INIT_DONE; 2480 done: 2481 if (rc != 0) 2482 port_full_uninit(pi); 2483 2484 return (rc); 2485 } 2486 2487 /* 2488 * Idempotent. 2489 */ 2490 static int 2491 port_full_uninit(struct port_info *pi) 2492 { 2493 struct adapter *sc = pi->adapter; 2494 int i; 2495 struct sge_rxq *rxq; 2496 struct sge_txq *txq; 2497 #ifndef TCP_OFFLOAD_DISABLE 2498 struct sge_ofld_rxq *ofld_rxq; 2499 struct sge_wrq *ofld_txq; 2500 #endif 2501 2502 if (pi->flags & PORT_INIT_DONE) { 2503 2504 /* Need to quiesce queues. XXX: ctrl queues? */ 2505 2506 for_each_txq(pi, i, txq) { 2507 quiesce_eq(sc, &txq->eq); 2508 } 2509 2510 #ifndef TCP_OFFLOAD_DISABLE 2511 for_each_ofld_txq(pi, i, ofld_txq) { 2512 quiesce_eq(sc, &ofld_txq->eq); 2513 } 2514 #endif 2515 2516 for_each_rxq(pi, i, rxq) { 2517 quiesce_iq(sc, &rxq->iq); 2518 quiesce_fl(sc, &rxq->fl); 2519 } 2520 2521 #ifndef TCP_OFFLOAD_DISABLE 2522 for_each_ofld_rxq(pi, i, ofld_rxq) { 2523 quiesce_iq(sc, &ofld_rxq->iq); 2524 quiesce_fl(sc, &ofld_rxq->fl); 2525 } 2526 #endif 2527 } 2528 2529 t4_teardown_port_queues(pi); 2530 pi->flags &= ~PORT_INIT_DONE; 2531 2532 return (0); 2533 } 2534 2535 static void 2536 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 2537 { 2538 EQ_LOCK(eq); 2539 eq->flags |= EQ_DOOMED; 2540 2541 /* 2542 * Wait for the response to a credit flush if one's 2543 * pending. 2544 */ 2545 while (eq->flags & EQ_CRFLUSHED) 2546 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 2547 EQ_UNLOCK(eq); 2548 2549 callout_drain(&eq->tx_callout); /* XXX: iffy */ 2550 pause("callout", 10); /* Still iffy */ 2551 2552 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 2553 } 2554 2555 static void 2556 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 2557 { 2558 (void) sc; /* unused */ 2559 2560 /* Synchronize with the interrupt handler */ 2561 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 2562 pause("iqfree", 1); 2563 } 2564 2565 static void 2566 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 2567 { 2568 mtx_lock(&sc->sfl_lock); 2569 FL_LOCK(fl); 2570 fl->flags |= FL_DOOMED; 2571 FL_UNLOCK(fl); 2572 mtx_unlock(&sc->sfl_lock); 2573 2574 callout_drain(&sc->sfl_callout); 2575 KASSERT((fl->flags & FL_STARVING) == 0, 2576 ("%s: still starving", __func__)); 2577 } 2578 2579 static int 2580 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 2581 driver_intr_t *handler, void *arg, char *name) 2582 { 2583 int rc; 2584 2585 irq->rid = rid; 2586 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 2587 RF_SHAREABLE | RF_ACTIVE); 2588 if (irq->res == NULL) { 2589 device_printf(sc->dev, 2590 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 2591 return (ENOMEM); 2592 } 2593 2594 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 2595 NULL, handler, arg, &irq->tag); 2596 if (rc != 0) { 2597 device_printf(sc->dev, 2598 "failed to setup interrupt for rid %d, name %s: %d\n", 2599 rid, name, rc); 2600 } else if (name) 2601 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 2602 2603 return (rc); 2604 } 2605 2606 static int 2607 t4_free_irq(struct adapter *sc, struct irq *irq) 2608 { 2609 if (irq->tag) 2610 bus_teardown_intr(sc->dev, irq->res, irq->tag); 2611 if (irq->res) 2612 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 2613 2614 bzero(irq, sizeof(*irq)); 2615 2616 return (0); 2617 } 2618 2619 static void 2620 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 2621 unsigned int end) 2622 { 2623 uint32_t *p = (uint32_t *)(buf + start); 2624 2625 for ( ; start <= end; start += sizeof(uint32_t)) 2626 *p++ = t4_read_reg(sc, start); 2627 } 2628 2629 static void 2630 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 2631 { 2632 int i; 2633 static const unsigned int reg_ranges[] = { 2634 0x1008, 0x1108, 2635 0x1180, 0x11b4, 2636 0x11fc, 0x123c, 2637 0x1300, 0x173c, 2638 0x1800, 0x18fc, 2639 0x3000, 0x30d8, 2640 0x30e0, 0x5924, 2641 0x5960, 0x59d4, 2642 0x5a00, 0x5af8, 2643 0x6000, 0x6098, 2644 0x6100, 0x6150, 2645 0x6200, 0x6208, 2646 0x6240, 0x6248, 2647 0x6280, 0x6338, 2648 0x6370, 0x638c, 2649 0x6400, 0x643c, 2650 0x6500, 0x6524, 2651 0x6a00, 0x6a38, 2652 0x6a60, 0x6a78, 2653 0x6b00, 0x6b84, 2654 0x6bf0, 0x6c84, 2655 0x6cf0, 0x6d84, 2656 0x6df0, 0x6e84, 2657 0x6ef0, 0x6f84, 2658 0x6ff0, 0x7084, 2659 0x70f0, 0x7184, 2660 0x71f0, 0x7284, 2661 0x72f0, 0x7384, 2662 0x73f0, 0x7450, 2663 0x7500, 0x7530, 2664 0x7600, 0x761c, 2665 0x7680, 0x76cc, 2666 0x7700, 0x7798, 2667 0x77c0, 0x77fc, 2668 0x7900, 0x79fc, 2669 0x7b00, 0x7c38, 2670 0x7d00, 0x7efc, 2671 0x8dc0, 0x8e1c, 2672 0x8e30, 0x8e78, 2673 0x8ea0, 0x8f6c, 2674 0x8fc0, 0x9074, 2675 0x90fc, 0x90fc, 2676 0x9400, 0x9458, 2677 0x9600, 0x96bc, 2678 0x9800, 0x9808, 2679 0x9820, 0x983c, 2680 0x9850, 0x9864, 2681 0x9c00, 0x9c6c, 2682 0x9c80, 0x9cec, 2683 0x9d00, 0x9d6c, 2684 0x9d80, 0x9dec, 2685 0x9e00, 0x9e6c, 2686 0x9e80, 0x9eec, 2687 0x9f00, 0x9f6c, 2688 0x9f80, 0x9fec, 2689 0xd004, 0xd03c, 2690 0xdfc0, 0xdfe0, 2691 0xe000, 0xea7c, 2692 0xf000, 0x11190, 2693 0x19040, 0x19124, 2694 0x19150, 0x191b0, 2695 0x191d0, 0x191e8, 2696 0x19238, 0x1924c, 2697 0x193f8, 0x19474, 2698 0x19490, 0x194f8, 2699 0x19800, 0x19f30, 2700 0x1a000, 0x1a06c, 2701 0x1a0b0, 0x1a120, 2702 0x1a128, 0x1a138, 2703 0x1a190, 0x1a1c4, 2704 0x1a1fc, 0x1a1fc, 2705 0x1e040, 0x1e04c, 2706 0x1e240, 0x1e28c, 2707 0x1e2c0, 0x1e2c0, 2708 0x1e2e0, 0x1e2e0, 2709 0x1e300, 0x1e384, 2710 0x1e3c0, 0x1e3c8, 2711 0x1e440, 0x1e44c, 2712 0x1e640, 0x1e68c, 2713 0x1e6c0, 0x1e6c0, 2714 0x1e6e0, 0x1e6e0, 2715 0x1e700, 0x1e784, 2716 0x1e7c0, 0x1e7c8, 2717 0x1e840, 0x1e84c, 2718 0x1ea40, 0x1ea8c, 2719 0x1eac0, 0x1eac0, 2720 0x1eae0, 0x1eae0, 2721 0x1eb00, 0x1eb84, 2722 0x1ebc0, 0x1ebc8, 2723 0x1ec40, 0x1ec4c, 2724 0x1ee40, 0x1ee8c, 2725 0x1eec0, 0x1eec0, 2726 0x1eee0, 0x1eee0, 2727 0x1ef00, 0x1ef84, 2728 0x1efc0, 0x1efc8, 2729 0x1f040, 0x1f04c, 2730 0x1f240, 0x1f28c, 2731 0x1f2c0, 0x1f2c0, 2732 0x1f2e0, 0x1f2e0, 2733 0x1f300, 0x1f384, 2734 0x1f3c0, 0x1f3c8, 2735 0x1f440, 0x1f44c, 2736 0x1f640, 0x1f68c, 2737 0x1f6c0, 0x1f6c0, 2738 0x1f6e0, 0x1f6e0, 2739 0x1f700, 0x1f784, 2740 0x1f7c0, 0x1f7c8, 2741 0x1f840, 0x1f84c, 2742 0x1fa40, 0x1fa8c, 2743 0x1fac0, 0x1fac0, 2744 0x1fae0, 0x1fae0, 2745 0x1fb00, 0x1fb84, 2746 0x1fbc0, 0x1fbc8, 2747 0x1fc40, 0x1fc4c, 2748 0x1fe40, 0x1fe8c, 2749 0x1fec0, 0x1fec0, 2750 0x1fee0, 0x1fee0, 2751 0x1ff00, 0x1ff84, 2752 0x1ffc0, 0x1ffc8, 2753 0x20000, 0x2002c, 2754 0x20100, 0x2013c, 2755 0x20190, 0x201c8, 2756 0x20200, 0x20318, 2757 0x20400, 0x20528, 2758 0x20540, 0x20614, 2759 0x21000, 0x21040, 2760 0x2104c, 0x21060, 2761 0x210c0, 0x210ec, 2762 0x21200, 0x21268, 2763 0x21270, 0x21284, 2764 0x212fc, 0x21388, 2765 0x21400, 0x21404, 2766 0x21500, 0x21518, 2767 0x2152c, 0x2153c, 2768 0x21550, 0x21554, 2769 0x21600, 0x21600, 2770 0x21608, 0x21628, 2771 0x21630, 0x2163c, 2772 0x21700, 0x2171c, 2773 0x21780, 0x2178c, 2774 0x21800, 0x21c38, 2775 0x21c80, 0x21d7c, 2776 0x21e00, 0x21e04, 2777 0x22000, 0x2202c, 2778 0x22100, 0x2213c, 2779 0x22190, 0x221c8, 2780 0x22200, 0x22318, 2781 0x22400, 0x22528, 2782 0x22540, 0x22614, 2783 0x23000, 0x23040, 2784 0x2304c, 0x23060, 2785 0x230c0, 0x230ec, 2786 0x23200, 0x23268, 2787 0x23270, 0x23284, 2788 0x232fc, 0x23388, 2789 0x23400, 0x23404, 2790 0x23500, 0x23518, 2791 0x2352c, 0x2353c, 2792 0x23550, 0x23554, 2793 0x23600, 0x23600, 2794 0x23608, 0x23628, 2795 0x23630, 0x2363c, 2796 0x23700, 0x2371c, 2797 0x23780, 0x2378c, 2798 0x23800, 0x23c38, 2799 0x23c80, 0x23d7c, 2800 0x23e00, 0x23e04, 2801 0x24000, 0x2402c, 2802 0x24100, 0x2413c, 2803 0x24190, 0x241c8, 2804 0x24200, 0x24318, 2805 0x24400, 0x24528, 2806 0x24540, 0x24614, 2807 0x25000, 0x25040, 2808 0x2504c, 0x25060, 2809 0x250c0, 0x250ec, 2810 0x25200, 0x25268, 2811 0x25270, 0x25284, 2812 0x252fc, 0x25388, 2813 0x25400, 0x25404, 2814 0x25500, 0x25518, 2815 0x2552c, 0x2553c, 2816 0x25550, 0x25554, 2817 0x25600, 0x25600, 2818 0x25608, 0x25628, 2819 0x25630, 0x2563c, 2820 0x25700, 0x2571c, 2821 0x25780, 0x2578c, 2822 0x25800, 0x25c38, 2823 0x25c80, 0x25d7c, 2824 0x25e00, 0x25e04, 2825 0x26000, 0x2602c, 2826 0x26100, 0x2613c, 2827 0x26190, 0x261c8, 2828 0x26200, 0x26318, 2829 0x26400, 0x26528, 2830 0x26540, 0x26614, 2831 0x27000, 0x27040, 2832 0x2704c, 0x27060, 2833 0x270c0, 0x270ec, 2834 0x27200, 0x27268, 2835 0x27270, 0x27284, 2836 0x272fc, 0x27388, 2837 0x27400, 0x27404, 2838 0x27500, 0x27518, 2839 0x2752c, 0x2753c, 2840 0x27550, 0x27554, 2841 0x27600, 0x27600, 2842 0x27608, 0x27628, 2843 0x27630, 0x2763c, 2844 0x27700, 0x2771c, 2845 0x27780, 0x2778c, 2846 0x27800, 0x27c38, 2847 0x27c80, 0x27d7c, 2848 0x27e00, 0x27e04 2849 }; 2850 2851 regs->version = 4 | (sc->params.rev << 10); 2852 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2) 2853 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 2854 } 2855 2856 static void 2857 cxgbe_tick(void *arg) 2858 { 2859 struct port_info *pi = arg; 2860 struct ifnet *ifp = pi->ifp; 2861 struct sge_txq *txq; 2862 int i, drops; 2863 struct port_stats *s = &pi->stats; 2864 2865 PORT_LOCK(pi); 2866 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2867 PORT_UNLOCK(pi); 2868 return; /* without scheduling another callout */ 2869 } 2870 2871 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 2872 2873 ifp->if_opackets = s->tx_frames - s->tx_pause; 2874 ifp->if_ipackets = s->rx_frames - s->rx_pause; 2875 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 2876 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 2877 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 2878 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 2879 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 2880 s->rx_ovflow3; 2881 2882 drops = s->tx_drop; 2883 for_each_txq(pi, i, txq) 2884 drops += txq->br->br_drops; 2885 ifp->if_snd.ifq_drops = drops; 2886 2887 ifp->if_oerrors = s->tx_error_frames; 2888 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 2889 s->rx_fcs_err + s->rx_len_err; 2890 2891 callout_schedule(&pi->tick, hz); 2892 PORT_UNLOCK(pi); 2893 } 2894 2895 static int 2896 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 2897 { 2898 #ifdef INVARIANTS 2899 panic("%s: opcode %02x on iq %p with payload %p", 2900 __func__, rss->opcode, iq, m); 2901 #else 2902 log(LOG_ERR, "%s: opcode %02x on iq %p with payload %p", 2903 __func__, rss->opcode, iq, m); 2904 m_freem(m); 2905 #endif 2906 return (EDOOFUS); 2907 } 2908 2909 int 2910 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 2911 { 2912 uintptr_t *loc, new; 2913 2914 if (opcode >= ARRAY_SIZE(sc->cpl_handler)) 2915 return (EINVAL); 2916 2917 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 2918 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 2919 atomic_store_rel_ptr(loc, new); 2920 2921 return (0); 2922 } 2923 2924 static int 2925 t4_sysctls(struct adapter *sc) 2926 { 2927 struct sysctl_ctx_list *ctx; 2928 struct sysctl_oid *oid; 2929 struct sysctl_oid_list *children, *c0; 2930 static char *caps[] = { 2931 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 2932 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 2933 "\20\1TOE", /* caps[2] toecaps */ 2934 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 2935 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 2936 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 2937 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 2938 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 2939 }; 2940 2941 ctx = device_get_sysctl_ctx(sc->dev); 2942 2943 /* 2944 * dev.t4nex.X. 2945 */ 2946 oid = device_get_sysctl_tree(sc->dev); 2947 c0 = children = SYSCTL_CHILDREN(oid); 2948 2949 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, 2950 &sc->params.nports, 0, "# of ports"); 2951 2952 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 2953 &sc->params.rev, 0, "chip hardware revision"); 2954 2955 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 2956 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 2957 2958 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 2959 CTLFLAG_RD, &t4_cfg_file, 0, "configuration file"); 2960 2961 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, 2962 &sc->cfcsum, 0, "config file checksum"); 2963 2964 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 2965 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 2966 sysctl_bitfield, "A", "available link capabilities"); 2967 2968 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 2969 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 2970 sysctl_bitfield, "A", "available NIC capabilities"); 2971 2972 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 2973 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 2974 sysctl_bitfield, "A", "available TCP offload capabilities"); 2975 2976 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 2977 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 2978 sysctl_bitfield, "A", "available RDMA capabilities"); 2979 2980 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 2981 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 2982 sysctl_bitfield, "A", "available iSCSI capabilities"); 2983 2984 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 2985 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 2986 sysctl_bitfield, "A", "available FCoE capabilities"); 2987 2988 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, 2989 &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)"); 2990 2991 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 2992 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 2993 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 2994 "interrupt holdoff timer values (us)"); 2995 2996 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 2997 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 2998 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 2999 "interrupt holdoff packet counter values"); 3000 3001 #ifdef SBUF_DRAIN 3002 /* 3003 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 3004 */ 3005 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 3006 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 3007 "logs and miscellaneous information"); 3008 children = SYSCTL_CHILDREN(oid); 3009 3010 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 3011 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3012 sysctl_cctrl, "A", "congestion control"); 3013 3014 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 3015 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3016 sysctl_cpl_stats, "A", "CPL statistics"); 3017 3018 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 3019 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3020 sysctl_ddp_stats, "A", "DDP statistics"); 3021 3022 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 3023 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3024 sysctl_devlog, "A", "firmware's device log"); 3025 3026 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 3027 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3028 sysctl_fcoe_stats, "A", "FCoE statistics"); 3029 3030 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 3031 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3032 sysctl_hw_sched, "A", "hardware scheduler "); 3033 3034 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 3035 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3036 sysctl_l2t, "A", "hardware L2 table"); 3037 3038 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 3039 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3040 sysctl_lb_stats, "A", "loopback statistics"); 3041 3042 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 3043 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3044 sysctl_meminfo, "A", "memory regions"); 3045 3046 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 3047 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3048 sysctl_path_mtus, "A", "path MTUs"); 3049 3050 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 3051 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3052 sysctl_pm_stats, "A", "PM statistics"); 3053 3054 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 3055 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3056 sysctl_rdma_stats, "A", "RDMA statistics"); 3057 3058 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 3059 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3060 sysctl_tcp_stats, "A", "TCP statistics"); 3061 3062 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 3063 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3064 sysctl_tids, "A", "TID information"); 3065 3066 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 3067 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3068 sysctl_tp_err_stats, "A", "TP error statistics"); 3069 3070 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 3071 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3072 sysctl_tx_rate, "A", "Tx rate"); 3073 #endif 3074 3075 #ifndef TCP_OFFLOAD_DISABLE 3076 if (is_offload(sc)) { 3077 /* 3078 * dev.t4nex.X.toe. 3079 */ 3080 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 3081 NULL, "TOE parameters"); 3082 children = SYSCTL_CHILDREN(oid); 3083 3084 sc->tt.sndbuf = 256 * 1024; 3085 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 3086 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 3087 3088 sc->tt.ddp = 0; 3089 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 3090 &sc->tt.ddp, 0, "DDP allowed"); 3091 sc->tt.indsz = M_INDICATESIZE; 3092 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 3093 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 3094 sc->tt.ddp_thres = 3*4096; 3095 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 3096 &sc->tt.ddp_thres, 0, "DDP threshold"); 3097 } 3098 #endif 3099 3100 3101 return (0); 3102 } 3103 3104 static int 3105 cxgbe_sysctls(struct port_info *pi) 3106 { 3107 struct sysctl_ctx_list *ctx; 3108 struct sysctl_oid *oid; 3109 struct sysctl_oid_list *children; 3110 3111 ctx = device_get_sysctl_ctx(pi->dev); 3112 3113 /* 3114 * dev.cxgbe.X. 3115 */ 3116 oid = device_get_sysctl_tree(pi->dev); 3117 children = SYSCTL_CHILDREN(oid); 3118 3119 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 3120 &pi->nrxq, 0, "# of rx queues"); 3121 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 3122 &pi->ntxq, 0, "# of tx queues"); 3123 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 3124 &pi->first_rxq, 0, "index of first rx queue"); 3125 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 3126 &pi->first_txq, 0, "index of first tx queue"); 3127 3128 #ifndef TCP_OFFLOAD_DISABLE 3129 if (is_offload(pi->adapter)) { 3130 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 3131 &pi->nofldrxq, 0, 3132 "# of rx queues for offloaded TCP connections"); 3133 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 3134 &pi->nofldtxq, 0, 3135 "# of tx queues for offloaded TCP connections"); 3136 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 3137 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 3138 "index of first TOE rx queue"); 3139 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 3140 CTLFLAG_RD, &pi->first_ofld_txq, 0, 3141 "index of first TOE tx queue"); 3142 } 3143 #endif 3144 3145 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 3146 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 3147 "holdoff timer index"); 3148 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 3149 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 3150 "holdoff packet counter index"); 3151 3152 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 3153 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 3154 "rx queue size"); 3155 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 3156 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 3157 "tx queue size"); 3158 3159 /* 3160 * dev.cxgbe.X.stats. 3161 */ 3162 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 3163 NULL, "port statistics"); 3164 children = SYSCTL_CHILDREN(oid); 3165 3166 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 3167 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 3168 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 3169 sysctl_handle_t4_reg64, "QU", desc) 3170 3171 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 3172 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 3173 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 3174 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 3175 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 3176 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 3177 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 3178 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 3179 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 3180 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 3181 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 3182 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 3183 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 3184 "# of tx frames in this range", 3185 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 3186 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 3187 "# of tx frames in this range", 3188 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 3189 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 3190 "# of tx frames in this range", 3191 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 3192 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 3193 "# of tx frames in this range", 3194 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 3195 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 3196 "# of tx frames in this range", 3197 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 3198 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 3199 "# of tx frames in this range", 3200 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 3201 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 3202 "# of tx frames in this range", 3203 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 3204 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 3205 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 3206 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 3207 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 3208 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 3209 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 3210 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 3211 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 3212 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 3213 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 3214 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 3215 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 3216 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 3217 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 3218 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 3219 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 3220 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 3221 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 3222 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 3223 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 3224 3225 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 3226 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 3227 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 3228 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 3229 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 3230 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 3231 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 3232 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 3233 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 3234 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 3235 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 3236 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 3237 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 3238 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 3239 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 3240 "# of frames received with bad FCS", 3241 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 3242 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 3243 "# of frames received with length error", 3244 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 3245 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 3246 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 3247 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 3248 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 3249 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 3250 "# of rx frames in this range", 3251 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 3252 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 3253 "# of rx frames in this range", 3254 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 3255 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 3256 "# of rx frames in this range", 3257 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 3258 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 3259 "# of rx frames in this range", 3260 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 3261 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 3262 "# of rx frames in this range", 3263 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 3264 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 3265 "# of rx frames in this range", 3266 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 3267 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 3268 "# of rx frames in this range", 3269 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 3270 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 3271 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 3272 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 3273 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 3274 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 3275 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 3276 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 3277 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 3278 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 3279 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 3280 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 3281 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 3282 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 3283 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 3284 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 3285 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 3286 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 3287 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 3288 3289 #undef SYSCTL_ADD_T4_REG64 3290 3291 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 3292 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 3293 &pi->stats.name, desc) 3294 3295 /* We get these from port_stats and they may be stale by upto 1s */ 3296 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 3297 "# drops due to buffer-group 0 overflows"); 3298 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 3299 "# drops due to buffer-group 1 overflows"); 3300 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 3301 "# drops due to buffer-group 2 overflows"); 3302 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 3303 "# drops due to buffer-group 3 overflows"); 3304 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 3305 "# of buffer-group 0 truncated packets"); 3306 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 3307 "# of buffer-group 1 truncated packets"); 3308 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 3309 "# of buffer-group 2 truncated packets"); 3310 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 3311 "# of buffer-group 3 truncated packets"); 3312 3313 #undef SYSCTL_ADD_T4_PORTSTAT 3314 3315 return (0); 3316 } 3317 3318 static int 3319 sysctl_int_array(SYSCTL_HANDLER_ARGS) 3320 { 3321 int rc, *i; 3322 struct sbuf sb; 3323 3324 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 3325 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 3326 sbuf_printf(&sb, "%d ", *i); 3327 sbuf_trim(&sb); 3328 sbuf_finish(&sb); 3329 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 3330 sbuf_delete(&sb); 3331 return (rc); 3332 } 3333 3334 static int 3335 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 3336 { 3337 int rc; 3338 struct sbuf *sb; 3339 3340 rc = sysctl_wire_old_buffer(req, 0); 3341 if (rc != 0) 3342 return(rc); 3343 3344 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 3345 if (sb == NULL) 3346 return (ENOMEM); 3347 3348 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 3349 rc = sbuf_finish(sb); 3350 sbuf_delete(sb); 3351 3352 return (rc); 3353 } 3354 3355 static int 3356 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 3357 { 3358 struct port_info *pi = arg1; 3359 struct adapter *sc = pi->adapter; 3360 int idx, rc, i; 3361 3362 idx = pi->tmr_idx; 3363 3364 rc = sysctl_handle_int(oidp, &idx, 0, req); 3365 if (rc != 0 || req->newptr == NULL) 3366 return (rc); 3367 3368 if (idx < 0 || idx >= SGE_NTIMERS) 3369 return (EINVAL); 3370 3371 ADAPTER_LOCK(sc); 3372 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3373 if (rc == 0) { 3374 struct sge_rxq *rxq; 3375 uint8_t v; 3376 3377 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 3378 for_each_rxq(pi, i, rxq) { 3379 #ifdef atomic_store_rel_8 3380 atomic_store_rel_8(&rxq->iq.intr_params, v); 3381 #else 3382 rxq->iq.intr_params = v; 3383 #endif 3384 } 3385 pi->tmr_idx = idx; 3386 } 3387 3388 ADAPTER_UNLOCK(sc); 3389 return (rc); 3390 } 3391 3392 static int 3393 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 3394 { 3395 struct port_info *pi = arg1; 3396 struct adapter *sc = pi->adapter; 3397 int idx, rc; 3398 3399 idx = pi->pktc_idx; 3400 3401 rc = sysctl_handle_int(oidp, &idx, 0, req); 3402 if (rc != 0 || req->newptr == NULL) 3403 return (rc); 3404 3405 if (idx < -1 || idx >= SGE_NCOUNTERS) 3406 return (EINVAL); 3407 3408 ADAPTER_LOCK(sc); 3409 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3410 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3411 rc = EBUSY; /* cannot be changed once the queues are created */ 3412 3413 if (rc == 0) 3414 pi->pktc_idx = idx; 3415 3416 ADAPTER_UNLOCK(sc); 3417 return (rc); 3418 } 3419 3420 static int 3421 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 3422 { 3423 struct port_info *pi = arg1; 3424 struct adapter *sc = pi->adapter; 3425 int qsize, rc; 3426 3427 qsize = pi->qsize_rxq; 3428 3429 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3430 if (rc != 0 || req->newptr == NULL) 3431 return (rc); 3432 3433 if (qsize < 128 || (qsize & 7)) 3434 return (EINVAL); 3435 3436 ADAPTER_LOCK(sc); 3437 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3438 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3439 rc = EBUSY; /* cannot be changed once the queues are created */ 3440 3441 if (rc == 0) 3442 pi->qsize_rxq = qsize; 3443 3444 ADAPTER_UNLOCK(sc); 3445 return (rc); 3446 } 3447 3448 static int 3449 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 3450 { 3451 struct port_info *pi = arg1; 3452 struct adapter *sc = pi->adapter; 3453 int qsize, rc; 3454 3455 qsize = pi->qsize_txq; 3456 3457 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3458 if (rc != 0 || req->newptr == NULL) 3459 return (rc); 3460 3461 if (qsize < 128) 3462 return (EINVAL); 3463 3464 ADAPTER_LOCK(sc); 3465 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3466 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3467 rc = EBUSY; /* cannot be changed once the queues are created */ 3468 3469 if (rc == 0) 3470 pi->qsize_txq = qsize; 3471 3472 ADAPTER_UNLOCK(sc); 3473 return (rc); 3474 } 3475 3476 static int 3477 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 3478 { 3479 struct adapter *sc = arg1; 3480 int reg = arg2; 3481 uint64_t val; 3482 3483 val = t4_read_reg64(sc, reg); 3484 3485 return (sysctl_handle_64(oidp, &val, 0, req)); 3486 } 3487 3488 #ifdef SBUF_DRAIN 3489 static int 3490 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 3491 { 3492 struct adapter *sc = arg1; 3493 struct sbuf *sb; 3494 int rc, i; 3495 uint16_t incr[NMTUS][NCCTRL_WIN]; 3496 static const char *dec_fac[] = { 3497 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 3498 "0.9375" 3499 }; 3500 3501 rc = sysctl_wire_old_buffer(req, 0); 3502 if (rc != 0) 3503 return (rc); 3504 3505 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3506 if (sb == NULL) 3507 return (ENOMEM); 3508 3509 t4_read_cong_tbl(sc, incr); 3510 3511 for (i = 0; i < NCCTRL_WIN; ++i) { 3512 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 3513 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 3514 incr[5][i], incr[6][i], incr[7][i]); 3515 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 3516 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 3517 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 3518 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 3519 } 3520 3521 rc = sbuf_finish(sb); 3522 sbuf_delete(sb); 3523 3524 return (rc); 3525 } 3526 3527 static int 3528 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 3529 { 3530 struct adapter *sc = arg1; 3531 struct sbuf *sb; 3532 int rc; 3533 struct tp_cpl_stats stats; 3534 3535 rc = sysctl_wire_old_buffer(req, 0); 3536 if (rc != 0) 3537 return (rc); 3538 3539 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3540 if (sb == NULL) 3541 return (ENOMEM); 3542 3543 t4_tp_get_cpl_stats(sc, &stats); 3544 3545 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 3546 "channel 3\n"); 3547 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 3548 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 3549 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 3550 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 3551 3552 rc = sbuf_finish(sb); 3553 sbuf_delete(sb); 3554 3555 return (rc); 3556 } 3557 3558 static int 3559 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 3560 { 3561 struct adapter *sc = arg1; 3562 struct sbuf *sb; 3563 int rc; 3564 struct tp_usm_stats stats; 3565 3566 rc = sysctl_wire_old_buffer(req, 0); 3567 if (rc != 0) 3568 return(rc); 3569 3570 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3571 if (sb == NULL) 3572 return (ENOMEM); 3573 3574 t4_get_usm_stats(sc, &stats); 3575 3576 sbuf_printf(sb, "Frames: %u\n", stats.frames); 3577 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 3578 sbuf_printf(sb, "Drops: %u", stats.drops); 3579 3580 rc = sbuf_finish(sb); 3581 sbuf_delete(sb); 3582 3583 return (rc); 3584 } 3585 3586 const char *devlog_level_strings[] = { 3587 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 3588 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 3589 [FW_DEVLOG_LEVEL_ERR] = "ERR", 3590 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 3591 [FW_DEVLOG_LEVEL_INFO] = "INFO", 3592 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 3593 }; 3594 3595 const char *devlog_facility_strings[] = { 3596 [FW_DEVLOG_FACILITY_CORE] = "CORE", 3597 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 3598 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 3599 [FW_DEVLOG_FACILITY_RES] = "RES", 3600 [FW_DEVLOG_FACILITY_HW] = "HW", 3601 [FW_DEVLOG_FACILITY_FLR] = "FLR", 3602 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 3603 [FW_DEVLOG_FACILITY_PHY] = "PHY", 3604 [FW_DEVLOG_FACILITY_MAC] = "MAC", 3605 [FW_DEVLOG_FACILITY_PORT] = "PORT", 3606 [FW_DEVLOG_FACILITY_VI] = "VI", 3607 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 3608 [FW_DEVLOG_FACILITY_ACL] = "ACL", 3609 [FW_DEVLOG_FACILITY_TM] = "TM", 3610 [FW_DEVLOG_FACILITY_QFC] = "QFC", 3611 [FW_DEVLOG_FACILITY_DCB] = "DCB", 3612 [FW_DEVLOG_FACILITY_ETH] = "ETH", 3613 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 3614 [FW_DEVLOG_FACILITY_RI] = "RI", 3615 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 3616 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 3617 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 3618 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 3619 }; 3620 3621 static int 3622 sysctl_devlog(SYSCTL_HANDLER_ARGS) 3623 { 3624 struct adapter *sc = arg1; 3625 struct devlog_params *dparams = &sc->params.devlog; 3626 struct fw_devlog_e *buf, *e; 3627 int i, j, rc, nentries, first = 0; 3628 struct sbuf *sb; 3629 uint64_t ftstamp = UINT64_MAX; 3630 3631 if (dparams->start == 0) 3632 return (ENXIO); 3633 3634 nentries = dparams->size / sizeof(struct fw_devlog_e); 3635 3636 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 3637 if (buf == NULL) 3638 return (ENOMEM); 3639 3640 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 3641 (void *)buf); 3642 if (rc != 0) 3643 goto done; 3644 3645 for (i = 0; i < nentries; i++) { 3646 e = &buf[i]; 3647 3648 if (e->timestamp == 0) 3649 break; /* end */ 3650 3651 e->timestamp = be64toh(e->timestamp); 3652 e->seqno = be32toh(e->seqno); 3653 for (j = 0; j < 8; j++) 3654 e->params[j] = be32toh(e->params[j]); 3655 3656 if (e->timestamp < ftstamp) { 3657 ftstamp = e->timestamp; 3658 first = i; 3659 } 3660 } 3661 3662 if (buf[first].timestamp == 0) 3663 goto done; /* nothing in the log */ 3664 3665 rc = sysctl_wire_old_buffer(req, 0); 3666 if (rc != 0) 3667 goto done; 3668 3669 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3670 if (sb == NULL) { 3671 rc = ENOMEM; 3672 goto done; 3673 } 3674 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 3675 "Seq#", "Tstamp", "Level", "Facility", "Message"); 3676 3677 i = first; 3678 do { 3679 e = &buf[i]; 3680 if (e->timestamp == 0) 3681 break; /* end */ 3682 3683 sbuf_printf(sb, "%10d %15ju %8s %8s ", 3684 e->seqno, e->timestamp, 3685 (e->level < ARRAY_SIZE(devlog_level_strings) ? 3686 devlog_level_strings[e->level] : "UNKNOWN"), 3687 (e->facility < ARRAY_SIZE(devlog_facility_strings) ? 3688 devlog_facility_strings[e->facility] : "UNKNOWN")); 3689 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 3690 e->params[2], e->params[3], e->params[4], 3691 e->params[5], e->params[6], e->params[7]); 3692 3693 if (++i == nentries) 3694 i = 0; 3695 } while (i != first); 3696 3697 rc = sbuf_finish(sb); 3698 sbuf_delete(sb); 3699 done: 3700 free(buf, M_CXGBE); 3701 return (rc); 3702 } 3703 3704 static int 3705 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 3706 { 3707 struct adapter *sc = arg1; 3708 struct sbuf *sb; 3709 int rc; 3710 struct tp_fcoe_stats stats[4]; 3711 3712 rc = sysctl_wire_old_buffer(req, 0); 3713 if (rc != 0) 3714 return (rc); 3715 3716 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3717 if (sb == NULL) 3718 return (ENOMEM); 3719 3720 t4_get_fcoe_stats(sc, 0, &stats[0]); 3721 t4_get_fcoe_stats(sc, 1, &stats[1]); 3722 t4_get_fcoe_stats(sc, 2, &stats[2]); 3723 t4_get_fcoe_stats(sc, 3, &stats[3]); 3724 3725 sbuf_printf(sb, " channel 0 channel 1 " 3726 "channel 2 channel 3\n"); 3727 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 3728 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 3729 stats[3].octetsDDP); 3730 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 3731 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 3732 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 3733 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 3734 stats[3].framesDrop); 3735 3736 rc = sbuf_finish(sb); 3737 sbuf_delete(sb); 3738 3739 return (rc); 3740 } 3741 3742 static int 3743 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 3744 { 3745 struct adapter *sc = arg1; 3746 struct sbuf *sb; 3747 int rc, i; 3748 unsigned int map, kbps, ipg, mode; 3749 unsigned int pace_tab[NTX_SCHED]; 3750 3751 rc = sysctl_wire_old_buffer(req, 0); 3752 if (rc != 0) 3753 return (rc); 3754 3755 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3756 if (sb == NULL) 3757 return (ENOMEM); 3758 3759 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 3760 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 3761 t4_read_pace_tbl(sc, pace_tab); 3762 3763 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 3764 "Class IPG (0.1 ns) Flow IPG (us)"); 3765 3766 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 3767 t4_get_tx_sched(sc, i, &kbps, &ipg); 3768 sbuf_printf(sb, "\n %u %-5s %u ", i, 3769 (mode & (1 << i)) ? "flow" : "class", map & 3); 3770 if (kbps) 3771 sbuf_printf(sb, "%9u ", kbps); 3772 else 3773 sbuf_printf(sb, " disabled "); 3774 3775 if (ipg) 3776 sbuf_printf(sb, "%13u ", ipg); 3777 else 3778 sbuf_printf(sb, " disabled "); 3779 3780 if (pace_tab[i]) 3781 sbuf_printf(sb, "%10u", pace_tab[i]); 3782 else 3783 sbuf_printf(sb, " disabled"); 3784 } 3785 3786 rc = sbuf_finish(sb); 3787 sbuf_delete(sb); 3788 3789 return (rc); 3790 } 3791 3792 static int 3793 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 3794 { 3795 struct adapter *sc = arg1; 3796 struct sbuf *sb; 3797 int rc, i, j; 3798 uint64_t *p0, *p1; 3799 struct lb_port_stats s[2]; 3800 static const char *stat_name[] = { 3801 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 3802 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 3803 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 3804 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 3805 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 3806 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 3807 "BG2FramesTrunc:", "BG3FramesTrunc:" 3808 }; 3809 3810 rc = sysctl_wire_old_buffer(req, 0); 3811 if (rc != 0) 3812 return (rc); 3813 3814 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3815 if (sb == NULL) 3816 return (ENOMEM); 3817 3818 memset(s, 0, sizeof(s)); 3819 3820 for (i = 0; i < 4; i += 2) { 3821 t4_get_lb_stats(sc, i, &s[0]); 3822 t4_get_lb_stats(sc, i + 1, &s[1]); 3823 3824 p0 = &s[0].octets; 3825 p1 = &s[1].octets; 3826 sbuf_printf(sb, "%s Loopback %u" 3827 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 3828 3829 for (j = 0; j < ARRAY_SIZE(stat_name); j++) 3830 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 3831 *p0++, *p1++); 3832 } 3833 3834 rc = sbuf_finish(sb); 3835 sbuf_delete(sb); 3836 3837 return (rc); 3838 } 3839 3840 struct mem_desc { 3841 unsigned int base; 3842 unsigned int limit; 3843 unsigned int idx; 3844 }; 3845 3846 static int 3847 mem_desc_cmp(const void *a, const void *b) 3848 { 3849 return ((const struct mem_desc *)a)->base - 3850 ((const struct mem_desc *)b)->base; 3851 } 3852 3853 static void 3854 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 3855 unsigned int to) 3856 { 3857 unsigned int size; 3858 3859 size = to - from + 1; 3860 if (size == 0) 3861 return; 3862 3863 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 3864 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 3865 } 3866 3867 static int 3868 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 3869 { 3870 struct adapter *sc = arg1; 3871 struct sbuf *sb; 3872 int rc, i, n; 3873 uint32_t lo, hi; 3874 static const char *memory[] = { "EDC0:", "EDC1:", "MC:" }; 3875 static const char *region[] = { 3876 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 3877 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 3878 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 3879 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 3880 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:", 3881 "ULPTX state:", "On-chip queues:" 3882 }; 3883 struct mem_desc avail[3]; 3884 struct mem_desc mem[ARRAY_SIZE(region) + 3]; /* up to 3 holes */ 3885 struct mem_desc *md = mem; 3886 3887 rc = sysctl_wire_old_buffer(req, 0); 3888 if (rc != 0) 3889 return (rc); 3890 3891 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3892 if (sb == NULL) 3893 return (ENOMEM); 3894 3895 for (i = 0; i < ARRAY_SIZE(mem); i++) { 3896 mem[i].limit = 0; 3897 mem[i].idx = i; 3898 } 3899 3900 /* Find and sort the populated memory ranges */ 3901 i = 0; 3902 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 3903 if (lo & F_EDRAM0_ENABLE) { 3904 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 3905 avail[i].base = G_EDRAM0_BASE(hi) << 20; 3906 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 3907 avail[i].idx = 0; 3908 i++; 3909 } 3910 if (lo & F_EDRAM1_ENABLE) { 3911 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 3912 avail[i].base = G_EDRAM1_BASE(hi) << 20; 3913 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 3914 avail[i].idx = 1; 3915 i++; 3916 } 3917 if (lo & F_EXT_MEM_ENABLE) { 3918 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 3919 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 3920 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20); 3921 avail[i].idx = 2; 3922 i++; 3923 } 3924 if (!i) /* no memory available */ 3925 return 0; 3926 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 3927 3928 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 3929 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 3930 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 3931 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 3932 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 3933 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 3934 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 3935 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 3936 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 3937 3938 /* the next few have explicit upper bounds */ 3939 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 3940 md->limit = md->base - 1 + 3941 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 3942 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 3943 md++; 3944 3945 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 3946 md->limit = md->base - 1 + 3947 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 3948 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 3949 md++; 3950 3951 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 3952 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 3953 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 3954 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 3955 } else { 3956 md->base = 0; 3957 md->idx = ARRAY_SIZE(region); /* hide it */ 3958 } 3959 md++; 3960 3961 #define ulp_region(reg) \ 3962 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 3963 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 3964 3965 ulp_region(RX_ISCSI); 3966 ulp_region(RX_TDDP); 3967 ulp_region(TX_TPT); 3968 ulp_region(RX_STAG); 3969 ulp_region(RX_RQ); 3970 ulp_region(RX_RQUDP); 3971 ulp_region(RX_PBL); 3972 ulp_region(TX_PBL); 3973 #undef ulp_region 3974 3975 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 3976 md->limit = md->base + sc->tids.ntids - 1; 3977 md++; 3978 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 3979 md->limit = md->base + sc->tids.ntids - 1; 3980 md++; 3981 3982 md->base = sc->vres.ocq.start; 3983 if (sc->vres.ocq.size) 3984 md->limit = md->base + sc->vres.ocq.size - 1; 3985 else 3986 md->idx = ARRAY_SIZE(region); /* hide it */ 3987 md++; 3988 3989 /* add any address-space holes, there can be up to 3 */ 3990 for (n = 0; n < i - 1; n++) 3991 if (avail[n].limit < avail[n + 1].base) 3992 (md++)->base = avail[n].limit; 3993 if (avail[n].limit) 3994 (md++)->base = avail[n].limit; 3995 3996 n = md - mem; 3997 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 3998 3999 for (lo = 0; lo < i; lo++) 4000 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 4001 avail[lo].limit - 1); 4002 4003 sbuf_printf(sb, "\n"); 4004 for (i = 0; i < n; i++) { 4005 if (mem[i].idx >= ARRAY_SIZE(region)) 4006 continue; /* skip holes */ 4007 if (!mem[i].limit) 4008 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 4009 mem_region_show(sb, region[mem[i].idx], mem[i].base, 4010 mem[i].limit); 4011 } 4012 4013 sbuf_printf(sb, "\n"); 4014 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 4015 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 4016 mem_region_show(sb, "uP RAM:", lo, hi); 4017 4018 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 4019 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 4020 mem_region_show(sb, "uP Extmem2:", lo, hi); 4021 4022 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 4023 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 4024 G_PMRXMAXPAGE(lo), 4025 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 4026 (lo & F_PMRXNUMCHN) ? 2 : 1); 4027 4028 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 4029 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 4030 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 4031 G_PMTXMAXPAGE(lo), 4032 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 4033 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 4034 sbuf_printf(sb, "%u p-structs\n", 4035 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 4036 4037 for (i = 0; i < 4; i++) { 4038 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 4039 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 4040 i, G_USED(lo), G_ALLOC(lo)); 4041 } 4042 for (i = 0; i < 4; i++) { 4043 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 4044 sbuf_printf(sb, 4045 "\nLoopback %d using %u pages out of %u allocated", 4046 i, G_USED(lo), G_ALLOC(lo)); 4047 } 4048 4049 rc = sbuf_finish(sb); 4050 sbuf_delete(sb); 4051 4052 return (rc); 4053 } 4054 4055 static int 4056 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 4057 { 4058 struct adapter *sc = arg1; 4059 struct sbuf *sb; 4060 int rc; 4061 uint16_t mtus[NMTUS]; 4062 4063 rc = sysctl_wire_old_buffer(req, 0); 4064 if (rc != 0) 4065 return (rc); 4066 4067 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4068 if (sb == NULL) 4069 return (ENOMEM); 4070 4071 t4_read_mtu_tbl(sc, mtus, NULL); 4072 4073 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 4074 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 4075 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 4076 mtus[14], mtus[15]); 4077 4078 rc = sbuf_finish(sb); 4079 sbuf_delete(sb); 4080 4081 return (rc); 4082 } 4083 4084 static int 4085 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 4086 { 4087 struct adapter *sc = arg1; 4088 struct sbuf *sb; 4089 int rc, i; 4090 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 4091 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 4092 static const char *pm_stats[] = { 4093 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 4094 }; 4095 4096 rc = sysctl_wire_old_buffer(req, 0); 4097 if (rc != 0) 4098 return (rc); 4099 4100 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4101 if (sb == NULL) 4102 return (ENOMEM); 4103 4104 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 4105 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 4106 4107 sbuf_printf(sb, " Tx count Tx cycles " 4108 "Rx count Rx cycles"); 4109 for (i = 0; i < PM_NSTATS; i++) 4110 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 4111 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 4112 4113 rc = sbuf_finish(sb); 4114 sbuf_delete(sb); 4115 4116 return (rc); 4117 } 4118 4119 static int 4120 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 4121 { 4122 struct adapter *sc = arg1; 4123 struct sbuf *sb; 4124 int rc; 4125 struct tp_rdma_stats stats; 4126 4127 rc = sysctl_wire_old_buffer(req, 0); 4128 if (rc != 0) 4129 return (rc); 4130 4131 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4132 if (sb == NULL) 4133 return (ENOMEM); 4134 4135 t4_tp_get_rdma_stats(sc, &stats); 4136 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 4137 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 4138 4139 rc = sbuf_finish(sb); 4140 sbuf_delete(sb); 4141 4142 return (rc); 4143 } 4144 4145 static int 4146 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 4147 { 4148 struct adapter *sc = arg1; 4149 struct sbuf *sb; 4150 int rc; 4151 struct tp_tcp_stats v4, v6; 4152 4153 rc = sysctl_wire_old_buffer(req, 0); 4154 if (rc != 0) 4155 return (rc); 4156 4157 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4158 if (sb == NULL) 4159 return (ENOMEM); 4160 4161 t4_tp_get_tcp_stats(sc, &v4, &v6); 4162 sbuf_printf(sb, 4163 " IP IPv6\n"); 4164 sbuf_printf(sb, "OutRsts: %20u %20u\n", 4165 v4.tcpOutRsts, v6.tcpOutRsts); 4166 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 4167 v4.tcpInSegs, v6.tcpInSegs); 4168 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 4169 v4.tcpOutSegs, v6.tcpOutSegs); 4170 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 4171 v4.tcpRetransSegs, v6.tcpRetransSegs); 4172 4173 rc = sbuf_finish(sb); 4174 sbuf_delete(sb); 4175 4176 return (rc); 4177 } 4178 4179 static int 4180 sysctl_tids(SYSCTL_HANDLER_ARGS) 4181 { 4182 struct adapter *sc = arg1; 4183 struct sbuf *sb; 4184 int rc; 4185 struct tid_info *t = &sc->tids; 4186 4187 rc = sysctl_wire_old_buffer(req, 0); 4188 if (rc != 0) 4189 return (rc); 4190 4191 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4192 if (sb == NULL) 4193 return (ENOMEM); 4194 4195 if (t->natids) { 4196 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 4197 t->atids_in_use); 4198 } 4199 4200 if (t->ntids) { 4201 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4202 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 4203 4204 if (b) { 4205 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 4206 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4207 t->ntids - 1); 4208 } else { 4209 sbuf_printf(sb, "TID range: %u-%u", 4210 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4211 t->ntids - 1); 4212 } 4213 } else 4214 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 4215 sbuf_printf(sb, ", in use: %u\n", 4216 atomic_load_acq_int(&t->tids_in_use)); 4217 } 4218 4219 if (t->nstids) { 4220 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 4221 t->stid_base + t->nstids - 1, t->stids_in_use); 4222 } 4223 4224 if (t->nftids) { 4225 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 4226 t->ftid_base + t->nftids - 1); 4227 } 4228 4229 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 4230 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 4231 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 4232 4233 rc = sbuf_finish(sb); 4234 sbuf_delete(sb); 4235 4236 return (rc); 4237 } 4238 4239 static int 4240 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 4241 { 4242 struct adapter *sc = arg1; 4243 struct sbuf *sb; 4244 int rc; 4245 struct tp_err_stats stats; 4246 4247 rc = sysctl_wire_old_buffer(req, 0); 4248 if (rc != 0) 4249 return (rc); 4250 4251 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4252 if (sb == NULL) 4253 return (ENOMEM); 4254 4255 t4_tp_get_err_stats(sc, &stats); 4256 4257 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4258 "channel 3\n"); 4259 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 4260 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 4261 stats.macInErrs[3]); 4262 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 4263 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 4264 stats.hdrInErrs[3]); 4265 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 4266 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 4267 stats.tcpInErrs[3]); 4268 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 4269 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 4270 stats.tcp6InErrs[3]); 4271 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 4272 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 4273 stats.tnlCongDrops[3]); 4274 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 4275 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 4276 stats.tnlTxDrops[3]); 4277 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 4278 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 4279 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 4280 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 4281 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 4282 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 4283 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 4284 stats.ofldNoNeigh, stats.ofldCongDefer); 4285 4286 rc = sbuf_finish(sb); 4287 sbuf_delete(sb); 4288 4289 return (rc); 4290 } 4291 4292 static int 4293 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 4294 { 4295 struct adapter *sc = arg1; 4296 struct sbuf *sb; 4297 int rc; 4298 u64 nrate[NCHAN], orate[NCHAN]; 4299 4300 rc = sysctl_wire_old_buffer(req, 0); 4301 if (rc != 0) 4302 return (rc); 4303 4304 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4305 if (sb == NULL) 4306 return (ENOMEM); 4307 4308 t4_get_chan_txrate(sc, nrate, orate); 4309 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4310 "channel 3\n"); 4311 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 4312 nrate[0], nrate[1], nrate[2], nrate[3]); 4313 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 4314 orate[0], orate[1], orate[2], orate[3]); 4315 4316 rc = sbuf_finish(sb); 4317 sbuf_delete(sb); 4318 4319 return (rc); 4320 } 4321 #endif 4322 4323 static inline void 4324 txq_start(struct ifnet *ifp, struct sge_txq *txq) 4325 { 4326 struct buf_ring *br; 4327 struct mbuf *m; 4328 4329 TXQ_LOCK_ASSERT_OWNED(txq); 4330 4331 br = txq->br; 4332 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 4333 if (m) 4334 t4_eth_tx(ifp, txq, m); 4335 } 4336 4337 void 4338 t4_tx_callout(void *arg) 4339 { 4340 struct sge_eq *eq = arg; 4341 struct adapter *sc; 4342 4343 if (EQ_TRYLOCK(eq) == 0) 4344 goto reschedule; 4345 4346 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 4347 EQ_UNLOCK(eq); 4348 reschedule: 4349 if (__predict_true(!(eq->flags && EQ_DOOMED))) 4350 callout_schedule(&eq->tx_callout, 1); 4351 return; 4352 } 4353 4354 EQ_LOCK_ASSERT_OWNED(eq); 4355 4356 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 4357 4358 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4359 struct sge_txq *txq = arg; 4360 struct port_info *pi = txq->ifp->if_softc; 4361 4362 sc = pi->adapter; 4363 } else { 4364 struct sge_wrq *wrq = arg; 4365 4366 sc = wrq->adapter; 4367 } 4368 4369 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 4370 } 4371 4372 EQ_UNLOCK(eq); 4373 } 4374 4375 void 4376 t4_tx_task(void *arg, int count) 4377 { 4378 struct sge_eq *eq = arg; 4379 4380 EQ_LOCK(eq); 4381 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4382 struct sge_txq *txq = arg; 4383 txq_start(txq->ifp, txq); 4384 } else { 4385 struct sge_wrq *wrq = arg; 4386 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 4387 } 4388 EQ_UNLOCK(eq); 4389 } 4390 4391 static uint32_t 4392 fconf_to_mode(uint32_t fconf) 4393 { 4394 uint32_t mode; 4395 4396 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 4397 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 4398 4399 if (fconf & F_FRAGMENTATION) 4400 mode |= T4_FILTER_IP_FRAGMENT; 4401 4402 if (fconf & F_MPSHITTYPE) 4403 mode |= T4_FILTER_MPS_HIT_TYPE; 4404 4405 if (fconf & F_MACMATCH) 4406 mode |= T4_FILTER_MAC_IDX; 4407 4408 if (fconf & F_ETHERTYPE) 4409 mode |= T4_FILTER_ETH_TYPE; 4410 4411 if (fconf & F_PROTOCOL) 4412 mode |= T4_FILTER_IP_PROTO; 4413 4414 if (fconf & F_TOS) 4415 mode |= T4_FILTER_IP_TOS; 4416 4417 if (fconf & F_VLAN) 4418 mode |= T4_FILTER_VLAN; 4419 4420 if (fconf & F_VNIC_ID) 4421 mode |= T4_FILTER_VNIC; 4422 4423 if (fconf & F_PORT) 4424 mode |= T4_FILTER_PORT; 4425 4426 if (fconf & F_FCOE) 4427 mode |= T4_FILTER_FCoE; 4428 4429 return (mode); 4430 } 4431 4432 static uint32_t 4433 mode_to_fconf(uint32_t mode) 4434 { 4435 uint32_t fconf = 0; 4436 4437 if (mode & T4_FILTER_IP_FRAGMENT) 4438 fconf |= F_FRAGMENTATION; 4439 4440 if (mode & T4_FILTER_MPS_HIT_TYPE) 4441 fconf |= F_MPSHITTYPE; 4442 4443 if (mode & T4_FILTER_MAC_IDX) 4444 fconf |= F_MACMATCH; 4445 4446 if (mode & T4_FILTER_ETH_TYPE) 4447 fconf |= F_ETHERTYPE; 4448 4449 if (mode & T4_FILTER_IP_PROTO) 4450 fconf |= F_PROTOCOL; 4451 4452 if (mode & T4_FILTER_IP_TOS) 4453 fconf |= F_TOS; 4454 4455 if (mode & T4_FILTER_VLAN) 4456 fconf |= F_VLAN; 4457 4458 if (mode & T4_FILTER_VNIC) 4459 fconf |= F_VNIC_ID; 4460 4461 if (mode & T4_FILTER_PORT) 4462 fconf |= F_PORT; 4463 4464 if (mode & T4_FILTER_FCoE) 4465 fconf |= F_FCOE; 4466 4467 return (fconf); 4468 } 4469 4470 static uint32_t 4471 fspec_to_fconf(struct t4_filter_specification *fs) 4472 { 4473 uint32_t fconf = 0; 4474 4475 if (fs->val.frag || fs->mask.frag) 4476 fconf |= F_FRAGMENTATION; 4477 4478 if (fs->val.matchtype || fs->mask.matchtype) 4479 fconf |= F_MPSHITTYPE; 4480 4481 if (fs->val.macidx || fs->mask.macidx) 4482 fconf |= F_MACMATCH; 4483 4484 if (fs->val.ethtype || fs->mask.ethtype) 4485 fconf |= F_ETHERTYPE; 4486 4487 if (fs->val.proto || fs->mask.proto) 4488 fconf |= F_PROTOCOL; 4489 4490 if (fs->val.tos || fs->mask.tos) 4491 fconf |= F_TOS; 4492 4493 if (fs->val.vlan_vld || fs->mask.vlan_vld) 4494 fconf |= F_VLAN; 4495 4496 if (fs->val.vnic_vld || fs->mask.vnic_vld) 4497 fconf |= F_VNIC_ID; 4498 4499 if (fs->val.iport || fs->mask.iport) 4500 fconf |= F_PORT; 4501 4502 if (fs->val.fcoe || fs->mask.fcoe) 4503 fconf |= F_FCOE; 4504 4505 return (fconf); 4506 } 4507 4508 static int 4509 get_filter_mode(struct adapter *sc, uint32_t *mode) 4510 { 4511 uint32_t fconf; 4512 4513 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 4514 A_TP_VLAN_PRI_MAP); 4515 4516 if (sc->filter_mode != fconf) { 4517 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 4518 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 4519 sc->filter_mode = fconf; 4520 } 4521 4522 *mode = fconf_to_mode(sc->filter_mode); 4523 4524 return (0); 4525 } 4526 4527 static int 4528 set_filter_mode(struct adapter *sc, uint32_t mode) 4529 { 4530 uint32_t fconf; 4531 int rc; 4532 4533 fconf = mode_to_fconf(mode); 4534 4535 ADAPTER_LOCK(sc); 4536 if (IS_BUSY(sc)) { 4537 rc = EAGAIN; 4538 goto done; 4539 } 4540 4541 if (sc->tids.ftids_in_use > 0) { 4542 rc = EBUSY; 4543 goto done; 4544 } 4545 4546 #ifndef TCP_OFFLOAD_DISABLE 4547 if (sc->offload_map) { 4548 rc = EBUSY; 4549 goto done; 4550 } 4551 #endif 4552 4553 #ifdef notyet 4554 rc = -t4_set_filter_mode(sc, fconf); 4555 if (rc == 0) 4556 sc->filter_mode = fconf; 4557 #else 4558 rc = ENOTSUP; 4559 #endif 4560 4561 done: 4562 ADAPTER_UNLOCK(sc); 4563 return (rc); 4564 } 4565 4566 static inline uint64_t 4567 get_filter_hits(struct adapter *sc, uint32_t fid) 4568 { 4569 uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4570 uint64_t hits; 4571 4572 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0), 4573 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 4574 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0)); 4575 hits = t4_read_reg64(sc, MEMWIN0_BASE + 16); 4576 4577 return (be64toh(hits)); 4578 } 4579 4580 static int 4581 get_filter(struct adapter *sc, struct t4_filter *t) 4582 { 4583 int i, nfilters = sc->tids.nftids; 4584 struct filter_entry *f; 4585 4586 ADAPTER_LOCK_ASSERT_OWNED(sc); 4587 4588 if (IS_BUSY(sc)) 4589 return (EAGAIN); 4590 4591 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 4592 t->idx >= nfilters) { 4593 t->idx = 0xffffffff; 4594 return (0); 4595 } 4596 4597 f = &sc->tids.ftid_tab[t->idx]; 4598 for (i = t->idx; i < nfilters; i++, f++) { 4599 if (f->valid) { 4600 t->idx = i; 4601 t->l2tidx = f->l2t ? f->l2t->idx : 0; 4602 t->smtidx = f->smtidx; 4603 if (f->fs.hitcnts) 4604 t->hits = get_filter_hits(sc, t->idx); 4605 else 4606 t->hits = UINT64_MAX; 4607 t->fs = f->fs; 4608 4609 return (0); 4610 } 4611 } 4612 4613 t->idx = 0xffffffff; 4614 return (0); 4615 } 4616 4617 static int 4618 set_filter(struct adapter *sc, struct t4_filter *t) 4619 { 4620 unsigned int nfilters, nports; 4621 struct filter_entry *f; 4622 int i; 4623 4624 ADAPTER_LOCK_ASSERT_OWNED(sc); 4625 4626 nfilters = sc->tids.nftids; 4627 nports = sc->params.nports; 4628 4629 if (nfilters == 0) 4630 return (ENOTSUP); 4631 4632 if (!(sc->flags & FULL_INIT_DONE)) 4633 return (EAGAIN); 4634 4635 if (t->idx >= nfilters) 4636 return (EINVAL); 4637 4638 /* Validate against the global filter mode */ 4639 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) 4640 return (E2BIG); 4641 4642 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) 4643 return (EINVAL); 4644 4645 if (t->fs.val.iport >= nports) 4646 return (EINVAL); 4647 4648 /* Can't specify an iq if not steering to it */ 4649 if (!t->fs.dirsteer && t->fs.iq) 4650 return (EINVAL); 4651 4652 /* IPv6 filter idx must be 4 aligned */ 4653 if (t->fs.type == 1 && 4654 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) 4655 return (EINVAL); 4656 4657 if (sc->tids.ftid_tab == NULL) { 4658 KASSERT(sc->tids.ftids_in_use == 0, 4659 ("%s: no memory allocated but filters_in_use > 0", 4660 __func__)); 4661 4662 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 4663 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 4664 if (sc->tids.ftid_tab == NULL) 4665 return (ENOMEM); 4666 } 4667 4668 for (i = 0; i < 4; i++) { 4669 f = &sc->tids.ftid_tab[t->idx + i]; 4670 4671 if (f->pending || f->valid) 4672 return (EBUSY); 4673 if (f->locked) 4674 return (EPERM); 4675 4676 if (t->fs.type == 0) 4677 break; 4678 } 4679 4680 f = &sc->tids.ftid_tab[t->idx]; 4681 f->fs = t->fs; 4682 4683 return set_filter_wr(sc, t->idx); 4684 } 4685 4686 static int 4687 del_filter(struct adapter *sc, struct t4_filter *t) 4688 { 4689 unsigned int nfilters; 4690 struct filter_entry *f; 4691 4692 ADAPTER_LOCK_ASSERT_OWNED(sc); 4693 4694 if (IS_BUSY(sc)) 4695 return (EAGAIN); 4696 4697 nfilters = sc->tids.nftids; 4698 4699 if (nfilters == 0) 4700 return (ENOTSUP); 4701 4702 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 4703 t->idx >= nfilters) 4704 return (EINVAL); 4705 4706 if (!(sc->flags & FULL_INIT_DONE)) 4707 return (EAGAIN); 4708 4709 f = &sc->tids.ftid_tab[t->idx]; 4710 4711 if (f->pending) 4712 return (EBUSY); 4713 if (f->locked) 4714 return (EPERM); 4715 4716 if (f->valid) { 4717 t->fs = f->fs; /* extra info for the caller */ 4718 return del_filter_wr(sc, t->idx); 4719 } 4720 4721 return (0); 4722 } 4723 4724 static void 4725 clear_filter(struct filter_entry *f) 4726 { 4727 if (f->l2t) 4728 t4_l2t_release(f->l2t); 4729 4730 bzero(f, sizeof (*f)); 4731 } 4732 4733 static int 4734 set_filter_wr(struct adapter *sc, int fidx) 4735 { 4736 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4737 struct mbuf *m; 4738 struct fw_filter_wr *fwr; 4739 unsigned int ftid; 4740 4741 ADAPTER_LOCK_ASSERT_OWNED(sc); 4742 4743 if (f->fs.newdmac || f->fs.newvlan) { 4744 /* This filter needs an L2T entry; allocate one. */ 4745 f->l2t = t4_l2t_alloc_switching(sc->l2t); 4746 if (f->l2t == NULL) 4747 return (EAGAIN); 4748 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 4749 f->fs.dmac)) { 4750 t4_l2t_release(f->l2t); 4751 f->l2t = NULL; 4752 return (ENOMEM); 4753 } 4754 } 4755 4756 ftid = sc->tids.ftid_base + fidx; 4757 4758 m = m_gethdr(M_NOWAIT, MT_DATA); 4759 if (m == NULL) 4760 return (ENOMEM); 4761 4762 fwr = mtod(m, struct fw_filter_wr *); 4763 m->m_len = m->m_pkthdr.len = sizeof(*fwr); 4764 bzero(fwr, sizeof (*fwr)); 4765 4766 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 4767 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 4768 fwr->tid_to_iq = 4769 htobe32(V_FW_FILTER_WR_TID(ftid) | 4770 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 4771 V_FW_FILTER_WR_NOREPLY(0) | 4772 V_FW_FILTER_WR_IQ(f->fs.iq)); 4773 fwr->del_filter_to_l2tix = 4774 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 4775 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 4776 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 4777 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 4778 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 4779 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 4780 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 4781 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 4782 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 4783 f->fs.newvlan == VLAN_REWRITE) | 4784 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 4785 f->fs.newvlan == VLAN_REWRITE) | 4786 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 4787 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 4788 V_FW_FILTER_WR_PRIO(f->fs.prio) | 4789 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 4790 fwr->ethtype = htobe16(f->fs.val.ethtype); 4791 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 4792 fwr->frag_to_ovlan_vldm = 4793 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 4794 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 4795 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 4796 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 4797 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 4798 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 4799 fwr->smac_sel = 0; 4800 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 4801 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 4802 fwr->maci_to_matchtypem = 4803 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 4804 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 4805 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 4806 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 4807 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 4808 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 4809 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 4810 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 4811 fwr->ptcl = f->fs.val.proto; 4812 fwr->ptclm = f->fs.mask.proto; 4813 fwr->ttyp = f->fs.val.tos; 4814 fwr->ttypm = f->fs.mask.tos; 4815 fwr->ivlan = htobe16(f->fs.val.vlan); 4816 fwr->ivlanm = htobe16(f->fs.mask.vlan); 4817 fwr->ovlan = htobe16(f->fs.val.vnic); 4818 fwr->ovlanm = htobe16(f->fs.mask.vnic); 4819 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 4820 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 4821 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 4822 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 4823 fwr->lp = htobe16(f->fs.val.dport); 4824 fwr->lpm = htobe16(f->fs.mask.dport); 4825 fwr->fp = htobe16(f->fs.val.sport); 4826 fwr->fpm = htobe16(f->fs.mask.sport); 4827 if (f->fs.newsmac) 4828 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 4829 4830 f->pending = 1; 4831 sc->tids.ftids_in_use++; 4832 4833 t4_mgmt_tx(sc, m); 4834 return (0); 4835 } 4836 4837 static int 4838 del_filter_wr(struct adapter *sc, int fidx) 4839 { 4840 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4841 struct mbuf *m; 4842 struct fw_filter_wr *fwr; 4843 unsigned int ftid; 4844 4845 ADAPTER_LOCK_ASSERT_OWNED(sc); 4846 4847 ftid = sc->tids.ftid_base + fidx; 4848 4849 m = m_gethdr(M_NOWAIT, MT_DATA); 4850 if (m == NULL) 4851 return (ENOMEM); 4852 4853 fwr = mtod(m, struct fw_filter_wr *); 4854 m->m_len = m->m_pkthdr.len = sizeof(*fwr); 4855 bzero(fwr, sizeof (*fwr)); 4856 4857 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 4858 4859 f->pending = 1; 4860 t4_mgmt_tx(sc, m); 4861 return (0); 4862 } 4863 4864 static int 4865 filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4866 { 4867 struct adapter *sc = iq->adapter; 4868 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 4869 unsigned int idx = GET_TID(rpl); 4870 4871 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 4872 rss->opcode)); 4873 4874 if (idx >= sc->tids.ftid_base && 4875 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 4876 unsigned int rc = G_COOKIE(rpl->cookie); 4877 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 4878 4879 ADAPTER_LOCK(sc); 4880 if (rc == FW_FILTER_WR_FLT_ADDED) { 4881 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 4882 f->pending = 0; /* asynchronous setup completed */ 4883 f->valid = 1; 4884 } else { 4885 if (rc != FW_FILTER_WR_FLT_DELETED) { 4886 /* Add or delete failed, display an error */ 4887 log(LOG_ERR, 4888 "filter %u setup failed with error %u\n", 4889 idx, rc); 4890 } 4891 4892 clear_filter(f); 4893 sc->tids.ftids_in_use--; 4894 } 4895 ADAPTER_UNLOCK(sc); 4896 } 4897 4898 return (0); 4899 } 4900 4901 static int 4902 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 4903 { 4904 int rc = EINVAL; 4905 4906 if (cntxt->cid > M_CTXTQID) 4907 return (rc); 4908 4909 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 4910 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 4911 return (rc); 4912 4913 if (sc->flags & FW_OK) { 4914 ADAPTER_LOCK(sc); /* Avoid parallel t4_wr_mbox */ 4915 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 4916 &cntxt->data[0]); 4917 ADAPTER_UNLOCK(sc); 4918 } 4919 4920 if (rc != 0) { 4921 /* Read via firmware failed or wasn't even attempted */ 4922 4923 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, 4924 &cntxt->data[0]); 4925 } 4926 4927 return (rc); 4928 } 4929 4930 static int 4931 read_card_mem(struct adapter *sc, struct t4_mem_range *mr) 4932 { 4933 uint32_t base, size, lo, hi, win, off, remaining, i, n; 4934 uint32_t *buf, *b; 4935 int rc; 4936 4937 /* reads are in multiples of 32 bits */ 4938 if (mr->addr & 3 || mr->len & 3 || mr->len == 0) 4939 return (EINVAL); 4940 4941 /* 4942 * We don't want to deal with potential holes so we mandate that the 4943 * requested region must lie entirely within one of the 3 memories. 4944 */ 4945 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 4946 if (lo & F_EDRAM0_ENABLE) { 4947 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 4948 base = G_EDRAM0_BASE(hi) << 20; 4949 size = G_EDRAM0_SIZE(hi) << 20; 4950 if (size > 0 && 4951 mr->addr >= base && mr->addr < base + size && 4952 mr->addr + mr->len <= base + size) 4953 goto proceed; 4954 } 4955 if (lo & F_EDRAM1_ENABLE) { 4956 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 4957 base = G_EDRAM1_BASE(hi) << 20; 4958 size = G_EDRAM1_SIZE(hi) << 20; 4959 if (size > 0 && 4960 mr->addr >= base && mr->addr < base + size && 4961 mr->addr + mr->len <= base + size) 4962 goto proceed; 4963 } 4964 if (lo & F_EXT_MEM_ENABLE) { 4965 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 4966 base = G_EXT_MEM_BASE(hi) << 20; 4967 size = G_EXT_MEM_SIZE(hi) << 20; 4968 if (size > 0 && 4969 mr->addr >= base && mr->addr < base + size && 4970 mr->addr + mr->len <= base + size) 4971 goto proceed; 4972 } 4973 return (ENXIO); 4974 4975 proceed: 4976 buf = b = malloc(mr->len, M_CXGBE, M_WAITOK); 4977 4978 /* 4979 * Position the PCIe window (we use memwin2) to the 16B aligned area 4980 * just at/before the requested region. 4981 */ 4982 win = mr->addr & ~0xf; 4983 off = mr->addr - win; /* offset of the requested region in the win */ 4984 remaining = mr->len; 4985 4986 while (remaining) { 4987 t4_write_reg(sc, 4988 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 4989 t4_read_reg(sc, 4990 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 4991 4992 /* number of bytes that we'll copy in the inner loop */ 4993 n = min(remaining, MEMWIN2_APERTURE - off); 4994 4995 for (i = 0; i < n; i += 4, remaining -= 4) 4996 *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i); 4997 4998 win += MEMWIN2_APERTURE; 4999 off = 0; 5000 } 5001 5002 rc = copyout(buf, mr->data, mr->len); 5003 free(buf, M_CXGBE); 5004 5005 return (rc); 5006 } 5007 5008 int 5009 t4_os_find_pci_capability(struct adapter *sc, int cap) 5010 { 5011 int i; 5012 5013 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 5014 } 5015 5016 int 5017 t4_os_pci_save_state(struct adapter *sc) 5018 { 5019 device_t dev; 5020 struct pci_devinfo *dinfo; 5021 5022 dev = sc->dev; 5023 dinfo = device_get_ivars(dev); 5024 5025 pci_cfg_save(dev, dinfo, 0); 5026 return (0); 5027 } 5028 5029 int 5030 t4_os_pci_restore_state(struct adapter *sc) 5031 { 5032 device_t dev; 5033 struct pci_devinfo *dinfo; 5034 5035 dev = sc->dev; 5036 dinfo = device_get_ivars(dev); 5037 5038 pci_cfg_restore(dev, dinfo); 5039 return (0); 5040 } 5041 5042 void 5043 t4_os_portmod_changed(const struct adapter *sc, int idx) 5044 { 5045 struct port_info *pi = sc->port[idx]; 5046 static const char *mod_str[] = { 5047 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 5048 }; 5049 5050 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 5051 if_printf(pi->ifp, "transceiver unplugged.\n"); 5052 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 5053 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 5054 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 5055 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 5056 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) { 5057 if_printf(pi->ifp, "%s transceiver inserted.\n", 5058 mod_str[pi->mod_type]); 5059 } else { 5060 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 5061 pi->mod_type); 5062 } 5063 } 5064 5065 void 5066 t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 5067 { 5068 struct port_info *pi = sc->port[idx]; 5069 struct ifnet *ifp = pi->ifp; 5070 5071 if (link_stat) { 5072 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 5073 if_link_state_change(ifp, LINK_STATE_UP); 5074 } else 5075 if_link_state_change(ifp, LINK_STATE_DOWN); 5076 } 5077 5078 void 5079 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 5080 { 5081 struct adapter *sc; 5082 5083 mtx_lock(&t4_list_lock); 5084 SLIST_FOREACH(sc, &t4_list, link) { 5085 /* 5086 * func should not make any assumptions about what state sc is 5087 * in - the only guarantee is that sc->sc_lock is a valid lock. 5088 */ 5089 func(sc, arg); 5090 } 5091 mtx_unlock(&t4_list_lock); 5092 } 5093 5094 static int 5095 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 5096 { 5097 return (0); 5098 } 5099 5100 static int 5101 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 5102 { 5103 return (0); 5104 } 5105 5106 static int 5107 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 5108 struct thread *td) 5109 { 5110 int rc; 5111 struct adapter *sc = dev->si_drv1; 5112 5113 rc = priv_check(td, PRIV_DRIVER); 5114 if (rc != 0) 5115 return (rc); 5116 5117 switch (cmd) { 5118 case CHELSIO_T4_GETREG: { 5119 struct t4_reg *edata = (struct t4_reg *)data; 5120 5121 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5122 return (EFAULT); 5123 5124 if (edata->size == 4) 5125 edata->val = t4_read_reg(sc, edata->addr); 5126 else if (edata->size == 8) 5127 edata->val = t4_read_reg64(sc, edata->addr); 5128 else 5129 return (EINVAL); 5130 5131 break; 5132 } 5133 case CHELSIO_T4_SETREG: { 5134 struct t4_reg *edata = (struct t4_reg *)data; 5135 5136 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5137 return (EFAULT); 5138 5139 if (edata->size == 4) { 5140 if (edata->val & 0xffffffff00000000) 5141 return (EINVAL); 5142 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 5143 } else if (edata->size == 8) 5144 t4_write_reg64(sc, edata->addr, edata->val); 5145 else 5146 return (EINVAL); 5147 break; 5148 } 5149 case CHELSIO_T4_REGDUMP: { 5150 struct t4_regdump *regs = (struct t4_regdump *)data; 5151 int reglen = T4_REGDUMP_SIZE; 5152 uint8_t *buf; 5153 5154 if (regs->len < reglen) { 5155 regs->len = reglen; /* hint to the caller */ 5156 return (ENOBUFS); 5157 } 5158 5159 regs->len = reglen; 5160 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 5161 t4_get_regs(sc, regs, buf); 5162 rc = copyout(buf, regs->data, reglen); 5163 free(buf, M_CXGBE); 5164 break; 5165 } 5166 case CHELSIO_T4_GET_FILTER_MODE: 5167 rc = get_filter_mode(sc, (uint32_t *)data); 5168 break; 5169 case CHELSIO_T4_SET_FILTER_MODE: 5170 rc = set_filter_mode(sc, *(uint32_t *)data); 5171 break; 5172 case CHELSIO_T4_GET_FILTER: 5173 ADAPTER_LOCK(sc); 5174 rc = get_filter(sc, (struct t4_filter *)data); 5175 ADAPTER_UNLOCK(sc); 5176 break; 5177 case CHELSIO_T4_SET_FILTER: 5178 ADAPTER_LOCK(sc); 5179 rc = set_filter(sc, (struct t4_filter *)data); 5180 ADAPTER_UNLOCK(sc); 5181 break; 5182 case CHELSIO_T4_DEL_FILTER: 5183 ADAPTER_LOCK(sc); 5184 rc = del_filter(sc, (struct t4_filter *)data); 5185 ADAPTER_UNLOCK(sc); 5186 break; 5187 case CHELSIO_T4_GET_SGE_CONTEXT: 5188 rc = get_sge_context(sc, (struct t4_sge_context *)data); 5189 break; 5190 case CHELSIO_T4_LOAD_FW: { 5191 struct t4_data *fw = (struct t4_data *)data; 5192 uint8_t *fw_data; 5193 5194 if (sc->flags & FULL_INIT_DONE) 5195 return (EBUSY); 5196 5197 fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT); 5198 if (fw_data == NULL) 5199 return (ENOMEM); 5200 5201 rc = copyin(fw->data, fw_data, fw->len); 5202 if (rc == 0) 5203 rc = -t4_load_fw(sc, fw_data, fw->len); 5204 5205 free(fw_data, M_CXGBE); 5206 break; 5207 } 5208 case CHELSIO_T4_GET_MEM: 5209 rc = read_card_mem(sc, (struct t4_mem_range *)data); 5210 break; 5211 default: 5212 rc = EINVAL; 5213 } 5214 5215 return (rc); 5216 } 5217 5218 #ifndef TCP_OFFLOAD_DISABLE 5219 static int 5220 toe_capability(struct port_info *pi, int enable) 5221 { 5222 int rc; 5223 struct adapter *sc = pi->adapter; 5224 5225 ADAPTER_LOCK_ASSERT_OWNED(sc); 5226 5227 if (!is_offload(sc)) 5228 return (ENODEV); 5229 5230 if (enable) { 5231 if (isset(&sc->offload_map, pi->port_id)) 5232 return (0); 5233 5234 if (sc->offload_map == 0) { 5235 rc = activate_uld(sc, ULD_TOM, &sc->tom); 5236 if (rc != 0) 5237 return (rc); 5238 } 5239 5240 setbit(&sc->offload_map, pi->port_id); 5241 } else { 5242 if (!isset(&sc->offload_map, pi->port_id)) 5243 return (0); 5244 5245 clrbit(&sc->offload_map, pi->port_id); 5246 5247 if (sc->offload_map == 0) { 5248 rc = deactivate_uld(&sc->tom); 5249 if (rc != 0) { 5250 setbit(&sc->offload_map, pi->port_id); 5251 return (rc); 5252 } 5253 } 5254 } 5255 5256 return (0); 5257 } 5258 5259 /* 5260 * Add an upper layer driver to the global list. 5261 */ 5262 int 5263 t4_register_uld(struct uld_info *ui) 5264 { 5265 int rc = 0; 5266 struct uld_info *u; 5267 5268 mtx_lock(&t4_uld_list_lock); 5269 SLIST_FOREACH(u, &t4_uld_list, link) { 5270 if (u->uld_id == ui->uld_id) { 5271 rc = EEXIST; 5272 goto done; 5273 } 5274 } 5275 5276 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 5277 ui->refcount = 0; 5278 done: 5279 mtx_unlock(&t4_uld_list_lock); 5280 return (rc); 5281 } 5282 5283 int 5284 t4_unregister_uld(struct uld_info *ui) 5285 { 5286 int rc = EINVAL; 5287 struct uld_info *u; 5288 5289 mtx_lock(&t4_uld_list_lock); 5290 5291 SLIST_FOREACH(u, &t4_uld_list, link) { 5292 if (u == ui) { 5293 if (ui->refcount > 0) { 5294 rc = EBUSY; 5295 goto done; 5296 } 5297 5298 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 5299 rc = 0; 5300 goto done; 5301 } 5302 } 5303 done: 5304 mtx_unlock(&t4_uld_list_lock); 5305 return (rc); 5306 } 5307 5308 static int 5309 activate_uld(struct adapter *sc, int id, struct uld_softc *usc) 5310 { 5311 int rc = EAGAIN; 5312 struct uld_info *ui; 5313 5314 mtx_lock(&t4_uld_list_lock); 5315 5316 SLIST_FOREACH(ui, &t4_uld_list, link) { 5317 if (ui->uld_id == id) { 5318 rc = ui->attach(sc, &usc->softc); 5319 if (rc == 0) { 5320 KASSERT(usc->softc != NULL, 5321 ("%s: ULD %d has no state", __func__, id)); 5322 ui->refcount++; 5323 usc->uld = ui; 5324 } 5325 goto done; 5326 } 5327 } 5328 done: 5329 mtx_unlock(&t4_uld_list_lock); 5330 5331 return (rc); 5332 } 5333 5334 static int 5335 deactivate_uld(struct uld_softc *usc) 5336 { 5337 int rc; 5338 5339 mtx_lock(&t4_uld_list_lock); 5340 5341 if (usc->uld == NULL || usc->softc == NULL) { 5342 rc = EINVAL; 5343 goto done; 5344 } 5345 5346 rc = usc->uld->detach(usc->softc); 5347 if (rc == 0) { 5348 KASSERT(usc->uld->refcount > 0, 5349 ("%s: ULD has bad refcount", __func__)); 5350 usc->uld->refcount--; 5351 usc->uld = NULL; 5352 usc->softc = NULL; 5353 } 5354 done: 5355 mtx_unlock(&t4_uld_list_lock); 5356 5357 return (rc); 5358 } 5359 #endif 5360 5361 /* 5362 * Come up with reasonable defaults for some of the tunables, provided they're 5363 * not set by the user (in which case we'll use the values as is). 5364 */ 5365 static void 5366 tweak_tunables(void) 5367 { 5368 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 5369 5370 if (t4_ntxq10g < 1) 5371 t4_ntxq10g = min(nc, NTXQ_10G); 5372 5373 if (t4_ntxq1g < 1) 5374 t4_ntxq1g = min(nc, NTXQ_1G); 5375 5376 if (t4_nrxq10g < 1) 5377 t4_nrxq10g = min(nc, NRXQ_10G); 5378 5379 if (t4_nrxq1g < 1) 5380 t4_nrxq1g = min(nc, NRXQ_1G); 5381 5382 #ifndef TCP_OFFLOAD_DISABLE 5383 if (t4_nofldtxq10g < 1) 5384 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 5385 5386 if (t4_nofldtxq1g < 1) 5387 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 5388 5389 if (t4_nofldrxq10g < 1) 5390 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 5391 5392 if (t4_nofldrxq1g < 1) 5393 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 5394 #endif 5395 5396 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 5397 t4_tmr_idx_10g = TMR_IDX_10G; 5398 5399 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 5400 t4_pktc_idx_10g = PKTC_IDX_10G; 5401 5402 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 5403 t4_tmr_idx_1g = TMR_IDX_1G; 5404 5405 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 5406 t4_pktc_idx_1g = PKTC_IDX_1G; 5407 5408 if (t4_qsize_txq < 128) 5409 t4_qsize_txq = 128; 5410 5411 if (t4_qsize_rxq < 128) 5412 t4_qsize_rxq = 128; 5413 while (t4_qsize_rxq & 7) 5414 t4_qsize_rxq++; 5415 5416 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 5417 } 5418 5419 static int 5420 t4_mod_event(module_t mod, int cmd, void *arg) 5421 { 5422 int rc = 0; 5423 5424 switch (cmd) { 5425 case MOD_LOAD: 5426 t4_sge_modload(); 5427 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 5428 SLIST_INIT(&t4_list); 5429 #ifndef TCP_OFFLOAD_DISABLE 5430 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 5431 SLIST_INIT(&t4_uld_list); 5432 #endif 5433 tweak_tunables(); 5434 break; 5435 5436 case MOD_UNLOAD: 5437 #ifndef TCP_OFFLOAD_DISABLE 5438 mtx_lock(&t4_uld_list_lock); 5439 if (!SLIST_EMPTY(&t4_uld_list)) { 5440 rc = EBUSY; 5441 mtx_unlock(&t4_uld_list_lock); 5442 break; 5443 } 5444 mtx_unlock(&t4_uld_list_lock); 5445 mtx_destroy(&t4_uld_list_lock); 5446 #endif 5447 mtx_lock(&t4_list_lock); 5448 if (!SLIST_EMPTY(&t4_list)) { 5449 rc = EBUSY; 5450 mtx_unlock(&t4_list_lock); 5451 break; 5452 } 5453 mtx_unlock(&t4_list_lock); 5454 mtx_destroy(&t4_list_lock); 5455 break; 5456 } 5457 5458 return (rc); 5459 } 5460 5461 static devclass_t t4_devclass; 5462 static devclass_t cxgbe_devclass; 5463 5464 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0); 5465 MODULE_VERSION(t4nex, 1); 5466 5467 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 5468 MODULE_VERSION(cxgbe, 1); 5469