1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 33 #include <sys/param.h> 34 #include <sys/conf.h> 35 #include <sys/priv.h> 36 #include <sys/kernel.h> 37 #include <sys/bus.h> 38 #include <sys/module.h> 39 #include <sys/malloc.h> 40 #include <sys/queue.h> 41 #include <sys/taskqueue.h> 42 #include <sys/pciio.h> 43 #include <dev/pci/pcireg.h> 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pci_private.h> 46 #include <sys/firmware.h> 47 #include <sys/sbuf.h> 48 #include <sys/smp.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/sysctl.h> 52 #include <net/ethernet.h> 53 #include <net/if.h> 54 #include <net/if_types.h> 55 #include <net/if_dl.h> 56 #include <net/if_vlan_var.h> 57 58 #include "common/common.h" 59 #include "common/t4_msg.h" 60 #include "common/t4_regs.h" 61 #include "common/t4_regs_values.h" 62 #include "t4_ioctl.h" 63 #include "t4_l2t.h" 64 65 /* T4 bus driver interface */ 66 static int t4_probe(device_t); 67 static int t4_attach(device_t); 68 static int t4_detach(device_t); 69 static device_method_t t4_methods[] = { 70 DEVMETHOD(device_probe, t4_probe), 71 DEVMETHOD(device_attach, t4_attach), 72 DEVMETHOD(device_detach, t4_detach), 73 74 DEVMETHOD_END 75 }; 76 static driver_t t4_driver = { 77 "t4nex", 78 t4_methods, 79 sizeof(struct adapter) 80 }; 81 82 83 /* T4 port (cxgbe) interface */ 84 static int cxgbe_probe(device_t); 85 static int cxgbe_attach(device_t); 86 static int cxgbe_detach(device_t); 87 static device_method_t cxgbe_methods[] = { 88 DEVMETHOD(device_probe, cxgbe_probe), 89 DEVMETHOD(device_attach, cxgbe_attach), 90 DEVMETHOD(device_detach, cxgbe_detach), 91 { 0, 0 } 92 }; 93 static driver_t cxgbe_driver = { 94 "cxgbe", 95 cxgbe_methods, 96 sizeof(struct port_info) 97 }; 98 99 static d_ioctl_t t4_ioctl; 100 static d_open_t t4_open; 101 static d_close_t t4_close; 102 103 static struct cdevsw t4_cdevsw = { 104 .d_version = D_VERSION, 105 .d_flags = 0, 106 .d_open = t4_open, 107 .d_close = t4_close, 108 .d_ioctl = t4_ioctl, 109 .d_name = "t4nex", 110 }; 111 112 /* ifnet + media interface */ 113 static void cxgbe_init(void *); 114 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 115 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 116 static void cxgbe_qflush(struct ifnet *); 117 static int cxgbe_media_change(struct ifnet *); 118 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 119 120 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services"); 121 122 static struct mtx t4_list_lock; 123 static SLIST_HEAD(, adapter) t4_list; 124 #ifndef TCP_OFFLOAD_DISABLE 125 static struct mtx t4_uld_list_lock; 126 static SLIST_HEAD(, uld_info) t4_uld_list; 127 #endif 128 129 /* 130 * Tunables. See tweak_tunables() too. 131 */ 132 133 /* 134 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 135 */ 136 #define NTXQ_10G 16 137 static int t4_ntxq10g = -1; 138 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 139 140 #define NRXQ_10G 8 141 static int t4_nrxq10g = -1; 142 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 143 144 #define NTXQ_1G 4 145 static int t4_ntxq1g = -1; 146 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 147 148 #define NRXQ_1G 2 149 static int t4_nrxq1g = -1; 150 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 151 152 #ifndef TCP_OFFLOAD_DISABLE 153 #define NOFLDTXQ_10G 8 154 static int t4_nofldtxq10g = -1; 155 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 156 157 #define NOFLDRXQ_10G 2 158 static int t4_nofldrxq10g = -1; 159 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 160 161 #define NOFLDTXQ_1G 2 162 static int t4_nofldtxq1g = -1; 163 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 164 165 #define NOFLDRXQ_1G 1 166 static int t4_nofldrxq1g = -1; 167 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 168 #endif 169 170 /* 171 * Holdoff parameters for 10G and 1G ports. 172 */ 173 #define TMR_IDX_10G 1 174 static int t4_tmr_idx_10g = TMR_IDX_10G; 175 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 176 177 #define PKTC_IDX_10G 2 178 static int t4_pktc_idx_10g = PKTC_IDX_10G; 179 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 180 181 #define TMR_IDX_1G 1 182 static int t4_tmr_idx_1g = TMR_IDX_1G; 183 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 184 185 #define PKTC_IDX_1G 2 186 static int t4_pktc_idx_1g = PKTC_IDX_1G; 187 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 188 189 /* 190 * Size (# of entries) of each tx and rx queue. 191 */ 192 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 193 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 194 195 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 196 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 197 198 /* 199 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 200 */ 201 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 202 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 203 204 /* 205 * Configuration file. 206 */ 207 static char t4_cfg_file[32] = "default"; 208 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 209 210 /* 211 * ASIC features that will be used. Disable the ones you don't want so that the 212 * chip resources aren't wasted on features that will not be used. 213 */ 214 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 215 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 216 217 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 218 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 219 220 static int t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 221 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 222 223 static int t4_rdmacaps_allowed = 0; 224 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 225 226 static int t4_iscsicaps_allowed = 0; 227 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 228 229 static int t4_fcoecaps_allowed = 0; 230 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 231 232 struct intrs_and_queues { 233 int intr_type; /* INTx, MSI, or MSI-X */ 234 int nirq; /* Number of vectors */ 235 int intr_flags; 236 int ntxq10g; /* # of NIC txq's for each 10G port */ 237 int nrxq10g; /* # of NIC rxq's for each 10G port */ 238 int ntxq1g; /* # of NIC txq's for each 1G port */ 239 int nrxq1g; /* # of NIC rxq's for each 1G port */ 240 #ifndef TCP_OFFLOAD_DISABLE 241 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 242 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 243 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 244 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 245 #endif 246 }; 247 248 struct filter_entry { 249 uint32_t valid:1; /* filter allocated and valid */ 250 uint32_t locked:1; /* filter is administratively locked */ 251 uint32_t pending:1; /* filter action is pending firmware reply */ 252 uint32_t smtidx:8; /* Source MAC Table index for smac */ 253 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 254 255 struct t4_filter_specification fs; 256 }; 257 258 enum { 259 XGMAC_MTU = (1 << 0), 260 XGMAC_PROMISC = (1 << 1), 261 XGMAC_ALLMULTI = (1 << 2), 262 XGMAC_VLANEX = (1 << 3), 263 XGMAC_UCADDR = (1 << 4), 264 XGMAC_MCADDRS = (1 << 5), 265 266 XGMAC_ALL = 0xffff 267 }; 268 269 static int map_bars(struct adapter *); 270 static void setup_memwin(struct adapter *); 271 static int cfg_itype_and_nqueues(struct adapter *, int, int, 272 struct intrs_and_queues *); 273 static int prep_firmware(struct adapter *); 274 static int upload_config_file(struct adapter *, const struct firmware *, 275 uint32_t *, uint32_t *); 276 static int partition_resources(struct adapter *, const struct firmware *); 277 static int get_params__pre_init(struct adapter *); 278 static int get_params__post_init(struct adapter *); 279 static void t4_set_desc(struct adapter *); 280 static void build_medialist(struct port_info *); 281 static int update_mac_settings(struct port_info *, int); 282 static int cxgbe_init_locked(struct port_info *); 283 static int cxgbe_init_synchronized(struct port_info *); 284 static int cxgbe_uninit_locked(struct port_info *); 285 static int cxgbe_uninit_synchronized(struct port_info *); 286 static int adapter_full_init(struct adapter *); 287 static int adapter_full_uninit(struct adapter *); 288 static int port_full_init(struct port_info *); 289 static int port_full_uninit(struct port_info *); 290 static void quiesce_eq(struct adapter *, struct sge_eq *); 291 static void quiesce_iq(struct adapter *, struct sge_iq *); 292 static void quiesce_fl(struct adapter *, struct sge_fl *); 293 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 294 driver_intr_t *, void *, char *); 295 static int t4_free_irq(struct adapter *, struct irq *); 296 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 297 unsigned int); 298 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 299 static void cxgbe_tick(void *); 300 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 301 struct mbuf *); 302 static int t4_sysctls(struct adapter *); 303 static int cxgbe_sysctls(struct port_info *); 304 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 305 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 306 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 307 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 308 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 309 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 310 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 311 #ifdef SBUF_DRAIN 312 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 313 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 314 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 315 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 316 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 317 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 318 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 319 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 320 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 321 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 322 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 323 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 324 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 325 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 326 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 327 #endif 328 static inline void txq_start(struct ifnet *, struct sge_txq *); 329 static uint32_t fconf_to_mode(uint32_t); 330 static uint32_t mode_to_fconf(uint32_t); 331 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 332 static int get_filter_mode(struct adapter *, uint32_t *); 333 static int set_filter_mode(struct adapter *, uint32_t); 334 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 335 static int get_filter(struct adapter *, struct t4_filter *); 336 static int set_filter(struct adapter *, struct t4_filter *); 337 static int del_filter(struct adapter *, struct t4_filter *); 338 static void clear_filter(struct filter_entry *); 339 static int set_filter_wr(struct adapter *, int); 340 static int del_filter_wr(struct adapter *, int); 341 static int filter_rpl(struct sge_iq *, const struct rss_header *, 342 struct mbuf *); 343 static int get_sge_context(struct adapter *, struct t4_sge_context *); 344 static int read_card_mem(struct adapter *, struct t4_mem_range *); 345 #ifndef TCP_OFFLOAD_DISABLE 346 static int toe_capability(struct port_info *, int); 347 static int activate_uld(struct adapter *, int, struct uld_softc *); 348 static int deactivate_uld(struct uld_softc *); 349 #endif 350 static int t4_mod_event(module_t, int, void *); 351 352 struct t4_pciids { 353 uint16_t device; 354 uint8_t mpf; 355 char *desc; 356 } t4_pciids[] = { 357 {0xa000, 0, "Chelsio Terminator 4 FPGA"}, 358 {0x4400, 4, "Chelsio T440-dbg"}, 359 {0x4401, 4, "Chelsio T420-CR"}, 360 {0x4402, 4, "Chelsio T422-CR"}, 361 {0x4403, 4, "Chelsio T440-CR"}, 362 {0x4404, 4, "Chelsio T420-BCH"}, 363 {0x4405, 4, "Chelsio T440-BCH"}, 364 {0x4406, 4, "Chelsio T440-CH"}, 365 {0x4407, 4, "Chelsio T420-SO"}, 366 {0x4408, 4, "Chelsio T420-CX"}, 367 {0x4409, 4, "Chelsio T420-BT"}, 368 {0x440a, 4, "Chelsio T404-BT"}, 369 }; 370 371 #ifndef TCP_OFFLOAD_DISABLE 372 /* This is used in service_iq() to get to the fl associated with an iq. */ 373 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 374 #endif 375 376 static int 377 t4_probe(device_t dev) 378 { 379 int i; 380 uint16_t v = pci_get_vendor(dev); 381 uint16_t d = pci_get_device(dev); 382 383 if (v != PCI_VENDOR_ID_CHELSIO) 384 return (ENXIO); 385 386 for (i = 0; i < ARRAY_SIZE(t4_pciids); i++) { 387 if (d == t4_pciids[i].device && 388 pci_get_function(dev) == t4_pciids[i].mpf) { 389 device_set_desc(dev, t4_pciids[i].desc); 390 return (BUS_PROBE_DEFAULT); 391 } 392 } 393 394 return (ENXIO); 395 } 396 397 static int 398 t4_attach(device_t dev) 399 { 400 struct adapter *sc; 401 int rc = 0, i, n10g, n1g, rqidx, tqidx; 402 struct intrs_and_queues iaq; 403 struct sge *s; 404 #ifndef TCP_OFFLOAD_DISABLE 405 int ofld_rqidx, ofld_tqidx; 406 #endif 407 408 sc = device_get_softc(dev); 409 sc->dev = dev; 410 sc->pf = pci_get_function(dev); 411 sc->mbox = sc->pf; 412 413 pci_enable_busmaster(dev); 414 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 415 uint32_t v; 416 417 pci_set_max_read_req(dev, 4096); 418 v = pci_read_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, 2); 419 v |= PCIM_EXP_CTL_RELAXED_ORD_ENABLE; 420 pci_write_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, v, 2); 421 } 422 423 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 424 device_get_nameunit(dev)); 425 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 426 mtx_lock(&t4_list_lock); 427 SLIST_INSERT_HEAD(&t4_list, sc, link); 428 mtx_unlock(&t4_list_lock); 429 430 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 431 TAILQ_INIT(&sc->sfl); 432 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 433 434 rc = map_bars(sc); 435 if (rc != 0) 436 goto done; /* error message displayed already */ 437 438 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 439 for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) 440 sc->cpl_handler[i] = cpl_not_handled; 441 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, filter_rpl); 442 443 /* Prepare the adapter for operation */ 444 rc = -t4_prep_adapter(sc); 445 if (rc != 0) { 446 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 447 goto done; 448 } 449 450 /* 451 * Do this really early, with the memory windows set up even before the 452 * character device. The userland tool's register i/o and mem read 453 * will work even in "recovery mode". 454 */ 455 setup_memwin(sc); 456 sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT, 457 GID_WHEEL, 0600, "%s", device_get_nameunit(dev)); 458 sc->cdev->si_drv1 = sc; 459 460 /* Go no further if recovery mode has been requested. */ 461 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 462 device_printf(dev, "recovery mode.\n"); 463 goto done; 464 } 465 466 /* Prepare the firmware for operation */ 467 rc = prep_firmware(sc); 468 if (rc != 0) 469 goto done; /* error message displayed already */ 470 471 rc = get_params__pre_init(sc); 472 if (rc != 0) 473 goto done; /* error message displayed already */ 474 475 rc = t4_sge_init(sc); 476 if (rc != 0) 477 goto done; /* error message displayed already */ 478 479 if (sc->flags & MASTER_PF) { 480 /* get basic stuff going */ 481 rc = -t4_fw_initialize(sc, sc->mbox); 482 if (rc != 0) { 483 device_printf(dev, "early init failed: %d.\n", rc); 484 goto done; 485 } 486 } 487 488 rc = get_params__post_init(sc); 489 if (rc != 0) 490 goto done; /* error message displayed already */ 491 492 if (sc->flags & MASTER_PF) { 493 494 /* final tweaks to some settings */ 495 496 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, 497 sc->params.b_wnd); 498 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 499 t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 | 500 F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3, 0); 501 t4_set_reg_field(sc, A_TP_PARA_REG5, 502 V_INDICATESIZE(M_INDICATESIZE) | 503 F_REARMDDPOFFSET | F_RESETDDPOFFSET, 504 V_INDICATESIZE(M_INDICATESIZE) | 505 F_REARMDDPOFFSET | F_RESETDDPOFFSET); 506 } else { 507 /* 508 * XXX: Verify that we can live with whatever the master driver 509 * has done so far, and hope that it doesn't change any global 510 * setting from underneath us in the future. 511 */ 512 } 513 514 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 515 A_TP_VLAN_PRI_MAP); 516 517 for (i = 0; i < NCHAN; i++) 518 sc->params.tp.tx_modq[i] = i; 519 520 rc = t4_create_dma_tag(sc); 521 if (rc != 0) 522 goto done; /* error message displayed already */ 523 524 /* 525 * First pass over all the ports - allocate VIs and initialize some 526 * basic parameters like mac address, port type, etc. We also figure 527 * out whether a port is 10G or 1G and use that information when 528 * calculating how many interrupts to attempt to allocate. 529 */ 530 n10g = n1g = 0; 531 for_each_port(sc, i) { 532 struct port_info *pi; 533 534 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 535 sc->port[i] = pi; 536 537 /* These must be set before t4_port_init */ 538 pi->adapter = sc; 539 pi->port_id = i; 540 541 /* Allocate the vi and initialize parameters like mac addr */ 542 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 543 if (rc != 0) { 544 device_printf(dev, "unable to initialize port %d: %d\n", 545 i, rc); 546 free(pi, M_CXGBE); 547 sc->port[i] = NULL; 548 goto done; 549 } 550 551 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 552 device_get_nameunit(dev), i); 553 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 554 555 if (is_10G_port(pi)) { 556 n10g++; 557 pi->tmr_idx = t4_tmr_idx_10g; 558 pi->pktc_idx = t4_pktc_idx_10g; 559 } else { 560 n1g++; 561 pi->tmr_idx = t4_tmr_idx_1g; 562 pi->pktc_idx = t4_pktc_idx_1g; 563 } 564 565 pi->xact_addr_filt = -1; 566 567 pi->qsize_rxq = t4_qsize_rxq; 568 pi->qsize_txq = t4_qsize_txq; 569 570 pi->dev = device_add_child(dev, "cxgbe", -1); 571 if (pi->dev == NULL) { 572 device_printf(dev, 573 "failed to add device for port %d.\n", i); 574 rc = ENXIO; 575 goto done; 576 } 577 device_set_softc(pi->dev, pi); 578 } 579 580 /* 581 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 582 */ 583 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 584 if (rc != 0) 585 goto done; /* error message displayed already */ 586 587 sc->intr_type = iaq.intr_type; 588 sc->intr_count = iaq.nirq; 589 sc->flags |= iaq.intr_flags; 590 591 s = &sc->sge; 592 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 593 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 594 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 595 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 596 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 597 598 #ifndef TCP_OFFLOAD_DISABLE 599 if (is_offload(sc)) { 600 601 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 602 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 603 s->neq += s->nofldtxq + s->nofldrxq; 604 s->niq += s->nofldrxq; 605 606 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 607 M_CXGBE, M_ZERO | M_WAITOK); 608 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 609 M_CXGBE, M_ZERO | M_WAITOK); 610 } 611 #endif 612 613 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 614 M_ZERO | M_WAITOK); 615 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 616 M_ZERO | M_WAITOK); 617 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 618 M_ZERO | M_WAITOK); 619 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 620 M_ZERO | M_WAITOK); 621 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 622 M_ZERO | M_WAITOK); 623 624 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 625 M_ZERO | M_WAITOK); 626 627 t4_init_l2t(sc, M_WAITOK); 628 629 /* 630 * Second pass over the ports. This time we know the number of rx and 631 * tx queues that each port should get. 632 */ 633 rqidx = tqidx = 0; 634 #ifndef TCP_OFFLOAD_DISABLE 635 ofld_rqidx = ofld_tqidx = 0; 636 #endif 637 for_each_port(sc, i) { 638 struct port_info *pi = sc->port[i]; 639 640 if (pi == NULL) 641 continue; 642 643 pi->first_rxq = rqidx; 644 pi->first_txq = tqidx; 645 if (is_10G_port(pi)) { 646 pi->nrxq = iaq.nrxq10g; 647 pi->ntxq = iaq.ntxq10g; 648 } else { 649 pi->nrxq = iaq.nrxq1g; 650 pi->ntxq = iaq.ntxq1g; 651 } 652 653 rqidx += pi->nrxq; 654 tqidx += pi->ntxq; 655 656 #ifndef TCP_OFFLOAD_DISABLE 657 if (is_offload(sc)) { 658 pi->first_ofld_rxq = ofld_rqidx; 659 pi->first_ofld_txq = ofld_tqidx; 660 if (is_10G_port(pi)) { 661 pi->nofldrxq = iaq.nofldrxq10g; 662 pi->nofldtxq = iaq.nofldtxq10g; 663 } else { 664 pi->nofldrxq = iaq.nofldrxq1g; 665 pi->nofldtxq = iaq.nofldtxq1g; 666 } 667 ofld_rqidx += pi->nofldrxq; 668 ofld_tqidx += pi->nofldtxq; 669 } 670 #endif 671 } 672 673 rc = bus_generic_attach(dev); 674 if (rc != 0) { 675 device_printf(dev, 676 "failed to attach all child ports: %d\n", rc); 677 goto done; 678 } 679 680 device_printf(dev, 681 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 682 sc->params.pci.width, sc->params.nports, sc->intr_count, 683 sc->intr_type == INTR_MSIX ? "MSI-X" : 684 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 685 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 686 687 t4_set_desc(sc); 688 689 done: 690 if (rc != 0 && sc->cdev) { 691 /* cdev was created and so cxgbetool works; recover that way. */ 692 device_printf(dev, 693 "error during attach, adapter is now in recovery mode.\n"); 694 rc = 0; 695 } 696 697 if (rc != 0) 698 t4_detach(dev); 699 else 700 t4_sysctls(sc); 701 702 return (rc); 703 } 704 705 /* 706 * Idempotent 707 */ 708 static int 709 t4_detach(device_t dev) 710 { 711 struct adapter *sc; 712 struct port_info *pi; 713 int i, rc; 714 715 sc = device_get_softc(dev); 716 717 if (sc->flags & FULL_INIT_DONE) 718 t4_intr_disable(sc); 719 720 if (sc->cdev) { 721 destroy_dev(sc->cdev); 722 sc->cdev = NULL; 723 } 724 725 rc = bus_generic_detach(dev); 726 if (rc) { 727 device_printf(dev, 728 "failed to detach child devices: %d\n", rc); 729 return (rc); 730 } 731 732 for (i = 0; i < MAX_NPORTS; i++) { 733 pi = sc->port[i]; 734 if (pi) { 735 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 736 if (pi->dev) 737 device_delete_child(dev, pi->dev); 738 739 mtx_destroy(&pi->pi_lock); 740 free(pi, M_CXGBE); 741 } 742 } 743 744 if (sc->flags & FULL_INIT_DONE) 745 adapter_full_uninit(sc); 746 747 if (sc->flags & FW_OK) 748 t4_fw_bye(sc, sc->mbox); 749 750 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 751 pci_release_msi(dev); 752 753 if (sc->regs_res) 754 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 755 sc->regs_res); 756 757 if (sc->msix_res) 758 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 759 sc->msix_res); 760 761 if (sc->l2t) 762 t4_free_l2t(sc->l2t); 763 764 #ifndef TCP_OFFLOAD_DISABLE 765 free(sc->sge.ofld_rxq, M_CXGBE); 766 free(sc->sge.ofld_txq, M_CXGBE); 767 #endif 768 free(sc->irq, M_CXGBE); 769 free(sc->sge.rxq, M_CXGBE); 770 free(sc->sge.txq, M_CXGBE); 771 free(sc->sge.ctrlq, M_CXGBE); 772 free(sc->sge.iqmap, M_CXGBE); 773 free(sc->sge.eqmap, M_CXGBE); 774 free(sc->tids.ftid_tab, M_CXGBE); 775 t4_destroy_dma_tag(sc); 776 if (mtx_initialized(&sc->sc_lock)) { 777 mtx_lock(&t4_list_lock); 778 SLIST_REMOVE(&t4_list, sc, adapter, link); 779 mtx_unlock(&t4_list_lock); 780 mtx_destroy(&sc->sc_lock); 781 } 782 783 if (mtx_initialized(&sc->sfl_lock)) 784 mtx_destroy(&sc->sfl_lock); 785 786 bzero(sc, sizeof(*sc)); 787 788 return (0); 789 } 790 791 792 static int 793 cxgbe_probe(device_t dev) 794 { 795 char buf[128]; 796 struct port_info *pi = device_get_softc(dev); 797 798 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 799 device_set_desc_copy(dev, buf); 800 801 return (BUS_PROBE_DEFAULT); 802 } 803 804 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 805 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 806 IFCAP_VLAN_HWTSO) 807 #define T4_CAP_ENABLE (T4_CAP & ~IFCAP_TSO6) 808 809 static int 810 cxgbe_attach(device_t dev) 811 { 812 struct port_info *pi = device_get_softc(dev); 813 struct ifnet *ifp; 814 815 /* Allocate an ifnet and set it up */ 816 ifp = if_alloc(IFT_ETHER); 817 if (ifp == NULL) { 818 device_printf(dev, "Cannot allocate ifnet\n"); 819 return (ENOMEM); 820 } 821 pi->ifp = ifp; 822 ifp->if_softc = pi; 823 824 callout_init(&pi->tick, CALLOUT_MPSAFE); 825 826 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 827 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 828 829 ifp->if_init = cxgbe_init; 830 ifp->if_ioctl = cxgbe_ioctl; 831 ifp->if_transmit = cxgbe_transmit; 832 ifp->if_qflush = cxgbe_qflush; 833 834 ifp->if_capabilities = T4_CAP; 835 #ifndef TCP_OFFLOAD_DISABLE 836 if (is_offload(pi->adapter)) 837 ifp->if_capabilities |= IFCAP_TOE4; 838 #endif 839 ifp->if_capenable = T4_CAP_ENABLE; 840 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; 841 842 /* Initialize ifmedia for this port */ 843 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 844 cxgbe_media_status); 845 build_medialist(pi); 846 847 ether_ifattach(ifp, pi->hw_addr); 848 849 #ifndef TCP_OFFLOAD_DISABLE 850 if (is_offload(pi->adapter)) { 851 device_printf(dev, 852 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 853 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 854 } else 855 #endif 856 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 857 858 cxgbe_sysctls(pi); 859 860 return (0); 861 } 862 863 static int 864 cxgbe_detach(device_t dev) 865 { 866 struct port_info *pi = device_get_softc(dev); 867 struct adapter *sc = pi->adapter; 868 struct ifnet *ifp = pi->ifp; 869 870 /* Tell if_ioctl and if_init that the port is going away */ 871 ADAPTER_LOCK(sc); 872 SET_DOOMED(pi); 873 wakeup(&sc->flags); 874 while (IS_BUSY(sc)) 875 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 876 SET_BUSY(sc); 877 ADAPTER_UNLOCK(sc); 878 879 PORT_LOCK(pi); 880 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 881 callout_stop(&pi->tick); 882 PORT_UNLOCK(pi); 883 callout_drain(&pi->tick); 884 885 /* Let detach proceed even if these fail. */ 886 cxgbe_uninit_synchronized(pi); 887 port_full_uninit(pi); 888 889 ifmedia_removeall(&pi->media); 890 ether_ifdetach(pi->ifp); 891 if_free(pi->ifp); 892 893 ADAPTER_LOCK(sc); 894 CLR_BUSY(sc); 895 wakeup_one(&sc->flags); 896 ADAPTER_UNLOCK(sc); 897 898 return (0); 899 } 900 901 static void 902 cxgbe_init(void *arg) 903 { 904 struct port_info *pi = arg; 905 struct adapter *sc = pi->adapter; 906 907 ADAPTER_LOCK(sc); 908 cxgbe_init_locked(pi); /* releases adapter lock */ 909 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 910 } 911 912 static int 913 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 914 { 915 int rc = 0, mtu, flags; 916 struct port_info *pi = ifp->if_softc; 917 struct adapter *sc = pi->adapter; 918 struct ifreq *ifr = (struct ifreq *)data; 919 uint32_t mask; 920 921 switch (cmd) { 922 case SIOCSIFMTU: 923 ADAPTER_LOCK(sc); 924 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 925 if (rc) { 926 fail: 927 ADAPTER_UNLOCK(sc); 928 return (rc); 929 } 930 931 mtu = ifr->ifr_mtu; 932 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 933 rc = EINVAL; 934 } else { 935 ifp->if_mtu = mtu; 936 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 937 t4_update_fl_bufsize(ifp); 938 PORT_LOCK(pi); 939 rc = update_mac_settings(pi, XGMAC_MTU); 940 PORT_UNLOCK(pi); 941 } 942 } 943 ADAPTER_UNLOCK(sc); 944 break; 945 946 case SIOCSIFFLAGS: 947 ADAPTER_LOCK(sc); 948 if (IS_DOOMED(pi)) { 949 rc = ENXIO; 950 goto fail; 951 } 952 if (ifp->if_flags & IFF_UP) { 953 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 954 flags = pi->if_flags; 955 if ((ifp->if_flags ^ flags) & 956 (IFF_PROMISC | IFF_ALLMULTI)) { 957 if (IS_BUSY(sc)) { 958 rc = EBUSY; 959 goto fail; 960 } 961 PORT_LOCK(pi); 962 rc = update_mac_settings(pi, 963 XGMAC_PROMISC | XGMAC_ALLMULTI); 964 PORT_UNLOCK(pi); 965 } 966 ADAPTER_UNLOCK(sc); 967 } else 968 rc = cxgbe_init_locked(pi); 969 pi->if_flags = ifp->if_flags; 970 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 971 rc = cxgbe_uninit_locked(pi); 972 else 973 ADAPTER_UNLOCK(sc); 974 975 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 976 break; 977 978 case SIOCADDMULTI: 979 case SIOCDELMULTI: /* these two can be called with a mutex held :-( */ 980 ADAPTER_LOCK(sc); 981 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 982 if (rc) 983 goto fail; 984 985 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 986 PORT_LOCK(pi); 987 rc = update_mac_settings(pi, XGMAC_MCADDRS); 988 PORT_UNLOCK(pi); 989 } 990 ADAPTER_UNLOCK(sc); 991 break; 992 993 case SIOCSIFCAP: 994 ADAPTER_LOCK(sc); 995 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 996 if (rc) 997 goto fail; 998 999 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1000 if (mask & IFCAP_TXCSUM) { 1001 ifp->if_capenable ^= IFCAP_TXCSUM; 1002 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1003 1004 if (IFCAP_TSO & ifp->if_capenable && 1005 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1006 ifp->if_capenable &= ~IFCAP_TSO; 1007 ifp->if_hwassist &= ~CSUM_TSO; 1008 if_printf(ifp, 1009 "tso disabled due to -txcsum.\n"); 1010 } 1011 } 1012 if (mask & IFCAP_RXCSUM) 1013 ifp->if_capenable ^= IFCAP_RXCSUM; 1014 if (mask & IFCAP_TSO4) { 1015 ifp->if_capenable ^= IFCAP_TSO4; 1016 1017 if (IFCAP_TSO & ifp->if_capenable) { 1018 if (IFCAP_TXCSUM & ifp->if_capenable) 1019 ifp->if_hwassist |= CSUM_TSO; 1020 else { 1021 ifp->if_capenable &= ~IFCAP_TSO; 1022 ifp->if_hwassist &= ~CSUM_TSO; 1023 if_printf(ifp, 1024 "enable txcsum first.\n"); 1025 rc = EAGAIN; 1026 goto fail; 1027 } 1028 } else 1029 ifp->if_hwassist &= ~CSUM_TSO; 1030 } 1031 if (mask & IFCAP_LRO) { 1032 #ifdef INET 1033 int i; 1034 struct sge_rxq *rxq; 1035 1036 ifp->if_capenable ^= IFCAP_LRO; 1037 for_each_rxq(pi, i, rxq) { 1038 if (ifp->if_capenable & IFCAP_LRO) 1039 rxq->iq.flags |= IQ_LRO_ENABLED; 1040 else 1041 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1042 } 1043 #endif 1044 } 1045 #ifndef TCP_OFFLOAD_DISABLE 1046 if (mask & IFCAP_TOE) { 1047 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1048 1049 rc = toe_capability(pi, enable); 1050 if (rc != 0) 1051 goto fail; 1052 1053 ifp->if_capenable ^= mask; 1054 } 1055 #endif 1056 if (mask & IFCAP_VLAN_HWTAGGING) { 1057 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1058 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1059 PORT_LOCK(pi); 1060 rc = update_mac_settings(pi, XGMAC_VLANEX); 1061 PORT_UNLOCK(pi); 1062 } 1063 } 1064 if (mask & IFCAP_VLAN_MTU) { 1065 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1066 1067 /* Need to find out how to disable auto-mtu-inflation */ 1068 } 1069 if (mask & IFCAP_VLAN_HWTSO) 1070 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1071 if (mask & IFCAP_VLAN_HWCSUM) 1072 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1073 1074 #ifdef VLAN_CAPABILITIES 1075 VLAN_CAPABILITIES(ifp); 1076 #endif 1077 ADAPTER_UNLOCK(sc); 1078 break; 1079 1080 case SIOCSIFMEDIA: 1081 case SIOCGIFMEDIA: 1082 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1083 break; 1084 1085 default: 1086 rc = ether_ioctl(ifp, cmd, data); 1087 } 1088 1089 return (rc); 1090 } 1091 1092 static int 1093 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1094 { 1095 struct port_info *pi = ifp->if_softc; 1096 struct adapter *sc = pi->adapter; 1097 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1098 struct buf_ring *br; 1099 int rc; 1100 1101 M_ASSERTPKTHDR(m); 1102 1103 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1104 m_freem(m); 1105 return (ENETDOWN); 1106 } 1107 1108 if (m->m_flags & M_FLOWID) 1109 txq += (m->m_pkthdr.flowid % pi->ntxq); 1110 br = txq->br; 1111 1112 if (TXQ_TRYLOCK(txq) == 0) { 1113 struct sge_eq *eq = &txq->eq; 1114 1115 /* 1116 * It is possible that t4_eth_tx finishes up and releases the 1117 * lock between the TRYLOCK above and the drbr_enqueue here. We 1118 * need to make sure that this mbuf doesn't just sit there in 1119 * the drbr. 1120 */ 1121 1122 rc = drbr_enqueue(ifp, br, m); 1123 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1124 !(eq->flags & EQ_DOOMED)) 1125 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1126 return (rc); 1127 } 1128 1129 /* 1130 * txq->m is the mbuf that is held up due to a temporary shortage of 1131 * resources and it should be put on the wire first. Then what's in 1132 * drbr and finally the mbuf that was just passed in to us. 1133 * 1134 * Return code should indicate the fate of the mbuf that was passed in 1135 * this time. 1136 */ 1137 1138 TXQ_LOCK_ASSERT_OWNED(txq); 1139 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1140 1141 /* Queued for transmission. */ 1142 1143 rc = drbr_enqueue(ifp, br, m); 1144 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1145 (void) t4_eth_tx(ifp, txq, m); 1146 TXQ_UNLOCK(txq); 1147 return (rc); 1148 } 1149 1150 /* Direct transmission. */ 1151 rc = t4_eth_tx(ifp, txq, m); 1152 if (rc != 0 && txq->m) 1153 rc = 0; /* held, will be transmitted soon (hopefully) */ 1154 1155 TXQ_UNLOCK(txq); 1156 return (rc); 1157 } 1158 1159 static void 1160 cxgbe_qflush(struct ifnet *ifp) 1161 { 1162 struct port_info *pi = ifp->if_softc; 1163 struct sge_txq *txq; 1164 int i; 1165 struct mbuf *m; 1166 1167 /* queues do not exist if !PORT_INIT_DONE. */ 1168 if (pi->flags & PORT_INIT_DONE) { 1169 for_each_txq(pi, i, txq) { 1170 TXQ_LOCK(txq); 1171 m_freem(txq->m); 1172 txq->m = NULL; 1173 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1174 m_freem(m); 1175 TXQ_UNLOCK(txq); 1176 } 1177 } 1178 if_qflush(ifp); 1179 } 1180 1181 static int 1182 cxgbe_media_change(struct ifnet *ifp) 1183 { 1184 struct port_info *pi = ifp->if_softc; 1185 1186 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1187 1188 return (EOPNOTSUPP); 1189 } 1190 1191 static void 1192 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1193 { 1194 struct port_info *pi = ifp->if_softc; 1195 struct ifmedia_entry *cur = pi->media.ifm_cur; 1196 int speed = pi->link_cfg.speed; 1197 int data = (pi->port_type << 8) | pi->mod_type; 1198 1199 if (cur->ifm_data != data) { 1200 build_medialist(pi); 1201 cur = pi->media.ifm_cur; 1202 } 1203 1204 ifmr->ifm_status = IFM_AVALID; 1205 if (!pi->link_cfg.link_ok) 1206 return; 1207 1208 ifmr->ifm_status |= IFM_ACTIVE; 1209 1210 /* active and current will differ iff current media is autoselect. */ 1211 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1212 return; 1213 1214 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1215 if (speed == SPEED_10000) 1216 ifmr->ifm_active |= IFM_10G_T; 1217 else if (speed == SPEED_1000) 1218 ifmr->ifm_active |= IFM_1000_T; 1219 else if (speed == SPEED_100) 1220 ifmr->ifm_active |= IFM_100_TX; 1221 else if (speed == SPEED_10) 1222 ifmr->ifm_active |= IFM_10_T; 1223 else 1224 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1225 speed)); 1226 } 1227 1228 void 1229 t4_fatal_err(struct adapter *sc) 1230 { 1231 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1232 t4_intr_disable(sc); 1233 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1234 device_get_nameunit(sc->dev)); 1235 } 1236 1237 static int 1238 map_bars(struct adapter *sc) 1239 { 1240 sc->regs_rid = PCIR_BAR(0); 1241 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1242 &sc->regs_rid, RF_ACTIVE); 1243 if (sc->regs_res == NULL) { 1244 device_printf(sc->dev, "cannot map registers.\n"); 1245 return (ENXIO); 1246 } 1247 sc->bt = rman_get_bustag(sc->regs_res); 1248 sc->bh = rman_get_bushandle(sc->regs_res); 1249 sc->mmio_len = rman_get_size(sc->regs_res); 1250 1251 sc->msix_rid = PCIR_BAR(4); 1252 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1253 &sc->msix_rid, RF_ACTIVE); 1254 if (sc->msix_res == NULL) { 1255 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1256 return (ENXIO); 1257 } 1258 1259 return (0); 1260 } 1261 1262 static void 1263 setup_memwin(struct adapter *sc) 1264 { 1265 u_long bar0; 1266 1267 bar0 = rman_get_start(sc->regs_res); 1268 1269 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1270 (bar0 + MEMWIN0_BASE) | V_BIR(0) | 1271 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1272 1273 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1274 (bar0 + MEMWIN1_BASE) | V_BIR(0) | 1275 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1276 1277 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1278 (bar0 + MEMWIN2_BASE) | V_BIR(0) | 1279 V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10)); 1280 } 1281 1282 static int 1283 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1284 struct intrs_and_queues *iaq) 1285 { 1286 int rc, itype, navail, nrxq10g, nrxq1g, n; 1287 int nofldrxq10g = 0, nofldrxq1g = 0; 1288 1289 bzero(iaq, sizeof(*iaq)); 1290 1291 iaq->ntxq10g = t4_ntxq10g; 1292 iaq->ntxq1g = t4_ntxq1g; 1293 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1294 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1295 #ifndef TCP_OFFLOAD_DISABLE 1296 iaq->nofldtxq10g = t4_nofldtxq10g; 1297 iaq->nofldtxq1g = t4_nofldtxq1g; 1298 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1299 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1300 #endif 1301 1302 for (itype = INTR_MSIX; itype; itype >>= 1) { 1303 1304 if ((itype & t4_intr_types) == 0) 1305 continue; /* not allowed */ 1306 1307 if (itype == INTR_MSIX) 1308 navail = pci_msix_count(sc->dev); 1309 else if (itype == INTR_MSI) 1310 navail = pci_msi_count(sc->dev); 1311 else 1312 navail = 1; 1313 restart: 1314 if (navail == 0) 1315 continue; 1316 1317 iaq->intr_type = itype; 1318 iaq->intr_flags = 0; 1319 1320 /* 1321 * Best option: an interrupt vector for errors, one for the 1322 * firmware event queue, and one each for each rxq (NIC as well 1323 * as offload). 1324 */ 1325 iaq->nirq = T4_EXTRA_INTR; 1326 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1327 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1328 if (iaq->nirq <= navail && 1329 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1330 iaq->intr_flags |= INTR_DIRECT; 1331 goto allocate; 1332 } 1333 1334 /* 1335 * Second best option: an interrupt vector for errors, one for 1336 * the firmware event queue, and one each for either NIC or 1337 * offload rxq's. 1338 */ 1339 iaq->nirq = T4_EXTRA_INTR; 1340 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1341 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1342 if (iaq->nirq <= navail && 1343 (itype != INTR_MSI || powerof2(iaq->nirq))) 1344 goto allocate; 1345 1346 /* 1347 * Next best option: an interrupt vector for errors, one for the 1348 * firmware event queue, and at least one per port. At this 1349 * point we know we'll have to downsize nrxq or nofldrxq to fit 1350 * what's available to us. 1351 */ 1352 iaq->nirq = T4_EXTRA_INTR; 1353 iaq->nirq += n10g + n1g; 1354 if (iaq->nirq <= navail) { 1355 int leftover = navail - iaq->nirq; 1356 1357 if (n10g > 0) { 1358 int target = max(nrxq10g, nofldrxq10g); 1359 1360 n = 1; 1361 while (n < target && leftover >= n10g) { 1362 leftover -= n10g; 1363 iaq->nirq += n10g; 1364 n++; 1365 } 1366 iaq->nrxq10g = min(n, nrxq10g); 1367 #ifndef TCP_OFFLOAD_DISABLE 1368 iaq->nofldrxq10g = min(n, nofldrxq10g); 1369 #endif 1370 } 1371 1372 if (n1g > 0) { 1373 int target = max(nrxq1g, nofldrxq1g); 1374 1375 n = 1; 1376 while (n < target && leftover >= n1g) { 1377 leftover -= n1g; 1378 iaq->nirq += n1g; 1379 n++; 1380 } 1381 iaq->nrxq1g = min(n, nrxq1g); 1382 #ifndef TCP_OFFLOAD_DISABLE 1383 iaq->nofldrxq1g = min(n, nofldrxq1g); 1384 #endif 1385 } 1386 1387 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1388 goto allocate; 1389 } 1390 1391 /* 1392 * Least desirable option: one interrupt vector for everything. 1393 */ 1394 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1395 #ifndef TCP_OFFLOAD_DISABLE 1396 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1397 #endif 1398 1399 allocate: 1400 navail = iaq->nirq; 1401 rc = 0; 1402 if (itype == INTR_MSIX) 1403 rc = pci_alloc_msix(sc->dev, &navail); 1404 else if (itype == INTR_MSI) 1405 rc = pci_alloc_msi(sc->dev, &navail); 1406 1407 if (rc == 0) { 1408 if (navail == iaq->nirq) 1409 return (0); 1410 1411 /* 1412 * Didn't get the number requested. Use whatever number 1413 * the kernel is willing to allocate (it's in navail). 1414 */ 1415 device_printf(sc->dev, "fewer vectors than requested, " 1416 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1417 itype, iaq->nirq, navail); 1418 pci_release_msi(sc->dev); 1419 goto restart; 1420 } 1421 1422 device_printf(sc->dev, 1423 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1424 itype, rc, iaq->nirq, navail); 1425 } 1426 1427 device_printf(sc->dev, 1428 "failed to find a usable interrupt type. " 1429 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1430 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1431 1432 return (ENXIO); 1433 } 1434 1435 /* 1436 * Install a compatible firmware (if required), establish contact with it (by 1437 * saying hello), and reset the device. If we end up as the master driver, 1438 * partition adapter resources by providing a configuration file to the 1439 * firmware. 1440 */ 1441 static int 1442 prep_firmware(struct adapter *sc) 1443 { 1444 const struct firmware *fw = NULL, *cfg = NULL, *default_cfg; 1445 int rc; 1446 enum dev_state state; 1447 1448 default_cfg = firmware_get(T4_CFGNAME); 1449 1450 /* Check firmware version and install a different one if necessary */ 1451 rc = t4_check_fw_version(sc); 1452 if (rc != 0) { 1453 uint32_t v = 0; 1454 1455 fw = firmware_get(T4_FWNAME); 1456 if (fw != NULL) { 1457 const struct fw_hdr *hdr = (const void *)fw->data; 1458 1459 v = ntohl(hdr->fw_ver); 1460 1461 /* 1462 * The firmware module will not be used if it isn't the 1463 * same major version as what the driver was compiled 1464 * with. 1465 */ 1466 if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) { 1467 device_printf(sc->dev, 1468 "Found firmware image but version %d " 1469 "can not be used with this driver (%d)\n", 1470 G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR); 1471 1472 firmware_put(fw, FIRMWARE_UNLOAD); 1473 fw = NULL; 1474 } 1475 } 1476 1477 if (fw == NULL && rc < 0) { 1478 device_printf(sc->dev, "No usable firmware. " 1479 "card has %d.%d.%d, driver compiled with %d.%d.%d", 1480 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1481 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1482 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1483 FW_VERSION_MAJOR, FW_VERSION_MINOR, 1484 FW_VERSION_MICRO); 1485 rc = EAGAIN; 1486 goto done; 1487 } 1488 1489 /* 1490 * Always upgrade, even for minor/micro/build mismatches. 1491 * Downgrade only for a major version mismatch or if 1492 * force_firmware_install was specified. 1493 */ 1494 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) { 1495 device_printf(sc->dev, 1496 "installing firmware %d.%d.%d.%d on card.\n", 1497 G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v), 1498 G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v)); 1499 1500 rc = -t4_load_fw(sc, fw->data, fw->datasize); 1501 if (rc != 0) { 1502 device_printf(sc->dev, 1503 "failed to install firmware: %d\n", rc); 1504 goto done; 1505 } else { 1506 /* refresh */ 1507 (void) t4_check_fw_version(sc); 1508 } 1509 } 1510 } 1511 1512 /* Contact firmware. */ 1513 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1514 if (rc < 0) { 1515 rc = -rc; 1516 device_printf(sc->dev, 1517 "failed to connect to the firmware: %d.\n", rc); 1518 goto done; 1519 } 1520 if (rc == sc->mbox) 1521 sc->flags |= MASTER_PF; 1522 1523 /* Reset device */ 1524 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1525 if (rc != 0) { 1526 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 1527 if (rc != ETIMEDOUT && rc != EIO) 1528 t4_fw_bye(sc, sc->mbox); 1529 goto done; 1530 } 1531 1532 /* Partition adapter resources as specified in the config file. */ 1533 if (sc->flags & MASTER_PF) { 1534 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) { 1535 char s[32]; 1536 1537 snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file); 1538 cfg = firmware_get(s); 1539 if (cfg == NULL) { 1540 device_printf(sc->dev, 1541 "unable to locate %s module, " 1542 "will use default config file.\n", s); 1543 } 1544 } 1545 1546 rc = partition_resources(sc, cfg ? cfg : default_cfg); 1547 if (rc != 0) 1548 goto done; /* error message displayed already */ 1549 } 1550 1551 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 1552 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1553 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1554 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1555 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1556 sc->flags |= FW_OK; 1557 1558 done: 1559 if (fw != NULL) 1560 firmware_put(fw, FIRMWARE_UNLOAD); 1561 if (cfg != NULL) 1562 firmware_put(cfg, FIRMWARE_UNLOAD); 1563 if (default_cfg != NULL) 1564 firmware_put(default_cfg, FIRMWARE_UNLOAD); 1565 1566 return (rc); 1567 } 1568 1569 #define FW_PARAM_DEV(param) \ 1570 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1571 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1572 #define FW_PARAM_PFVF(param) \ 1573 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1574 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1575 1576 /* 1577 * Upload configuration file to card's memory. 1578 */ 1579 static int 1580 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt, 1581 uint32_t *ma) 1582 { 1583 int rc, i; 1584 uint32_t param, val, mtype, maddr, bar, off, win, remaining; 1585 const uint32_t *b; 1586 1587 /* Figure out where the firmware wants us to upload it. */ 1588 param = FW_PARAM_DEV(CF); 1589 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1590 if (rc != 0) { 1591 /* Firmwares without config file support will fail this way */ 1592 device_printf(sc->dev, 1593 "failed to query config file location: %d.\n", rc); 1594 return (rc); 1595 } 1596 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1597 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1598 1599 if (maddr & 3) { 1600 device_printf(sc->dev, 1601 "cannot upload config file (type %u, addr %x).\n", 1602 mtype, maddr); 1603 return (EFAULT); 1604 } 1605 1606 /* Translate mtype/maddr to an address suitable for the PCIe window */ 1607 val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1608 val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE; 1609 switch (mtype) { 1610 case FW_MEMTYPE_CF_EDC0: 1611 if (!(val & F_EDRAM0_ENABLE)) 1612 goto err; 1613 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1614 maddr += G_EDRAM0_BASE(bar) << 20; 1615 break; 1616 1617 case FW_MEMTYPE_CF_EDC1: 1618 if (!(val & F_EDRAM1_ENABLE)) 1619 goto err; 1620 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1621 maddr += G_EDRAM1_BASE(bar) << 20; 1622 break; 1623 1624 case FW_MEMTYPE_CF_EXTMEM: 1625 if (!(val & F_EXT_MEM_ENABLE)) 1626 goto err; 1627 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1628 maddr += G_EXT_MEM_BASE(bar) << 20; 1629 break; 1630 1631 default: 1632 err: 1633 device_printf(sc->dev, 1634 "cannot upload config file (type %u, enabled %u).\n", 1635 mtype, val); 1636 return (EFAULT); 1637 } 1638 1639 /* 1640 * Position the PCIe window (we use memwin2) to the 16B aligned area 1641 * just at/before the upload location. 1642 */ 1643 win = maddr & ~0xf; 1644 off = maddr - win; /* offset from the start of the window. */ 1645 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 1646 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 1647 1648 remaining = fw->datasize; 1649 if (remaining > FLASH_CFG_MAX_SIZE || 1650 remaining > MEMWIN2_APERTURE - off) { 1651 device_printf(sc->dev, "cannot upload config file all at once " 1652 "(size %u, max %u, room %u).\n", 1653 remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off); 1654 return (EFBIG); 1655 } 1656 1657 /* 1658 * XXX: sheer laziness. We deliberately added 4 bytes of useless 1659 * stuffing/comments at the end of the config file so it's ok to simply 1660 * throw away the last remaining bytes when the config file is not an 1661 * exact multiple of 4. 1662 */ 1663 b = fw->data; 1664 for (i = 0; remaining >= 4; i += 4, remaining -= 4) 1665 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++); 1666 1667 return (rc); 1668 } 1669 1670 /* 1671 * Partition chip resources for use between various PFs, VFs, etc. This is done 1672 * by uploading the firmware configuration file to the adapter and instructing 1673 * the firmware to process it. 1674 */ 1675 static int 1676 partition_resources(struct adapter *sc, const struct firmware *cfg) 1677 { 1678 int rc; 1679 struct fw_caps_config_cmd caps; 1680 uint32_t mtype, maddr, finicsum, cfcsum; 1681 1682 rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT; 1683 if (rc != 0) { 1684 mtype = FW_MEMTYPE_CF_FLASH; 1685 maddr = t4_flash_cfg_addr(sc); 1686 } 1687 1688 bzero(&caps, sizeof(caps)); 1689 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1690 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1691 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1692 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1693 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1694 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1695 if (rc != 0) { 1696 device_printf(sc->dev, 1697 "failed to pre-process config file: %d.\n", rc); 1698 return (rc); 1699 } 1700 1701 finicsum = be32toh(caps.finicsum); 1702 cfcsum = be32toh(caps.cfcsum); 1703 if (finicsum != cfcsum) { 1704 device_printf(sc->dev, 1705 "WARNING: config file checksum mismatch: %08x %08x\n", 1706 finicsum, cfcsum); 1707 } 1708 sc->cfcsum = cfcsum; 1709 1710 #define LIMIT_CAPS(x) do { \ 1711 caps.x &= htobe16(t4_##x##_allowed); \ 1712 sc->x = htobe16(caps.x); \ 1713 } while (0) 1714 1715 /* 1716 * Let the firmware know what features will (not) be used so it can tune 1717 * things accordingly. 1718 */ 1719 LIMIT_CAPS(linkcaps); 1720 LIMIT_CAPS(niccaps); 1721 LIMIT_CAPS(toecaps); 1722 LIMIT_CAPS(rdmacaps); 1723 LIMIT_CAPS(iscsicaps); 1724 LIMIT_CAPS(fcoecaps); 1725 #undef LIMIT_CAPS 1726 1727 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1728 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1729 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1730 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 1731 if (rc != 0) { 1732 device_printf(sc->dev, 1733 "failed to process config file: %d.\n", rc); 1734 return (rc); 1735 } 1736 1737 return (0); 1738 } 1739 1740 /* 1741 * Retrieve parameters that are needed (or nice to have) prior to calling 1742 * t4_sge_init and t4_fw_initialize. 1743 */ 1744 static int 1745 get_params__pre_init(struct adapter *sc) 1746 { 1747 int rc; 1748 uint32_t param[2], val[2]; 1749 struct fw_devlog_cmd cmd; 1750 struct devlog_params *dlog = &sc->params.devlog; 1751 1752 param[0] = FW_PARAM_DEV(PORTVEC); 1753 param[1] = FW_PARAM_DEV(CCLK); 1754 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1755 if (rc != 0) { 1756 device_printf(sc->dev, 1757 "failed to query parameters (pre_init): %d.\n", rc); 1758 return (rc); 1759 } 1760 1761 sc->params.portvec = val[0]; 1762 sc->params.nports = 0; 1763 while (val[0]) { 1764 sc->params.nports++; 1765 val[0] &= val[0] - 1; 1766 } 1767 1768 sc->params.vpd.cclk = val[1]; 1769 1770 /* Read device log parameters. */ 1771 bzero(&cmd, sizeof(cmd)); 1772 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1773 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1774 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 1775 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 1776 if (rc != 0) { 1777 device_printf(sc->dev, 1778 "failed to get devlog parameters: %d.\n", rc); 1779 bzero(dlog, sizeof (*dlog)); 1780 rc = 0; /* devlog isn't critical for device operation */ 1781 } else { 1782 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 1783 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1784 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1785 dlog->size = be32toh(cmd.memsize_devlog); 1786 } 1787 1788 return (rc); 1789 } 1790 1791 /* 1792 * Retrieve various parameters that are of interest to the driver. The device 1793 * has been initialized by the firmware at this point. 1794 */ 1795 static int 1796 get_params__post_init(struct adapter *sc) 1797 { 1798 int rc; 1799 uint32_t param[7], val[7]; 1800 struct fw_caps_config_cmd caps; 1801 1802 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1803 param[1] = FW_PARAM_PFVF(EQ_START); 1804 param[2] = FW_PARAM_PFVF(FILTER_START); 1805 param[3] = FW_PARAM_PFVF(FILTER_END); 1806 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); 1807 if (rc != 0) { 1808 device_printf(sc->dev, 1809 "failed to query parameters (post_init): %d.\n", rc); 1810 return (rc); 1811 } 1812 1813 sc->sge.iq_start = val[0]; 1814 sc->sge.eq_start = val[1]; 1815 sc->tids.ftid_base = val[2]; 1816 sc->tids.nftids = val[3] - val[2] + 1; 1817 1818 /* get capabilites */ 1819 bzero(&caps, sizeof(caps)); 1820 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1821 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1822 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1823 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1824 if (rc != 0) { 1825 device_printf(sc->dev, 1826 "failed to get card capabilities: %d.\n", rc); 1827 return (rc); 1828 } 1829 1830 if (caps.toecaps) { 1831 /* query offload-related parameters */ 1832 param[0] = FW_PARAM_DEV(NTID); 1833 param[1] = FW_PARAM_PFVF(SERVER_START); 1834 param[2] = FW_PARAM_PFVF(SERVER_END); 1835 param[3] = FW_PARAM_PFVF(TDDP_START); 1836 param[4] = FW_PARAM_PFVF(TDDP_END); 1837 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1838 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1839 if (rc != 0) { 1840 device_printf(sc->dev, 1841 "failed to query TOE parameters: %d.\n", rc); 1842 return (rc); 1843 } 1844 sc->tids.ntids = val[0]; 1845 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1846 sc->tids.stid_base = val[1]; 1847 sc->tids.nstids = val[2] - val[1] + 1; 1848 sc->vres.ddp.start = val[3]; 1849 sc->vres.ddp.size = val[4] - val[3] + 1; 1850 sc->params.ofldq_wr_cred = val[5]; 1851 sc->params.offload = 1; 1852 } 1853 if (caps.rdmacaps) { 1854 param[0] = FW_PARAM_PFVF(STAG_START); 1855 param[1] = FW_PARAM_PFVF(STAG_END); 1856 param[2] = FW_PARAM_PFVF(RQ_START); 1857 param[3] = FW_PARAM_PFVF(RQ_END); 1858 param[4] = FW_PARAM_PFVF(PBL_START); 1859 param[5] = FW_PARAM_PFVF(PBL_END); 1860 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1861 if (rc != 0) { 1862 device_printf(sc->dev, 1863 "failed to query RDMA parameters(1): %d.\n", rc); 1864 return (rc); 1865 } 1866 sc->vres.stag.start = val[0]; 1867 sc->vres.stag.size = val[1] - val[0] + 1; 1868 sc->vres.rq.start = val[2]; 1869 sc->vres.rq.size = val[3] - val[2] + 1; 1870 sc->vres.pbl.start = val[4]; 1871 sc->vres.pbl.size = val[5] - val[4] + 1; 1872 1873 param[0] = FW_PARAM_PFVF(SQRQ_START); 1874 param[1] = FW_PARAM_PFVF(SQRQ_END); 1875 param[2] = FW_PARAM_PFVF(CQ_START); 1876 param[3] = FW_PARAM_PFVF(CQ_END); 1877 param[4] = FW_PARAM_PFVF(OCQ_START); 1878 param[5] = FW_PARAM_PFVF(OCQ_END); 1879 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 1880 if (rc != 0) { 1881 device_printf(sc->dev, 1882 "failed to query RDMA parameters(2): %d.\n", rc); 1883 return (rc); 1884 } 1885 sc->vres.qp.start = val[0]; 1886 sc->vres.qp.size = val[1] - val[0] + 1; 1887 sc->vres.cq.start = val[2]; 1888 sc->vres.cq.size = val[3] - val[2] + 1; 1889 sc->vres.ocq.start = val[4]; 1890 sc->vres.ocq.size = val[5] - val[4] + 1; 1891 } 1892 if (caps.iscsicaps) { 1893 param[0] = FW_PARAM_PFVF(ISCSI_START); 1894 param[1] = FW_PARAM_PFVF(ISCSI_END); 1895 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1896 if (rc != 0) { 1897 device_printf(sc->dev, 1898 "failed to query iSCSI parameters: %d.\n", rc); 1899 return (rc); 1900 } 1901 sc->vres.iscsi.start = val[0]; 1902 sc->vres.iscsi.size = val[1] - val[0] + 1; 1903 } 1904 1905 /* These are finalized by FW initialization, load their values now */ 1906 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1907 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1908 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1909 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1910 1911 return (rc); 1912 } 1913 1914 #undef FW_PARAM_PFVF 1915 #undef FW_PARAM_DEV 1916 1917 static void 1918 t4_set_desc(struct adapter *sc) 1919 { 1920 char buf[128]; 1921 struct adapter_params *p = &sc->params; 1922 1923 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 1924 p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec); 1925 1926 device_set_desc_copy(sc->dev, buf); 1927 } 1928 1929 static void 1930 build_medialist(struct port_info *pi) 1931 { 1932 struct ifmedia *media = &pi->media; 1933 int data, m; 1934 1935 PORT_LOCK(pi); 1936 1937 ifmedia_removeall(media); 1938 1939 m = IFM_ETHER | IFM_FDX; 1940 data = (pi->port_type << 8) | pi->mod_type; 1941 1942 switch(pi->port_type) { 1943 case FW_PORT_TYPE_BT_XFI: 1944 ifmedia_add(media, m | IFM_10G_T, data, NULL); 1945 break; 1946 1947 case FW_PORT_TYPE_BT_XAUI: 1948 ifmedia_add(media, m | IFM_10G_T, data, NULL); 1949 /* fall through */ 1950 1951 case FW_PORT_TYPE_BT_SGMII: 1952 ifmedia_add(media, m | IFM_1000_T, data, NULL); 1953 ifmedia_add(media, m | IFM_100_TX, data, NULL); 1954 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 1955 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 1956 break; 1957 1958 case FW_PORT_TYPE_CX4: 1959 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 1960 ifmedia_set(media, m | IFM_10G_CX4); 1961 break; 1962 1963 case FW_PORT_TYPE_SFP: 1964 case FW_PORT_TYPE_FIBER_XFI: 1965 case FW_PORT_TYPE_FIBER_XAUI: 1966 switch (pi->mod_type) { 1967 1968 case FW_PORT_MOD_TYPE_LR: 1969 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 1970 ifmedia_set(media, m | IFM_10G_LR); 1971 break; 1972 1973 case FW_PORT_MOD_TYPE_SR: 1974 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 1975 ifmedia_set(media, m | IFM_10G_SR); 1976 break; 1977 1978 case FW_PORT_MOD_TYPE_LRM: 1979 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 1980 ifmedia_set(media, m | IFM_10G_LRM); 1981 break; 1982 1983 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 1984 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 1985 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 1986 ifmedia_set(media, m | IFM_10G_TWINAX); 1987 break; 1988 1989 case FW_PORT_MOD_TYPE_NONE: 1990 m &= ~IFM_FDX; 1991 ifmedia_add(media, m | IFM_NONE, data, NULL); 1992 ifmedia_set(media, m | IFM_NONE); 1993 break; 1994 1995 case FW_PORT_MOD_TYPE_NA: 1996 case FW_PORT_MOD_TYPE_ER: 1997 default: 1998 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 1999 ifmedia_set(media, m | IFM_UNKNOWN); 2000 break; 2001 } 2002 break; 2003 2004 case FW_PORT_TYPE_KX4: 2005 case FW_PORT_TYPE_KX: 2006 case FW_PORT_TYPE_KR: 2007 default: 2008 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2009 ifmedia_set(media, m | IFM_UNKNOWN); 2010 break; 2011 } 2012 2013 PORT_UNLOCK(pi); 2014 } 2015 2016 #define FW_MAC_EXACT_CHUNK 7 2017 2018 /* 2019 * Program the port's XGMAC based on parameters in ifnet. The caller also 2020 * indicates which parameters should be programmed (the rest are left alone). 2021 */ 2022 static int 2023 update_mac_settings(struct port_info *pi, int flags) 2024 { 2025 int rc; 2026 struct ifnet *ifp = pi->ifp; 2027 struct adapter *sc = pi->adapter; 2028 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2029 2030 PORT_LOCK_ASSERT_OWNED(pi); 2031 KASSERT(flags, ("%s: not told what to update.", __func__)); 2032 2033 if (flags & XGMAC_MTU) 2034 mtu = ifp->if_mtu; 2035 2036 if (flags & XGMAC_PROMISC) 2037 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2038 2039 if (flags & XGMAC_ALLMULTI) 2040 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2041 2042 if (flags & XGMAC_VLANEX) 2043 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2044 2045 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2046 vlanex, false); 2047 if (rc) { 2048 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2049 return (rc); 2050 } 2051 2052 if (flags & XGMAC_UCADDR) { 2053 uint8_t ucaddr[ETHER_ADDR_LEN]; 2054 2055 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2056 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2057 ucaddr, true, true); 2058 if (rc < 0) { 2059 rc = -rc; 2060 if_printf(ifp, "change_mac failed: %d\n", rc); 2061 return (rc); 2062 } else { 2063 pi->xact_addr_filt = rc; 2064 rc = 0; 2065 } 2066 } 2067 2068 if (flags & XGMAC_MCADDRS) { 2069 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2070 int del = 1; 2071 uint64_t hash = 0; 2072 struct ifmultiaddr *ifma; 2073 int i = 0, j; 2074 2075 if_maddr_rlock(ifp); 2076 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2077 if (ifma->ifma_addr->sa_family == AF_LINK) 2078 continue; 2079 mcaddr[i++] = 2080 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2081 2082 if (i == FW_MAC_EXACT_CHUNK) { 2083 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2084 del, i, mcaddr, NULL, &hash, 0); 2085 if (rc < 0) { 2086 rc = -rc; 2087 for (j = 0; j < i; j++) { 2088 if_printf(ifp, 2089 "failed to add mc address" 2090 " %02x:%02x:%02x:" 2091 "%02x:%02x:%02x rc=%d\n", 2092 mcaddr[j][0], mcaddr[j][1], 2093 mcaddr[j][2], mcaddr[j][3], 2094 mcaddr[j][4], mcaddr[j][5], 2095 rc); 2096 } 2097 goto mcfail; 2098 } 2099 del = 0; 2100 i = 0; 2101 } 2102 } 2103 if (i > 0) { 2104 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2105 del, i, mcaddr, NULL, &hash, 0); 2106 if (rc < 0) { 2107 rc = -rc; 2108 for (j = 0; j < i; j++) { 2109 if_printf(ifp, 2110 "failed to add mc address" 2111 " %02x:%02x:%02x:" 2112 "%02x:%02x:%02x rc=%d\n", 2113 mcaddr[j][0], mcaddr[j][1], 2114 mcaddr[j][2], mcaddr[j][3], 2115 mcaddr[j][4], mcaddr[j][5], 2116 rc); 2117 } 2118 goto mcfail; 2119 } 2120 } 2121 2122 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2123 if (rc != 0) 2124 if_printf(ifp, "failed to set mc address hash: %d", rc); 2125 mcfail: 2126 if_maddr_runlock(ifp); 2127 } 2128 2129 return (rc); 2130 } 2131 2132 static int 2133 cxgbe_init_locked(struct port_info *pi) 2134 { 2135 struct adapter *sc = pi->adapter; 2136 int rc = 0; 2137 2138 ADAPTER_LOCK_ASSERT_OWNED(sc); 2139 2140 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2141 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) { 2142 rc = EINTR; 2143 goto done; 2144 } 2145 } 2146 if (IS_DOOMED(pi)) { 2147 rc = ENXIO; 2148 goto done; 2149 } 2150 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2151 2152 /* Give up the adapter lock, port init code can sleep. */ 2153 SET_BUSY(sc); 2154 ADAPTER_UNLOCK(sc); 2155 2156 rc = cxgbe_init_synchronized(pi); 2157 2158 done: 2159 ADAPTER_LOCK(sc); 2160 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2161 CLR_BUSY(sc); 2162 wakeup_one(&sc->flags); 2163 ADAPTER_UNLOCK(sc); 2164 return (rc); 2165 } 2166 2167 static int 2168 cxgbe_init_synchronized(struct port_info *pi) 2169 { 2170 struct adapter *sc = pi->adapter; 2171 struct ifnet *ifp = pi->ifp; 2172 int rc = 0; 2173 2174 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2175 2176 if (isset(&sc->open_device_map, pi->port_id)) { 2177 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2178 ("mismatch between open_device_map and if_drv_flags")); 2179 return (0); /* already running */ 2180 } 2181 2182 if (!(sc->flags & FULL_INIT_DONE) && 2183 ((rc = adapter_full_init(sc)) != 0)) 2184 return (rc); /* error message displayed already */ 2185 2186 if (!(pi->flags & PORT_INIT_DONE) && 2187 ((rc = port_full_init(pi)) != 0)) 2188 return (rc); /* error message displayed already */ 2189 2190 PORT_LOCK(pi); 2191 rc = update_mac_settings(pi, XGMAC_ALL); 2192 PORT_UNLOCK(pi); 2193 if (rc) 2194 goto done; /* error message displayed already */ 2195 2196 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2197 if (rc != 0) { 2198 if_printf(ifp, "start_link failed: %d\n", rc); 2199 goto done; 2200 } 2201 2202 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2203 if (rc != 0) { 2204 if_printf(ifp, "enable_vi failed: %d\n", rc); 2205 goto done; 2206 } 2207 2208 /* all ok */ 2209 setbit(&sc->open_device_map, pi->port_id); 2210 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2211 2212 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2213 done: 2214 if (rc != 0) 2215 cxgbe_uninit_synchronized(pi); 2216 2217 return (rc); 2218 } 2219 2220 static int 2221 cxgbe_uninit_locked(struct port_info *pi) 2222 { 2223 struct adapter *sc = pi->adapter; 2224 int rc; 2225 2226 ADAPTER_LOCK_ASSERT_OWNED(sc); 2227 2228 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2229 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) { 2230 rc = EINTR; 2231 goto done; 2232 } 2233 } 2234 if (IS_DOOMED(pi)) { 2235 rc = ENXIO; 2236 goto done; 2237 } 2238 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2239 SET_BUSY(sc); 2240 ADAPTER_UNLOCK(sc); 2241 2242 rc = cxgbe_uninit_synchronized(pi); 2243 2244 ADAPTER_LOCK(sc); 2245 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2246 CLR_BUSY(sc); 2247 wakeup_one(&sc->flags); 2248 done: 2249 ADAPTER_UNLOCK(sc); 2250 return (rc); 2251 } 2252 2253 /* 2254 * Idempotent. 2255 */ 2256 static int 2257 cxgbe_uninit_synchronized(struct port_info *pi) 2258 { 2259 struct adapter *sc = pi->adapter; 2260 struct ifnet *ifp = pi->ifp; 2261 int rc; 2262 2263 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2264 2265 /* 2266 * Disable the VI so that all its data in either direction is discarded 2267 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2268 * tick) intact as the TP can deliver negative advice or data that it's 2269 * holding in its RAM (for an offloaded connection) even after the VI is 2270 * disabled. 2271 */ 2272 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2273 if (rc) { 2274 if_printf(ifp, "disable_vi failed: %d\n", rc); 2275 return (rc); 2276 } 2277 2278 clrbit(&sc->open_device_map, pi->port_id); 2279 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2280 2281 pi->link_cfg.link_ok = 0; 2282 pi->link_cfg.speed = 0; 2283 t4_os_link_changed(sc, pi->port_id, 0); 2284 2285 return (0); 2286 } 2287 2288 #define T4_ALLOC_IRQ(sc, irq, rid, handler, arg, name) do { \ 2289 rc = t4_alloc_irq(sc, irq, rid, handler, arg, name); \ 2290 if (rc != 0) \ 2291 goto done; \ 2292 } while (0) 2293 2294 static int 2295 adapter_full_init(struct adapter *sc) 2296 { 2297 int rc, i, rid, p, q; 2298 char s[8]; 2299 struct irq *irq; 2300 struct port_info *pi; 2301 struct sge_rxq *rxq; 2302 #ifndef TCP_OFFLOAD_DISABLE 2303 struct sge_ofld_rxq *ofld_rxq; 2304 #endif 2305 2306 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2307 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2308 ("%s: FULL_INIT_DONE already", __func__)); 2309 2310 /* 2311 * queues that belong to the adapter (not any particular port). 2312 */ 2313 rc = t4_setup_adapter_queues(sc); 2314 if (rc != 0) 2315 goto done; 2316 2317 for (i = 0; i < ARRAY_SIZE(sc->tq); i++) { 2318 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 2319 taskqueue_thread_enqueue, &sc->tq[i]); 2320 if (sc->tq[i] == NULL) { 2321 device_printf(sc->dev, 2322 "failed to allocate task queue %d\n", i); 2323 rc = ENOMEM; 2324 goto done; 2325 } 2326 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 2327 device_get_nameunit(sc->dev), i); 2328 } 2329 2330 /* 2331 * Setup interrupts. 2332 */ 2333 irq = &sc->irq[0]; 2334 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2335 if (sc->intr_count == 1) { 2336 KASSERT(!(sc->flags & INTR_DIRECT), 2337 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2338 2339 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_all, sc, "all"); 2340 } else { 2341 /* Multiple interrupts. */ 2342 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2343 ("%s: too few intr.", __func__)); 2344 2345 /* The first one is always error intr */ 2346 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_err, sc, "err"); 2347 irq++; 2348 rid++; 2349 2350 /* The second one is always the firmware event queue */ 2351 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 2352 irq++; 2353 rid++; 2354 2355 /* 2356 * Note that if INTR_DIRECT is not set then either the NIC rx 2357 * queues or (exclusive or) the TOE rx queueus will be taking 2358 * direct interrupts. 2359 * 2360 * There is no need to check for is_offload(sc) as nofldrxq 2361 * will be 0 if offload is disabled. 2362 */ 2363 for_each_port(sc, p) { 2364 pi = sc->port[p]; 2365 2366 #ifndef TCP_OFFLOAD_DISABLE 2367 /* 2368 * Skip over the NIC queues if they aren't taking direct 2369 * interrupts. 2370 */ 2371 if (!(sc->flags & INTR_DIRECT) && 2372 pi->nofldrxq > pi->nrxq) 2373 goto ofld_queues; 2374 #endif 2375 rxq = &sc->sge.rxq[pi->first_rxq]; 2376 for (q = 0; q < pi->nrxq; q++, rxq++) { 2377 snprintf(s, sizeof(s), "%d.%d", p, q); 2378 T4_ALLOC_IRQ(sc, irq, rid, t4_intr, rxq, s); 2379 irq++; 2380 rid++; 2381 } 2382 2383 #ifndef TCP_OFFLOAD_DISABLE 2384 /* 2385 * Skip over the offload queues if they aren't taking 2386 * direct interrupts. 2387 */ 2388 if (!(sc->flags & INTR_DIRECT)) 2389 continue; 2390 ofld_queues: 2391 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2392 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2393 snprintf(s, sizeof(s), "%d,%d", p, q); 2394 T4_ALLOC_IRQ(sc, irq, rid, t4_intr, ofld_rxq, s); 2395 irq++; 2396 rid++; 2397 } 2398 #endif 2399 } 2400 } 2401 2402 t4_intr_enable(sc); 2403 sc->flags |= FULL_INIT_DONE; 2404 done: 2405 if (rc != 0) 2406 adapter_full_uninit(sc); 2407 2408 return (rc); 2409 } 2410 #undef T4_ALLOC_IRQ 2411 2412 static int 2413 adapter_full_uninit(struct adapter *sc) 2414 { 2415 int i; 2416 2417 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2418 2419 t4_teardown_adapter_queues(sc); 2420 2421 for (i = 0; i < sc->intr_count; i++) 2422 t4_free_irq(sc, &sc->irq[i]); 2423 2424 for (i = 0; i < ARRAY_SIZE(sc->tq) && sc->tq[i]; i++) { 2425 taskqueue_free(sc->tq[i]); 2426 sc->tq[i] = NULL; 2427 } 2428 2429 sc->flags &= ~FULL_INIT_DONE; 2430 2431 return (0); 2432 } 2433 2434 static int 2435 port_full_init(struct port_info *pi) 2436 { 2437 struct adapter *sc = pi->adapter; 2438 struct ifnet *ifp = pi->ifp; 2439 uint16_t *rss; 2440 struct sge_rxq *rxq; 2441 int rc, i; 2442 2443 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2444 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 2445 ("%s: PORT_INIT_DONE already", __func__)); 2446 2447 sysctl_ctx_init(&pi->ctx); 2448 pi->flags |= PORT_SYSCTL_CTX; 2449 2450 /* 2451 * Allocate tx/rx/fl queues for this port. 2452 */ 2453 rc = t4_setup_port_queues(pi); 2454 if (rc != 0) 2455 goto done; /* error message displayed already */ 2456 2457 /* 2458 * Setup RSS for this port. 2459 */ 2460 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 2461 M_ZERO | M_WAITOK); 2462 for_each_rxq(pi, i, rxq) { 2463 rss[i] = rxq->iq.abs_id; 2464 } 2465 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2466 pi->rss_size, rss, pi->nrxq); 2467 free(rss, M_CXGBE); 2468 if (rc != 0) { 2469 if_printf(ifp, "rss_config failed: %d\n", rc); 2470 goto done; 2471 } 2472 2473 pi->flags |= PORT_INIT_DONE; 2474 done: 2475 if (rc != 0) 2476 port_full_uninit(pi); 2477 2478 return (rc); 2479 } 2480 2481 /* 2482 * Idempotent. 2483 */ 2484 static int 2485 port_full_uninit(struct port_info *pi) 2486 { 2487 struct adapter *sc = pi->adapter; 2488 int i; 2489 struct sge_rxq *rxq; 2490 struct sge_txq *txq; 2491 #ifndef TCP_OFFLOAD_DISABLE 2492 struct sge_ofld_rxq *ofld_rxq; 2493 struct sge_wrq *ofld_txq; 2494 #endif 2495 2496 if (pi->flags & PORT_INIT_DONE) { 2497 2498 /* Need to quiesce queues. XXX: ctrl queues? */ 2499 2500 for_each_txq(pi, i, txq) { 2501 quiesce_eq(sc, &txq->eq); 2502 } 2503 2504 #ifndef TCP_OFFLOAD_DISABLE 2505 for_each_ofld_txq(pi, i, ofld_txq) { 2506 quiesce_eq(sc, &ofld_txq->eq); 2507 } 2508 #endif 2509 2510 for_each_rxq(pi, i, rxq) { 2511 quiesce_iq(sc, &rxq->iq); 2512 quiesce_fl(sc, &rxq->fl); 2513 } 2514 2515 #ifndef TCP_OFFLOAD_DISABLE 2516 for_each_ofld_rxq(pi, i, ofld_rxq) { 2517 quiesce_iq(sc, &ofld_rxq->iq); 2518 quiesce_fl(sc, &ofld_rxq->fl); 2519 } 2520 #endif 2521 } 2522 2523 t4_teardown_port_queues(pi); 2524 pi->flags &= ~PORT_INIT_DONE; 2525 2526 return (0); 2527 } 2528 2529 static void 2530 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 2531 { 2532 EQ_LOCK(eq); 2533 eq->flags |= EQ_DOOMED; 2534 2535 /* 2536 * Wait for the response to a credit flush if one's 2537 * pending. 2538 */ 2539 while (eq->flags & EQ_CRFLUSHED) 2540 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 2541 EQ_UNLOCK(eq); 2542 2543 callout_drain(&eq->tx_callout); /* XXX: iffy */ 2544 pause("callout", 10); /* Still iffy */ 2545 2546 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 2547 } 2548 2549 static void 2550 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 2551 { 2552 (void) sc; /* unused */ 2553 2554 /* Synchronize with the interrupt handler */ 2555 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 2556 pause("iqfree", 1); 2557 } 2558 2559 static void 2560 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 2561 { 2562 mtx_lock(&sc->sfl_lock); 2563 FL_LOCK(fl); 2564 fl->flags |= FL_DOOMED; 2565 FL_UNLOCK(fl); 2566 mtx_unlock(&sc->sfl_lock); 2567 2568 callout_drain(&sc->sfl_callout); 2569 KASSERT((fl->flags & FL_STARVING) == 0, 2570 ("%s: still starving", __func__)); 2571 } 2572 2573 static int 2574 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 2575 driver_intr_t *handler, void *arg, char *name) 2576 { 2577 int rc; 2578 2579 irq->rid = rid; 2580 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 2581 RF_SHAREABLE | RF_ACTIVE); 2582 if (irq->res == NULL) { 2583 device_printf(sc->dev, 2584 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 2585 return (ENOMEM); 2586 } 2587 2588 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 2589 NULL, handler, arg, &irq->tag); 2590 if (rc != 0) { 2591 device_printf(sc->dev, 2592 "failed to setup interrupt for rid %d, name %s: %d\n", 2593 rid, name, rc); 2594 } else if (name) 2595 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 2596 2597 return (rc); 2598 } 2599 2600 static int 2601 t4_free_irq(struct adapter *sc, struct irq *irq) 2602 { 2603 if (irq->tag) 2604 bus_teardown_intr(sc->dev, irq->res, irq->tag); 2605 if (irq->res) 2606 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 2607 2608 bzero(irq, sizeof(*irq)); 2609 2610 return (0); 2611 } 2612 2613 static void 2614 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 2615 unsigned int end) 2616 { 2617 uint32_t *p = (uint32_t *)(buf + start); 2618 2619 for ( ; start <= end; start += sizeof(uint32_t)) 2620 *p++ = t4_read_reg(sc, start); 2621 } 2622 2623 static void 2624 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 2625 { 2626 int i; 2627 static const unsigned int reg_ranges[] = { 2628 0x1008, 0x1108, 2629 0x1180, 0x11b4, 2630 0x11fc, 0x123c, 2631 0x1300, 0x173c, 2632 0x1800, 0x18fc, 2633 0x3000, 0x30d8, 2634 0x30e0, 0x5924, 2635 0x5960, 0x59d4, 2636 0x5a00, 0x5af8, 2637 0x6000, 0x6098, 2638 0x6100, 0x6150, 2639 0x6200, 0x6208, 2640 0x6240, 0x6248, 2641 0x6280, 0x6338, 2642 0x6370, 0x638c, 2643 0x6400, 0x643c, 2644 0x6500, 0x6524, 2645 0x6a00, 0x6a38, 2646 0x6a60, 0x6a78, 2647 0x6b00, 0x6b84, 2648 0x6bf0, 0x6c84, 2649 0x6cf0, 0x6d84, 2650 0x6df0, 0x6e84, 2651 0x6ef0, 0x6f84, 2652 0x6ff0, 0x7084, 2653 0x70f0, 0x7184, 2654 0x71f0, 0x7284, 2655 0x72f0, 0x7384, 2656 0x73f0, 0x7450, 2657 0x7500, 0x7530, 2658 0x7600, 0x761c, 2659 0x7680, 0x76cc, 2660 0x7700, 0x7798, 2661 0x77c0, 0x77fc, 2662 0x7900, 0x79fc, 2663 0x7b00, 0x7c38, 2664 0x7d00, 0x7efc, 2665 0x8dc0, 0x8e1c, 2666 0x8e30, 0x8e78, 2667 0x8ea0, 0x8f6c, 2668 0x8fc0, 0x9074, 2669 0x90fc, 0x90fc, 2670 0x9400, 0x9458, 2671 0x9600, 0x96bc, 2672 0x9800, 0x9808, 2673 0x9820, 0x983c, 2674 0x9850, 0x9864, 2675 0x9c00, 0x9c6c, 2676 0x9c80, 0x9cec, 2677 0x9d00, 0x9d6c, 2678 0x9d80, 0x9dec, 2679 0x9e00, 0x9e6c, 2680 0x9e80, 0x9eec, 2681 0x9f00, 0x9f6c, 2682 0x9f80, 0x9fec, 2683 0xd004, 0xd03c, 2684 0xdfc0, 0xdfe0, 2685 0xe000, 0xea7c, 2686 0xf000, 0x11190, 2687 0x19040, 0x19124, 2688 0x19150, 0x191b0, 2689 0x191d0, 0x191e8, 2690 0x19238, 0x1924c, 2691 0x193f8, 0x19474, 2692 0x19490, 0x194f8, 2693 0x19800, 0x19f30, 2694 0x1a000, 0x1a06c, 2695 0x1a0b0, 0x1a120, 2696 0x1a128, 0x1a138, 2697 0x1a190, 0x1a1c4, 2698 0x1a1fc, 0x1a1fc, 2699 0x1e040, 0x1e04c, 2700 0x1e240, 0x1e28c, 2701 0x1e2c0, 0x1e2c0, 2702 0x1e2e0, 0x1e2e0, 2703 0x1e300, 0x1e384, 2704 0x1e3c0, 0x1e3c8, 2705 0x1e440, 0x1e44c, 2706 0x1e640, 0x1e68c, 2707 0x1e6c0, 0x1e6c0, 2708 0x1e6e0, 0x1e6e0, 2709 0x1e700, 0x1e784, 2710 0x1e7c0, 0x1e7c8, 2711 0x1e840, 0x1e84c, 2712 0x1ea40, 0x1ea8c, 2713 0x1eac0, 0x1eac0, 2714 0x1eae0, 0x1eae0, 2715 0x1eb00, 0x1eb84, 2716 0x1ebc0, 0x1ebc8, 2717 0x1ec40, 0x1ec4c, 2718 0x1ee40, 0x1ee8c, 2719 0x1eec0, 0x1eec0, 2720 0x1eee0, 0x1eee0, 2721 0x1ef00, 0x1ef84, 2722 0x1efc0, 0x1efc8, 2723 0x1f040, 0x1f04c, 2724 0x1f240, 0x1f28c, 2725 0x1f2c0, 0x1f2c0, 2726 0x1f2e0, 0x1f2e0, 2727 0x1f300, 0x1f384, 2728 0x1f3c0, 0x1f3c8, 2729 0x1f440, 0x1f44c, 2730 0x1f640, 0x1f68c, 2731 0x1f6c0, 0x1f6c0, 2732 0x1f6e0, 0x1f6e0, 2733 0x1f700, 0x1f784, 2734 0x1f7c0, 0x1f7c8, 2735 0x1f840, 0x1f84c, 2736 0x1fa40, 0x1fa8c, 2737 0x1fac0, 0x1fac0, 2738 0x1fae0, 0x1fae0, 2739 0x1fb00, 0x1fb84, 2740 0x1fbc0, 0x1fbc8, 2741 0x1fc40, 0x1fc4c, 2742 0x1fe40, 0x1fe8c, 2743 0x1fec0, 0x1fec0, 2744 0x1fee0, 0x1fee0, 2745 0x1ff00, 0x1ff84, 2746 0x1ffc0, 0x1ffc8, 2747 0x20000, 0x2002c, 2748 0x20100, 0x2013c, 2749 0x20190, 0x201c8, 2750 0x20200, 0x20318, 2751 0x20400, 0x20528, 2752 0x20540, 0x20614, 2753 0x21000, 0x21040, 2754 0x2104c, 0x21060, 2755 0x210c0, 0x210ec, 2756 0x21200, 0x21268, 2757 0x21270, 0x21284, 2758 0x212fc, 0x21388, 2759 0x21400, 0x21404, 2760 0x21500, 0x21518, 2761 0x2152c, 0x2153c, 2762 0x21550, 0x21554, 2763 0x21600, 0x21600, 2764 0x21608, 0x21628, 2765 0x21630, 0x2163c, 2766 0x21700, 0x2171c, 2767 0x21780, 0x2178c, 2768 0x21800, 0x21c38, 2769 0x21c80, 0x21d7c, 2770 0x21e00, 0x21e04, 2771 0x22000, 0x2202c, 2772 0x22100, 0x2213c, 2773 0x22190, 0x221c8, 2774 0x22200, 0x22318, 2775 0x22400, 0x22528, 2776 0x22540, 0x22614, 2777 0x23000, 0x23040, 2778 0x2304c, 0x23060, 2779 0x230c0, 0x230ec, 2780 0x23200, 0x23268, 2781 0x23270, 0x23284, 2782 0x232fc, 0x23388, 2783 0x23400, 0x23404, 2784 0x23500, 0x23518, 2785 0x2352c, 0x2353c, 2786 0x23550, 0x23554, 2787 0x23600, 0x23600, 2788 0x23608, 0x23628, 2789 0x23630, 0x2363c, 2790 0x23700, 0x2371c, 2791 0x23780, 0x2378c, 2792 0x23800, 0x23c38, 2793 0x23c80, 0x23d7c, 2794 0x23e00, 0x23e04, 2795 0x24000, 0x2402c, 2796 0x24100, 0x2413c, 2797 0x24190, 0x241c8, 2798 0x24200, 0x24318, 2799 0x24400, 0x24528, 2800 0x24540, 0x24614, 2801 0x25000, 0x25040, 2802 0x2504c, 0x25060, 2803 0x250c0, 0x250ec, 2804 0x25200, 0x25268, 2805 0x25270, 0x25284, 2806 0x252fc, 0x25388, 2807 0x25400, 0x25404, 2808 0x25500, 0x25518, 2809 0x2552c, 0x2553c, 2810 0x25550, 0x25554, 2811 0x25600, 0x25600, 2812 0x25608, 0x25628, 2813 0x25630, 0x2563c, 2814 0x25700, 0x2571c, 2815 0x25780, 0x2578c, 2816 0x25800, 0x25c38, 2817 0x25c80, 0x25d7c, 2818 0x25e00, 0x25e04, 2819 0x26000, 0x2602c, 2820 0x26100, 0x2613c, 2821 0x26190, 0x261c8, 2822 0x26200, 0x26318, 2823 0x26400, 0x26528, 2824 0x26540, 0x26614, 2825 0x27000, 0x27040, 2826 0x2704c, 0x27060, 2827 0x270c0, 0x270ec, 2828 0x27200, 0x27268, 2829 0x27270, 0x27284, 2830 0x272fc, 0x27388, 2831 0x27400, 0x27404, 2832 0x27500, 0x27518, 2833 0x2752c, 0x2753c, 2834 0x27550, 0x27554, 2835 0x27600, 0x27600, 2836 0x27608, 0x27628, 2837 0x27630, 0x2763c, 2838 0x27700, 0x2771c, 2839 0x27780, 0x2778c, 2840 0x27800, 0x27c38, 2841 0x27c80, 0x27d7c, 2842 0x27e00, 0x27e04 2843 }; 2844 2845 regs->version = 4 | (sc->params.rev << 10); 2846 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2) 2847 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 2848 } 2849 2850 static void 2851 cxgbe_tick(void *arg) 2852 { 2853 struct port_info *pi = arg; 2854 struct ifnet *ifp = pi->ifp; 2855 struct sge_txq *txq; 2856 int i, drops; 2857 struct port_stats *s = &pi->stats; 2858 2859 PORT_LOCK(pi); 2860 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2861 PORT_UNLOCK(pi); 2862 return; /* without scheduling another callout */ 2863 } 2864 2865 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 2866 2867 ifp->if_opackets = s->tx_frames - s->tx_pause; 2868 ifp->if_ipackets = s->rx_frames - s->rx_pause; 2869 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 2870 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 2871 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 2872 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 2873 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 2874 s->rx_ovflow3; 2875 2876 drops = s->tx_drop; 2877 for_each_txq(pi, i, txq) 2878 drops += txq->br->br_drops; 2879 ifp->if_snd.ifq_drops = drops; 2880 2881 ifp->if_oerrors = s->tx_error_frames; 2882 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 2883 s->rx_fcs_err + s->rx_len_err; 2884 2885 callout_schedule(&pi->tick, hz); 2886 PORT_UNLOCK(pi); 2887 } 2888 2889 static int 2890 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 2891 { 2892 #ifdef INVARIANTS 2893 panic("%s: opcode %02x on iq %p with payload %p", 2894 __func__, rss->opcode, iq, m); 2895 #else 2896 log(LOG_ERR, "%s: opcode %02x on iq %p with payload %p", 2897 __func__, rss->opcode, iq, m); 2898 m_freem(m); 2899 #endif 2900 return (EDOOFUS); 2901 } 2902 2903 int 2904 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 2905 { 2906 uintptr_t *loc, new; 2907 2908 if (opcode >= ARRAY_SIZE(sc->cpl_handler)) 2909 return (EINVAL); 2910 2911 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 2912 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 2913 atomic_store_rel_ptr(loc, new); 2914 2915 return (0); 2916 } 2917 2918 static int 2919 t4_sysctls(struct adapter *sc) 2920 { 2921 struct sysctl_ctx_list *ctx; 2922 struct sysctl_oid *oid; 2923 struct sysctl_oid_list *children, *c0; 2924 static char *caps[] = { 2925 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 2926 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 2927 "\20\1TOE", /* caps[2] toecaps */ 2928 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 2929 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 2930 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 2931 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 2932 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 2933 }; 2934 2935 ctx = device_get_sysctl_ctx(sc->dev); 2936 2937 /* 2938 * dev.t4nex.X. 2939 */ 2940 oid = device_get_sysctl_tree(sc->dev); 2941 c0 = children = SYSCTL_CHILDREN(oid); 2942 2943 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, 2944 &sc->params.nports, 0, "# of ports"); 2945 2946 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 2947 &sc->params.rev, 0, "chip hardware revision"); 2948 2949 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 2950 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 2951 2952 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 2953 CTLFLAG_RD, &t4_cfg_file, 0, "configuration file"); 2954 2955 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, 2956 &sc->cfcsum, 0, "config file checksum"); 2957 2958 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 2959 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 2960 sysctl_bitfield, "A", "available link capabilities"); 2961 2962 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 2963 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 2964 sysctl_bitfield, "A", "available NIC capabilities"); 2965 2966 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 2967 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 2968 sysctl_bitfield, "A", "available TCP offload capabilities"); 2969 2970 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 2971 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 2972 sysctl_bitfield, "A", "available RDMA capabilities"); 2973 2974 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 2975 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 2976 sysctl_bitfield, "A", "available iSCSI capabilities"); 2977 2978 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 2979 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 2980 sysctl_bitfield, "A", "available FCoE capabilities"); 2981 2982 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, 2983 &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)"); 2984 2985 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 2986 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 2987 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 2988 "interrupt holdoff timer values (us)"); 2989 2990 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 2991 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 2992 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 2993 "interrupt holdoff packet counter values"); 2994 2995 #ifdef SBUF_DRAIN 2996 /* 2997 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 2998 */ 2999 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 3000 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 3001 "logs and miscellaneous information"); 3002 children = SYSCTL_CHILDREN(oid); 3003 3004 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 3005 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3006 sysctl_cctrl, "A", "congestion control"); 3007 3008 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 3009 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3010 sysctl_cpl_stats, "A", "CPL statistics"); 3011 3012 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 3013 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3014 sysctl_ddp_stats, "A", "DDP statistics"); 3015 3016 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 3017 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3018 sysctl_devlog, "A", "firmware's device log"); 3019 3020 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 3021 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3022 sysctl_fcoe_stats, "A", "FCoE statistics"); 3023 3024 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 3025 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3026 sysctl_hw_sched, "A", "hardware scheduler "); 3027 3028 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 3029 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3030 sysctl_l2t, "A", "hardware L2 table"); 3031 3032 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 3033 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3034 sysctl_lb_stats, "A", "loopback statistics"); 3035 3036 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 3037 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3038 sysctl_meminfo, "A", "memory regions"); 3039 3040 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 3041 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3042 sysctl_path_mtus, "A", "path MTUs"); 3043 3044 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 3045 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3046 sysctl_pm_stats, "A", "PM statistics"); 3047 3048 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 3049 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3050 sysctl_rdma_stats, "A", "RDMA statistics"); 3051 3052 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 3053 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3054 sysctl_tcp_stats, "A", "TCP statistics"); 3055 3056 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 3057 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3058 sysctl_tids, "A", "TID information"); 3059 3060 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 3061 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3062 sysctl_tp_err_stats, "A", "TP error statistics"); 3063 3064 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 3065 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3066 sysctl_tx_rate, "A", "Tx rate"); 3067 #endif 3068 3069 #ifndef TCP_OFFLOAD_DISABLE 3070 if (is_offload(sc)) { 3071 /* 3072 * dev.t4nex.X.toe. 3073 */ 3074 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 3075 NULL, "TOE parameters"); 3076 children = SYSCTL_CHILDREN(oid); 3077 3078 sc->tt.sndbuf = 256 * 1024; 3079 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 3080 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 3081 3082 sc->tt.ddp = 0; 3083 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 3084 &sc->tt.ddp, 0, "DDP allowed"); 3085 sc->tt.indsz = M_INDICATESIZE; 3086 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 3087 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 3088 sc->tt.ddp_thres = 3*4096; 3089 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 3090 &sc->tt.ddp_thres, 0, "DDP threshold"); 3091 } 3092 #endif 3093 3094 3095 return (0); 3096 } 3097 3098 static int 3099 cxgbe_sysctls(struct port_info *pi) 3100 { 3101 struct sysctl_ctx_list *ctx; 3102 struct sysctl_oid *oid; 3103 struct sysctl_oid_list *children; 3104 3105 ctx = device_get_sysctl_ctx(pi->dev); 3106 3107 /* 3108 * dev.cxgbe.X. 3109 */ 3110 oid = device_get_sysctl_tree(pi->dev); 3111 children = SYSCTL_CHILDREN(oid); 3112 3113 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 3114 &pi->nrxq, 0, "# of rx queues"); 3115 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 3116 &pi->ntxq, 0, "# of tx queues"); 3117 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 3118 &pi->first_rxq, 0, "index of first rx queue"); 3119 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 3120 &pi->first_txq, 0, "index of first tx queue"); 3121 3122 #ifndef TCP_OFFLOAD_DISABLE 3123 if (is_offload(pi->adapter)) { 3124 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 3125 &pi->nofldrxq, 0, 3126 "# of rx queues for offloaded TCP connections"); 3127 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 3128 &pi->nofldtxq, 0, 3129 "# of tx queues for offloaded TCP connections"); 3130 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 3131 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 3132 "index of first TOE rx queue"); 3133 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 3134 CTLFLAG_RD, &pi->first_ofld_txq, 0, 3135 "index of first TOE tx queue"); 3136 } 3137 #endif 3138 3139 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 3140 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 3141 "holdoff timer index"); 3142 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 3143 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 3144 "holdoff packet counter index"); 3145 3146 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 3147 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 3148 "rx queue size"); 3149 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 3150 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 3151 "tx queue size"); 3152 3153 /* 3154 * dev.cxgbe.X.stats. 3155 */ 3156 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 3157 NULL, "port statistics"); 3158 children = SYSCTL_CHILDREN(oid); 3159 3160 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 3161 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 3162 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 3163 sysctl_handle_t4_reg64, "QU", desc) 3164 3165 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 3166 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 3167 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 3168 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 3169 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 3170 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 3171 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 3172 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 3173 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 3174 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 3175 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 3176 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 3177 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 3178 "# of tx frames in this range", 3179 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 3180 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 3181 "# of tx frames in this range", 3182 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 3183 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 3184 "# of tx frames in this range", 3185 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 3186 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 3187 "# of tx frames in this range", 3188 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 3189 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 3190 "# of tx frames in this range", 3191 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 3192 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 3193 "# of tx frames in this range", 3194 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 3195 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 3196 "# of tx frames in this range", 3197 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 3198 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 3199 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 3200 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 3201 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 3202 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 3203 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 3204 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 3205 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 3206 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 3207 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 3208 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 3209 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 3210 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 3211 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 3212 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 3213 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 3214 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 3215 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 3216 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 3217 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 3218 3219 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 3220 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 3221 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 3222 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 3223 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 3224 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 3225 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 3226 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 3227 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 3228 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 3229 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 3230 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 3231 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 3232 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 3233 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 3234 "# of frames received with bad FCS", 3235 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 3236 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 3237 "# of frames received with length error", 3238 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 3239 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 3240 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 3241 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 3242 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 3243 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 3244 "# of rx frames in this range", 3245 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 3246 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 3247 "# of rx frames in this range", 3248 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 3249 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 3250 "# of rx frames in this range", 3251 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 3252 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 3253 "# of rx frames in this range", 3254 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 3255 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 3256 "# of rx frames in this range", 3257 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 3258 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 3259 "# of rx frames in this range", 3260 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 3261 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 3262 "# of rx frames in this range", 3263 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 3264 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 3265 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 3266 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 3267 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 3268 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 3269 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 3270 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 3271 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 3272 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 3273 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 3274 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 3275 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 3276 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 3277 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 3278 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 3279 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 3280 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 3281 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 3282 3283 #undef SYSCTL_ADD_T4_REG64 3284 3285 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 3286 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 3287 &pi->stats.name, desc) 3288 3289 /* We get these from port_stats and they may be stale by upto 1s */ 3290 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 3291 "# drops due to buffer-group 0 overflows"); 3292 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 3293 "# drops due to buffer-group 1 overflows"); 3294 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 3295 "# drops due to buffer-group 2 overflows"); 3296 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 3297 "# drops due to buffer-group 3 overflows"); 3298 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 3299 "# of buffer-group 0 truncated packets"); 3300 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 3301 "# of buffer-group 1 truncated packets"); 3302 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 3303 "# of buffer-group 2 truncated packets"); 3304 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 3305 "# of buffer-group 3 truncated packets"); 3306 3307 #undef SYSCTL_ADD_T4_PORTSTAT 3308 3309 return (0); 3310 } 3311 3312 static int 3313 sysctl_int_array(SYSCTL_HANDLER_ARGS) 3314 { 3315 int rc, *i; 3316 struct sbuf sb; 3317 3318 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 3319 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 3320 sbuf_printf(&sb, "%d ", *i); 3321 sbuf_trim(&sb); 3322 sbuf_finish(&sb); 3323 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 3324 sbuf_delete(&sb); 3325 return (rc); 3326 } 3327 3328 static int 3329 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 3330 { 3331 int rc; 3332 struct sbuf *sb; 3333 3334 rc = sysctl_wire_old_buffer(req, 0); 3335 if (rc != 0) 3336 return(rc); 3337 3338 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 3339 if (sb == NULL) 3340 return (ENOMEM); 3341 3342 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 3343 rc = sbuf_finish(sb); 3344 sbuf_delete(sb); 3345 3346 return (rc); 3347 } 3348 3349 static int 3350 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 3351 { 3352 struct port_info *pi = arg1; 3353 struct adapter *sc = pi->adapter; 3354 int idx, rc, i; 3355 3356 idx = pi->tmr_idx; 3357 3358 rc = sysctl_handle_int(oidp, &idx, 0, req); 3359 if (rc != 0 || req->newptr == NULL) 3360 return (rc); 3361 3362 if (idx < 0 || idx >= SGE_NTIMERS) 3363 return (EINVAL); 3364 3365 ADAPTER_LOCK(sc); 3366 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3367 if (rc == 0) { 3368 struct sge_rxq *rxq; 3369 uint8_t v; 3370 3371 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 3372 for_each_rxq(pi, i, rxq) { 3373 #ifdef atomic_store_rel_8 3374 atomic_store_rel_8(&rxq->iq.intr_params, v); 3375 #else 3376 rxq->iq.intr_params = v; 3377 #endif 3378 } 3379 pi->tmr_idx = idx; 3380 } 3381 3382 ADAPTER_UNLOCK(sc); 3383 return (rc); 3384 } 3385 3386 static int 3387 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 3388 { 3389 struct port_info *pi = arg1; 3390 struct adapter *sc = pi->adapter; 3391 int idx, rc; 3392 3393 idx = pi->pktc_idx; 3394 3395 rc = sysctl_handle_int(oidp, &idx, 0, req); 3396 if (rc != 0 || req->newptr == NULL) 3397 return (rc); 3398 3399 if (idx < -1 || idx >= SGE_NCOUNTERS) 3400 return (EINVAL); 3401 3402 ADAPTER_LOCK(sc); 3403 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3404 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3405 rc = EBUSY; /* cannot be changed once the queues are created */ 3406 3407 if (rc == 0) 3408 pi->pktc_idx = idx; 3409 3410 ADAPTER_UNLOCK(sc); 3411 return (rc); 3412 } 3413 3414 static int 3415 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 3416 { 3417 struct port_info *pi = arg1; 3418 struct adapter *sc = pi->adapter; 3419 int qsize, rc; 3420 3421 qsize = pi->qsize_rxq; 3422 3423 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3424 if (rc != 0 || req->newptr == NULL) 3425 return (rc); 3426 3427 if (qsize < 128 || (qsize & 7)) 3428 return (EINVAL); 3429 3430 ADAPTER_LOCK(sc); 3431 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3432 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3433 rc = EBUSY; /* cannot be changed once the queues are created */ 3434 3435 if (rc == 0) 3436 pi->qsize_rxq = qsize; 3437 3438 ADAPTER_UNLOCK(sc); 3439 return (rc); 3440 } 3441 3442 static int 3443 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 3444 { 3445 struct port_info *pi = arg1; 3446 struct adapter *sc = pi->adapter; 3447 int qsize, rc; 3448 3449 qsize = pi->qsize_txq; 3450 3451 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3452 if (rc != 0 || req->newptr == NULL) 3453 return (rc); 3454 3455 if (qsize < 128) 3456 return (EINVAL); 3457 3458 ADAPTER_LOCK(sc); 3459 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3460 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3461 rc = EBUSY; /* cannot be changed once the queues are created */ 3462 3463 if (rc == 0) 3464 pi->qsize_txq = qsize; 3465 3466 ADAPTER_UNLOCK(sc); 3467 return (rc); 3468 } 3469 3470 static int 3471 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 3472 { 3473 struct adapter *sc = arg1; 3474 int reg = arg2; 3475 uint64_t val; 3476 3477 val = t4_read_reg64(sc, reg); 3478 3479 return (sysctl_handle_64(oidp, &val, 0, req)); 3480 } 3481 3482 #ifdef SBUF_DRAIN 3483 static int 3484 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 3485 { 3486 struct adapter *sc = arg1; 3487 struct sbuf *sb; 3488 int rc, i; 3489 uint16_t incr[NMTUS][NCCTRL_WIN]; 3490 static const char *dec_fac[] = { 3491 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 3492 "0.9375" 3493 }; 3494 3495 rc = sysctl_wire_old_buffer(req, 0); 3496 if (rc != 0) 3497 return (rc); 3498 3499 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3500 if (sb == NULL) 3501 return (ENOMEM); 3502 3503 t4_read_cong_tbl(sc, incr); 3504 3505 for (i = 0; i < NCCTRL_WIN; ++i) { 3506 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 3507 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 3508 incr[5][i], incr[6][i], incr[7][i]); 3509 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 3510 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 3511 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 3512 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 3513 } 3514 3515 rc = sbuf_finish(sb); 3516 sbuf_delete(sb); 3517 3518 return (rc); 3519 } 3520 3521 static int 3522 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 3523 { 3524 struct adapter *sc = arg1; 3525 struct sbuf *sb; 3526 int rc; 3527 struct tp_cpl_stats stats; 3528 3529 rc = sysctl_wire_old_buffer(req, 0); 3530 if (rc != 0) 3531 return (rc); 3532 3533 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3534 if (sb == NULL) 3535 return (ENOMEM); 3536 3537 t4_tp_get_cpl_stats(sc, &stats); 3538 3539 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 3540 "channel 3\n"); 3541 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 3542 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 3543 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 3544 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 3545 3546 rc = sbuf_finish(sb); 3547 sbuf_delete(sb); 3548 3549 return (rc); 3550 } 3551 3552 static int 3553 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 3554 { 3555 struct adapter *sc = arg1; 3556 struct sbuf *sb; 3557 int rc; 3558 struct tp_usm_stats stats; 3559 3560 rc = sysctl_wire_old_buffer(req, 0); 3561 if (rc != 0) 3562 return(rc); 3563 3564 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3565 if (sb == NULL) 3566 return (ENOMEM); 3567 3568 t4_get_usm_stats(sc, &stats); 3569 3570 sbuf_printf(sb, "Frames: %u\n", stats.frames); 3571 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 3572 sbuf_printf(sb, "Drops: %u", stats.drops); 3573 3574 rc = sbuf_finish(sb); 3575 sbuf_delete(sb); 3576 3577 return (rc); 3578 } 3579 3580 const char *devlog_level_strings[] = { 3581 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 3582 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 3583 [FW_DEVLOG_LEVEL_ERR] = "ERR", 3584 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 3585 [FW_DEVLOG_LEVEL_INFO] = "INFO", 3586 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 3587 }; 3588 3589 const char *devlog_facility_strings[] = { 3590 [FW_DEVLOG_FACILITY_CORE] = "CORE", 3591 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 3592 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 3593 [FW_DEVLOG_FACILITY_RES] = "RES", 3594 [FW_DEVLOG_FACILITY_HW] = "HW", 3595 [FW_DEVLOG_FACILITY_FLR] = "FLR", 3596 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 3597 [FW_DEVLOG_FACILITY_PHY] = "PHY", 3598 [FW_DEVLOG_FACILITY_MAC] = "MAC", 3599 [FW_DEVLOG_FACILITY_PORT] = "PORT", 3600 [FW_DEVLOG_FACILITY_VI] = "VI", 3601 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 3602 [FW_DEVLOG_FACILITY_ACL] = "ACL", 3603 [FW_DEVLOG_FACILITY_TM] = "TM", 3604 [FW_DEVLOG_FACILITY_QFC] = "QFC", 3605 [FW_DEVLOG_FACILITY_DCB] = "DCB", 3606 [FW_DEVLOG_FACILITY_ETH] = "ETH", 3607 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 3608 [FW_DEVLOG_FACILITY_RI] = "RI", 3609 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 3610 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 3611 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 3612 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 3613 }; 3614 3615 static int 3616 sysctl_devlog(SYSCTL_HANDLER_ARGS) 3617 { 3618 struct adapter *sc = arg1; 3619 struct devlog_params *dparams = &sc->params.devlog; 3620 struct fw_devlog_e *buf, *e; 3621 int i, j, rc, nentries, first = 0; 3622 struct sbuf *sb; 3623 uint64_t ftstamp = UINT64_MAX; 3624 3625 if (dparams->start == 0) 3626 return (ENXIO); 3627 3628 nentries = dparams->size / sizeof(struct fw_devlog_e); 3629 3630 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 3631 if (buf == NULL) 3632 return (ENOMEM); 3633 3634 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 3635 (void *)buf); 3636 if (rc != 0) 3637 goto done; 3638 3639 for (i = 0; i < nentries; i++) { 3640 e = &buf[i]; 3641 3642 if (e->timestamp == 0) 3643 break; /* end */ 3644 3645 e->timestamp = be64toh(e->timestamp); 3646 e->seqno = be32toh(e->seqno); 3647 for (j = 0; j < 8; j++) 3648 e->params[j] = be32toh(e->params[j]); 3649 3650 if (e->timestamp < ftstamp) { 3651 ftstamp = e->timestamp; 3652 first = i; 3653 } 3654 } 3655 3656 if (buf[first].timestamp == 0) 3657 goto done; /* nothing in the log */ 3658 3659 rc = sysctl_wire_old_buffer(req, 0); 3660 if (rc != 0) 3661 goto done; 3662 3663 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3664 if (sb == NULL) { 3665 rc = ENOMEM; 3666 goto done; 3667 } 3668 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 3669 "Seq#", "Tstamp", "Level", "Facility", "Message"); 3670 3671 i = first; 3672 do { 3673 e = &buf[i]; 3674 if (e->timestamp == 0) 3675 break; /* end */ 3676 3677 sbuf_printf(sb, "%10d %15ju %8s %8s ", 3678 e->seqno, e->timestamp, 3679 (e->level < ARRAY_SIZE(devlog_level_strings) ? 3680 devlog_level_strings[e->level] : "UNKNOWN"), 3681 (e->facility < ARRAY_SIZE(devlog_facility_strings) ? 3682 devlog_facility_strings[e->facility] : "UNKNOWN")); 3683 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 3684 e->params[2], e->params[3], e->params[4], 3685 e->params[5], e->params[6], e->params[7]); 3686 3687 if (++i == nentries) 3688 i = 0; 3689 } while (i != first); 3690 3691 rc = sbuf_finish(sb); 3692 sbuf_delete(sb); 3693 done: 3694 free(buf, M_CXGBE); 3695 return (rc); 3696 } 3697 3698 static int 3699 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 3700 { 3701 struct adapter *sc = arg1; 3702 struct sbuf *sb; 3703 int rc; 3704 struct tp_fcoe_stats stats[4]; 3705 3706 rc = sysctl_wire_old_buffer(req, 0); 3707 if (rc != 0) 3708 return (rc); 3709 3710 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3711 if (sb == NULL) 3712 return (ENOMEM); 3713 3714 t4_get_fcoe_stats(sc, 0, &stats[0]); 3715 t4_get_fcoe_stats(sc, 1, &stats[1]); 3716 t4_get_fcoe_stats(sc, 2, &stats[2]); 3717 t4_get_fcoe_stats(sc, 3, &stats[3]); 3718 3719 sbuf_printf(sb, " channel 0 channel 1 " 3720 "channel 2 channel 3\n"); 3721 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 3722 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 3723 stats[3].octetsDDP); 3724 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 3725 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 3726 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 3727 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 3728 stats[3].framesDrop); 3729 3730 rc = sbuf_finish(sb); 3731 sbuf_delete(sb); 3732 3733 return (rc); 3734 } 3735 3736 static int 3737 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 3738 { 3739 struct adapter *sc = arg1; 3740 struct sbuf *sb; 3741 int rc, i; 3742 unsigned int map, kbps, ipg, mode; 3743 unsigned int pace_tab[NTX_SCHED]; 3744 3745 rc = sysctl_wire_old_buffer(req, 0); 3746 if (rc != 0) 3747 return (rc); 3748 3749 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3750 if (sb == NULL) 3751 return (ENOMEM); 3752 3753 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 3754 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 3755 t4_read_pace_tbl(sc, pace_tab); 3756 3757 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 3758 "Class IPG (0.1 ns) Flow IPG (us)"); 3759 3760 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 3761 t4_get_tx_sched(sc, i, &kbps, &ipg); 3762 sbuf_printf(sb, "\n %u %-5s %u ", i, 3763 (mode & (1 << i)) ? "flow" : "class", map & 3); 3764 if (kbps) 3765 sbuf_printf(sb, "%9u ", kbps); 3766 else 3767 sbuf_printf(sb, " disabled "); 3768 3769 if (ipg) 3770 sbuf_printf(sb, "%13u ", ipg); 3771 else 3772 sbuf_printf(sb, " disabled "); 3773 3774 if (pace_tab[i]) 3775 sbuf_printf(sb, "%10u", pace_tab[i]); 3776 else 3777 sbuf_printf(sb, " disabled"); 3778 } 3779 3780 rc = sbuf_finish(sb); 3781 sbuf_delete(sb); 3782 3783 return (rc); 3784 } 3785 3786 static int 3787 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 3788 { 3789 struct adapter *sc = arg1; 3790 struct sbuf *sb; 3791 int rc, i, j; 3792 uint64_t *p0, *p1; 3793 struct lb_port_stats s[2]; 3794 static const char *stat_name[] = { 3795 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 3796 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 3797 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 3798 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 3799 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 3800 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 3801 "BG2FramesTrunc:", "BG3FramesTrunc:" 3802 }; 3803 3804 rc = sysctl_wire_old_buffer(req, 0); 3805 if (rc != 0) 3806 return (rc); 3807 3808 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3809 if (sb == NULL) 3810 return (ENOMEM); 3811 3812 memset(s, 0, sizeof(s)); 3813 3814 for (i = 0; i < 4; i += 2) { 3815 t4_get_lb_stats(sc, i, &s[0]); 3816 t4_get_lb_stats(sc, i + 1, &s[1]); 3817 3818 p0 = &s[0].octets; 3819 p1 = &s[1].octets; 3820 sbuf_printf(sb, "%s Loopback %u" 3821 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 3822 3823 for (j = 0; j < ARRAY_SIZE(stat_name); j++) 3824 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 3825 *p0++, *p1++); 3826 } 3827 3828 rc = sbuf_finish(sb); 3829 sbuf_delete(sb); 3830 3831 return (rc); 3832 } 3833 3834 struct mem_desc { 3835 unsigned int base; 3836 unsigned int limit; 3837 unsigned int idx; 3838 }; 3839 3840 static int 3841 mem_desc_cmp(const void *a, const void *b) 3842 { 3843 return ((const struct mem_desc *)a)->base - 3844 ((const struct mem_desc *)b)->base; 3845 } 3846 3847 static void 3848 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 3849 unsigned int to) 3850 { 3851 unsigned int size; 3852 3853 size = to - from + 1; 3854 if (size == 0) 3855 return; 3856 3857 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 3858 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 3859 } 3860 3861 static int 3862 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 3863 { 3864 struct adapter *sc = arg1; 3865 struct sbuf *sb; 3866 int rc, i, n; 3867 uint32_t lo, hi; 3868 static const char *memory[] = { "EDC0:", "EDC1:", "MC:" }; 3869 static const char *region[] = { 3870 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 3871 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 3872 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 3873 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 3874 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:", 3875 "ULPTX state:", "On-chip queues:" 3876 }; 3877 struct mem_desc avail[3]; 3878 struct mem_desc mem[ARRAY_SIZE(region) + 3]; /* up to 3 holes */ 3879 struct mem_desc *md = mem; 3880 3881 rc = sysctl_wire_old_buffer(req, 0); 3882 if (rc != 0) 3883 return (rc); 3884 3885 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3886 if (sb == NULL) 3887 return (ENOMEM); 3888 3889 for (i = 0; i < ARRAY_SIZE(mem); i++) { 3890 mem[i].limit = 0; 3891 mem[i].idx = i; 3892 } 3893 3894 /* Find and sort the populated memory ranges */ 3895 i = 0; 3896 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 3897 if (lo & F_EDRAM0_ENABLE) { 3898 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 3899 avail[i].base = G_EDRAM0_BASE(hi) << 20; 3900 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 3901 avail[i].idx = 0; 3902 i++; 3903 } 3904 if (lo & F_EDRAM1_ENABLE) { 3905 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 3906 avail[i].base = G_EDRAM1_BASE(hi) << 20; 3907 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 3908 avail[i].idx = 1; 3909 i++; 3910 } 3911 if (lo & F_EXT_MEM_ENABLE) { 3912 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 3913 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 3914 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20); 3915 avail[i].idx = 2; 3916 i++; 3917 } 3918 if (!i) /* no memory available */ 3919 return 0; 3920 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 3921 3922 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 3923 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 3924 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 3925 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 3926 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 3927 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 3928 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 3929 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 3930 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 3931 3932 /* the next few have explicit upper bounds */ 3933 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 3934 md->limit = md->base - 1 + 3935 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 3936 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 3937 md++; 3938 3939 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 3940 md->limit = md->base - 1 + 3941 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 3942 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 3943 md++; 3944 3945 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 3946 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 3947 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 3948 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 3949 } else { 3950 md->base = 0; 3951 md->idx = ARRAY_SIZE(region); /* hide it */ 3952 } 3953 md++; 3954 3955 #define ulp_region(reg) \ 3956 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 3957 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 3958 3959 ulp_region(RX_ISCSI); 3960 ulp_region(RX_TDDP); 3961 ulp_region(TX_TPT); 3962 ulp_region(RX_STAG); 3963 ulp_region(RX_RQ); 3964 ulp_region(RX_RQUDP); 3965 ulp_region(RX_PBL); 3966 ulp_region(TX_PBL); 3967 #undef ulp_region 3968 3969 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 3970 md->limit = md->base + sc->tids.ntids - 1; 3971 md++; 3972 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 3973 md->limit = md->base + sc->tids.ntids - 1; 3974 md++; 3975 3976 md->base = sc->vres.ocq.start; 3977 if (sc->vres.ocq.size) 3978 md->limit = md->base + sc->vres.ocq.size - 1; 3979 else 3980 md->idx = ARRAY_SIZE(region); /* hide it */ 3981 md++; 3982 3983 /* add any address-space holes, there can be up to 3 */ 3984 for (n = 0; n < i - 1; n++) 3985 if (avail[n].limit < avail[n + 1].base) 3986 (md++)->base = avail[n].limit; 3987 if (avail[n].limit) 3988 (md++)->base = avail[n].limit; 3989 3990 n = md - mem; 3991 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 3992 3993 for (lo = 0; lo < i; lo++) 3994 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 3995 avail[lo].limit - 1); 3996 3997 sbuf_printf(sb, "\n"); 3998 for (i = 0; i < n; i++) { 3999 if (mem[i].idx >= ARRAY_SIZE(region)) 4000 continue; /* skip holes */ 4001 if (!mem[i].limit) 4002 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 4003 mem_region_show(sb, region[mem[i].idx], mem[i].base, 4004 mem[i].limit); 4005 } 4006 4007 sbuf_printf(sb, "\n"); 4008 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 4009 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 4010 mem_region_show(sb, "uP RAM:", lo, hi); 4011 4012 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 4013 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 4014 mem_region_show(sb, "uP Extmem2:", lo, hi); 4015 4016 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 4017 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 4018 G_PMRXMAXPAGE(lo), 4019 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 4020 (lo & F_PMRXNUMCHN) ? 2 : 1); 4021 4022 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 4023 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 4024 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 4025 G_PMTXMAXPAGE(lo), 4026 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 4027 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 4028 sbuf_printf(sb, "%u p-structs\n", 4029 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 4030 4031 for (i = 0; i < 4; i++) { 4032 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 4033 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 4034 i, G_USED(lo), G_ALLOC(lo)); 4035 } 4036 for (i = 0; i < 4; i++) { 4037 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 4038 sbuf_printf(sb, 4039 "\nLoopback %d using %u pages out of %u allocated", 4040 i, G_USED(lo), G_ALLOC(lo)); 4041 } 4042 4043 rc = sbuf_finish(sb); 4044 sbuf_delete(sb); 4045 4046 return (rc); 4047 } 4048 4049 static int 4050 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 4051 { 4052 struct adapter *sc = arg1; 4053 struct sbuf *sb; 4054 int rc; 4055 uint16_t mtus[NMTUS]; 4056 4057 rc = sysctl_wire_old_buffer(req, 0); 4058 if (rc != 0) 4059 return (rc); 4060 4061 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4062 if (sb == NULL) 4063 return (ENOMEM); 4064 4065 t4_read_mtu_tbl(sc, mtus, NULL); 4066 4067 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 4068 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 4069 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 4070 mtus[14], mtus[15]); 4071 4072 rc = sbuf_finish(sb); 4073 sbuf_delete(sb); 4074 4075 return (rc); 4076 } 4077 4078 static int 4079 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 4080 { 4081 struct adapter *sc = arg1; 4082 struct sbuf *sb; 4083 int rc, i; 4084 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 4085 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 4086 static const char *pm_stats[] = { 4087 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 4088 }; 4089 4090 rc = sysctl_wire_old_buffer(req, 0); 4091 if (rc != 0) 4092 return (rc); 4093 4094 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4095 if (sb == NULL) 4096 return (ENOMEM); 4097 4098 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 4099 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 4100 4101 sbuf_printf(sb, " Tx count Tx cycles " 4102 "Rx count Rx cycles"); 4103 for (i = 0; i < PM_NSTATS; i++) 4104 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 4105 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 4106 4107 rc = sbuf_finish(sb); 4108 sbuf_delete(sb); 4109 4110 return (rc); 4111 } 4112 4113 static int 4114 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 4115 { 4116 struct adapter *sc = arg1; 4117 struct sbuf *sb; 4118 int rc; 4119 struct tp_rdma_stats stats; 4120 4121 rc = sysctl_wire_old_buffer(req, 0); 4122 if (rc != 0) 4123 return (rc); 4124 4125 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4126 if (sb == NULL) 4127 return (ENOMEM); 4128 4129 t4_tp_get_rdma_stats(sc, &stats); 4130 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 4131 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 4132 4133 rc = sbuf_finish(sb); 4134 sbuf_delete(sb); 4135 4136 return (rc); 4137 } 4138 4139 static int 4140 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 4141 { 4142 struct adapter *sc = arg1; 4143 struct sbuf *sb; 4144 int rc; 4145 struct tp_tcp_stats v4, v6; 4146 4147 rc = sysctl_wire_old_buffer(req, 0); 4148 if (rc != 0) 4149 return (rc); 4150 4151 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4152 if (sb == NULL) 4153 return (ENOMEM); 4154 4155 t4_tp_get_tcp_stats(sc, &v4, &v6); 4156 sbuf_printf(sb, 4157 " IP IPv6\n"); 4158 sbuf_printf(sb, "OutRsts: %20u %20u\n", 4159 v4.tcpOutRsts, v6.tcpOutRsts); 4160 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 4161 v4.tcpInSegs, v6.tcpInSegs); 4162 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 4163 v4.tcpOutSegs, v6.tcpOutSegs); 4164 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 4165 v4.tcpRetransSegs, v6.tcpRetransSegs); 4166 4167 rc = sbuf_finish(sb); 4168 sbuf_delete(sb); 4169 4170 return (rc); 4171 } 4172 4173 static int 4174 sysctl_tids(SYSCTL_HANDLER_ARGS) 4175 { 4176 struct adapter *sc = arg1; 4177 struct sbuf *sb; 4178 int rc; 4179 struct tid_info *t = &sc->tids; 4180 4181 rc = sysctl_wire_old_buffer(req, 0); 4182 if (rc != 0) 4183 return (rc); 4184 4185 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4186 if (sb == NULL) 4187 return (ENOMEM); 4188 4189 if (t->natids) { 4190 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 4191 t->atids_in_use); 4192 } 4193 4194 if (t->ntids) { 4195 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4196 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 4197 4198 if (b) { 4199 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 4200 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4201 t->ntids - 1); 4202 } else { 4203 sbuf_printf(sb, "TID range: %u-%u", 4204 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4205 t->ntids - 1); 4206 } 4207 } else 4208 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 4209 sbuf_printf(sb, ", in use: %u\n", 4210 atomic_load_acq_int(&t->tids_in_use)); 4211 } 4212 4213 if (t->nstids) { 4214 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 4215 t->stid_base + t->nstids - 1, t->stids_in_use); 4216 } 4217 4218 if (t->nftids) { 4219 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 4220 t->ftid_base + t->nftids - 1); 4221 } 4222 4223 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 4224 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 4225 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 4226 4227 rc = sbuf_finish(sb); 4228 sbuf_delete(sb); 4229 4230 return (rc); 4231 } 4232 4233 static int 4234 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 4235 { 4236 struct adapter *sc = arg1; 4237 struct sbuf *sb; 4238 int rc; 4239 struct tp_err_stats stats; 4240 4241 rc = sysctl_wire_old_buffer(req, 0); 4242 if (rc != 0) 4243 return (rc); 4244 4245 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4246 if (sb == NULL) 4247 return (ENOMEM); 4248 4249 t4_tp_get_err_stats(sc, &stats); 4250 4251 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4252 "channel 3\n"); 4253 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 4254 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 4255 stats.macInErrs[3]); 4256 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 4257 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 4258 stats.hdrInErrs[3]); 4259 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 4260 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 4261 stats.tcpInErrs[3]); 4262 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 4263 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 4264 stats.tcp6InErrs[3]); 4265 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 4266 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 4267 stats.tnlCongDrops[3]); 4268 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 4269 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 4270 stats.tnlTxDrops[3]); 4271 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 4272 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 4273 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 4274 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 4275 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 4276 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 4277 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 4278 stats.ofldNoNeigh, stats.ofldCongDefer); 4279 4280 rc = sbuf_finish(sb); 4281 sbuf_delete(sb); 4282 4283 return (rc); 4284 } 4285 4286 static int 4287 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 4288 { 4289 struct adapter *sc = arg1; 4290 struct sbuf *sb; 4291 int rc; 4292 u64 nrate[NCHAN], orate[NCHAN]; 4293 4294 rc = sysctl_wire_old_buffer(req, 0); 4295 if (rc != 0) 4296 return (rc); 4297 4298 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4299 if (sb == NULL) 4300 return (ENOMEM); 4301 4302 t4_get_chan_txrate(sc, nrate, orate); 4303 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4304 "channel 3\n"); 4305 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 4306 nrate[0], nrate[1], nrate[2], nrate[3]); 4307 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 4308 orate[0], orate[1], orate[2], orate[3]); 4309 4310 rc = sbuf_finish(sb); 4311 sbuf_delete(sb); 4312 4313 return (rc); 4314 } 4315 #endif 4316 4317 static inline void 4318 txq_start(struct ifnet *ifp, struct sge_txq *txq) 4319 { 4320 struct buf_ring *br; 4321 struct mbuf *m; 4322 4323 TXQ_LOCK_ASSERT_OWNED(txq); 4324 4325 br = txq->br; 4326 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 4327 if (m) 4328 t4_eth_tx(ifp, txq, m); 4329 } 4330 4331 void 4332 t4_tx_callout(void *arg) 4333 { 4334 struct sge_eq *eq = arg; 4335 struct adapter *sc; 4336 4337 if (EQ_TRYLOCK(eq) == 0) 4338 goto reschedule; 4339 4340 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 4341 EQ_UNLOCK(eq); 4342 reschedule: 4343 if (__predict_true(!(eq->flags && EQ_DOOMED))) 4344 callout_schedule(&eq->tx_callout, 1); 4345 return; 4346 } 4347 4348 EQ_LOCK_ASSERT_OWNED(eq); 4349 4350 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 4351 4352 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4353 struct sge_txq *txq = arg; 4354 struct port_info *pi = txq->ifp->if_softc; 4355 4356 sc = pi->adapter; 4357 } else { 4358 struct sge_wrq *wrq = arg; 4359 4360 sc = wrq->adapter; 4361 } 4362 4363 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 4364 } 4365 4366 EQ_UNLOCK(eq); 4367 } 4368 4369 void 4370 t4_tx_task(void *arg, int count) 4371 { 4372 struct sge_eq *eq = arg; 4373 4374 EQ_LOCK(eq); 4375 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4376 struct sge_txq *txq = arg; 4377 txq_start(txq->ifp, txq); 4378 } else { 4379 struct sge_wrq *wrq = arg; 4380 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 4381 } 4382 EQ_UNLOCK(eq); 4383 } 4384 4385 static uint32_t 4386 fconf_to_mode(uint32_t fconf) 4387 { 4388 uint32_t mode; 4389 4390 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 4391 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 4392 4393 if (fconf & F_FRAGMENTATION) 4394 mode |= T4_FILTER_IP_FRAGMENT; 4395 4396 if (fconf & F_MPSHITTYPE) 4397 mode |= T4_FILTER_MPS_HIT_TYPE; 4398 4399 if (fconf & F_MACMATCH) 4400 mode |= T4_FILTER_MAC_IDX; 4401 4402 if (fconf & F_ETHERTYPE) 4403 mode |= T4_FILTER_ETH_TYPE; 4404 4405 if (fconf & F_PROTOCOL) 4406 mode |= T4_FILTER_IP_PROTO; 4407 4408 if (fconf & F_TOS) 4409 mode |= T4_FILTER_IP_TOS; 4410 4411 if (fconf & F_VLAN) 4412 mode |= T4_FILTER_VLAN; 4413 4414 if (fconf & F_VNIC_ID) 4415 mode |= T4_FILTER_VNIC; 4416 4417 if (fconf & F_PORT) 4418 mode |= T4_FILTER_PORT; 4419 4420 if (fconf & F_FCOE) 4421 mode |= T4_FILTER_FCoE; 4422 4423 return (mode); 4424 } 4425 4426 static uint32_t 4427 mode_to_fconf(uint32_t mode) 4428 { 4429 uint32_t fconf = 0; 4430 4431 if (mode & T4_FILTER_IP_FRAGMENT) 4432 fconf |= F_FRAGMENTATION; 4433 4434 if (mode & T4_FILTER_MPS_HIT_TYPE) 4435 fconf |= F_MPSHITTYPE; 4436 4437 if (mode & T4_FILTER_MAC_IDX) 4438 fconf |= F_MACMATCH; 4439 4440 if (mode & T4_FILTER_ETH_TYPE) 4441 fconf |= F_ETHERTYPE; 4442 4443 if (mode & T4_FILTER_IP_PROTO) 4444 fconf |= F_PROTOCOL; 4445 4446 if (mode & T4_FILTER_IP_TOS) 4447 fconf |= F_TOS; 4448 4449 if (mode & T4_FILTER_VLAN) 4450 fconf |= F_VLAN; 4451 4452 if (mode & T4_FILTER_VNIC) 4453 fconf |= F_VNIC_ID; 4454 4455 if (mode & T4_FILTER_PORT) 4456 fconf |= F_PORT; 4457 4458 if (mode & T4_FILTER_FCoE) 4459 fconf |= F_FCOE; 4460 4461 return (fconf); 4462 } 4463 4464 static uint32_t 4465 fspec_to_fconf(struct t4_filter_specification *fs) 4466 { 4467 uint32_t fconf = 0; 4468 4469 if (fs->val.frag || fs->mask.frag) 4470 fconf |= F_FRAGMENTATION; 4471 4472 if (fs->val.matchtype || fs->mask.matchtype) 4473 fconf |= F_MPSHITTYPE; 4474 4475 if (fs->val.macidx || fs->mask.macidx) 4476 fconf |= F_MACMATCH; 4477 4478 if (fs->val.ethtype || fs->mask.ethtype) 4479 fconf |= F_ETHERTYPE; 4480 4481 if (fs->val.proto || fs->mask.proto) 4482 fconf |= F_PROTOCOL; 4483 4484 if (fs->val.tos || fs->mask.tos) 4485 fconf |= F_TOS; 4486 4487 if (fs->val.vlan_vld || fs->mask.vlan_vld) 4488 fconf |= F_VLAN; 4489 4490 if (fs->val.vnic_vld || fs->mask.vnic_vld) 4491 fconf |= F_VNIC_ID; 4492 4493 if (fs->val.iport || fs->mask.iport) 4494 fconf |= F_PORT; 4495 4496 if (fs->val.fcoe || fs->mask.fcoe) 4497 fconf |= F_FCOE; 4498 4499 return (fconf); 4500 } 4501 4502 static int 4503 get_filter_mode(struct adapter *sc, uint32_t *mode) 4504 { 4505 uint32_t fconf; 4506 4507 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 4508 A_TP_VLAN_PRI_MAP); 4509 4510 if (sc->filter_mode != fconf) { 4511 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 4512 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 4513 sc->filter_mode = fconf; 4514 } 4515 4516 *mode = fconf_to_mode(sc->filter_mode); 4517 4518 return (0); 4519 } 4520 4521 static int 4522 set_filter_mode(struct adapter *sc, uint32_t mode) 4523 { 4524 uint32_t fconf; 4525 int rc; 4526 4527 fconf = mode_to_fconf(mode); 4528 4529 ADAPTER_LOCK(sc); 4530 if (IS_BUSY(sc)) { 4531 rc = EAGAIN; 4532 goto done; 4533 } 4534 4535 if (sc->tids.ftids_in_use > 0) { 4536 rc = EBUSY; 4537 goto done; 4538 } 4539 4540 #ifndef TCP_OFFLOAD_DISABLE 4541 if (sc->offload_map) { 4542 rc = EBUSY; 4543 goto done; 4544 } 4545 #endif 4546 4547 #ifdef notyet 4548 rc = -t4_set_filter_mode(sc, fconf); 4549 if (rc == 0) 4550 sc->filter_mode = fconf; 4551 #else 4552 rc = ENOTSUP; 4553 #endif 4554 4555 done: 4556 ADAPTER_UNLOCK(sc); 4557 return (rc); 4558 } 4559 4560 static inline uint64_t 4561 get_filter_hits(struct adapter *sc, uint32_t fid) 4562 { 4563 uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4564 uint64_t hits; 4565 4566 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0), 4567 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 4568 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0)); 4569 hits = t4_read_reg64(sc, MEMWIN0_BASE + 16); 4570 4571 return (be64toh(hits)); 4572 } 4573 4574 static int 4575 get_filter(struct adapter *sc, struct t4_filter *t) 4576 { 4577 int i, nfilters = sc->tids.nftids; 4578 struct filter_entry *f; 4579 4580 ADAPTER_LOCK_ASSERT_OWNED(sc); 4581 4582 if (IS_BUSY(sc)) 4583 return (EAGAIN); 4584 4585 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 4586 t->idx >= nfilters) { 4587 t->idx = 0xffffffff; 4588 return (0); 4589 } 4590 4591 f = &sc->tids.ftid_tab[t->idx]; 4592 for (i = t->idx; i < nfilters; i++, f++) { 4593 if (f->valid) { 4594 t->idx = i; 4595 t->l2tidx = f->l2t ? f->l2t->idx : 0; 4596 t->smtidx = f->smtidx; 4597 if (f->fs.hitcnts) 4598 t->hits = get_filter_hits(sc, t->idx); 4599 else 4600 t->hits = UINT64_MAX; 4601 t->fs = f->fs; 4602 4603 return (0); 4604 } 4605 } 4606 4607 t->idx = 0xffffffff; 4608 return (0); 4609 } 4610 4611 static int 4612 set_filter(struct adapter *sc, struct t4_filter *t) 4613 { 4614 unsigned int nfilters, nports; 4615 struct filter_entry *f; 4616 int i; 4617 4618 ADAPTER_LOCK_ASSERT_OWNED(sc); 4619 4620 nfilters = sc->tids.nftids; 4621 nports = sc->params.nports; 4622 4623 if (nfilters == 0) 4624 return (ENOTSUP); 4625 4626 if (!(sc->flags & FULL_INIT_DONE)) 4627 return (EAGAIN); 4628 4629 if (t->idx >= nfilters) 4630 return (EINVAL); 4631 4632 /* Validate against the global filter mode */ 4633 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) 4634 return (E2BIG); 4635 4636 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) 4637 return (EINVAL); 4638 4639 if (t->fs.val.iport >= nports) 4640 return (EINVAL); 4641 4642 /* Can't specify an iq if not steering to it */ 4643 if (!t->fs.dirsteer && t->fs.iq) 4644 return (EINVAL); 4645 4646 /* IPv6 filter idx must be 4 aligned */ 4647 if (t->fs.type == 1 && 4648 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) 4649 return (EINVAL); 4650 4651 if (sc->tids.ftid_tab == NULL) { 4652 KASSERT(sc->tids.ftids_in_use == 0, 4653 ("%s: no memory allocated but filters_in_use > 0", 4654 __func__)); 4655 4656 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 4657 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 4658 if (sc->tids.ftid_tab == NULL) 4659 return (ENOMEM); 4660 } 4661 4662 for (i = 0; i < 4; i++) { 4663 f = &sc->tids.ftid_tab[t->idx + i]; 4664 4665 if (f->pending || f->valid) 4666 return (EBUSY); 4667 if (f->locked) 4668 return (EPERM); 4669 4670 if (t->fs.type == 0) 4671 break; 4672 } 4673 4674 f = &sc->tids.ftid_tab[t->idx]; 4675 f->fs = t->fs; 4676 4677 return set_filter_wr(sc, t->idx); 4678 } 4679 4680 static int 4681 del_filter(struct adapter *sc, struct t4_filter *t) 4682 { 4683 unsigned int nfilters; 4684 struct filter_entry *f; 4685 4686 ADAPTER_LOCK_ASSERT_OWNED(sc); 4687 4688 if (IS_BUSY(sc)) 4689 return (EAGAIN); 4690 4691 nfilters = sc->tids.nftids; 4692 4693 if (nfilters == 0) 4694 return (ENOTSUP); 4695 4696 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 4697 t->idx >= nfilters) 4698 return (EINVAL); 4699 4700 if (!(sc->flags & FULL_INIT_DONE)) 4701 return (EAGAIN); 4702 4703 f = &sc->tids.ftid_tab[t->idx]; 4704 4705 if (f->pending) 4706 return (EBUSY); 4707 if (f->locked) 4708 return (EPERM); 4709 4710 if (f->valid) { 4711 t->fs = f->fs; /* extra info for the caller */ 4712 return del_filter_wr(sc, t->idx); 4713 } 4714 4715 return (0); 4716 } 4717 4718 static void 4719 clear_filter(struct filter_entry *f) 4720 { 4721 if (f->l2t) 4722 t4_l2t_release(f->l2t); 4723 4724 bzero(f, sizeof (*f)); 4725 } 4726 4727 static int 4728 set_filter_wr(struct adapter *sc, int fidx) 4729 { 4730 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4731 struct mbuf *m; 4732 struct fw_filter_wr *fwr; 4733 unsigned int ftid; 4734 4735 ADAPTER_LOCK_ASSERT_OWNED(sc); 4736 4737 if (f->fs.newdmac || f->fs.newvlan) { 4738 /* This filter needs an L2T entry; allocate one. */ 4739 f->l2t = t4_l2t_alloc_switching(sc->l2t); 4740 if (f->l2t == NULL) 4741 return (EAGAIN); 4742 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 4743 f->fs.dmac)) { 4744 t4_l2t_release(f->l2t); 4745 f->l2t = NULL; 4746 return (ENOMEM); 4747 } 4748 } 4749 4750 ftid = sc->tids.ftid_base + fidx; 4751 4752 m = m_gethdr(M_NOWAIT, MT_DATA); 4753 if (m == NULL) 4754 return (ENOMEM); 4755 4756 fwr = mtod(m, struct fw_filter_wr *); 4757 m->m_len = m->m_pkthdr.len = sizeof(*fwr); 4758 bzero(fwr, sizeof (*fwr)); 4759 4760 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 4761 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 4762 fwr->tid_to_iq = 4763 htobe32(V_FW_FILTER_WR_TID(ftid) | 4764 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 4765 V_FW_FILTER_WR_NOREPLY(0) | 4766 V_FW_FILTER_WR_IQ(f->fs.iq)); 4767 fwr->del_filter_to_l2tix = 4768 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 4769 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 4770 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 4771 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 4772 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 4773 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 4774 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 4775 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 4776 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 4777 f->fs.newvlan == VLAN_REWRITE) | 4778 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 4779 f->fs.newvlan == VLAN_REWRITE) | 4780 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 4781 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 4782 V_FW_FILTER_WR_PRIO(f->fs.prio) | 4783 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 4784 fwr->ethtype = htobe16(f->fs.val.ethtype); 4785 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 4786 fwr->frag_to_ovlan_vldm = 4787 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 4788 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 4789 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 4790 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 4791 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 4792 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 4793 fwr->smac_sel = 0; 4794 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 4795 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 4796 fwr->maci_to_matchtypem = 4797 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 4798 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 4799 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 4800 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 4801 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 4802 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 4803 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 4804 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 4805 fwr->ptcl = f->fs.val.proto; 4806 fwr->ptclm = f->fs.mask.proto; 4807 fwr->ttyp = f->fs.val.tos; 4808 fwr->ttypm = f->fs.mask.tos; 4809 fwr->ivlan = htobe16(f->fs.val.vlan); 4810 fwr->ivlanm = htobe16(f->fs.mask.vlan); 4811 fwr->ovlan = htobe16(f->fs.val.vnic); 4812 fwr->ovlanm = htobe16(f->fs.mask.vnic); 4813 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 4814 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 4815 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 4816 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 4817 fwr->lp = htobe16(f->fs.val.dport); 4818 fwr->lpm = htobe16(f->fs.mask.dport); 4819 fwr->fp = htobe16(f->fs.val.sport); 4820 fwr->fpm = htobe16(f->fs.mask.sport); 4821 if (f->fs.newsmac) 4822 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 4823 4824 f->pending = 1; 4825 sc->tids.ftids_in_use++; 4826 4827 t4_mgmt_tx(sc, m); 4828 return (0); 4829 } 4830 4831 static int 4832 del_filter_wr(struct adapter *sc, int fidx) 4833 { 4834 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4835 struct mbuf *m; 4836 struct fw_filter_wr *fwr; 4837 unsigned int ftid; 4838 4839 ADAPTER_LOCK_ASSERT_OWNED(sc); 4840 4841 ftid = sc->tids.ftid_base + fidx; 4842 4843 m = m_gethdr(M_NOWAIT, MT_DATA); 4844 if (m == NULL) 4845 return (ENOMEM); 4846 4847 fwr = mtod(m, struct fw_filter_wr *); 4848 m->m_len = m->m_pkthdr.len = sizeof(*fwr); 4849 bzero(fwr, sizeof (*fwr)); 4850 4851 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 4852 4853 f->pending = 1; 4854 t4_mgmt_tx(sc, m); 4855 return (0); 4856 } 4857 4858 static int 4859 filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4860 { 4861 struct adapter *sc = iq->adapter; 4862 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 4863 unsigned int idx = GET_TID(rpl); 4864 4865 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 4866 rss->opcode)); 4867 4868 if (idx >= sc->tids.ftid_base && 4869 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 4870 unsigned int rc = G_COOKIE(rpl->cookie); 4871 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 4872 4873 ADAPTER_LOCK(sc); 4874 if (rc == FW_FILTER_WR_FLT_ADDED) { 4875 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 4876 f->pending = 0; /* asynchronous setup completed */ 4877 f->valid = 1; 4878 } else { 4879 if (rc != FW_FILTER_WR_FLT_DELETED) { 4880 /* Add or delete failed, display an error */ 4881 log(LOG_ERR, 4882 "filter %u setup failed with error %u\n", 4883 idx, rc); 4884 } 4885 4886 clear_filter(f); 4887 sc->tids.ftids_in_use--; 4888 } 4889 ADAPTER_UNLOCK(sc); 4890 } 4891 4892 return (0); 4893 } 4894 4895 static int 4896 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 4897 { 4898 int rc = EINVAL; 4899 4900 if (cntxt->cid > M_CTXTQID) 4901 return (rc); 4902 4903 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 4904 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 4905 return (rc); 4906 4907 if (sc->flags & FW_OK) { 4908 ADAPTER_LOCK(sc); /* Avoid parallel t4_wr_mbox */ 4909 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 4910 &cntxt->data[0]); 4911 ADAPTER_UNLOCK(sc); 4912 } 4913 4914 if (rc != 0) { 4915 /* Read via firmware failed or wasn't even attempted */ 4916 4917 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, 4918 &cntxt->data[0]); 4919 } 4920 4921 return (rc); 4922 } 4923 4924 static int 4925 read_card_mem(struct adapter *sc, struct t4_mem_range *mr) 4926 { 4927 uint32_t base, size, lo, hi, win, off, remaining, i, n; 4928 uint32_t *buf, *b; 4929 int rc; 4930 4931 /* reads are in multiples of 32 bits */ 4932 if (mr->addr & 3 || mr->len & 3 || mr->len == 0) 4933 return (EINVAL); 4934 4935 /* 4936 * We don't want to deal with potential holes so we mandate that the 4937 * requested region must lie entirely within one of the 3 memories. 4938 */ 4939 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 4940 if (lo & F_EDRAM0_ENABLE) { 4941 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 4942 base = G_EDRAM0_BASE(hi) << 20; 4943 size = G_EDRAM0_SIZE(hi) << 20; 4944 if (size > 0 && 4945 mr->addr >= base && mr->addr < base + size && 4946 mr->addr + mr->len <= base + size) 4947 goto proceed; 4948 } 4949 if (lo & F_EDRAM1_ENABLE) { 4950 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 4951 base = G_EDRAM1_BASE(hi) << 20; 4952 size = G_EDRAM1_SIZE(hi) << 20; 4953 if (size > 0 && 4954 mr->addr >= base && mr->addr < base + size && 4955 mr->addr + mr->len <= base + size) 4956 goto proceed; 4957 } 4958 if (lo & F_EXT_MEM_ENABLE) { 4959 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 4960 base = G_EXT_MEM_BASE(hi) << 20; 4961 size = G_EXT_MEM_SIZE(hi) << 20; 4962 if (size > 0 && 4963 mr->addr >= base && mr->addr < base + size && 4964 mr->addr + mr->len <= base + size) 4965 goto proceed; 4966 } 4967 return (ENXIO); 4968 4969 proceed: 4970 buf = b = malloc(mr->len, M_CXGBE, M_WAITOK); 4971 4972 /* 4973 * Position the PCIe window (we use memwin2) to the 16B aligned area 4974 * just at/before the requested region. 4975 */ 4976 win = mr->addr & ~0xf; 4977 off = mr->addr - win; /* offset of the requested region in the win */ 4978 remaining = mr->len; 4979 4980 while (remaining) { 4981 t4_write_reg(sc, 4982 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 4983 t4_read_reg(sc, 4984 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 4985 4986 /* number of bytes that we'll copy in the inner loop */ 4987 n = min(remaining, MEMWIN2_APERTURE - off); 4988 4989 for (i = 0; i < n; i += 4, remaining -= 4) 4990 *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i); 4991 4992 win += MEMWIN2_APERTURE; 4993 off = 0; 4994 } 4995 4996 rc = copyout(buf, mr->data, mr->len); 4997 free(buf, M_CXGBE); 4998 4999 return (rc); 5000 } 5001 5002 int 5003 t4_os_find_pci_capability(struct adapter *sc, int cap) 5004 { 5005 int i; 5006 5007 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 5008 } 5009 5010 int 5011 t4_os_pci_save_state(struct adapter *sc) 5012 { 5013 device_t dev; 5014 struct pci_devinfo *dinfo; 5015 5016 dev = sc->dev; 5017 dinfo = device_get_ivars(dev); 5018 5019 pci_cfg_save(dev, dinfo, 0); 5020 return (0); 5021 } 5022 5023 int 5024 t4_os_pci_restore_state(struct adapter *sc) 5025 { 5026 device_t dev; 5027 struct pci_devinfo *dinfo; 5028 5029 dev = sc->dev; 5030 dinfo = device_get_ivars(dev); 5031 5032 pci_cfg_restore(dev, dinfo); 5033 return (0); 5034 } 5035 5036 void 5037 t4_os_portmod_changed(const struct adapter *sc, int idx) 5038 { 5039 struct port_info *pi = sc->port[idx]; 5040 static const char *mod_str[] = { 5041 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 5042 }; 5043 5044 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 5045 if_printf(pi->ifp, "transceiver unplugged.\n"); 5046 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 5047 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 5048 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 5049 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 5050 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) { 5051 if_printf(pi->ifp, "%s transceiver inserted.\n", 5052 mod_str[pi->mod_type]); 5053 } else { 5054 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 5055 pi->mod_type); 5056 } 5057 } 5058 5059 void 5060 t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 5061 { 5062 struct port_info *pi = sc->port[idx]; 5063 struct ifnet *ifp = pi->ifp; 5064 5065 if (link_stat) { 5066 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 5067 if_link_state_change(ifp, LINK_STATE_UP); 5068 } else 5069 if_link_state_change(ifp, LINK_STATE_DOWN); 5070 } 5071 5072 void 5073 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 5074 { 5075 struct adapter *sc; 5076 5077 mtx_lock(&t4_list_lock); 5078 SLIST_FOREACH(sc, &t4_list, link) { 5079 /* 5080 * func should not make any assumptions about what state sc is 5081 * in - the only guarantee is that sc->sc_lock is a valid lock. 5082 */ 5083 func(sc, arg); 5084 } 5085 mtx_unlock(&t4_list_lock); 5086 } 5087 5088 static int 5089 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 5090 { 5091 return (0); 5092 } 5093 5094 static int 5095 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 5096 { 5097 return (0); 5098 } 5099 5100 static int 5101 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 5102 struct thread *td) 5103 { 5104 int rc; 5105 struct adapter *sc = dev->si_drv1; 5106 5107 rc = priv_check(td, PRIV_DRIVER); 5108 if (rc != 0) 5109 return (rc); 5110 5111 switch (cmd) { 5112 case CHELSIO_T4_GETREG: { 5113 struct t4_reg *edata = (struct t4_reg *)data; 5114 5115 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5116 return (EFAULT); 5117 5118 if (edata->size == 4) 5119 edata->val = t4_read_reg(sc, edata->addr); 5120 else if (edata->size == 8) 5121 edata->val = t4_read_reg64(sc, edata->addr); 5122 else 5123 return (EINVAL); 5124 5125 break; 5126 } 5127 case CHELSIO_T4_SETREG: { 5128 struct t4_reg *edata = (struct t4_reg *)data; 5129 5130 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5131 return (EFAULT); 5132 5133 if (edata->size == 4) { 5134 if (edata->val & 0xffffffff00000000) 5135 return (EINVAL); 5136 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 5137 } else if (edata->size == 8) 5138 t4_write_reg64(sc, edata->addr, edata->val); 5139 else 5140 return (EINVAL); 5141 break; 5142 } 5143 case CHELSIO_T4_REGDUMP: { 5144 struct t4_regdump *regs = (struct t4_regdump *)data; 5145 int reglen = T4_REGDUMP_SIZE; 5146 uint8_t *buf; 5147 5148 if (regs->len < reglen) { 5149 regs->len = reglen; /* hint to the caller */ 5150 return (ENOBUFS); 5151 } 5152 5153 regs->len = reglen; 5154 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 5155 t4_get_regs(sc, regs, buf); 5156 rc = copyout(buf, regs->data, reglen); 5157 free(buf, M_CXGBE); 5158 break; 5159 } 5160 case CHELSIO_T4_GET_FILTER_MODE: 5161 rc = get_filter_mode(sc, (uint32_t *)data); 5162 break; 5163 case CHELSIO_T4_SET_FILTER_MODE: 5164 rc = set_filter_mode(sc, *(uint32_t *)data); 5165 break; 5166 case CHELSIO_T4_GET_FILTER: 5167 ADAPTER_LOCK(sc); 5168 rc = get_filter(sc, (struct t4_filter *)data); 5169 ADAPTER_UNLOCK(sc); 5170 break; 5171 case CHELSIO_T4_SET_FILTER: 5172 ADAPTER_LOCK(sc); 5173 rc = set_filter(sc, (struct t4_filter *)data); 5174 ADAPTER_UNLOCK(sc); 5175 break; 5176 case CHELSIO_T4_DEL_FILTER: 5177 ADAPTER_LOCK(sc); 5178 rc = del_filter(sc, (struct t4_filter *)data); 5179 ADAPTER_UNLOCK(sc); 5180 break; 5181 case CHELSIO_T4_GET_SGE_CONTEXT: 5182 rc = get_sge_context(sc, (struct t4_sge_context *)data); 5183 break; 5184 case CHELSIO_T4_LOAD_FW: { 5185 struct t4_data *fw = (struct t4_data *)data; 5186 uint8_t *fw_data; 5187 5188 if (sc->flags & FULL_INIT_DONE) 5189 return (EBUSY); 5190 5191 fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT); 5192 if (fw_data == NULL) 5193 return (ENOMEM); 5194 5195 rc = copyin(fw->data, fw_data, fw->len); 5196 if (rc == 0) 5197 rc = -t4_load_fw(sc, fw_data, fw->len); 5198 5199 free(fw_data, M_CXGBE); 5200 break; 5201 } 5202 case CHELSIO_T4_GET_MEM: 5203 rc = read_card_mem(sc, (struct t4_mem_range *)data); 5204 break; 5205 default: 5206 rc = EINVAL; 5207 } 5208 5209 return (rc); 5210 } 5211 5212 #ifndef TCP_OFFLOAD_DISABLE 5213 static int 5214 toe_capability(struct port_info *pi, int enable) 5215 { 5216 int rc; 5217 struct adapter *sc = pi->adapter; 5218 5219 ADAPTER_LOCK_ASSERT_OWNED(sc); 5220 5221 if (!is_offload(sc)) 5222 return (ENODEV); 5223 5224 if (enable) { 5225 if (isset(&sc->offload_map, pi->port_id)) 5226 return (0); 5227 5228 if (sc->offload_map == 0) { 5229 rc = activate_uld(sc, ULD_TOM, &sc->tom); 5230 if (rc != 0) 5231 return (rc); 5232 } 5233 5234 setbit(&sc->offload_map, pi->port_id); 5235 } else { 5236 if (!isset(&sc->offload_map, pi->port_id)) 5237 return (0); 5238 5239 clrbit(&sc->offload_map, pi->port_id); 5240 5241 if (sc->offload_map == 0) { 5242 rc = deactivate_uld(&sc->tom); 5243 if (rc != 0) { 5244 setbit(&sc->offload_map, pi->port_id); 5245 return (rc); 5246 } 5247 } 5248 } 5249 5250 return (0); 5251 } 5252 5253 /* 5254 * Add an upper layer driver to the global list. 5255 */ 5256 int 5257 t4_register_uld(struct uld_info *ui) 5258 { 5259 int rc = 0; 5260 struct uld_info *u; 5261 5262 mtx_lock(&t4_uld_list_lock); 5263 SLIST_FOREACH(u, &t4_uld_list, link) { 5264 if (u->uld_id == ui->uld_id) { 5265 rc = EEXIST; 5266 goto done; 5267 } 5268 } 5269 5270 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 5271 ui->refcount = 0; 5272 done: 5273 mtx_unlock(&t4_uld_list_lock); 5274 return (rc); 5275 } 5276 5277 int 5278 t4_unregister_uld(struct uld_info *ui) 5279 { 5280 int rc = EINVAL; 5281 struct uld_info *u; 5282 5283 mtx_lock(&t4_uld_list_lock); 5284 5285 SLIST_FOREACH(u, &t4_uld_list, link) { 5286 if (u == ui) { 5287 if (ui->refcount > 0) { 5288 rc = EBUSY; 5289 goto done; 5290 } 5291 5292 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 5293 rc = 0; 5294 goto done; 5295 } 5296 } 5297 done: 5298 mtx_unlock(&t4_uld_list_lock); 5299 return (rc); 5300 } 5301 5302 static int 5303 activate_uld(struct adapter *sc, int id, struct uld_softc *usc) 5304 { 5305 int rc = EAGAIN; 5306 struct uld_info *ui; 5307 5308 mtx_lock(&t4_uld_list_lock); 5309 5310 SLIST_FOREACH(ui, &t4_uld_list, link) { 5311 if (ui->uld_id == id) { 5312 rc = ui->attach(sc, &usc->softc); 5313 if (rc == 0) { 5314 KASSERT(usc->softc != NULL, 5315 ("%s: ULD %d has no state", __func__, id)); 5316 ui->refcount++; 5317 usc->uld = ui; 5318 } 5319 goto done; 5320 } 5321 } 5322 done: 5323 mtx_unlock(&t4_uld_list_lock); 5324 5325 return (rc); 5326 } 5327 5328 static int 5329 deactivate_uld(struct uld_softc *usc) 5330 { 5331 int rc; 5332 5333 mtx_lock(&t4_uld_list_lock); 5334 5335 if (usc->uld == NULL || usc->softc == NULL) { 5336 rc = EINVAL; 5337 goto done; 5338 } 5339 5340 rc = usc->uld->detach(usc->softc); 5341 if (rc == 0) { 5342 KASSERT(usc->uld->refcount > 0, 5343 ("%s: ULD has bad refcount", __func__)); 5344 usc->uld->refcount--; 5345 usc->uld = NULL; 5346 usc->softc = NULL; 5347 } 5348 done: 5349 mtx_unlock(&t4_uld_list_lock); 5350 5351 return (rc); 5352 } 5353 #endif 5354 5355 /* 5356 * Come up with reasonable defaults for some of the tunables, provided they're 5357 * not set by the user (in which case we'll use the values as is). 5358 */ 5359 static void 5360 tweak_tunables(void) 5361 { 5362 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 5363 5364 if (t4_ntxq10g < 1) 5365 t4_ntxq10g = min(nc, NTXQ_10G); 5366 5367 if (t4_ntxq1g < 1) 5368 t4_ntxq1g = min(nc, NTXQ_1G); 5369 5370 if (t4_nrxq10g < 1) 5371 t4_nrxq10g = min(nc, NRXQ_10G); 5372 5373 if (t4_nrxq1g < 1) 5374 t4_nrxq1g = min(nc, NRXQ_1G); 5375 5376 #ifndef TCP_OFFLOAD_DISABLE 5377 if (t4_nofldtxq10g < 1) 5378 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 5379 5380 if (t4_nofldtxq1g < 1) 5381 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 5382 5383 if (t4_nofldrxq10g < 1) 5384 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 5385 5386 if (t4_nofldrxq1g < 1) 5387 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 5388 #endif 5389 5390 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 5391 t4_tmr_idx_10g = TMR_IDX_10G; 5392 5393 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 5394 t4_pktc_idx_10g = PKTC_IDX_10G; 5395 5396 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 5397 t4_tmr_idx_1g = TMR_IDX_1G; 5398 5399 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 5400 t4_pktc_idx_1g = PKTC_IDX_1G; 5401 5402 if (t4_qsize_txq < 128) 5403 t4_qsize_txq = 128; 5404 5405 if (t4_qsize_rxq < 128) 5406 t4_qsize_rxq = 128; 5407 while (t4_qsize_rxq & 7) 5408 t4_qsize_rxq++; 5409 5410 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 5411 } 5412 5413 static int 5414 t4_mod_event(module_t mod, int cmd, void *arg) 5415 { 5416 int rc = 0; 5417 5418 switch (cmd) { 5419 case MOD_LOAD: 5420 t4_sge_modload(); 5421 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 5422 SLIST_INIT(&t4_list); 5423 #ifndef TCP_OFFLOAD_DISABLE 5424 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 5425 SLIST_INIT(&t4_uld_list); 5426 #endif 5427 tweak_tunables(); 5428 break; 5429 5430 case MOD_UNLOAD: 5431 #ifndef TCP_OFFLOAD_DISABLE 5432 mtx_lock(&t4_uld_list_lock); 5433 if (!SLIST_EMPTY(&t4_uld_list)) { 5434 rc = EBUSY; 5435 mtx_unlock(&t4_uld_list_lock); 5436 break; 5437 } 5438 mtx_unlock(&t4_uld_list_lock); 5439 mtx_destroy(&t4_uld_list_lock); 5440 #endif 5441 mtx_lock(&t4_list_lock); 5442 if (!SLIST_EMPTY(&t4_list)) { 5443 rc = EBUSY; 5444 mtx_unlock(&t4_list_lock); 5445 break; 5446 } 5447 mtx_unlock(&t4_list_lock); 5448 mtx_destroy(&t4_list_lock); 5449 break; 5450 } 5451 5452 return (rc); 5453 } 5454 5455 static devclass_t t4_devclass; 5456 static devclass_t cxgbe_devclass; 5457 5458 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0); 5459 MODULE_VERSION(t4nex, 1); 5460 5461 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 5462 MODULE_VERSION(cxgbe, 1); 5463