1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 33 #include <sys/param.h> 34 #include <sys/conf.h> 35 #include <sys/priv.h> 36 #include <sys/kernel.h> 37 #include <sys/bus.h> 38 #include <sys/module.h> 39 #include <sys/malloc.h> 40 #include <sys/queue.h> 41 #include <sys/taskqueue.h> 42 #include <sys/pciio.h> 43 #include <dev/pci/pcireg.h> 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pci_private.h> 46 #include <sys/firmware.h> 47 #include <sys/sbuf.h> 48 #include <sys/smp.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/sysctl.h> 52 #include <net/ethernet.h> 53 #include <net/if.h> 54 #include <net/if_types.h> 55 #include <net/if_dl.h> 56 #include <net/if_vlan_var.h> 57 58 #include "common/common.h" 59 #include "common/t4_msg.h" 60 #include "common/t4_regs.h" 61 #include "common/t4_regs_values.h" 62 #include "t4_ioctl.h" 63 #include "t4_l2t.h" 64 65 /* T4 bus driver interface */ 66 static int t4_probe(device_t); 67 static int t4_attach(device_t); 68 static int t4_detach(device_t); 69 static device_method_t t4_methods[] = { 70 DEVMETHOD(device_probe, t4_probe), 71 DEVMETHOD(device_attach, t4_attach), 72 DEVMETHOD(device_detach, t4_detach), 73 74 DEVMETHOD_END 75 }; 76 static driver_t t4_driver = { 77 "t4nex", 78 t4_methods, 79 sizeof(struct adapter) 80 }; 81 82 83 /* T4 port (cxgbe) interface */ 84 static int cxgbe_probe(device_t); 85 static int cxgbe_attach(device_t); 86 static int cxgbe_detach(device_t); 87 static device_method_t cxgbe_methods[] = { 88 DEVMETHOD(device_probe, cxgbe_probe), 89 DEVMETHOD(device_attach, cxgbe_attach), 90 DEVMETHOD(device_detach, cxgbe_detach), 91 { 0, 0 } 92 }; 93 static driver_t cxgbe_driver = { 94 "cxgbe", 95 cxgbe_methods, 96 sizeof(struct port_info) 97 }; 98 99 static d_ioctl_t t4_ioctl; 100 static d_open_t t4_open; 101 static d_close_t t4_close; 102 103 static struct cdevsw t4_cdevsw = { 104 .d_version = D_VERSION, 105 .d_flags = 0, 106 .d_open = t4_open, 107 .d_close = t4_close, 108 .d_ioctl = t4_ioctl, 109 .d_name = "t4nex", 110 }; 111 112 /* ifnet + media interface */ 113 static void cxgbe_init(void *); 114 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 115 static void cxgbe_start(struct ifnet *); 116 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 117 static void cxgbe_qflush(struct ifnet *); 118 static int cxgbe_media_change(struct ifnet *); 119 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 120 121 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services"); 122 123 static struct mtx t4_list_lock; 124 static SLIST_HEAD(, adapter) t4_list; 125 #ifndef TCP_OFFLOAD_DISABLE 126 static struct mtx t4_uld_list_lock; 127 static SLIST_HEAD(, uld_info) t4_uld_list; 128 #endif 129 130 /* 131 * Tunables. See tweak_tunables() too. 132 */ 133 134 /* 135 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 136 */ 137 #define NTXQ_10G 16 138 static int t4_ntxq10g = -1; 139 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 140 141 #define NRXQ_10G 8 142 static int t4_nrxq10g = -1; 143 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 144 145 #define NTXQ_1G 4 146 static int t4_ntxq1g = -1; 147 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 148 149 #define NRXQ_1G 2 150 static int t4_nrxq1g = -1; 151 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 152 153 #ifndef TCP_OFFLOAD_DISABLE 154 #define NOFLDTXQ_10G 8 155 static int t4_nofldtxq10g = -1; 156 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 157 158 #define NOFLDRXQ_10G 2 159 static int t4_nofldrxq10g = -1; 160 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 161 162 #define NOFLDTXQ_1G 2 163 static int t4_nofldtxq1g = -1; 164 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 165 166 #define NOFLDRXQ_1G 1 167 static int t4_nofldrxq1g = -1; 168 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 169 #endif 170 171 /* 172 * Holdoff parameters for 10G and 1G ports. 173 */ 174 #define TMR_IDX_10G 1 175 static int t4_tmr_idx_10g = TMR_IDX_10G; 176 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 177 178 #define PKTC_IDX_10G 2 179 static int t4_pktc_idx_10g = PKTC_IDX_10G; 180 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 181 182 #define TMR_IDX_1G 1 183 static int t4_tmr_idx_1g = TMR_IDX_1G; 184 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 185 186 #define PKTC_IDX_1G 2 187 static int t4_pktc_idx_1g = PKTC_IDX_1G; 188 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 189 190 /* 191 * Size (# of entries) of each tx and rx queue. 192 */ 193 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 194 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 195 196 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 197 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 198 199 /* 200 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 201 */ 202 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 203 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 204 205 /* 206 * Configuration file. 207 */ 208 static char t4_cfg_file[32] = "default"; 209 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 210 211 /* 212 * ASIC features that will be used. Disable the ones you don't want so that the 213 * chip resources aren't wasted on features that will not be used. 214 */ 215 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 216 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 217 218 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 219 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 220 221 static int t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 222 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 223 224 static int t4_rdmacaps_allowed = 0; 225 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 226 227 static int t4_iscsicaps_allowed = 0; 228 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 229 230 static int t4_fcoecaps_allowed = 0; 231 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 232 233 struct intrs_and_queues { 234 int intr_type; /* INTx, MSI, or MSI-X */ 235 int nirq; /* Number of vectors */ 236 int intr_flags; 237 int ntxq10g; /* # of NIC txq's for each 10G port */ 238 int nrxq10g; /* # of NIC rxq's for each 10G port */ 239 int ntxq1g; /* # of NIC txq's for each 1G port */ 240 int nrxq1g; /* # of NIC rxq's for each 1G port */ 241 #ifndef TCP_OFFLOAD_DISABLE 242 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 243 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 244 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 245 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 246 #endif 247 }; 248 249 struct filter_entry { 250 uint32_t valid:1; /* filter allocated and valid */ 251 uint32_t locked:1; /* filter is administratively locked */ 252 uint32_t pending:1; /* filter action is pending firmware reply */ 253 uint32_t smtidx:8; /* Source MAC Table index for smac */ 254 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 255 256 struct t4_filter_specification fs; 257 }; 258 259 enum { 260 XGMAC_MTU = (1 << 0), 261 XGMAC_PROMISC = (1 << 1), 262 XGMAC_ALLMULTI = (1 << 2), 263 XGMAC_VLANEX = (1 << 3), 264 XGMAC_UCADDR = (1 << 4), 265 XGMAC_MCADDRS = (1 << 5), 266 267 XGMAC_ALL = 0xffff 268 }; 269 270 static int map_bars(struct adapter *); 271 static void setup_memwin(struct adapter *); 272 static int cfg_itype_and_nqueues(struct adapter *, int, int, 273 struct intrs_and_queues *); 274 static int prep_firmware(struct adapter *); 275 static int upload_config_file(struct adapter *, const struct firmware *, 276 uint32_t *, uint32_t *); 277 static int partition_resources(struct adapter *, const struct firmware *); 278 static int get_params__pre_init(struct adapter *); 279 static int get_params__post_init(struct adapter *); 280 static void t4_set_desc(struct adapter *); 281 static void build_medialist(struct port_info *); 282 static int update_mac_settings(struct port_info *, int); 283 static int cxgbe_init_locked(struct port_info *); 284 static int cxgbe_init_synchronized(struct port_info *); 285 static int cxgbe_uninit_locked(struct port_info *); 286 static int cxgbe_uninit_synchronized(struct port_info *); 287 static int adapter_full_init(struct adapter *); 288 static int adapter_full_uninit(struct adapter *); 289 static int port_full_init(struct port_info *); 290 static int port_full_uninit(struct port_info *); 291 static void quiesce_eq(struct adapter *, struct sge_eq *); 292 static void quiesce_iq(struct adapter *, struct sge_iq *); 293 static void quiesce_fl(struct adapter *, struct sge_fl *); 294 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 295 driver_intr_t *, void *, char *); 296 static int t4_free_irq(struct adapter *, struct irq *); 297 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 298 unsigned int); 299 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 300 static void cxgbe_tick(void *); 301 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 302 struct mbuf *); 303 static int t4_sysctls(struct adapter *); 304 static int cxgbe_sysctls(struct port_info *); 305 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 306 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 307 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 308 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 309 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 310 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 311 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 312 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 313 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 314 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 315 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 316 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 317 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 318 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 319 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 320 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 321 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 322 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 323 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 324 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 325 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 326 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 327 static inline void txq_start(struct ifnet *, struct sge_txq *); 328 static uint32_t fconf_to_mode(uint32_t); 329 static uint32_t mode_to_fconf(uint32_t); 330 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 331 static int get_filter_mode(struct adapter *, uint32_t *); 332 static int set_filter_mode(struct adapter *, uint32_t); 333 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 334 static int get_filter(struct adapter *, struct t4_filter *); 335 static int set_filter(struct adapter *, struct t4_filter *); 336 static int del_filter(struct adapter *, struct t4_filter *); 337 static void clear_filter(struct filter_entry *); 338 static int set_filter_wr(struct adapter *, int); 339 static int del_filter_wr(struct adapter *, int); 340 static int filter_rpl(struct sge_iq *, const struct rss_header *, 341 struct mbuf *); 342 static int get_sge_context(struct adapter *, struct t4_sge_context *); 343 static int read_card_mem(struct adapter *, struct t4_mem_range *); 344 #ifndef TCP_OFFLOAD_DISABLE 345 static int toe_capability(struct port_info *, int); 346 static int activate_uld(struct adapter *, int, struct uld_softc *); 347 static int deactivate_uld(struct uld_softc *); 348 #endif 349 static int t4_mod_event(module_t, int, void *); 350 351 struct t4_pciids { 352 uint16_t device; 353 uint8_t mpf; 354 char *desc; 355 } t4_pciids[] = { 356 {0xa000, 0, "Chelsio Terminator 4 FPGA"}, 357 {0x4400, 4, "Chelsio T440-dbg"}, 358 {0x4401, 4, "Chelsio T420-CR"}, 359 {0x4402, 4, "Chelsio T422-CR"}, 360 {0x4403, 4, "Chelsio T440-CR"}, 361 {0x4404, 4, "Chelsio T420-BCH"}, 362 {0x4405, 4, "Chelsio T440-BCH"}, 363 {0x4406, 4, "Chelsio T440-CH"}, 364 {0x4407, 4, "Chelsio T420-SO"}, 365 {0x4408, 4, "Chelsio T420-CX"}, 366 {0x4409, 4, "Chelsio T420-BT"}, 367 {0x440a, 4, "Chelsio T404-BT"}, 368 }; 369 370 #ifndef TCP_OFFLOAD_DISABLE 371 /* This is used in service_iq() to get to the fl associated with an iq. */ 372 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 373 #endif 374 375 static int 376 t4_probe(device_t dev) 377 { 378 int i; 379 uint16_t v = pci_get_vendor(dev); 380 uint16_t d = pci_get_device(dev); 381 382 if (v != PCI_VENDOR_ID_CHELSIO) 383 return (ENXIO); 384 385 for (i = 0; i < ARRAY_SIZE(t4_pciids); i++) { 386 if (d == t4_pciids[i].device && 387 pci_get_function(dev) == t4_pciids[i].mpf) { 388 device_set_desc(dev, t4_pciids[i].desc); 389 return (BUS_PROBE_DEFAULT); 390 } 391 } 392 393 return (ENXIO); 394 } 395 396 static int 397 t4_attach(device_t dev) 398 { 399 struct adapter *sc; 400 int rc = 0, i, n10g, n1g, rqidx, tqidx; 401 struct intrs_and_queues iaq; 402 struct sge *s; 403 #ifndef TCP_OFFLOAD_DISABLE 404 int ofld_rqidx, ofld_tqidx; 405 #endif 406 407 sc = device_get_softc(dev); 408 sc->dev = dev; 409 sc->pf = pci_get_function(dev); 410 sc->mbox = sc->pf; 411 412 pci_enable_busmaster(dev); 413 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 414 uint32_t v; 415 416 pci_set_max_read_req(dev, 4096); 417 v = pci_read_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, 2); 418 v |= PCIM_EXP_CTL_RELAXED_ORD_ENABLE; 419 pci_write_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, v, 2); 420 } 421 422 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 423 device_get_nameunit(dev)); 424 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 425 mtx_lock(&t4_list_lock); 426 SLIST_INSERT_HEAD(&t4_list, sc, link); 427 mtx_unlock(&t4_list_lock); 428 429 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 430 TAILQ_INIT(&sc->sfl); 431 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 432 433 rc = map_bars(sc); 434 if (rc != 0) 435 goto done; /* error message displayed already */ 436 437 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 438 for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) 439 sc->cpl_handler[i] = cpl_not_handled; 440 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, filter_rpl); 441 442 /* Prepare the adapter for operation */ 443 rc = -t4_prep_adapter(sc); 444 if (rc != 0) { 445 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 446 goto done; 447 } 448 449 /* 450 * Do this really early, with the memory windows set up even before the 451 * character device. The userland tool's register i/o and mem read 452 * will work even in "recovery mode". 453 */ 454 setup_memwin(sc); 455 sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT, 456 GID_WHEEL, 0600, "%s", device_get_nameunit(dev)); 457 sc->cdev->si_drv1 = sc; 458 459 /* Go no further if recovery mode has been requested. */ 460 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 461 device_printf(dev, "recovery mode.\n"); 462 goto done; 463 } 464 465 /* Prepare the firmware for operation */ 466 rc = prep_firmware(sc); 467 if (rc != 0) 468 goto done; /* error message displayed already */ 469 470 rc = get_params__pre_init(sc); 471 if (rc != 0) 472 goto done; /* error message displayed already */ 473 474 rc = t4_sge_init(sc); 475 if (rc != 0) 476 goto done; /* error message displayed already */ 477 478 if (sc->flags & MASTER_PF) { 479 /* get basic stuff going */ 480 rc = -t4_fw_initialize(sc, sc->mbox); 481 if (rc != 0) { 482 device_printf(dev, "early init failed: %d.\n", rc); 483 goto done; 484 } 485 } 486 487 rc = get_params__post_init(sc); 488 if (rc != 0) 489 goto done; /* error message displayed already */ 490 491 if (sc->flags & MASTER_PF) { 492 493 /* final tweaks to some settings */ 494 495 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, 496 sc->params.b_wnd); 497 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 498 t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 | 499 F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3, 0); 500 t4_set_reg_field(sc, A_TP_PARA_REG5, 501 V_INDICATESIZE(M_INDICATESIZE) | 502 F_REARMDDPOFFSET | F_RESETDDPOFFSET, 503 V_INDICATESIZE(M_INDICATESIZE) | 504 F_REARMDDPOFFSET | F_RESETDDPOFFSET); 505 } else { 506 /* 507 * XXX: Verify that we can live with whatever the master driver 508 * has done so far, and hope that it doesn't change any global 509 * setting from underneath us in the future. 510 */ 511 } 512 513 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 514 A_TP_VLAN_PRI_MAP); 515 516 for (i = 0; i < NCHAN; i++) 517 sc->params.tp.tx_modq[i] = i; 518 519 rc = t4_create_dma_tag(sc); 520 if (rc != 0) 521 goto done; /* error message displayed already */ 522 523 /* 524 * First pass over all the ports - allocate VIs and initialize some 525 * basic parameters like mac address, port type, etc. We also figure 526 * out whether a port is 10G or 1G and use that information when 527 * calculating how many interrupts to attempt to allocate. 528 */ 529 n10g = n1g = 0; 530 for_each_port(sc, i) { 531 struct port_info *pi; 532 533 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 534 sc->port[i] = pi; 535 536 /* These must be set before t4_port_init */ 537 pi->adapter = sc; 538 pi->port_id = i; 539 540 /* Allocate the vi and initialize parameters like mac addr */ 541 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 542 if (rc != 0) { 543 device_printf(dev, "unable to initialize port %d: %d\n", 544 i, rc); 545 free(pi, M_CXGBE); 546 sc->port[i] = NULL; 547 goto done; 548 } 549 550 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 551 device_get_nameunit(dev), i); 552 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 553 554 if (is_10G_port(pi)) { 555 n10g++; 556 pi->tmr_idx = t4_tmr_idx_10g; 557 pi->pktc_idx = t4_pktc_idx_10g; 558 } else { 559 n1g++; 560 pi->tmr_idx = t4_tmr_idx_1g; 561 pi->pktc_idx = t4_pktc_idx_1g; 562 } 563 564 pi->xact_addr_filt = -1; 565 566 pi->qsize_rxq = t4_qsize_rxq; 567 pi->qsize_txq = t4_qsize_txq; 568 569 pi->dev = device_add_child(dev, "cxgbe", -1); 570 if (pi->dev == NULL) { 571 device_printf(dev, 572 "failed to add device for port %d.\n", i); 573 rc = ENXIO; 574 goto done; 575 } 576 device_set_softc(pi->dev, pi); 577 } 578 579 /* 580 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 581 */ 582 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 583 if (rc != 0) 584 goto done; /* error message displayed already */ 585 586 sc->intr_type = iaq.intr_type; 587 sc->intr_count = iaq.nirq; 588 sc->flags |= iaq.intr_flags; 589 590 s = &sc->sge; 591 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 592 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 593 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 594 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 595 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 596 597 #ifndef TCP_OFFLOAD_DISABLE 598 if (is_offload(sc)) { 599 600 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 601 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 602 s->neq += s->nofldtxq + s->nofldrxq; 603 s->niq += s->nofldrxq; 604 605 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 606 M_CXGBE, M_ZERO | M_WAITOK); 607 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 608 M_CXGBE, M_ZERO | M_WAITOK); 609 } 610 #endif 611 612 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 613 M_ZERO | M_WAITOK); 614 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 615 M_ZERO | M_WAITOK); 616 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 617 M_ZERO | M_WAITOK); 618 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 619 M_ZERO | M_WAITOK); 620 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 621 M_ZERO | M_WAITOK); 622 623 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 624 M_ZERO | M_WAITOK); 625 626 t4_init_l2t(sc, M_WAITOK); 627 628 /* 629 * Second pass over the ports. This time we know the number of rx and 630 * tx queues that each port should get. 631 */ 632 rqidx = tqidx = 0; 633 #ifndef TCP_OFFLOAD_DISABLE 634 ofld_rqidx = ofld_tqidx = 0; 635 #endif 636 for_each_port(sc, i) { 637 struct port_info *pi = sc->port[i]; 638 639 if (pi == NULL) 640 continue; 641 642 pi->first_rxq = rqidx; 643 pi->first_txq = tqidx; 644 if (is_10G_port(pi)) { 645 pi->nrxq = iaq.nrxq10g; 646 pi->ntxq = iaq.ntxq10g; 647 } else { 648 pi->nrxq = iaq.nrxq1g; 649 pi->ntxq = iaq.ntxq1g; 650 } 651 652 rqidx += pi->nrxq; 653 tqidx += pi->ntxq; 654 655 #ifndef TCP_OFFLOAD_DISABLE 656 if (is_offload(sc)) { 657 pi->first_ofld_rxq = ofld_rqidx; 658 pi->first_ofld_txq = ofld_tqidx; 659 if (is_10G_port(pi)) { 660 pi->nofldrxq = iaq.nofldrxq10g; 661 pi->nofldtxq = iaq.nofldtxq10g; 662 } else { 663 pi->nofldrxq = iaq.nofldrxq1g; 664 pi->nofldtxq = iaq.nofldtxq1g; 665 } 666 ofld_rqidx += pi->nofldrxq; 667 ofld_tqidx += pi->nofldtxq; 668 } 669 #endif 670 } 671 672 rc = bus_generic_attach(dev); 673 if (rc != 0) { 674 device_printf(dev, 675 "failed to attach all child ports: %d\n", rc); 676 goto done; 677 } 678 679 device_printf(dev, 680 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 681 sc->params.pci.width, sc->params.nports, sc->intr_count, 682 sc->intr_type == INTR_MSIX ? "MSI-X" : 683 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 684 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 685 686 t4_set_desc(sc); 687 688 done: 689 if (rc != 0 && sc->cdev) { 690 /* cdev was created and so cxgbetool works; recover that way. */ 691 device_printf(dev, 692 "error during attach, adapter is now in recovery mode.\n"); 693 rc = 0; 694 } 695 696 if (rc != 0) 697 t4_detach(dev); 698 else 699 t4_sysctls(sc); 700 701 return (rc); 702 } 703 704 /* 705 * Idempotent 706 */ 707 static int 708 t4_detach(device_t dev) 709 { 710 struct adapter *sc; 711 struct port_info *pi; 712 int i, rc; 713 714 sc = device_get_softc(dev); 715 716 if (sc->flags & FULL_INIT_DONE) 717 t4_intr_disable(sc); 718 719 if (sc->cdev) { 720 destroy_dev(sc->cdev); 721 sc->cdev = NULL; 722 } 723 724 rc = bus_generic_detach(dev); 725 if (rc) { 726 device_printf(dev, 727 "failed to detach child devices: %d\n", rc); 728 return (rc); 729 } 730 731 for (i = 0; i < MAX_NPORTS; i++) { 732 pi = sc->port[i]; 733 if (pi) { 734 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 735 if (pi->dev) 736 device_delete_child(dev, pi->dev); 737 738 mtx_destroy(&pi->pi_lock); 739 free(pi, M_CXGBE); 740 } 741 } 742 743 if (sc->flags & FULL_INIT_DONE) 744 adapter_full_uninit(sc); 745 746 if (sc->flags & FW_OK) 747 t4_fw_bye(sc, sc->mbox); 748 749 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 750 pci_release_msi(dev); 751 752 if (sc->regs_res) 753 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 754 sc->regs_res); 755 756 if (sc->msix_res) 757 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 758 sc->msix_res); 759 760 if (sc->l2t) 761 t4_free_l2t(sc->l2t); 762 763 #ifndef TCP_OFFLOAD_DISABLE 764 free(sc->sge.ofld_rxq, M_CXGBE); 765 free(sc->sge.ofld_txq, M_CXGBE); 766 #endif 767 free(sc->irq, M_CXGBE); 768 free(sc->sge.rxq, M_CXGBE); 769 free(sc->sge.txq, M_CXGBE); 770 free(sc->sge.ctrlq, M_CXGBE); 771 free(sc->sge.iqmap, M_CXGBE); 772 free(sc->sge.eqmap, M_CXGBE); 773 free(sc->tids.ftid_tab, M_CXGBE); 774 t4_destroy_dma_tag(sc); 775 if (mtx_initialized(&sc->sc_lock)) { 776 mtx_lock(&t4_list_lock); 777 SLIST_REMOVE(&t4_list, sc, adapter, link); 778 mtx_unlock(&t4_list_lock); 779 mtx_destroy(&sc->sc_lock); 780 } 781 782 if (mtx_initialized(&sc->sfl_lock)) 783 mtx_destroy(&sc->sfl_lock); 784 785 bzero(sc, sizeof(*sc)); 786 787 return (0); 788 } 789 790 791 static int 792 cxgbe_probe(device_t dev) 793 { 794 char buf[128]; 795 struct port_info *pi = device_get_softc(dev); 796 797 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 798 device_set_desc_copy(dev, buf); 799 800 return (BUS_PROBE_DEFAULT); 801 } 802 803 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 804 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 805 IFCAP_VLAN_HWTSO) 806 #define T4_CAP_ENABLE (T4_CAP & ~IFCAP_TSO6) 807 808 static int 809 cxgbe_attach(device_t dev) 810 { 811 struct port_info *pi = device_get_softc(dev); 812 struct ifnet *ifp; 813 814 /* Allocate an ifnet and set it up */ 815 ifp = if_alloc(IFT_ETHER); 816 if (ifp == NULL) { 817 device_printf(dev, "Cannot allocate ifnet\n"); 818 return (ENOMEM); 819 } 820 pi->ifp = ifp; 821 ifp->if_softc = pi; 822 823 callout_init(&pi->tick, CALLOUT_MPSAFE); 824 825 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 826 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 827 828 ifp->if_init = cxgbe_init; 829 ifp->if_ioctl = cxgbe_ioctl; 830 ifp->if_start = cxgbe_start; 831 ifp->if_transmit = cxgbe_transmit; 832 ifp->if_qflush = cxgbe_qflush; 833 834 ifp->if_snd.ifq_drv_maxlen = 1024; 835 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 836 IFQ_SET_READY(&ifp->if_snd); 837 838 ifp->if_capabilities = T4_CAP; 839 #ifndef TCP_OFFLOAD_DISABLE 840 if (is_offload(pi->adapter)) 841 ifp->if_capabilities |= IFCAP_TOE4; 842 #endif 843 ifp->if_capenable = T4_CAP_ENABLE; 844 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; 845 846 /* Initialize ifmedia for this port */ 847 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 848 cxgbe_media_status); 849 build_medialist(pi); 850 851 ether_ifattach(ifp, pi->hw_addr); 852 853 #ifndef TCP_OFFLOAD_DISABLE 854 if (is_offload(pi->adapter)) { 855 device_printf(dev, 856 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 857 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 858 } else 859 #endif 860 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 861 862 cxgbe_sysctls(pi); 863 864 return (0); 865 } 866 867 static int 868 cxgbe_detach(device_t dev) 869 { 870 struct port_info *pi = device_get_softc(dev); 871 struct adapter *sc = pi->adapter; 872 struct ifnet *ifp = pi->ifp; 873 874 /* Tell if_ioctl and if_init that the port is going away */ 875 ADAPTER_LOCK(sc); 876 SET_DOOMED(pi); 877 wakeup(&sc->flags); 878 while (IS_BUSY(sc)) 879 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 880 SET_BUSY(sc); 881 ADAPTER_UNLOCK(sc); 882 883 PORT_LOCK(pi); 884 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 885 callout_stop(&pi->tick); 886 PORT_UNLOCK(pi); 887 callout_drain(&pi->tick); 888 889 /* Let detach proceed even if these fail. */ 890 cxgbe_uninit_synchronized(pi); 891 port_full_uninit(pi); 892 893 ifmedia_removeall(&pi->media); 894 ether_ifdetach(pi->ifp); 895 if_free(pi->ifp); 896 897 ADAPTER_LOCK(sc); 898 CLR_BUSY(sc); 899 wakeup_one(&sc->flags); 900 ADAPTER_UNLOCK(sc); 901 902 return (0); 903 } 904 905 static void 906 cxgbe_init(void *arg) 907 { 908 struct port_info *pi = arg; 909 struct adapter *sc = pi->adapter; 910 911 ADAPTER_LOCK(sc); 912 cxgbe_init_locked(pi); /* releases adapter lock */ 913 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 914 } 915 916 static int 917 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 918 { 919 int rc = 0, mtu, flags; 920 struct port_info *pi = ifp->if_softc; 921 struct adapter *sc = pi->adapter; 922 struct ifreq *ifr = (struct ifreq *)data; 923 uint32_t mask; 924 925 switch (cmd) { 926 case SIOCSIFMTU: 927 ADAPTER_LOCK(sc); 928 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 929 if (rc) { 930 fail: 931 ADAPTER_UNLOCK(sc); 932 return (rc); 933 } 934 935 mtu = ifr->ifr_mtu; 936 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 937 rc = EINVAL; 938 } else { 939 ifp->if_mtu = mtu; 940 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 941 t4_update_fl_bufsize(ifp); 942 PORT_LOCK(pi); 943 rc = update_mac_settings(pi, XGMAC_MTU); 944 PORT_UNLOCK(pi); 945 } 946 } 947 ADAPTER_UNLOCK(sc); 948 break; 949 950 case SIOCSIFFLAGS: 951 ADAPTER_LOCK(sc); 952 if (IS_DOOMED(pi)) { 953 rc = ENXIO; 954 goto fail; 955 } 956 if (ifp->if_flags & IFF_UP) { 957 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 958 flags = pi->if_flags; 959 if ((ifp->if_flags ^ flags) & 960 (IFF_PROMISC | IFF_ALLMULTI)) { 961 if (IS_BUSY(sc)) { 962 rc = EBUSY; 963 goto fail; 964 } 965 PORT_LOCK(pi); 966 rc = update_mac_settings(pi, 967 XGMAC_PROMISC | XGMAC_ALLMULTI); 968 PORT_UNLOCK(pi); 969 } 970 ADAPTER_UNLOCK(sc); 971 } else 972 rc = cxgbe_init_locked(pi); 973 pi->if_flags = ifp->if_flags; 974 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 975 rc = cxgbe_uninit_locked(pi); 976 else 977 ADAPTER_UNLOCK(sc); 978 979 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 980 break; 981 982 case SIOCADDMULTI: 983 case SIOCDELMULTI: /* these two can be called with a mutex held :-( */ 984 ADAPTER_LOCK(sc); 985 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 986 if (rc) 987 goto fail; 988 989 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 990 PORT_LOCK(pi); 991 rc = update_mac_settings(pi, XGMAC_MCADDRS); 992 PORT_UNLOCK(pi); 993 } 994 ADAPTER_UNLOCK(sc); 995 break; 996 997 case SIOCSIFCAP: 998 ADAPTER_LOCK(sc); 999 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1000 if (rc) 1001 goto fail; 1002 1003 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1004 if (mask & IFCAP_TXCSUM) { 1005 ifp->if_capenable ^= IFCAP_TXCSUM; 1006 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1007 1008 if (IFCAP_TSO & ifp->if_capenable && 1009 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1010 ifp->if_capenable &= ~IFCAP_TSO; 1011 ifp->if_hwassist &= ~CSUM_TSO; 1012 if_printf(ifp, 1013 "tso disabled due to -txcsum.\n"); 1014 } 1015 } 1016 if (mask & IFCAP_RXCSUM) 1017 ifp->if_capenable ^= IFCAP_RXCSUM; 1018 if (mask & IFCAP_TSO4) { 1019 ifp->if_capenable ^= IFCAP_TSO4; 1020 1021 if (IFCAP_TSO & ifp->if_capenable) { 1022 if (IFCAP_TXCSUM & ifp->if_capenable) 1023 ifp->if_hwassist |= CSUM_TSO; 1024 else { 1025 ifp->if_capenable &= ~IFCAP_TSO; 1026 ifp->if_hwassist &= ~CSUM_TSO; 1027 if_printf(ifp, 1028 "enable txcsum first.\n"); 1029 rc = EAGAIN; 1030 goto fail; 1031 } 1032 } else 1033 ifp->if_hwassist &= ~CSUM_TSO; 1034 } 1035 if (mask & IFCAP_LRO) { 1036 #ifdef INET 1037 int i; 1038 struct sge_rxq *rxq; 1039 1040 ifp->if_capenable ^= IFCAP_LRO; 1041 for_each_rxq(pi, i, rxq) { 1042 if (ifp->if_capenable & IFCAP_LRO) 1043 rxq->iq.flags |= IQ_LRO_ENABLED; 1044 else 1045 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1046 } 1047 #endif 1048 } 1049 #ifndef TCP_OFFLOAD_DISABLE 1050 if (mask & IFCAP_TOE) { 1051 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1052 1053 rc = toe_capability(pi, enable); 1054 if (rc != 0) 1055 goto fail; 1056 1057 ifp->if_capenable ^= mask; 1058 } 1059 #endif 1060 if (mask & IFCAP_VLAN_HWTAGGING) { 1061 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1062 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1063 PORT_LOCK(pi); 1064 rc = update_mac_settings(pi, XGMAC_VLANEX); 1065 PORT_UNLOCK(pi); 1066 } 1067 } 1068 if (mask & IFCAP_VLAN_MTU) { 1069 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1070 1071 /* Need to find out how to disable auto-mtu-inflation */ 1072 } 1073 if (mask & IFCAP_VLAN_HWTSO) 1074 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1075 if (mask & IFCAP_VLAN_HWCSUM) 1076 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1077 1078 #ifdef VLAN_CAPABILITIES 1079 VLAN_CAPABILITIES(ifp); 1080 #endif 1081 ADAPTER_UNLOCK(sc); 1082 break; 1083 1084 case SIOCSIFMEDIA: 1085 case SIOCGIFMEDIA: 1086 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1087 break; 1088 1089 default: 1090 rc = ether_ioctl(ifp, cmd, data); 1091 } 1092 1093 return (rc); 1094 } 1095 1096 static void 1097 cxgbe_start(struct ifnet *ifp) 1098 { 1099 struct port_info *pi = ifp->if_softc; 1100 struct sge_txq *txq; 1101 int i; 1102 1103 for_each_txq(pi, i, txq) { 1104 if (TXQ_TRYLOCK(txq)) { 1105 txq_start(ifp, txq); 1106 TXQ_UNLOCK(txq); 1107 } 1108 } 1109 } 1110 1111 static int 1112 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1113 { 1114 struct port_info *pi = ifp->if_softc; 1115 struct adapter *sc = pi->adapter; 1116 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1117 struct buf_ring *br; 1118 int rc; 1119 1120 M_ASSERTPKTHDR(m); 1121 1122 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1123 m_freem(m); 1124 return (ENETDOWN); 1125 } 1126 1127 if (m->m_flags & M_FLOWID) 1128 txq += (m->m_pkthdr.flowid % pi->ntxq); 1129 br = txq->br; 1130 1131 if (TXQ_TRYLOCK(txq) == 0) { 1132 struct sge_eq *eq = &txq->eq; 1133 1134 /* 1135 * It is possible that t4_eth_tx finishes up and releases the 1136 * lock between the TRYLOCK above and the drbr_enqueue here. We 1137 * need to make sure that this mbuf doesn't just sit there in 1138 * the drbr. 1139 */ 1140 1141 rc = drbr_enqueue(ifp, br, m); 1142 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1143 !(eq->flags & EQ_DOOMED)) 1144 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1145 return (rc); 1146 } 1147 1148 /* 1149 * txq->m is the mbuf that is held up due to a temporary shortage of 1150 * resources and it should be put on the wire first. Then what's in 1151 * drbr and finally the mbuf that was just passed in to us. 1152 * 1153 * Return code should indicate the fate of the mbuf that was passed in 1154 * this time. 1155 */ 1156 1157 TXQ_LOCK_ASSERT_OWNED(txq); 1158 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1159 1160 /* Queued for transmission. */ 1161 1162 rc = drbr_enqueue(ifp, br, m); 1163 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1164 (void) t4_eth_tx(ifp, txq, m); 1165 TXQ_UNLOCK(txq); 1166 return (rc); 1167 } 1168 1169 /* Direct transmission. */ 1170 rc = t4_eth_tx(ifp, txq, m); 1171 if (rc != 0 && txq->m) 1172 rc = 0; /* held, will be transmitted soon (hopefully) */ 1173 1174 TXQ_UNLOCK(txq); 1175 return (rc); 1176 } 1177 1178 static void 1179 cxgbe_qflush(struct ifnet *ifp) 1180 { 1181 struct port_info *pi = ifp->if_softc; 1182 struct sge_txq *txq; 1183 int i; 1184 struct mbuf *m; 1185 1186 /* queues do not exist if !PORT_INIT_DONE. */ 1187 if (pi->flags & PORT_INIT_DONE) { 1188 for_each_txq(pi, i, txq) { 1189 TXQ_LOCK(txq); 1190 m_freem(txq->m); 1191 txq->m = NULL; 1192 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1193 m_freem(m); 1194 TXQ_UNLOCK(txq); 1195 } 1196 } 1197 if_qflush(ifp); 1198 } 1199 1200 static int 1201 cxgbe_media_change(struct ifnet *ifp) 1202 { 1203 struct port_info *pi = ifp->if_softc; 1204 1205 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1206 1207 return (EOPNOTSUPP); 1208 } 1209 1210 static void 1211 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1212 { 1213 struct port_info *pi = ifp->if_softc; 1214 struct ifmedia_entry *cur = pi->media.ifm_cur; 1215 int speed = pi->link_cfg.speed; 1216 int data = (pi->port_type << 8) | pi->mod_type; 1217 1218 if (cur->ifm_data != data) { 1219 build_medialist(pi); 1220 cur = pi->media.ifm_cur; 1221 } 1222 1223 ifmr->ifm_status = IFM_AVALID; 1224 if (!pi->link_cfg.link_ok) 1225 return; 1226 1227 ifmr->ifm_status |= IFM_ACTIVE; 1228 1229 /* active and current will differ iff current media is autoselect. */ 1230 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1231 return; 1232 1233 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1234 if (speed == SPEED_10000) 1235 ifmr->ifm_active |= IFM_10G_T; 1236 else if (speed == SPEED_1000) 1237 ifmr->ifm_active |= IFM_1000_T; 1238 else if (speed == SPEED_100) 1239 ifmr->ifm_active |= IFM_100_TX; 1240 else if (speed == SPEED_10) 1241 ifmr->ifm_active |= IFM_10_T; 1242 else 1243 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1244 speed)); 1245 } 1246 1247 void 1248 t4_fatal_err(struct adapter *sc) 1249 { 1250 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1251 t4_intr_disable(sc); 1252 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1253 device_get_nameunit(sc->dev)); 1254 } 1255 1256 static int 1257 map_bars(struct adapter *sc) 1258 { 1259 sc->regs_rid = PCIR_BAR(0); 1260 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1261 &sc->regs_rid, RF_ACTIVE); 1262 if (sc->regs_res == NULL) { 1263 device_printf(sc->dev, "cannot map registers.\n"); 1264 return (ENXIO); 1265 } 1266 sc->bt = rman_get_bustag(sc->regs_res); 1267 sc->bh = rman_get_bushandle(sc->regs_res); 1268 sc->mmio_len = rman_get_size(sc->regs_res); 1269 1270 sc->msix_rid = PCIR_BAR(4); 1271 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1272 &sc->msix_rid, RF_ACTIVE); 1273 if (sc->msix_res == NULL) { 1274 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1275 return (ENXIO); 1276 } 1277 1278 return (0); 1279 } 1280 1281 static void 1282 setup_memwin(struct adapter *sc) 1283 { 1284 u_long bar0; 1285 1286 bar0 = rman_get_start(sc->regs_res); 1287 1288 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1289 (bar0 + MEMWIN0_BASE) | V_BIR(0) | 1290 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1291 1292 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1293 (bar0 + MEMWIN1_BASE) | V_BIR(0) | 1294 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1295 1296 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1297 (bar0 + MEMWIN2_BASE) | V_BIR(0) | 1298 V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10)); 1299 } 1300 1301 static int 1302 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1303 struct intrs_and_queues *iaq) 1304 { 1305 int rc, itype, navail, nrxq10g, nrxq1g, n; 1306 int nofldrxq10g = 0, nofldrxq1g = 0; 1307 1308 bzero(iaq, sizeof(*iaq)); 1309 1310 iaq->ntxq10g = t4_ntxq10g; 1311 iaq->ntxq1g = t4_ntxq1g; 1312 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1313 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1314 #ifndef TCP_OFFLOAD_DISABLE 1315 iaq->nofldtxq10g = t4_nofldtxq10g; 1316 iaq->nofldtxq1g = t4_nofldtxq1g; 1317 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1318 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1319 #endif 1320 1321 for (itype = INTR_MSIX; itype; itype >>= 1) { 1322 1323 if ((itype & t4_intr_types) == 0) 1324 continue; /* not allowed */ 1325 1326 if (itype == INTR_MSIX) 1327 navail = pci_msix_count(sc->dev); 1328 else if (itype == INTR_MSI) 1329 navail = pci_msi_count(sc->dev); 1330 else 1331 navail = 1; 1332 restart: 1333 if (navail == 0) 1334 continue; 1335 1336 iaq->intr_type = itype; 1337 iaq->intr_flags = 0; 1338 1339 /* 1340 * Best option: an interrupt vector for errors, one for the 1341 * firmware event queue, and one each for each rxq (NIC as well 1342 * as offload). 1343 */ 1344 iaq->nirq = T4_EXTRA_INTR; 1345 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1346 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1347 if (iaq->nirq <= navail && 1348 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1349 iaq->intr_flags |= INTR_DIRECT; 1350 goto allocate; 1351 } 1352 1353 /* 1354 * Second best option: an interrupt vector for errors, one for 1355 * the firmware event queue, and one each for either NIC or 1356 * offload rxq's. 1357 */ 1358 iaq->nirq = T4_EXTRA_INTR; 1359 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1360 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1361 if (iaq->nirq <= navail && 1362 (itype != INTR_MSI || powerof2(iaq->nirq))) 1363 goto allocate; 1364 1365 /* 1366 * Next best option: an interrupt vector for errors, one for the 1367 * firmware event queue, and at least one per port. At this 1368 * point we know we'll have to downsize nrxq or nofldrxq to fit 1369 * what's available to us. 1370 */ 1371 iaq->nirq = T4_EXTRA_INTR; 1372 iaq->nirq += n10g + n1g; 1373 if (iaq->nirq <= navail) { 1374 int leftover = navail - iaq->nirq; 1375 1376 if (n10g > 0) { 1377 int target = max(nrxq10g, nofldrxq10g); 1378 1379 n = 1; 1380 while (n < target && leftover >= n10g) { 1381 leftover -= n10g; 1382 iaq->nirq += n10g; 1383 n++; 1384 } 1385 iaq->nrxq10g = min(n, nrxq10g); 1386 #ifndef TCP_OFFLOAD_DISABLE 1387 iaq->nofldrxq10g = min(n, nofldrxq10g); 1388 #endif 1389 } 1390 1391 if (n1g > 0) { 1392 int target = max(nrxq1g, nofldrxq1g); 1393 1394 n = 1; 1395 while (n < target && leftover >= n1g) { 1396 leftover -= n1g; 1397 iaq->nirq += n1g; 1398 n++; 1399 } 1400 iaq->nrxq1g = min(n, nrxq1g); 1401 #ifndef TCP_OFFLOAD_DISABLE 1402 iaq->nofldrxq1g = min(n, nofldrxq1g); 1403 #endif 1404 } 1405 1406 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1407 goto allocate; 1408 } 1409 1410 /* 1411 * Least desirable option: one interrupt vector for everything. 1412 */ 1413 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1414 #ifndef TCP_OFFLOAD_DISABLE 1415 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1416 #endif 1417 1418 allocate: 1419 navail = iaq->nirq; 1420 rc = 0; 1421 if (itype == INTR_MSIX) 1422 rc = pci_alloc_msix(sc->dev, &navail); 1423 else if (itype == INTR_MSI) 1424 rc = pci_alloc_msi(sc->dev, &navail); 1425 1426 if (rc == 0) { 1427 if (navail == iaq->nirq) 1428 return (0); 1429 1430 /* 1431 * Didn't get the number requested. Use whatever number 1432 * the kernel is willing to allocate (it's in navail). 1433 */ 1434 device_printf(sc->dev, "fewer vectors than requested, " 1435 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1436 itype, iaq->nirq, navail); 1437 pci_release_msi(sc->dev); 1438 goto restart; 1439 } 1440 1441 device_printf(sc->dev, 1442 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1443 itype, rc, iaq->nirq, navail); 1444 } 1445 1446 device_printf(sc->dev, 1447 "failed to find a usable interrupt type. " 1448 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1449 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1450 1451 return (ENXIO); 1452 } 1453 1454 /* 1455 * Install a compatible firmware (if required), establish contact with it (by 1456 * saying hello), and reset the device. If we end up as the master driver, 1457 * partition adapter resources by providing a configuration file to the 1458 * firmware. 1459 */ 1460 static int 1461 prep_firmware(struct adapter *sc) 1462 { 1463 const struct firmware *fw = NULL, *cfg = NULL, *default_cfg; 1464 int rc; 1465 enum dev_state state; 1466 1467 default_cfg = firmware_get(T4_CFGNAME); 1468 1469 /* Check firmware version and install a different one if necessary */ 1470 rc = t4_check_fw_version(sc); 1471 if (rc != 0) { 1472 uint32_t v = 0; 1473 1474 fw = firmware_get(T4_FWNAME); 1475 if (fw != NULL) { 1476 const struct fw_hdr *hdr = (const void *)fw->data; 1477 1478 v = ntohl(hdr->fw_ver); 1479 1480 /* 1481 * The firmware module will not be used if it isn't the 1482 * same major version as what the driver was compiled 1483 * with. 1484 */ 1485 if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) { 1486 device_printf(sc->dev, 1487 "Found firmware image but version %d " 1488 "can not be used with this driver (%d)\n", 1489 G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR); 1490 1491 firmware_put(fw, FIRMWARE_UNLOAD); 1492 fw = NULL; 1493 } 1494 } 1495 1496 if (fw == NULL && rc < 0) { 1497 device_printf(sc->dev, "No usable firmware. " 1498 "card has %d.%d.%d, driver compiled with %d.%d.%d", 1499 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1500 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1501 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1502 FW_VERSION_MAJOR, FW_VERSION_MINOR, 1503 FW_VERSION_MICRO); 1504 rc = EAGAIN; 1505 goto done; 1506 } 1507 1508 /* 1509 * Always upgrade, even for minor/micro/build mismatches. 1510 * Downgrade only for a major version mismatch or if 1511 * force_firmware_install was specified. 1512 */ 1513 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) { 1514 device_printf(sc->dev, 1515 "installing firmware %d.%d.%d.%d on card.\n", 1516 G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v), 1517 G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v)); 1518 1519 rc = -t4_load_fw(sc, fw->data, fw->datasize); 1520 if (rc != 0) { 1521 device_printf(sc->dev, 1522 "failed to install firmware: %d\n", rc); 1523 goto done; 1524 } else { 1525 /* refresh */ 1526 (void) t4_check_fw_version(sc); 1527 } 1528 } 1529 } 1530 1531 /* Contact firmware. */ 1532 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1533 if (rc < 0) { 1534 rc = -rc; 1535 device_printf(sc->dev, 1536 "failed to connect to the firmware: %d.\n", rc); 1537 goto done; 1538 } 1539 if (rc == sc->mbox) 1540 sc->flags |= MASTER_PF; 1541 1542 /* Reset device */ 1543 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1544 if (rc != 0) { 1545 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 1546 if (rc != ETIMEDOUT && rc != EIO) 1547 t4_fw_bye(sc, sc->mbox); 1548 goto done; 1549 } 1550 1551 /* Partition adapter resources as specified in the config file. */ 1552 if (sc->flags & MASTER_PF) { 1553 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) { 1554 char s[32]; 1555 1556 snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file); 1557 cfg = firmware_get(s); 1558 if (cfg == NULL) { 1559 device_printf(sc->dev, 1560 "unable to locate %s module, " 1561 "will use default config file.\n", s); 1562 } 1563 } 1564 1565 rc = partition_resources(sc, cfg ? cfg : default_cfg); 1566 if (rc != 0) 1567 goto done; /* error message displayed already */ 1568 } 1569 1570 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 1571 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1572 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1573 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1574 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1575 sc->flags |= FW_OK; 1576 1577 done: 1578 if (fw != NULL) 1579 firmware_put(fw, FIRMWARE_UNLOAD); 1580 if (cfg != NULL) 1581 firmware_put(cfg, FIRMWARE_UNLOAD); 1582 if (default_cfg != NULL) 1583 firmware_put(default_cfg, FIRMWARE_UNLOAD); 1584 1585 return (rc); 1586 } 1587 1588 #define FW_PARAM_DEV(param) \ 1589 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1590 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1591 #define FW_PARAM_PFVF(param) \ 1592 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1593 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1594 1595 /* 1596 * Upload configuration file to card's memory. 1597 */ 1598 static int 1599 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt, 1600 uint32_t *ma) 1601 { 1602 int rc, i; 1603 uint32_t param, val, mtype, maddr, bar, off, win, remaining; 1604 const uint32_t *b; 1605 1606 /* Figure out where the firmware wants us to upload it. */ 1607 param = FW_PARAM_DEV(CF); 1608 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1609 if (rc != 0) { 1610 /* Firmwares without config file support will fail this way */ 1611 device_printf(sc->dev, 1612 "failed to query config file location: %d.\n", rc); 1613 return (rc); 1614 } 1615 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1616 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1617 1618 if (maddr & 3) { 1619 device_printf(sc->dev, 1620 "cannot upload config file (type %u, addr %x).\n", 1621 mtype, maddr); 1622 return (EFAULT); 1623 } 1624 1625 /* Translate mtype/maddr to an address suitable for the PCIe window */ 1626 val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1627 val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE; 1628 switch (mtype) { 1629 case FW_MEMTYPE_CF_EDC0: 1630 if (!(val & F_EDRAM0_ENABLE)) 1631 goto err; 1632 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1633 maddr += G_EDRAM0_BASE(bar) << 20; 1634 break; 1635 1636 case FW_MEMTYPE_CF_EDC1: 1637 if (!(val & F_EDRAM1_ENABLE)) 1638 goto err; 1639 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1640 maddr += G_EDRAM1_BASE(bar) << 20; 1641 break; 1642 1643 case FW_MEMTYPE_CF_EXTMEM: 1644 if (!(val & F_EXT_MEM_ENABLE)) 1645 goto err; 1646 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1647 maddr += G_EXT_MEM_BASE(bar) << 20; 1648 break; 1649 1650 default: 1651 err: 1652 device_printf(sc->dev, 1653 "cannot upload config file (type %u, enabled %u).\n", 1654 mtype, val); 1655 return (EFAULT); 1656 } 1657 1658 /* 1659 * Position the PCIe window (we use memwin2) to the 16B aligned area 1660 * just at/before the upload location. 1661 */ 1662 win = maddr & ~0xf; 1663 off = maddr - win; /* offset from the start of the window. */ 1664 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 1665 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 1666 1667 remaining = fw->datasize; 1668 if (remaining > FLASH_CFG_MAX_SIZE || 1669 remaining > MEMWIN2_APERTURE - off) { 1670 device_printf(sc->dev, "cannot upload config file all at once " 1671 "(size %u, max %u, room %u).\n", 1672 remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off); 1673 return (EFBIG); 1674 } 1675 1676 /* 1677 * XXX: sheer laziness. We deliberately added 4 bytes of useless 1678 * stuffing/comments at the end of the config file so it's ok to simply 1679 * throw away the last remaining bytes when the config file is not an 1680 * exact multiple of 4. 1681 */ 1682 b = fw->data; 1683 for (i = 0; remaining >= 4; i += 4, remaining -= 4) 1684 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++); 1685 1686 return (rc); 1687 } 1688 1689 /* 1690 * Partition chip resources for use between various PFs, VFs, etc. This is done 1691 * by uploading the firmware configuration file to the adapter and instructing 1692 * the firmware to process it. 1693 */ 1694 static int 1695 partition_resources(struct adapter *sc, const struct firmware *cfg) 1696 { 1697 int rc; 1698 struct fw_caps_config_cmd caps; 1699 uint32_t mtype, maddr, finicsum, cfcsum; 1700 1701 rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT; 1702 if (rc != 0) { 1703 mtype = FW_MEMTYPE_CF_FLASH; 1704 maddr = t4_flash_cfg_addr(sc); 1705 } 1706 1707 bzero(&caps, sizeof(caps)); 1708 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1709 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1710 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1711 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1712 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1713 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1714 if (rc != 0) { 1715 device_printf(sc->dev, 1716 "failed to pre-process config file: %d.\n", rc); 1717 return (rc); 1718 } 1719 1720 finicsum = be32toh(caps.finicsum); 1721 cfcsum = be32toh(caps.cfcsum); 1722 if (finicsum != cfcsum) { 1723 device_printf(sc->dev, 1724 "WARNING: config file checksum mismatch: %08x %08x\n", 1725 finicsum, cfcsum); 1726 } 1727 sc->cfcsum = cfcsum; 1728 1729 #define LIMIT_CAPS(x) do { \ 1730 caps.x &= htobe16(t4_##x##_allowed); \ 1731 sc->x = htobe16(caps.x); \ 1732 } while (0) 1733 1734 /* 1735 * Let the firmware know what features will (not) be used so it can tune 1736 * things accordingly. 1737 */ 1738 LIMIT_CAPS(linkcaps); 1739 LIMIT_CAPS(niccaps); 1740 LIMIT_CAPS(toecaps); 1741 LIMIT_CAPS(rdmacaps); 1742 LIMIT_CAPS(iscsicaps); 1743 LIMIT_CAPS(fcoecaps); 1744 #undef LIMIT_CAPS 1745 1746 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1747 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1748 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1749 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 1750 if (rc != 0) { 1751 device_printf(sc->dev, 1752 "failed to process config file: %d.\n", rc); 1753 return (rc); 1754 } 1755 1756 return (0); 1757 } 1758 1759 /* 1760 * Retrieve parameters that are needed (or nice to have) prior to calling 1761 * t4_sge_init and t4_fw_initialize. 1762 */ 1763 static int 1764 get_params__pre_init(struct adapter *sc) 1765 { 1766 int rc; 1767 uint32_t param[2], val[2]; 1768 struct fw_devlog_cmd cmd; 1769 struct devlog_params *dlog = &sc->params.devlog; 1770 1771 param[0] = FW_PARAM_DEV(PORTVEC); 1772 param[1] = FW_PARAM_DEV(CCLK); 1773 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1774 if (rc != 0) { 1775 device_printf(sc->dev, 1776 "failed to query parameters (pre_init): %d.\n", rc); 1777 return (rc); 1778 } 1779 1780 sc->params.portvec = val[0]; 1781 sc->params.nports = 0; 1782 while (val[0]) { 1783 sc->params.nports++; 1784 val[0] &= val[0] - 1; 1785 } 1786 1787 sc->params.vpd.cclk = val[1]; 1788 1789 /* Read device log parameters. */ 1790 bzero(&cmd, sizeof(cmd)); 1791 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1792 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1793 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 1794 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 1795 if (rc != 0) { 1796 device_printf(sc->dev, 1797 "failed to get devlog parameters: %d.\n", rc); 1798 bzero(dlog, sizeof (*dlog)); 1799 rc = 0; /* devlog isn't critical for device operation */ 1800 } else { 1801 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 1802 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1803 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1804 dlog->size = be32toh(cmd.memsize_devlog); 1805 } 1806 1807 return (rc); 1808 } 1809 1810 /* 1811 * Retrieve various parameters that are of interest to the driver. The device 1812 * has been initialized by the firmware at this point. 1813 */ 1814 static int 1815 get_params__post_init(struct adapter *sc) 1816 { 1817 int rc; 1818 uint32_t param[7], val[7]; 1819 struct fw_caps_config_cmd caps; 1820 1821 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1822 param[1] = FW_PARAM_PFVF(EQ_START); 1823 param[2] = FW_PARAM_PFVF(FILTER_START); 1824 param[3] = FW_PARAM_PFVF(FILTER_END); 1825 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); 1826 if (rc != 0) { 1827 device_printf(sc->dev, 1828 "failed to query parameters (post_init): %d.\n", rc); 1829 return (rc); 1830 } 1831 1832 sc->sge.iq_start = val[0]; 1833 sc->sge.eq_start = val[1]; 1834 sc->tids.ftid_base = val[2]; 1835 sc->tids.nftids = val[3] - val[2] + 1; 1836 1837 /* get capabilites */ 1838 bzero(&caps, sizeof(caps)); 1839 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1840 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1841 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1842 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1843 if (rc != 0) { 1844 device_printf(sc->dev, 1845 "failed to get card capabilities: %d.\n", rc); 1846 return (rc); 1847 } 1848 1849 if (caps.toecaps) { 1850 /* query offload-related parameters */ 1851 param[0] = FW_PARAM_DEV(NTID); 1852 param[1] = FW_PARAM_PFVF(SERVER_START); 1853 param[2] = FW_PARAM_PFVF(SERVER_END); 1854 param[3] = FW_PARAM_PFVF(TDDP_START); 1855 param[4] = FW_PARAM_PFVF(TDDP_END); 1856 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1857 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1858 if (rc != 0) { 1859 device_printf(sc->dev, 1860 "failed to query TOE parameters: %d.\n", rc); 1861 return (rc); 1862 } 1863 sc->tids.ntids = val[0]; 1864 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1865 sc->tids.stid_base = val[1]; 1866 sc->tids.nstids = val[2] - val[1] + 1; 1867 sc->vres.ddp.start = val[3]; 1868 sc->vres.ddp.size = val[4] - val[3] + 1; 1869 sc->params.ofldq_wr_cred = val[5]; 1870 sc->params.offload = 1; 1871 } 1872 if (caps.rdmacaps) { 1873 param[0] = FW_PARAM_PFVF(STAG_START); 1874 param[1] = FW_PARAM_PFVF(STAG_END); 1875 param[2] = FW_PARAM_PFVF(RQ_START); 1876 param[3] = FW_PARAM_PFVF(RQ_END); 1877 param[4] = FW_PARAM_PFVF(PBL_START); 1878 param[5] = FW_PARAM_PFVF(PBL_END); 1879 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1880 if (rc != 0) { 1881 device_printf(sc->dev, 1882 "failed to query RDMA parameters(1): %d.\n", rc); 1883 return (rc); 1884 } 1885 sc->vres.stag.start = val[0]; 1886 sc->vres.stag.size = val[1] - val[0] + 1; 1887 sc->vres.rq.start = val[2]; 1888 sc->vres.rq.size = val[3] - val[2] + 1; 1889 sc->vres.pbl.start = val[4]; 1890 sc->vres.pbl.size = val[5] - val[4] + 1; 1891 1892 param[0] = FW_PARAM_PFVF(SQRQ_START); 1893 param[1] = FW_PARAM_PFVF(SQRQ_END); 1894 param[2] = FW_PARAM_PFVF(CQ_START); 1895 param[3] = FW_PARAM_PFVF(CQ_END); 1896 param[4] = FW_PARAM_PFVF(OCQ_START); 1897 param[5] = FW_PARAM_PFVF(OCQ_END); 1898 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 1899 if (rc != 0) { 1900 device_printf(sc->dev, 1901 "failed to query RDMA parameters(2): %d.\n", rc); 1902 return (rc); 1903 } 1904 sc->vres.qp.start = val[0]; 1905 sc->vres.qp.size = val[1] - val[0] + 1; 1906 sc->vres.cq.start = val[2]; 1907 sc->vres.cq.size = val[3] - val[2] + 1; 1908 sc->vres.ocq.start = val[4]; 1909 sc->vres.ocq.size = val[5] - val[4] + 1; 1910 } 1911 if (caps.iscsicaps) { 1912 param[0] = FW_PARAM_PFVF(ISCSI_START); 1913 param[1] = FW_PARAM_PFVF(ISCSI_END); 1914 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1915 if (rc != 0) { 1916 device_printf(sc->dev, 1917 "failed to query iSCSI parameters: %d.\n", rc); 1918 return (rc); 1919 } 1920 sc->vres.iscsi.start = val[0]; 1921 sc->vres.iscsi.size = val[1] - val[0] + 1; 1922 } 1923 1924 /* These are finalized by FW initialization, load their values now */ 1925 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1926 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1927 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1928 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1929 1930 return (rc); 1931 } 1932 1933 #undef FW_PARAM_PFVF 1934 #undef FW_PARAM_DEV 1935 1936 static void 1937 t4_set_desc(struct adapter *sc) 1938 { 1939 char buf[128]; 1940 struct adapter_params *p = &sc->params; 1941 1942 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 1943 p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec); 1944 1945 device_set_desc_copy(sc->dev, buf); 1946 } 1947 1948 static void 1949 build_medialist(struct port_info *pi) 1950 { 1951 struct ifmedia *media = &pi->media; 1952 int data, m; 1953 1954 PORT_LOCK(pi); 1955 1956 ifmedia_removeall(media); 1957 1958 m = IFM_ETHER | IFM_FDX; 1959 data = (pi->port_type << 8) | pi->mod_type; 1960 1961 switch(pi->port_type) { 1962 case FW_PORT_TYPE_BT_XFI: 1963 ifmedia_add(media, m | IFM_10G_T, data, NULL); 1964 break; 1965 1966 case FW_PORT_TYPE_BT_XAUI: 1967 ifmedia_add(media, m | IFM_10G_T, data, NULL); 1968 /* fall through */ 1969 1970 case FW_PORT_TYPE_BT_SGMII: 1971 ifmedia_add(media, m | IFM_1000_T, data, NULL); 1972 ifmedia_add(media, m | IFM_100_TX, data, NULL); 1973 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 1974 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 1975 break; 1976 1977 case FW_PORT_TYPE_CX4: 1978 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 1979 ifmedia_set(media, m | IFM_10G_CX4); 1980 break; 1981 1982 case FW_PORT_TYPE_SFP: 1983 case FW_PORT_TYPE_FIBER_XFI: 1984 case FW_PORT_TYPE_FIBER_XAUI: 1985 switch (pi->mod_type) { 1986 1987 case FW_PORT_MOD_TYPE_LR: 1988 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 1989 ifmedia_set(media, m | IFM_10G_LR); 1990 break; 1991 1992 case FW_PORT_MOD_TYPE_SR: 1993 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 1994 ifmedia_set(media, m | IFM_10G_SR); 1995 break; 1996 1997 case FW_PORT_MOD_TYPE_LRM: 1998 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 1999 ifmedia_set(media, m | IFM_10G_LRM); 2000 break; 2001 2002 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2003 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2004 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2005 ifmedia_set(media, m | IFM_10G_TWINAX); 2006 break; 2007 2008 case FW_PORT_MOD_TYPE_NONE: 2009 m &= ~IFM_FDX; 2010 ifmedia_add(media, m | IFM_NONE, data, NULL); 2011 ifmedia_set(media, m | IFM_NONE); 2012 break; 2013 2014 case FW_PORT_MOD_TYPE_NA: 2015 case FW_PORT_MOD_TYPE_ER: 2016 default: 2017 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2018 ifmedia_set(media, m | IFM_UNKNOWN); 2019 break; 2020 } 2021 break; 2022 2023 case FW_PORT_TYPE_KX4: 2024 case FW_PORT_TYPE_KX: 2025 case FW_PORT_TYPE_KR: 2026 default: 2027 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2028 ifmedia_set(media, m | IFM_UNKNOWN); 2029 break; 2030 } 2031 2032 PORT_UNLOCK(pi); 2033 } 2034 2035 /* 2036 * Program the port's XGMAC based on parameters in ifnet. The caller also 2037 * indicates which parameters should be programmed (the rest are left alone). 2038 */ 2039 static int 2040 update_mac_settings(struct port_info *pi, int flags) 2041 { 2042 int rc; 2043 struct ifnet *ifp = pi->ifp; 2044 struct adapter *sc = pi->adapter; 2045 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2046 2047 PORT_LOCK_ASSERT_OWNED(pi); 2048 KASSERT(flags, ("%s: not told what to update.", __func__)); 2049 2050 if (flags & XGMAC_MTU) 2051 mtu = ifp->if_mtu; 2052 2053 if (flags & XGMAC_PROMISC) 2054 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2055 2056 if (flags & XGMAC_ALLMULTI) 2057 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2058 2059 if (flags & XGMAC_VLANEX) 2060 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2061 2062 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2063 vlanex, false); 2064 if (rc) { 2065 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2066 return (rc); 2067 } 2068 2069 if (flags & XGMAC_UCADDR) { 2070 uint8_t ucaddr[ETHER_ADDR_LEN]; 2071 2072 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2073 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2074 ucaddr, true, true); 2075 if (rc < 0) { 2076 rc = -rc; 2077 if_printf(ifp, "change_mac failed: %d\n", rc); 2078 return (rc); 2079 } else { 2080 pi->xact_addr_filt = rc; 2081 rc = 0; 2082 } 2083 } 2084 2085 if (flags & XGMAC_MCADDRS) { 2086 const uint8_t *mcaddr; 2087 int del = 1; 2088 uint64_t hash = 0; 2089 struct ifmultiaddr *ifma; 2090 2091 if_maddr_rlock(ifp); 2092 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2093 if (ifma->ifma_addr->sa_family != AF_LINK) 2094 continue; 2095 mcaddr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2096 2097 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, del, 1, 2098 &mcaddr, NULL, &hash, 0); 2099 if (rc < 0) { 2100 rc = -rc; 2101 if_printf(ifp, "failed to add mc address" 2102 " %02x:%02x:%02x:%02x:%02x:%02x rc=%d\n", 2103 mcaddr[0], mcaddr[1], mcaddr[2], mcaddr[3], 2104 mcaddr[4], mcaddr[5], rc); 2105 goto mcfail; 2106 } 2107 del = 0; 2108 } 2109 2110 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2111 if (rc != 0) 2112 if_printf(ifp, "failed to set mc address hash: %d", rc); 2113 mcfail: 2114 if_maddr_runlock(ifp); 2115 } 2116 2117 return (rc); 2118 } 2119 2120 static int 2121 cxgbe_init_locked(struct port_info *pi) 2122 { 2123 struct adapter *sc = pi->adapter; 2124 int rc = 0; 2125 2126 ADAPTER_LOCK_ASSERT_OWNED(sc); 2127 2128 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2129 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) { 2130 rc = EINTR; 2131 goto done; 2132 } 2133 } 2134 if (IS_DOOMED(pi)) { 2135 rc = ENXIO; 2136 goto done; 2137 } 2138 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2139 2140 /* Give up the adapter lock, port init code can sleep. */ 2141 SET_BUSY(sc); 2142 ADAPTER_UNLOCK(sc); 2143 2144 rc = cxgbe_init_synchronized(pi); 2145 2146 done: 2147 ADAPTER_LOCK(sc); 2148 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2149 CLR_BUSY(sc); 2150 wakeup_one(&sc->flags); 2151 ADAPTER_UNLOCK(sc); 2152 return (rc); 2153 } 2154 2155 static int 2156 cxgbe_init_synchronized(struct port_info *pi) 2157 { 2158 struct adapter *sc = pi->adapter; 2159 struct ifnet *ifp = pi->ifp; 2160 int rc = 0; 2161 2162 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2163 2164 if (isset(&sc->open_device_map, pi->port_id)) { 2165 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2166 ("mismatch between open_device_map and if_drv_flags")); 2167 return (0); /* already running */ 2168 } 2169 2170 if (!(sc->flags & FULL_INIT_DONE) && 2171 ((rc = adapter_full_init(sc)) != 0)) 2172 return (rc); /* error message displayed already */ 2173 2174 if (!(pi->flags & PORT_INIT_DONE) && 2175 ((rc = port_full_init(pi)) != 0)) 2176 return (rc); /* error message displayed already */ 2177 2178 PORT_LOCK(pi); 2179 rc = update_mac_settings(pi, XGMAC_ALL); 2180 PORT_UNLOCK(pi); 2181 if (rc) 2182 goto done; /* error message displayed already */ 2183 2184 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2185 if (rc != 0) { 2186 if_printf(ifp, "start_link failed: %d\n", rc); 2187 goto done; 2188 } 2189 2190 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2191 if (rc != 0) { 2192 if_printf(ifp, "enable_vi failed: %d\n", rc); 2193 goto done; 2194 } 2195 2196 /* all ok */ 2197 setbit(&sc->open_device_map, pi->port_id); 2198 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2199 2200 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2201 done: 2202 if (rc != 0) 2203 cxgbe_uninit_synchronized(pi); 2204 2205 return (rc); 2206 } 2207 2208 static int 2209 cxgbe_uninit_locked(struct port_info *pi) 2210 { 2211 struct adapter *sc = pi->adapter; 2212 int rc; 2213 2214 ADAPTER_LOCK_ASSERT_OWNED(sc); 2215 2216 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2217 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) { 2218 rc = EINTR; 2219 goto done; 2220 } 2221 } 2222 if (IS_DOOMED(pi)) { 2223 rc = ENXIO; 2224 goto done; 2225 } 2226 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2227 SET_BUSY(sc); 2228 ADAPTER_UNLOCK(sc); 2229 2230 rc = cxgbe_uninit_synchronized(pi); 2231 2232 ADAPTER_LOCK(sc); 2233 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2234 CLR_BUSY(sc); 2235 wakeup_one(&sc->flags); 2236 done: 2237 ADAPTER_UNLOCK(sc); 2238 return (rc); 2239 } 2240 2241 /* 2242 * Idempotent. 2243 */ 2244 static int 2245 cxgbe_uninit_synchronized(struct port_info *pi) 2246 { 2247 struct adapter *sc = pi->adapter; 2248 struct ifnet *ifp = pi->ifp; 2249 int rc; 2250 2251 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2252 2253 /* 2254 * Disable the VI so that all its data in either direction is discarded 2255 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2256 * tick) intact as the TP can deliver negative advice or data that it's 2257 * holding in its RAM (for an offloaded connection) even after the VI is 2258 * disabled. 2259 */ 2260 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2261 if (rc) { 2262 if_printf(ifp, "disable_vi failed: %d\n", rc); 2263 return (rc); 2264 } 2265 2266 clrbit(&sc->open_device_map, pi->port_id); 2267 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2268 2269 pi->link_cfg.link_ok = 0; 2270 pi->link_cfg.speed = 0; 2271 t4_os_link_changed(sc, pi->port_id, 0); 2272 2273 return (0); 2274 } 2275 2276 #define T4_ALLOC_IRQ(sc, irq, rid, handler, arg, name) do { \ 2277 rc = t4_alloc_irq(sc, irq, rid, handler, arg, name); \ 2278 if (rc != 0) \ 2279 goto done; \ 2280 } while (0) 2281 2282 static int 2283 adapter_full_init(struct adapter *sc) 2284 { 2285 int rc, i, rid, p, q; 2286 char s[8]; 2287 struct irq *irq; 2288 struct port_info *pi; 2289 struct sge_rxq *rxq; 2290 #ifndef TCP_OFFLOAD_DISABLE 2291 struct sge_ofld_rxq *ofld_rxq; 2292 #endif 2293 2294 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2295 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2296 ("%s: FULL_INIT_DONE already", __func__)); 2297 2298 /* 2299 * queues that belong to the adapter (not any particular port). 2300 */ 2301 rc = t4_setup_adapter_queues(sc); 2302 if (rc != 0) 2303 goto done; 2304 2305 for (i = 0; i < ARRAY_SIZE(sc->tq); i++) { 2306 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 2307 taskqueue_thread_enqueue, &sc->tq[i]); 2308 if (sc->tq[i] == NULL) { 2309 device_printf(sc->dev, 2310 "failed to allocate task queue %d\n", i); 2311 rc = ENOMEM; 2312 goto done; 2313 } 2314 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 2315 device_get_nameunit(sc->dev), i); 2316 } 2317 2318 /* 2319 * Setup interrupts. 2320 */ 2321 irq = &sc->irq[0]; 2322 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2323 if (sc->intr_count == 1) { 2324 KASSERT(!(sc->flags & INTR_DIRECT), 2325 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2326 2327 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_all, sc, "all"); 2328 } else { 2329 /* Multiple interrupts. */ 2330 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2331 ("%s: too few intr.", __func__)); 2332 2333 /* The first one is always error intr */ 2334 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_err, sc, "err"); 2335 irq++; 2336 rid++; 2337 2338 /* The second one is always the firmware event queue */ 2339 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 2340 irq++; 2341 rid++; 2342 2343 /* 2344 * Note that if INTR_DIRECT is not set then either the NIC rx 2345 * queues or (exclusive or) the TOE rx queueus will be taking 2346 * direct interrupts. 2347 * 2348 * There is no need to check for is_offload(sc) as nofldrxq 2349 * will be 0 if offload is disabled. 2350 */ 2351 for_each_port(sc, p) { 2352 pi = sc->port[p]; 2353 2354 #ifndef TCP_OFFLOAD_DISABLE 2355 /* 2356 * Skip over the NIC queues if they aren't taking direct 2357 * interrupts. 2358 */ 2359 if (!(sc->flags & INTR_DIRECT) && 2360 pi->nofldrxq > pi->nrxq) 2361 goto ofld_queues; 2362 #endif 2363 rxq = &sc->sge.rxq[pi->first_rxq]; 2364 for (q = 0; q < pi->nrxq; q++, rxq++) { 2365 snprintf(s, sizeof(s), "%d.%d", p, q); 2366 T4_ALLOC_IRQ(sc, irq, rid, t4_intr, rxq, s); 2367 irq++; 2368 rid++; 2369 } 2370 2371 #ifndef TCP_OFFLOAD_DISABLE 2372 /* 2373 * Skip over the offload queues if they aren't taking 2374 * direct interrupts. 2375 */ 2376 if (!(sc->flags & INTR_DIRECT)) 2377 continue; 2378 ofld_queues: 2379 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2380 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2381 snprintf(s, sizeof(s), "%d,%d", p, q); 2382 T4_ALLOC_IRQ(sc, irq, rid, t4_intr, ofld_rxq, s); 2383 irq++; 2384 rid++; 2385 } 2386 #endif 2387 } 2388 } 2389 2390 t4_intr_enable(sc); 2391 sc->flags |= FULL_INIT_DONE; 2392 done: 2393 if (rc != 0) 2394 adapter_full_uninit(sc); 2395 2396 return (rc); 2397 } 2398 #undef T4_ALLOC_IRQ 2399 2400 static int 2401 adapter_full_uninit(struct adapter *sc) 2402 { 2403 int i; 2404 2405 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2406 2407 t4_teardown_adapter_queues(sc); 2408 2409 for (i = 0; i < sc->intr_count; i++) 2410 t4_free_irq(sc, &sc->irq[i]); 2411 2412 for (i = 0; i < ARRAY_SIZE(sc->tq) && sc->tq[i]; i++) { 2413 taskqueue_free(sc->tq[i]); 2414 sc->tq[i] = NULL; 2415 } 2416 2417 sc->flags &= ~FULL_INIT_DONE; 2418 2419 return (0); 2420 } 2421 2422 static int 2423 port_full_init(struct port_info *pi) 2424 { 2425 struct adapter *sc = pi->adapter; 2426 struct ifnet *ifp = pi->ifp; 2427 uint16_t *rss; 2428 struct sge_rxq *rxq; 2429 int rc, i; 2430 2431 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2432 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 2433 ("%s: PORT_INIT_DONE already", __func__)); 2434 2435 sysctl_ctx_init(&pi->ctx); 2436 pi->flags |= PORT_SYSCTL_CTX; 2437 2438 /* 2439 * Allocate tx/rx/fl queues for this port. 2440 */ 2441 rc = t4_setup_port_queues(pi); 2442 if (rc != 0) 2443 goto done; /* error message displayed already */ 2444 2445 /* 2446 * Setup RSS for this port. 2447 */ 2448 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 2449 M_ZERO | M_WAITOK); 2450 for_each_rxq(pi, i, rxq) { 2451 rss[i] = rxq->iq.abs_id; 2452 } 2453 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2454 pi->rss_size, rss, pi->nrxq); 2455 free(rss, M_CXGBE); 2456 if (rc != 0) { 2457 if_printf(ifp, "rss_config failed: %d\n", rc); 2458 goto done; 2459 } 2460 2461 pi->flags |= PORT_INIT_DONE; 2462 done: 2463 if (rc != 0) 2464 port_full_uninit(pi); 2465 2466 return (rc); 2467 } 2468 2469 /* 2470 * Idempotent. 2471 */ 2472 static int 2473 port_full_uninit(struct port_info *pi) 2474 { 2475 struct adapter *sc = pi->adapter; 2476 int i; 2477 struct sge_rxq *rxq; 2478 struct sge_txq *txq; 2479 #ifndef TCP_OFFLOAD_DISABLE 2480 struct sge_ofld_rxq *ofld_rxq; 2481 struct sge_wrq *ofld_txq; 2482 #endif 2483 2484 if (pi->flags & PORT_INIT_DONE) { 2485 2486 /* Need to quiesce queues. XXX: ctrl queues? */ 2487 2488 for_each_txq(pi, i, txq) { 2489 quiesce_eq(sc, &txq->eq); 2490 } 2491 2492 #ifndef TCP_OFFLOAD_DISABLE 2493 for_each_ofld_txq(pi, i, ofld_txq) { 2494 quiesce_eq(sc, &ofld_txq->eq); 2495 } 2496 #endif 2497 2498 for_each_rxq(pi, i, rxq) { 2499 quiesce_iq(sc, &rxq->iq); 2500 quiesce_fl(sc, &rxq->fl); 2501 } 2502 2503 #ifndef TCP_OFFLOAD_DISABLE 2504 for_each_ofld_rxq(pi, i, ofld_rxq) { 2505 quiesce_iq(sc, &ofld_rxq->iq); 2506 quiesce_fl(sc, &ofld_rxq->fl); 2507 } 2508 #endif 2509 } 2510 2511 t4_teardown_port_queues(pi); 2512 pi->flags &= ~PORT_INIT_DONE; 2513 2514 return (0); 2515 } 2516 2517 static void 2518 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 2519 { 2520 EQ_LOCK(eq); 2521 eq->flags |= EQ_DOOMED; 2522 2523 /* 2524 * Wait for the response to a credit flush if one's 2525 * pending. 2526 */ 2527 while (eq->flags & EQ_CRFLUSHED) 2528 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 2529 EQ_UNLOCK(eq); 2530 2531 callout_drain(&eq->tx_callout); /* XXX: iffy */ 2532 pause("callout", 10); /* Still iffy */ 2533 2534 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 2535 } 2536 2537 static void 2538 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 2539 { 2540 (void) sc; /* unused */ 2541 2542 /* Synchronize with the interrupt handler */ 2543 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 2544 pause("iqfree", 1); 2545 } 2546 2547 static void 2548 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 2549 { 2550 mtx_lock(&sc->sfl_lock); 2551 FL_LOCK(fl); 2552 fl->flags |= FL_DOOMED; 2553 FL_UNLOCK(fl); 2554 mtx_unlock(&sc->sfl_lock); 2555 2556 callout_drain(&sc->sfl_callout); 2557 KASSERT((fl->flags & FL_STARVING) == 0, 2558 ("%s: still starving", __func__)); 2559 } 2560 2561 static int 2562 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 2563 driver_intr_t *handler, void *arg, char *name) 2564 { 2565 int rc; 2566 2567 irq->rid = rid; 2568 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 2569 RF_SHAREABLE | RF_ACTIVE); 2570 if (irq->res == NULL) { 2571 device_printf(sc->dev, 2572 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 2573 return (ENOMEM); 2574 } 2575 2576 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 2577 NULL, handler, arg, &irq->tag); 2578 if (rc != 0) { 2579 device_printf(sc->dev, 2580 "failed to setup interrupt for rid %d, name %s: %d\n", 2581 rid, name, rc); 2582 } else if (name) 2583 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 2584 2585 return (rc); 2586 } 2587 2588 static int 2589 t4_free_irq(struct adapter *sc, struct irq *irq) 2590 { 2591 if (irq->tag) 2592 bus_teardown_intr(sc->dev, irq->res, irq->tag); 2593 if (irq->res) 2594 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 2595 2596 bzero(irq, sizeof(*irq)); 2597 2598 return (0); 2599 } 2600 2601 static void 2602 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 2603 unsigned int end) 2604 { 2605 uint32_t *p = (uint32_t *)(buf + start); 2606 2607 for ( ; start <= end; start += sizeof(uint32_t)) 2608 *p++ = t4_read_reg(sc, start); 2609 } 2610 2611 static void 2612 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 2613 { 2614 int i; 2615 static const unsigned int reg_ranges[] = { 2616 0x1008, 0x1108, 2617 0x1180, 0x11b4, 2618 0x11fc, 0x123c, 2619 0x1300, 0x173c, 2620 0x1800, 0x18fc, 2621 0x3000, 0x30d8, 2622 0x30e0, 0x5924, 2623 0x5960, 0x59d4, 2624 0x5a00, 0x5af8, 2625 0x6000, 0x6098, 2626 0x6100, 0x6150, 2627 0x6200, 0x6208, 2628 0x6240, 0x6248, 2629 0x6280, 0x6338, 2630 0x6370, 0x638c, 2631 0x6400, 0x643c, 2632 0x6500, 0x6524, 2633 0x6a00, 0x6a38, 2634 0x6a60, 0x6a78, 2635 0x6b00, 0x6b84, 2636 0x6bf0, 0x6c84, 2637 0x6cf0, 0x6d84, 2638 0x6df0, 0x6e84, 2639 0x6ef0, 0x6f84, 2640 0x6ff0, 0x7084, 2641 0x70f0, 0x7184, 2642 0x71f0, 0x7284, 2643 0x72f0, 0x7384, 2644 0x73f0, 0x7450, 2645 0x7500, 0x7530, 2646 0x7600, 0x761c, 2647 0x7680, 0x76cc, 2648 0x7700, 0x7798, 2649 0x77c0, 0x77fc, 2650 0x7900, 0x79fc, 2651 0x7b00, 0x7c38, 2652 0x7d00, 0x7efc, 2653 0x8dc0, 0x8e1c, 2654 0x8e30, 0x8e78, 2655 0x8ea0, 0x8f6c, 2656 0x8fc0, 0x9074, 2657 0x90fc, 0x90fc, 2658 0x9400, 0x9458, 2659 0x9600, 0x96bc, 2660 0x9800, 0x9808, 2661 0x9820, 0x983c, 2662 0x9850, 0x9864, 2663 0x9c00, 0x9c6c, 2664 0x9c80, 0x9cec, 2665 0x9d00, 0x9d6c, 2666 0x9d80, 0x9dec, 2667 0x9e00, 0x9e6c, 2668 0x9e80, 0x9eec, 2669 0x9f00, 0x9f6c, 2670 0x9f80, 0x9fec, 2671 0xd004, 0xd03c, 2672 0xdfc0, 0xdfe0, 2673 0xe000, 0xea7c, 2674 0xf000, 0x11190, 2675 0x19040, 0x19124, 2676 0x19150, 0x191b0, 2677 0x191d0, 0x191e8, 2678 0x19238, 0x1924c, 2679 0x193f8, 0x19474, 2680 0x19490, 0x194f8, 2681 0x19800, 0x19f30, 2682 0x1a000, 0x1a06c, 2683 0x1a0b0, 0x1a120, 2684 0x1a128, 0x1a138, 2685 0x1a190, 0x1a1c4, 2686 0x1a1fc, 0x1a1fc, 2687 0x1e040, 0x1e04c, 2688 0x1e240, 0x1e28c, 2689 0x1e2c0, 0x1e2c0, 2690 0x1e2e0, 0x1e2e0, 2691 0x1e300, 0x1e384, 2692 0x1e3c0, 0x1e3c8, 2693 0x1e440, 0x1e44c, 2694 0x1e640, 0x1e68c, 2695 0x1e6c0, 0x1e6c0, 2696 0x1e6e0, 0x1e6e0, 2697 0x1e700, 0x1e784, 2698 0x1e7c0, 0x1e7c8, 2699 0x1e840, 0x1e84c, 2700 0x1ea40, 0x1ea8c, 2701 0x1eac0, 0x1eac0, 2702 0x1eae0, 0x1eae0, 2703 0x1eb00, 0x1eb84, 2704 0x1ebc0, 0x1ebc8, 2705 0x1ec40, 0x1ec4c, 2706 0x1ee40, 0x1ee8c, 2707 0x1eec0, 0x1eec0, 2708 0x1eee0, 0x1eee0, 2709 0x1ef00, 0x1ef84, 2710 0x1efc0, 0x1efc8, 2711 0x1f040, 0x1f04c, 2712 0x1f240, 0x1f28c, 2713 0x1f2c0, 0x1f2c0, 2714 0x1f2e0, 0x1f2e0, 2715 0x1f300, 0x1f384, 2716 0x1f3c0, 0x1f3c8, 2717 0x1f440, 0x1f44c, 2718 0x1f640, 0x1f68c, 2719 0x1f6c0, 0x1f6c0, 2720 0x1f6e0, 0x1f6e0, 2721 0x1f700, 0x1f784, 2722 0x1f7c0, 0x1f7c8, 2723 0x1f840, 0x1f84c, 2724 0x1fa40, 0x1fa8c, 2725 0x1fac0, 0x1fac0, 2726 0x1fae0, 0x1fae0, 2727 0x1fb00, 0x1fb84, 2728 0x1fbc0, 0x1fbc8, 2729 0x1fc40, 0x1fc4c, 2730 0x1fe40, 0x1fe8c, 2731 0x1fec0, 0x1fec0, 2732 0x1fee0, 0x1fee0, 2733 0x1ff00, 0x1ff84, 2734 0x1ffc0, 0x1ffc8, 2735 0x20000, 0x2002c, 2736 0x20100, 0x2013c, 2737 0x20190, 0x201c8, 2738 0x20200, 0x20318, 2739 0x20400, 0x20528, 2740 0x20540, 0x20614, 2741 0x21000, 0x21040, 2742 0x2104c, 0x21060, 2743 0x210c0, 0x210ec, 2744 0x21200, 0x21268, 2745 0x21270, 0x21284, 2746 0x212fc, 0x21388, 2747 0x21400, 0x21404, 2748 0x21500, 0x21518, 2749 0x2152c, 0x2153c, 2750 0x21550, 0x21554, 2751 0x21600, 0x21600, 2752 0x21608, 0x21628, 2753 0x21630, 0x2163c, 2754 0x21700, 0x2171c, 2755 0x21780, 0x2178c, 2756 0x21800, 0x21c38, 2757 0x21c80, 0x21d7c, 2758 0x21e00, 0x21e04, 2759 0x22000, 0x2202c, 2760 0x22100, 0x2213c, 2761 0x22190, 0x221c8, 2762 0x22200, 0x22318, 2763 0x22400, 0x22528, 2764 0x22540, 0x22614, 2765 0x23000, 0x23040, 2766 0x2304c, 0x23060, 2767 0x230c0, 0x230ec, 2768 0x23200, 0x23268, 2769 0x23270, 0x23284, 2770 0x232fc, 0x23388, 2771 0x23400, 0x23404, 2772 0x23500, 0x23518, 2773 0x2352c, 0x2353c, 2774 0x23550, 0x23554, 2775 0x23600, 0x23600, 2776 0x23608, 0x23628, 2777 0x23630, 0x2363c, 2778 0x23700, 0x2371c, 2779 0x23780, 0x2378c, 2780 0x23800, 0x23c38, 2781 0x23c80, 0x23d7c, 2782 0x23e00, 0x23e04, 2783 0x24000, 0x2402c, 2784 0x24100, 0x2413c, 2785 0x24190, 0x241c8, 2786 0x24200, 0x24318, 2787 0x24400, 0x24528, 2788 0x24540, 0x24614, 2789 0x25000, 0x25040, 2790 0x2504c, 0x25060, 2791 0x250c0, 0x250ec, 2792 0x25200, 0x25268, 2793 0x25270, 0x25284, 2794 0x252fc, 0x25388, 2795 0x25400, 0x25404, 2796 0x25500, 0x25518, 2797 0x2552c, 0x2553c, 2798 0x25550, 0x25554, 2799 0x25600, 0x25600, 2800 0x25608, 0x25628, 2801 0x25630, 0x2563c, 2802 0x25700, 0x2571c, 2803 0x25780, 0x2578c, 2804 0x25800, 0x25c38, 2805 0x25c80, 0x25d7c, 2806 0x25e00, 0x25e04, 2807 0x26000, 0x2602c, 2808 0x26100, 0x2613c, 2809 0x26190, 0x261c8, 2810 0x26200, 0x26318, 2811 0x26400, 0x26528, 2812 0x26540, 0x26614, 2813 0x27000, 0x27040, 2814 0x2704c, 0x27060, 2815 0x270c0, 0x270ec, 2816 0x27200, 0x27268, 2817 0x27270, 0x27284, 2818 0x272fc, 0x27388, 2819 0x27400, 0x27404, 2820 0x27500, 0x27518, 2821 0x2752c, 0x2753c, 2822 0x27550, 0x27554, 2823 0x27600, 0x27600, 2824 0x27608, 0x27628, 2825 0x27630, 0x2763c, 2826 0x27700, 0x2771c, 2827 0x27780, 0x2778c, 2828 0x27800, 0x27c38, 2829 0x27c80, 0x27d7c, 2830 0x27e00, 0x27e04 2831 }; 2832 2833 regs->version = 4 | (sc->params.rev << 10); 2834 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2) 2835 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 2836 } 2837 2838 static void 2839 cxgbe_tick(void *arg) 2840 { 2841 struct port_info *pi = arg; 2842 struct ifnet *ifp = pi->ifp; 2843 struct sge_txq *txq; 2844 int i, drops; 2845 struct port_stats *s = &pi->stats; 2846 2847 PORT_LOCK(pi); 2848 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2849 PORT_UNLOCK(pi); 2850 return; /* without scheduling another callout */ 2851 } 2852 2853 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 2854 2855 ifp->if_opackets = s->tx_frames - s->tx_pause; 2856 ifp->if_ipackets = s->rx_frames - s->rx_pause; 2857 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 2858 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 2859 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 2860 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 2861 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 2862 s->rx_ovflow3; 2863 2864 drops = s->tx_drop; 2865 for_each_txq(pi, i, txq) 2866 drops += txq->br->br_drops; 2867 ifp->if_snd.ifq_drops = drops; 2868 2869 ifp->if_oerrors = s->tx_error_frames; 2870 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 2871 s->rx_fcs_err + s->rx_len_err; 2872 2873 callout_schedule(&pi->tick, hz); 2874 PORT_UNLOCK(pi); 2875 } 2876 2877 static int 2878 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 2879 { 2880 #ifdef INVARIANTS 2881 panic("%s: opcode %02x on iq %p with payload %p", 2882 __func__, rss->opcode, iq, m); 2883 #else 2884 log(LOG_ERR, "%s: opcode %02x on iq %p with payload %p", 2885 __func__, rss->opcode, iq, m); 2886 m_freem(m); 2887 #endif 2888 return (EDOOFUS); 2889 } 2890 2891 int 2892 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 2893 { 2894 uintptr_t *loc, new; 2895 2896 if (opcode >= ARRAY_SIZE(sc->cpl_handler)) 2897 return (EINVAL); 2898 2899 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 2900 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 2901 atomic_store_rel_ptr(loc, new); 2902 2903 return (0); 2904 } 2905 2906 static int 2907 t4_sysctls(struct adapter *sc) 2908 { 2909 struct sysctl_ctx_list *ctx; 2910 struct sysctl_oid *oid; 2911 struct sysctl_oid_list *children, *c0; 2912 static char *caps[] = { 2913 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 2914 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 2915 "\20\1TOE", /* caps[2] toecaps */ 2916 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 2917 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 2918 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 2919 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 2920 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 2921 }; 2922 2923 ctx = device_get_sysctl_ctx(sc->dev); 2924 2925 /* 2926 * dev.t4nex.X. 2927 */ 2928 oid = device_get_sysctl_tree(sc->dev); 2929 c0 = children = SYSCTL_CHILDREN(oid); 2930 2931 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, 2932 &sc->params.nports, 0, "# of ports"); 2933 2934 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 2935 &sc->params.rev, 0, "chip hardware revision"); 2936 2937 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 2938 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 2939 2940 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 2941 CTLFLAG_RD, &t4_cfg_file, 0, "configuration file"); 2942 2943 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, 2944 &sc->cfcsum, 0, "config file checksum"); 2945 2946 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 2947 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 2948 sysctl_bitfield, "A", "available link capabilities"); 2949 2950 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 2951 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 2952 sysctl_bitfield, "A", "available NIC capabilities"); 2953 2954 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 2955 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 2956 sysctl_bitfield, "A", "available TCP offload capabilities"); 2957 2958 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 2959 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 2960 sysctl_bitfield, "A", "available RDMA capabilities"); 2961 2962 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 2963 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 2964 sysctl_bitfield, "A", "available iSCSI capabilities"); 2965 2966 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 2967 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 2968 sysctl_bitfield, "A", "available FCoE capabilities"); 2969 2970 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, 2971 &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)"); 2972 2973 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 2974 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 2975 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 2976 "interrupt holdoff timer values (us)"); 2977 2978 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 2979 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 2980 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 2981 "interrupt holdoff packet counter values"); 2982 2983 /* 2984 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 2985 */ 2986 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 2987 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 2988 "logs and miscellaneous information"); 2989 children = SYSCTL_CHILDREN(oid); 2990 2991 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 2992 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 2993 sysctl_cctrl, "A", "congestion control"); 2994 2995 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 2996 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 2997 sysctl_cpl_stats, "A", "CPL statistics"); 2998 2999 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 3000 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3001 sysctl_ddp_stats, "A", "DDP statistics"); 3002 3003 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 3004 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3005 sysctl_devlog, "A", "firmware's device log"); 3006 3007 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 3008 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3009 sysctl_fcoe_stats, "A", "FCoE statistics"); 3010 3011 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 3012 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3013 sysctl_hw_sched, "A", "hardware scheduler "); 3014 3015 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 3016 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3017 sysctl_l2t, "A", "hardware L2 table"); 3018 3019 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 3020 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3021 sysctl_lb_stats, "A", "loopback statistics"); 3022 3023 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 3024 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3025 sysctl_meminfo, "A", "memory regions"); 3026 3027 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 3028 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3029 sysctl_path_mtus, "A", "path MTUs"); 3030 3031 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 3032 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3033 sysctl_pm_stats, "A", "PM statistics"); 3034 3035 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 3036 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3037 sysctl_rdma_stats, "A", "RDMA statistics"); 3038 3039 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 3040 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3041 sysctl_tcp_stats, "A", "TCP statistics"); 3042 3043 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 3044 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3045 sysctl_tids, "A", "TID information"); 3046 3047 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 3048 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3049 sysctl_tp_err_stats, "A", "TP error statistics"); 3050 3051 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 3052 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3053 sysctl_tx_rate, "A", "Tx rate"); 3054 3055 #ifndef TCP_OFFLOAD_DISABLE 3056 if (is_offload(sc)) { 3057 /* 3058 * dev.t4nex.X.toe. 3059 */ 3060 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 3061 NULL, "TOE parameters"); 3062 children = SYSCTL_CHILDREN(oid); 3063 3064 sc->tt.sndbuf = 256 * 1024; 3065 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 3066 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 3067 3068 sc->tt.ddp = 0; 3069 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 3070 &sc->tt.ddp, 0, "DDP allowed"); 3071 sc->tt.indsz = M_INDICATESIZE; 3072 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 3073 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 3074 sc->tt.ddp_thres = 3*4096; 3075 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 3076 &sc->tt.ddp_thres, 0, "DDP threshold"); 3077 } 3078 #endif 3079 3080 3081 return (0); 3082 } 3083 3084 static int 3085 cxgbe_sysctls(struct port_info *pi) 3086 { 3087 struct sysctl_ctx_list *ctx; 3088 struct sysctl_oid *oid; 3089 struct sysctl_oid_list *children; 3090 3091 ctx = device_get_sysctl_ctx(pi->dev); 3092 3093 /* 3094 * dev.cxgbe.X. 3095 */ 3096 oid = device_get_sysctl_tree(pi->dev); 3097 children = SYSCTL_CHILDREN(oid); 3098 3099 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 3100 &pi->nrxq, 0, "# of rx queues"); 3101 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 3102 &pi->ntxq, 0, "# of tx queues"); 3103 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 3104 &pi->first_rxq, 0, "index of first rx queue"); 3105 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 3106 &pi->first_txq, 0, "index of first tx queue"); 3107 3108 #ifndef TCP_OFFLOAD_DISABLE 3109 if (is_offload(pi->adapter)) { 3110 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 3111 &pi->nofldrxq, 0, 3112 "# of rx queues for offloaded TCP connections"); 3113 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 3114 &pi->nofldtxq, 0, 3115 "# of tx queues for offloaded TCP connections"); 3116 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 3117 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 3118 "index of first TOE rx queue"); 3119 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 3120 CTLFLAG_RD, &pi->first_ofld_txq, 0, 3121 "index of first TOE tx queue"); 3122 } 3123 #endif 3124 3125 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 3126 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 3127 "holdoff timer index"); 3128 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 3129 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 3130 "holdoff packet counter index"); 3131 3132 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 3133 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 3134 "rx queue size"); 3135 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 3136 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 3137 "tx queue size"); 3138 3139 /* 3140 * dev.cxgbe.X.stats. 3141 */ 3142 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 3143 NULL, "port statistics"); 3144 children = SYSCTL_CHILDREN(oid); 3145 3146 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 3147 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 3148 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 3149 sysctl_handle_t4_reg64, "QU", desc) 3150 3151 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 3152 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 3153 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 3154 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 3155 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 3156 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 3157 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 3158 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 3159 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 3160 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 3161 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 3162 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 3163 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 3164 "# of tx frames in this range", 3165 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 3166 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 3167 "# of tx frames in this range", 3168 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 3169 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 3170 "# of tx frames in this range", 3171 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 3172 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 3173 "# of tx frames in this range", 3174 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 3175 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 3176 "# of tx frames in this range", 3177 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 3178 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 3179 "# of tx frames in this range", 3180 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 3181 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 3182 "# of tx frames in this range", 3183 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 3184 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 3185 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 3186 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 3187 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 3188 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 3189 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 3190 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 3191 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 3192 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 3193 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 3194 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 3195 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 3196 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 3197 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 3198 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 3199 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 3200 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 3201 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 3202 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 3203 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 3204 3205 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 3206 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 3207 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 3208 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 3209 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 3210 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 3211 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 3212 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 3213 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 3214 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 3215 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 3216 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 3217 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 3218 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 3219 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 3220 "# of frames received with bad FCS", 3221 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 3222 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 3223 "# of frames received with length error", 3224 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 3225 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 3226 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 3227 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 3228 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 3229 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 3230 "# of rx frames in this range", 3231 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 3232 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 3233 "# of rx frames in this range", 3234 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 3235 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 3236 "# of rx frames in this range", 3237 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 3238 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 3239 "# of rx frames in this range", 3240 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 3241 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 3242 "# of rx frames in this range", 3243 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 3244 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 3245 "# of rx frames in this range", 3246 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 3247 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 3248 "# of rx frames in this range", 3249 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 3250 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 3251 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 3252 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 3253 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 3254 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 3255 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 3256 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 3257 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 3258 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 3259 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 3260 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 3261 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 3262 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 3263 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 3264 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 3265 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 3266 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 3267 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 3268 3269 #undef SYSCTL_ADD_T4_REG64 3270 3271 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 3272 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 3273 &pi->stats.name, desc) 3274 3275 /* We get these from port_stats and they may be stale by upto 1s */ 3276 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 3277 "# drops due to buffer-group 0 overflows"); 3278 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 3279 "# drops due to buffer-group 1 overflows"); 3280 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 3281 "# drops due to buffer-group 2 overflows"); 3282 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 3283 "# drops due to buffer-group 3 overflows"); 3284 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 3285 "# of buffer-group 0 truncated packets"); 3286 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 3287 "# of buffer-group 1 truncated packets"); 3288 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 3289 "# of buffer-group 2 truncated packets"); 3290 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 3291 "# of buffer-group 3 truncated packets"); 3292 3293 #undef SYSCTL_ADD_T4_PORTSTAT 3294 3295 return (0); 3296 } 3297 3298 static int 3299 sysctl_int_array(SYSCTL_HANDLER_ARGS) 3300 { 3301 int rc, *i; 3302 struct sbuf sb; 3303 3304 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 3305 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 3306 sbuf_printf(&sb, "%d ", *i); 3307 sbuf_trim(&sb); 3308 sbuf_finish(&sb); 3309 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 3310 sbuf_delete(&sb); 3311 return (rc); 3312 } 3313 3314 static int 3315 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 3316 { 3317 int rc; 3318 struct sbuf *sb; 3319 3320 rc = sysctl_wire_old_buffer(req, 0); 3321 if (rc != 0) 3322 return(rc); 3323 3324 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 3325 if (sb == NULL) 3326 return (ENOMEM); 3327 3328 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 3329 rc = sbuf_finish(sb); 3330 sbuf_delete(sb); 3331 3332 return (rc); 3333 } 3334 3335 static int 3336 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 3337 { 3338 struct port_info *pi = arg1; 3339 struct adapter *sc = pi->adapter; 3340 int idx, rc, i; 3341 3342 idx = pi->tmr_idx; 3343 3344 rc = sysctl_handle_int(oidp, &idx, 0, req); 3345 if (rc != 0 || req->newptr == NULL) 3346 return (rc); 3347 3348 if (idx < 0 || idx >= SGE_NTIMERS) 3349 return (EINVAL); 3350 3351 ADAPTER_LOCK(sc); 3352 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3353 if (rc == 0) { 3354 struct sge_rxq *rxq; 3355 uint8_t v; 3356 3357 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 3358 for_each_rxq(pi, i, rxq) { 3359 #ifdef atomic_store_rel_8 3360 atomic_store_rel_8(&rxq->iq.intr_params, v); 3361 #else 3362 rxq->iq.intr_params = v; 3363 #endif 3364 } 3365 pi->tmr_idx = idx; 3366 } 3367 3368 ADAPTER_UNLOCK(sc); 3369 return (rc); 3370 } 3371 3372 static int 3373 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 3374 { 3375 struct port_info *pi = arg1; 3376 struct adapter *sc = pi->adapter; 3377 int idx, rc; 3378 3379 idx = pi->pktc_idx; 3380 3381 rc = sysctl_handle_int(oidp, &idx, 0, req); 3382 if (rc != 0 || req->newptr == NULL) 3383 return (rc); 3384 3385 if (idx < -1 || idx >= SGE_NCOUNTERS) 3386 return (EINVAL); 3387 3388 ADAPTER_LOCK(sc); 3389 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3390 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3391 rc = EBUSY; /* cannot be changed once the queues are created */ 3392 3393 if (rc == 0) 3394 pi->pktc_idx = idx; 3395 3396 ADAPTER_UNLOCK(sc); 3397 return (rc); 3398 } 3399 3400 static int 3401 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 3402 { 3403 struct port_info *pi = arg1; 3404 struct adapter *sc = pi->adapter; 3405 int qsize, rc; 3406 3407 qsize = pi->qsize_rxq; 3408 3409 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3410 if (rc != 0 || req->newptr == NULL) 3411 return (rc); 3412 3413 if (qsize < 128 || (qsize & 7)) 3414 return (EINVAL); 3415 3416 ADAPTER_LOCK(sc); 3417 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3418 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3419 rc = EBUSY; /* cannot be changed once the queues are created */ 3420 3421 if (rc == 0) 3422 pi->qsize_rxq = qsize; 3423 3424 ADAPTER_UNLOCK(sc); 3425 return (rc); 3426 } 3427 3428 static int 3429 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 3430 { 3431 struct port_info *pi = arg1; 3432 struct adapter *sc = pi->adapter; 3433 int qsize, rc; 3434 3435 qsize = pi->qsize_txq; 3436 3437 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3438 if (rc != 0 || req->newptr == NULL) 3439 return (rc); 3440 3441 if (qsize < 128) 3442 return (EINVAL); 3443 3444 ADAPTER_LOCK(sc); 3445 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3446 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3447 rc = EBUSY; /* cannot be changed once the queues are created */ 3448 3449 if (rc == 0) 3450 pi->qsize_txq = qsize; 3451 3452 ADAPTER_UNLOCK(sc); 3453 return (rc); 3454 } 3455 3456 static int 3457 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 3458 { 3459 struct adapter *sc = arg1; 3460 int reg = arg2; 3461 uint64_t val; 3462 3463 val = t4_read_reg64(sc, reg); 3464 3465 return (sysctl_handle_64(oidp, &val, 0, req)); 3466 } 3467 3468 static int 3469 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 3470 { 3471 struct adapter *sc = arg1; 3472 struct sbuf *sb; 3473 int rc, i; 3474 uint16_t incr[NMTUS][NCCTRL_WIN]; 3475 static const char *dec_fac[] = { 3476 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 3477 "0.9375" 3478 }; 3479 3480 rc = sysctl_wire_old_buffer(req, 0); 3481 if (rc != 0) 3482 return (rc); 3483 3484 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3485 if (sb == NULL) 3486 return (ENOMEM); 3487 3488 t4_read_cong_tbl(sc, incr); 3489 3490 for (i = 0; i < NCCTRL_WIN; ++i) { 3491 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 3492 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 3493 incr[5][i], incr[6][i], incr[7][i]); 3494 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 3495 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 3496 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 3497 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 3498 } 3499 3500 rc = sbuf_finish(sb); 3501 sbuf_delete(sb); 3502 3503 return (rc); 3504 } 3505 3506 static int 3507 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 3508 { 3509 struct adapter *sc = arg1; 3510 struct sbuf *sb; 3511 int rc; 3512 struct tp_cpl_stats stats; 3513 3514 rc = sysctl_wire_old_buffer(req, 0); 3515 if (rc != 0) 3516 return (rc); 3517 3518 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3519 if (sb == NULL) 3520 return (ENOMEM); 3521 3522 t4_tp_get_cpl_stats(sc, &stats); 3523 3524 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 3525 "channel 3\n"); 3526 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 3527 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 3528 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 3529 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 3530 3531 rc = sbuf_finish(sb); 3532 sbuf_delete(sb); 3533 3534 return (rc); 3535 } 3536 3537 static int 3538 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 3539 { 3540 struct adapter *sc = arg1; 3541 struct sbuf *sb; 3542 int rc; 3543 struct tp_usm_stats stats; 3544 3545 rc = sysctl_wire_old_buffer(req, 0); 3546 if (rc != 0) 3547 return(rc); 3548 3549 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3550 if (sb == NULL) 3551 return (ENOMEM); 3552 3553 t4_get_usm_stats(sc, &stats); 3554 3555 sbuf_printf(sb, "Frames: %u\n", stats.frames); 3556 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 3557 sbuf_printf(sb, "Drops: %u", stats.drops); 3558 3559 rc = sbuf_finish(sb); 3560 sbuf_delete(sb); 3561 3562 return (rc); 3563 } 3564 3565 const char *devlog_level_strings[] = { 3566 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 3567 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 3568 [FW_DEVLOG_LEVEL_ERR] = "ERR", 3569 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 3570 [FW_DEVLOG_LEVEL_INFO] = "INFO", 3571 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 3572 }; 3573 3574 const char *devlog_facility_strings[] = { 3575 [FW_DEVLOG_FACILITY_CORE] = "CORE", 3576 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 3577 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 3578 [FW_DEVLOG_FACILITY_RES] = "RES", 3579 [FW_DEVLOG_FACILITY_HW] = "HW", 3580 [FW_DEVLOG_FACILITY_FLR] = "FLR", 3581 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 3582 [FW_DEVLOG_FACILITY_PHY] = "PHY", 3583 [FW_DEVLOG_FACILITY_MAC] = "MAC", 3584 [FW_DEVLOG_FACILITY_PORT] = "PORT", 3585 [FW_DEVLOG_FACILITY_VI] = "VI", 3586 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 3587 [FW_DEVLOG_FACILITY_ACL] = "ACL", 3588 [FW_DEVLOG_FACILITY_TM] = "TM", 3589 [FW_DEVLOG_FACILITY_QFC] = "QFC", 3590 [FW_DEVLOG_FACILITY_DCB] = "DCB", 3591 [FW_DEVLOG_FACILITY_ETH] = "ETH", 3592 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 3593 [FW_DEVLOG_FACILITY_RI] = "RI", 3594 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 3595 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 3596 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 3597 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 3598 }; 3599 3600 static int 3601 sysctl_devlog(SYSCTL_HANDLER_ARGS) 3602 { 3603 struct adapter *sc = arg1; 3604 struct devlog_params *dparams = &sc->params.devlog; 3605 struct fw_devlog_e *buf, *e; 3606 int i, j, rc, nentries, first = 0; 3607 struct sbuf *sb; 3608 uint64_t ftstamp = UINT64_MAX; 3609 3610 if (dparams->start == 0) 3611 return (ENXIO); 3612 3613 nentries = dparams->size / sizeof(struct fw_devlog_e); 3614 3615 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 3616 if (buf == NULL) 3617 return (ENOMEM); 3618 3619 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 3620 (void *)buf); 3621 if (rc != 0) 3622 goto done; 3623 3624 for (i = 0; i < nentries; i++) { 3625 e = &buf[i]; 3626 3627 if (e->timestamp == 0) 3628 break; /* end */ 3629 3630 e->timestamp = be64toh(e->timestamp); 3631 e->seqno = be32toh(e->seqno); 3632 for (j = 0; j < 8; j++) 3633 e->params[j] = be32toh(e->params[j]); 3634 3635 if (e->timestamp < ftstamp) { 3636 ftstamp = e->timestamp; 3637 first = i; 3638 } 3639 } 3640 3641 if (buf[first].timestamp == 0) 3642 goto done; /* nothing in the log */ 3643 3644 rc = sysctl_wire_old_buffer(req, 0); 3645 if (rc != 0) 3646 goto done; 3647 3648 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3649 if (sb == NULL) { 3650 rc = ENOMEM; 3651 goto done; 3652 } 3653 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 3654 "Seq#", "Tstamp", "Level", "Facility", "Message"); 3655 3656 i = first; 3657 do { 3658 e = &buf[i]; 3659 if (e->timestamp == 0) 3660 break; /* end */ 3661 3662 sbuf_printf(sb, "%10d %15ju %8s %8s ", 3663 e->seqno, e->timestamp, 3664 (e->level < ARRAY_SIZE(devlog_level_strings) ? 3665 devlog_level_strings[e->level] : "UNKNOWN"), 3666 (e->facility < ARRAY_SIZE(devlog_facility_strings) ? 3667 devlog_facility_strings[e->facility] : "UNKNOWN")); 3668 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 3669 e->params[2], e->params[3], e->params[4], 3670 e->params[5], e->params[6], e->params[7]); 3671 3672 if (++i == nentries) 3673 i = 0; 3674 } while (i != first); 3675 3676 rc = sbuf_finish(sb); 3677 sbuf_delete(sb); 3678 done: 3679 free(buf, M_CXGBE); 3680 return (rc); 3681 } 3682 3683 static int 3684 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 3685 { 3686 struct adapter *sc = arg1; 3687 struct sbuf *sb; 3688 int rc; 3689 struct tp_fcoe_stats stats[4]; 3690 3691 rc = sysctl_wire_old_buffer(req, 0); 3692 if (rc != 0) 3693 return (rc); 3694 3695 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3696 if (sb == NULL) 3697 return (ENOMEM); 3698 3699 t4_get_fcoe_stats(sc, 0, &stats[0]); 3700 t4_get_fcoe_stats(sc, 1, &stats[1]); 3701 t4_get_fcoe_stats(sc, 2, &stats[2]); 3702 t4_get_fcoe_stats(sc, 3, &stats[3]); 3703 3704 sbuf_printf(sb, " channel 0 channel 1 " 3705 "channel 2 channel 3\n"); 3706 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 3707 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 3708 stats[3].octetsDDP); 3709 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 3710 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 3711 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 3712 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 3713 stats[3].framesDrop); 3714 3715 rc = sbuf_finish(sb); 3716 sbuf_delete(sb); 3717 3718 return (rc); 3719 } 3720 3721 static int 3722 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 3723 { 3724 struct adapter *sc = arg1; 3725 struct sbuf *sb; 3726 int rc, i; 3727 unsigned int map, kbps, ipg, mode; 3728 unsigned int pace_tab[NTX_SCHED]; 3729 3730 rc = sysctl_wire_old_buffer(req, 0); 3731 if (rc != 0) 3732 return (rc); 3733 3734 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3735 if (sb == NULL) 3736 return (ENOMEM); 3737 3738 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 3739 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 3740 t4_read_pace_tbl(sc, pace_tab); 3741 3742 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 3743 "Class IPG (0.1 ns) Flow IPG (us)"); 3744 3745 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 3746 t4_get_tx_sched(sc, i, &kbps, &ipg); 3747 sbuf_printf(sb, "\n %u %-5s %u ", i, 3748 (mode & (1 << i)) ? "flow" : "class", map & 3); 3749 if (kbps) 3750 sbuf_printf(sb, "%9u ", kbps); 3751 else 3752 sbuf_printf(sb, " disabled "); 3753 3754 if (ipg) 3755 sbuf_printf(sb, "%13u ", ipg); 3756 else 3757 sbuf_printf(sb, " disabled "); 3758 3759 if (pace_tab[i]) 3760 sbuf_printf(sb, "%10u", pace_tab[i]); 3761 else 3762 sbuf_printf(sb, " disabled"); 3763 } 3764 3765 rc = sbuf_finish(sb); 3766 sbuf_delete(sb); 3767 3768 return (rc); 3769 } 3770 3771 static int 3772 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 3773 { 3774 struct adapter *sc = arg1; 3775 struct sbuf *sb; 3776 int rc, i, j; 3777 uint64_t *p0, *p1; 3778 struct lb_port_stats s[2]; 3779 static const char *stat_name[] = { 3780 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 3781 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 3782 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 3783 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 3784 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 3785 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 3786 "BG2FramesTrunc:", "BG3FramesTrunc:" 3787 }; 3788 3789 rc = sysctl_wire_old_buffer(req, 0); 3790 if (rc != 0) 3791 return (rc); 3792 3793 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3794 if (sb == NULL) 3795 return (ENOMEM); 3796 3797 memset(s, 0, sizeof(s)); 3798 3799 for (i = 0; i < 4; i += 2) { 3800 t4_get_lb_stats(sc, i, &s[0]); 3801 t4_get_lb_stats(sc, i + 1, &s[1]); 3802 3803 p0 = &s[0].octets; 3804 p1 = &s[1].octets; 3805 sbuf_printf(sb, "%s Loopback %u" 3806 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 3807 3808 for (j = 0; j < ARRAY_SIZE(stat_name); j++) 3809 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 3810 *p0++, *p1++); 3811 } 3812 3813 rc = sbuf_finish(sb); 3814 sbuf_delete(sb); 3815 3816 return (rc); 3817 } 3818 3819 struct mem_desc { 3820 unsigned int base; 3821 unsigned int limit; 3822 unsigned int idx; 3823 }; 3824 3825 static int 3826 mem_desc_cmp(const void *a, const void *b) 3827 { 3828 return ((const struct mem_desc *)a)->base - 3829 ((const struct mem_desc *)b)->base; 3830 } 3831 3832 static void 3833 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 3834 unsigned int to) 3835 { 3836 unsigned int size; 3837 3838 size = to - from + 1; 3839 if (size == 0) 3840 return; 3841 3842 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 3843 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 3844 } 3845 3846 static int 3847 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 3848 { 3849 struct adapter *sc = arg1; 3850 struct sbuf *sb; 3851 int rc, i, n; 3852 uint32_t lo, hi; 3853 static const char *memory[] = { "EDC0:", "EDC1:", "MC:" }; 3854 static const char *region[] = { 3855 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 3856 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 3857 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 3858 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 3859 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:", 3860 "ULPTX state:", "On-chip queues:" 3861 }; 3862 struct mem_desc avail[3]; 3863 struct mem_desc mem[ARRAY_SIZE(region) + 3]; /* up to 3 holes */ 3864 struct mem_desc *md = mem; 3865 3866 rc = sysctl_wire_old_buffer(req, 0); 3867 if (rc != 0) 3868 return (rc); 3869 3870 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3871 if (sb == NULL) 3872 return (ENOMEM); 3873 3874 for (i = 0; i < ARRAY_SIZE(mem); i++) { 3875 mem[i].limit = 0; 3876 mem[i].idx = i; 3877 } 3878 3879 /* Find and sort the populated memory ranges */ 3880 i = 0; 3881 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 3882 if (lo & F_EDRAM0_ENABLE) { 3883 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 3884 avail[i].base = G_EDRAM0_BASE(hi) << 20; 3885 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 3886 avail[i].idx = 0; 3887 i++; 3888 } 3889 if (lo & F_EDRAM1_ENABLE) { 3890 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 3891 avail[i].base = G_EDRAM1_BASE(hi) << 20; 3892 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 3893 avail[i].idx = 1; 3894 i++; 3895 } 3896 if (lo & F_EXT_MEM_ENABLE) { 3897 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 3898 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 3899 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20); 3900 avail[i].idx = 2; 3901 i++; 3902 } 3903 if (!i) /* no memory available */ 3904 return 0; 3905 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 3906 3907 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 3908 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 3909 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 3910 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 3911 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 3912 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 3913 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 3914 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 3915 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 3916 3917 /* the next few have explicit upper bounds */ 3918 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 3919 md->limit = md->base - 1 + 3920 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 3921 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 3922 md++; 3923 3924 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 3925 md->limit = md->base - 1 + 3926 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 3927 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 3928 md++; 3929 3930 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 3931 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 3932 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 3933 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 3934 } else { 3935 md->base = 0; 3936 md->idx = ARRAY_SIZE(region); /* hide it */ 3937 } 3938 md++; 3939 3940 #define ulp_region(reg) \ 3941 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 3942 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 3943 3944 ulp_region(RX_ISCSI); 3945 ulp_region(RX_TDDP); 3946 ulp_region(TX_TPT); 3947 ulp_region(RX_STAG); 3948 ulp_region(RX_RQ); 3949 ulp_region(RX_RQUDP); 3950 ulp_region(RX_PBL); 3951 ulp_region(TX_PBL); 3952 #undef ulp_region 3953 3954 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 3955 md->limit = md->base + sc->tids.ntids - 1; 3956 md++; 3957 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 3958 md->limit = md->base + sc->tids.ntids - 1; 3959 md++; 3960 3961 md->base = sc->vres.ocq.start; 3962 if (sc->vres.ocq.size) 3963 md->limit = md->base + sc->vres.ocq.size - 1; 3964 else 3965 md->idx = ARRAY_SIZE(region); /* hide it */ 3966 md++; 3967 3968 /* add any address-space holes, there can be up to 3 */ 3969 for (n = 0; n < i - 1; n++) 3970 if (avail[n].limit < avail[n + 1].base) 3971 (md++)->base = avail[n].limit; 3972 if (avail[n].limit) 3973 (md++)->base = avail[n].limit; 3974 3975 n = md - mem; 3976 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 3977 3978 for (lo = 0; lo < i; lo++) 3979 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 3980 avail[lo].limit - 1); 3981 3982 sbuf_printf(sb, "\n"); 3983 for (i = 0; i < n; i++) { 3984 if (mem[i].idx >= ARRAY_SIZE(region)) 3985 continue; /* skip holes */ 3986 if (!mem[i].limit) 3987 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 3988 mem_region_show(sb, region[mem[i].idx], mem[i].base, 3989 mem[i].limit); 3990 } 3991 3992 sbuf_printf(sb, "\n"); 3993 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 3994 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 3995 mem_region_show(sb, "uP RAM:", lo, hi); 3996 3997 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 3998 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 3999 mem_region_show(sb, "uP Extmem2:", lo, hi); 4000 4001 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 4002 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 4003 G_PMRXMAXPAGE(lo), 4004 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 4005 (lo & F_PMRXNUMCHN) ? 2 : 1); 4006 4007 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 4008 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 4009 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 4010 G_PMTXMAXPAGE(lo), 4011 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 4012 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 4013 sbuf_printf(sb, "%u p-structs\n", 4014 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 4015 4016 for (i = 0; i < 4; i++) { 4017 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 4018 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 4019 i, G_USED(lo), G_ALLOC(lo)); 4020 } 4021 for (i = 0; i < 4; i++) { 4022 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 4023 sbuf_printf(sb, 4024 "\nLoopback %d using %u pages out of %u allocated", 4025 i, G_USED(lo), G_ALLOC(lo)); 4026 } 4027 4028 rc = sbuf_finish(sb); 4029 sbuf_delete(sb); 4030 4031 return (rc); 4032 } 4033 4034 static int 4035 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 4036 { 4037 struct adapter *sc = arg1; 4038 struct sbuf *sb; 4039 int rc; 4040 uint16_t mtus[NMTUS]; 4041 4042 rc = sysctl_wire_old_buffer(req, 0); 4043 if (rc != 0) 4044 return (rc); 4045 4046 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4047 if (sb == NULL) 4048 return (ENOMEM); 4049 4050 t4_read_mtu_tbl(sc, mtus, NULL); 4051 4052 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 4053 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 4054 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 4055 mtus[14], mtus[15]); 4056 4057 rc = sbuf_finish(sb); 4058 sbuf_delete(sb); 4059 4060 return (rc); 4061 } 4062 4063 static int 4064 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 4065 { 4066 struct adapter *sc = arg1; 4067 struct sbuf *sb; 4068 int rc, i; 4069 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 4070 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 4071 static const char *pm_stats[] = { 4072 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 4073 }; 4074 4075 rc = sysctl_wire_old_buffer(req, 0); 4076 if (rc != 0) 4077 return (rc); 4078 4079 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4080 if (sb == NULL) 4081 return (ENOMEM); 4082 4083 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 4084 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 4085 4086 sbuf_printf(sb, " Tx count Tx cycles " 4087 "Rx count Rx cycles"); 4088 for (i = 0; i < PM_NSTATS; i++) 4089 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 4090 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 4091 4092 rc = sbuf_finish(sb); 4093 sbuf_delete(sb); 4094 4095 return (rc); 4096 } 4097 4098 static int 4099 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 4100 { 4101 struct adapter *sc = arg1; 4102 struct sbuf *sb; 4103 int rc; 4104 struct tp_rdma_stats stats; 4105 4106 rc = sysctl_wire_old_buffer(req, 0); 4107 if (rc != 0) 4108 return (rc); 4109 4110 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4111 if (sb == NULL) 4112 return (ENOMEM); 4113 4114 t4_tp_get_rdma_stats(sc, &stats); 4115 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 4116 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 4117 4118 rc = sbuf_finish(sb); 4119 sbuf_delete(sb); 4120 4121 return (rc); 4122 } 4123 4124 static int 4125 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 4126 { 4127 struct adapter *sc = arg1; 4128 struct sbuf *sb; 4129 int rc; 4130 struct tp_tcp_stats v4, v6; 4131 4132 rc = sysctl_wire_old_buffer(req, 0); 4133 if (rc != 0) 4134 return (rc); 4135 4136 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4137 if (sb == NULL) 4138 return (ENOMEM); 4139 4140 t4_tp_get_tcp_stats(sc, &v4, &v6); 4141 sbuf_printf(sb, 4142 " IP IPv6\n"); 4143 sbuf_printf(sb, "OutRsts: %20u %20u\n", 4144 v4.tcpOutRsts, v6.tcpOutRsts); 4145 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 4146 v4.tcpInSegs, v6.tcpInSegs); 4147 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 4148 v4.tcpOutSegs, v6.tcpOutSegs); 4149 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 4150 v4.tcpRetransSegs, v6.tcpRetransSegs); 4151 4152 rc = sbuf_finish(sb); 4153 sbuf_delete(sb); 4154 4155 return (rc); 4156 } 4157 4158 static int 4159 sysctl_tids(SYSCTL_HANDLER_ARGS) 4160 { 4161 struct adapter *sc = arg1; 4162 struct sbuf *sb; 4163 int rc; 4164 struct tid_info *t = &sc->tids; 4165 4166 rc = sysctl_wire_old_buffer(req, 0); 4167 if (rc != 0) 4168 return (rc); 4169 4170 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4171 if (sb == NULL) 4172 return (ENOMEM); 4173 4174 if (t->natids) { 4175 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 4176 t->atids_in_use); 4177 } 4178 4179 if (t->ntids) { 4180 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4181 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 4182 4183 if (b) { 4184 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 4185 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4186 t->ntids - 1); 4187 } else { 4188 sbuf_printf(sb, "TID range: %u-%u", 4189 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4190 t->ntids - 1); 4191 } 4192 } else 4193 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 4194 sbuf_printf(sb, ", in use: %u\n", 4195 atomic_load_acq_int(&t->tids_in_use)); 4196 } 4197 4198 if (t->nstids) { 4199 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 4200 t->stid_base + t->nstids - 1, t->stids_in_use); 4201 } 4202 4203 if (t->nftids) { 4204 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 4205 t->ftid_base + t->nftids - 1); 4206 } 4207 4208 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 4209 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 4210 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 4211 4212 rc = sbuf_finish(sb); 4213 sbuf_delete(sb); 4214 4215 return (rc); 4216 } 4217 4218 static int 4219 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 4220 { 4221 struct adapter *sc = arg1; 4222 struct sbuf *sb; 4223 int rc; 4224 struct tp_err_stats stats; 4225 4226 rc = sysctl_wire_old_buffer(req, 0); 4227 if (rc != 0) 4228 return (rc); 4229 4230 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4231 if (sb == NULL) 4232 return (ENOMEM); 4233 4234 t4_tp_get_err_stats(sc, &stats); 4235 4236 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4237 "channel 3\n"); 4238 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 4239 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 4240 stats.macInErrs[3]); 4241 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 4242 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 4243 stats.hdrInErrs[3]); 4244 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 4245 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 4246 stats.tcpInErrs[3]); 4247 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 4248 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 4249 stats.tcp6InErrs[3]); 4250 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 4251 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 4252 stats.tnlCongDrops[3]); 4253 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 4254 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 4255 stats.tnlTxDrops[3]); 4256 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 4257 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 4258 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 4259 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 4260 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 4261 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 4262 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 4263 stats.ofldNoNeigh, stats.ofldCongDefer); 4264 4265 rc = sbuf_finish(sb); 4266 sbuf_delete(sb); 4267 4268 return (rc); 4269 } 4270 4271 static int 4272 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 4273 { 4274 struct adapter *sc = arg1; 4275 struct sbuf *sb; 4276 int rc; 4277 u64 nrate[NCHAN], orate[NCHAN]; 4278 4279 rc = sysctl_wire_old_buffer(req, 0); 4280 if (rc != 0) 4281 return (rc); 4282 4283 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4284 if (sb == NULL) 4285 return (ENOMEM); 4286 4287 t4_get_chan_txrate(sc, nrate, orate); 4288 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4289 "channel 3\n"); 4290 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 4291 nrate[0], nrate[1], nrate[2], nrate[3]); 4292 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 4293 orate[0], orate[1], orate[2], orate[3]); 4294 4295 rc = sbuf_finish(sb); 4296 sbuf_delete(sb); 4297 4298 return (rc); 4299 } 4300 4301 static inline void 4302 txq_start(struct ifnet *ifp, struct sge_txq *txq) 4303 { 4304 struct buf_ring *br; 4305 struct mbuf *m; 4306 4307 TXQ_LOCK_ASSERT_OWNED(txq); 4308 4309 br = txq->br; 4310 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 4311 if (m) 4312 t4_eth_tx(ifp, txq, m); 4313 } 4314 4315 void 4316 t4_tx_callout(void *arg) 4317 { 4318 struct sge_eq *eq = arg; 4319 struct adapter *sc; 4320 4321 if (EQ_TRYLOCK(eq) == 0) 4322 goto reschedule; 4323 4324 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 4325 EQ_UNLOCK(eq); 4326 reschedule: 4327 if (__predict_true(!(eq->flags && EQ_DOOMED))) 4328 callout_schedule(&eq->tx_callout, 1); 4329 return; 4330 } 4331 4332 EQ_LOCK_ASSERT_OWNED(eq); 4333 4334 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 4335 4336 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4337 struct sge_txq *txq = arg; 4338 struct port_info *pi = txq->ifp->if_softc; 4339 4340 sc = pi->adapter; 4341 } else { 4342 struct sge_wrq *wrq = arg; 4343 4344 sc = wrq->adapter; 4345 } 4346 4347 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 4348 } 4349 4350 EQ_UNLOCK(eq); 4351 } 4352 4353 void 4354 t4_tx_task(void *arg, int count) 4355 { 4356 struct sge_eq *eq = arg; 4357 4358 EQ_LOCK(eq); 4359 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4360 struct sge_txq *txq = arg; 4361 txq_start(txq->ifp, txq); 4362 } else { 4363 struct sge_wrq *wrq = arg; 4364 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 4365 } 4366 EQ_UNLOCK(eq); 4367 } 4368 4369 static uint32_t 4370 fconf_to_mode(uint32_t fconf) 4371 { 4372 uint32_t mode; 4373 4374 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 4375 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 4376 4377 if (fconf & F_FRAGMENTATION) 4378 mode |= T4_FILTER_IP_FRAGMENT; 4379 4380 if (fconf & F_MPSHITTYPE) 4381 mode |= T4_FILTER_MPS_HIT_TYPE; 4382 4383 if (fconf & F_MACMATCH) 4384 mode |= T4_FILTER_MAC_IDX; 4385 4386 if (fconf & F_ETHERTYPE) 4387 mode |= T4_FILTER_ETH_TYPE; 4388 4389 if (fconf & F_PROTOCOL) 4390 mode |= T4_FILTER_IP_PROTO; 4391 4392 if (fconf & F_TOS) 4393 mode |= T4_FILTER_IP_TOS; 4394 4395 if (fconf & F_VLAN) 4396 mode |= T4_FILTER_VLAN; 4397 4398 if (fconf & F_VNIC_ID) 4399 mode |= T4_FILTER_VNIC; 4400 4401 if (fconf & F_PORT) 4402 mode |= T4_FILTER_PORT; 4403 4404 if (fconf & F_FCOE) 4405 mode |= T4_FILTER_FCoE; 4406 4407 return (mode); 4408 } 4409 4410 static uint32_t 4411 mode_to_fconf(uint32_t mode) 4412 { 4413 uint32_t fconf = 0; 4414 4415 if (mode & T4_FILTER_IP_FRAGMENT) 4416 fconf |= F_FRAGMENTATION; 4417 4418 if (mode & T4_FILTER_MPS_HIT_TYPE) 4419 fconf |= F_MPSHITTYPE; 4420 4421 if (mode & T4_FILTER_MAC_IDX) 4422 fconf |= F_MACMATCH; 4423 4424 if (mode & T4_FILTER_ETH_TYPE) 4425 fconf |= F_ETHERTYPE; 4426 4427 if (mode & T4_FILTER_IP_PROTO) 4428 fconf |= F_PROTOCOL; 4429 4430 if (mode & T4_FILTER_IP_TOS) 4431 fconf |= F_TOS; 4432 4433 if (mode & T4_FILTER_VLAN) 4434 fconf |= F_VLAN; 4435 4436 if (mode & T4_FILTER_VNIC) 4437 fconf |= F_VNIC_ID; 4438 4439 if (mode & T4_FILTER_PORT) 4440 fconf |= F_PORT; 4441 4442 if (mode & T4_FILTER_FCoE) 4443 fconf |= F_FCOE; 4444 4445 return (fconf); 4446 } 4447 4448 static uint32_t 4449 fspec_to_fconf(struct t4_filter_specification *fs) 4450 { 4451 uint32_t fconf = 0; 4452 4453 if (fs->val.frag || fs->mask.frag) 4454 fconf |= F_FRAGMENTATION; 4455 4456 if (fs->val.matchtype || fs->mask.matchtype) 4457 fconf |= F_MPSHITTYPE; 4458 4459 if (fs->val.macidx || fs->mask.macidx) 4460 fconf |= F_MACMATCH; 4461 4462 if (fs->val.ethtype || fs->mask.ethtype) 4463 fconf |= F_ETHERTYPE; 4464 4465 if (fs->val.proto || fs->mask.proto) 4466 fconf |= F_PROTOCOL; 4467 4468 if (fs->val.tos || fs->mask.tos) 4469 fconf |= F_TOS; 4470 4471 if (fs->val.vlan_vld || fs->mask.vlan_vld) 4472 fconf |= F_VLAN; 4473 4474 if (fs->val.vnic_vld || fs->mask.vnic_vld) 4475 fconf |= F_VNIC_ID; 4476 4477 if (fs->val.iport || fs->mask.iport) 4478 fconf |= F_PORT; 4479 4480 if (fs->val.fcoe || fs->mask.fcoe) 4481 fconf |= F_FCOE; 4482 4483 return (fconf); 4484 } 4485 4486 static int 4487 get_filter_mode(struct adapter *sc, uint32_t *mode) 4488 { 4489 uint32_t fconf; 4490 4491 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 4492 A_TP_VLAN_PRI_MAP); 4493 4494 if (sc->filter_mode != fconf) { 4495 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 4496 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 4497 sc->filter_mode = fconf; 4498 } 4499 4500 *mode = fconf_to_mode(sc->filter_mode); 4501 4502 return (0); 4503 } 4504 4505 static int 4506 set_filter_mode(struct adapter *sc, uint32_t mode) 4507 { 4508 uint32_t fconf; 4509 int rc; 4510 4511 fconf = mode_to_fconf(mode); 4512 4513 ADAPTER_LOCK(sc); 4514 if (IS_BUSY(sc)) { 4515 rc = EAGAIN; 4516 goto done; 4517 } 4518 4519 if (sc->tids.ftids_in_use > 0) { 4520 rc = EBUSY; 4521 goto done; 4522 } 4523 4524 #ifndef TCP_OFFLOAD_DISABLE 4525 if (sc->offload_map) { 4526 rc = EBUSY; 4527 goto done; 4528 } 4529 #endif 4530 4531 #ifdef notyet 4532 rc = -t4_set_filter_mode(sc, fconf); 4533 if (rc == 0) 4534 sc->filter_mode = fconf; 4535 #else 4536 rc = ENOTSUP; 4537 #endif 4538 4539 done: 4540 ADAPTER_UNLOCK(sc); 4541 return (rc); 4542 } 4543 4544 static inline uint64_t 4545 get_filter_hits(struct adapter *sc, uint32_t fid) 4546 { 4547 uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4548 uint64_t hits; 4549 4550 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0), 4551 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 4552 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0)); 4553 hits = t4_read_reg64(sc, MEMWIN0_BASE + 16); 4554 4555 return (be64toh(hits)); 4556 } 4557 4558 static int 4559 get_filter(struct adapter *sc, struct t4_filter *t) 4560 { 4561 int i, nfilters = sc->tids.nftids; 4562 struct filter_entry *f; 4563 4564 ADAPTER_LOCK_ASSERT_OWNED(sc); 4565 4566 if (IS_BUSY(sc)) 4567 return (EAGAIN); 4568 4569 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 4570 t->idx >= nfilters) { 4571 t->idx = 0xffffffff; 4572 return (0); 4573 } 4574 4575 f = &sc->tids.ftid_tab[t->idx]; 4576 for (i = t->idx; i < nfilters; i++, f++) { 4577 if (f->valid) { 4578 t->idx = i; 4579 t->l2tidx = f->l2t ? f->l2t->idx : 0; 4580 t->smtidx = f->smtidx; 4581 if (f->fs.hitcnts) 4582 t->hits = get_filter_hits(sc, t->idx); 4583 else 4584 t->hits = UINT64_MAX; 4585 t->fs = f->fs; 4586 4587 return (0); 4588 } 4589 } 4590 4591 t->idx = 0xffffffff; 4592 return (0); 4593 } 4594 4595 static int 4596 set_filter(struct adapter *sc, struct t4_filter *t) 4597 { 4598 unsigned int nfilters, nports; 4599 struct filter_entry *f; 4600 int i; 4601 4602 ADAPTER_LOCK_ASSERT_OWNED(sc); 4603 4604 nfilters = sc->tids.nftids; 4605 nports = sc->params.nports; 4606 4607 if (nfilters == 0) 4608 return (ENOTSUP); 4609 4610 if (!(sc->flags & FULL_INIT_DONE)) 4611 return (EAGAIN); 4612 4613 if (t->idx >= nfilters) 4614 return (EINVAL); 4615 4616 /* Validate against the global filter mode */ 4617 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) 4618 return (E2BIG); 4619 4620 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) 4621 return (EINVAL); 4622 4623 if (t->fs.val.iport >= nports) 4624 return (EINVAL); 4625 4626 /* Can't specify an iq if not steering to it */ 4627 if (!t->fs.dirsteer && t->fs.iq) 4628 return (EINVAL); 4629 4630 /* IPv6 filter idx must be 4 aligned */ 4631 if (t->fs.type == 1 && 4632 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) 4633 return (EINVAL); 4634 4635 if (sc->tids.ftid_tab == NULL) { 4636 KASSERT(sc->tids.ftids_in_use == 0, 4637 ("%s: no memory allocated but filters_in_use > 0", 4638 __func__)); 4639 4640 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 4641 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 4642 if (sc->tids.ftid_tab == NULL) 4643 return (ENOMEM); 4644 } 4645 4646 for (i = 0; i < 4; i++) { 4647 f = &sc->tids.ftid_tab[t->idx + i]; 4648 4649 if (f->pending || f->valid) 4650 return (EBUSY); 4651 if (f->locked) 4652 return (EPERM); 4653 4654 if (t->fs.type == 0) 4655 break; 4656 } 4657 4658 f = &sc->tids.ftid_tab[t->idx]; 4659 f->fs = t->fs; 4660 4661 return set_filter_wr(sc, t->idx); 4662 } 4663 4664 static int 4665 del_filter(struct adapter *sc, struct t4_filter *t) 4666 { 4667 unsigned int nfilters; 4668 struct filter_entry *f; 4669 4670 ADAPTER_LOCK_ASSERT_OWNED(sc); 4671 4672 if (IS_BUSY(sc)) 4673 return (EAGAIN); 4674 4675 nfilters = sc->tids.nftids; 4676 4677 if (nfilters == 0) 4678 return (ENOTSUP); 4679 4680 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 4681 t->idx >= nfilters) 4682 return (EINVAL); 4683 4684 if (!(sc->flags & FULL_INIT_DONE)) 4685 return (EAGAIN); 4686 4687 f = &sc->tids.ftid_tab[t->idx]; 4688 4689 if (f->pending) 4690 return (EBUSY); 4691 if (f->locked) 4692 return (EPERM); 4693 4694 if (f->valid) { 4695 t->fs = f->fs; /* extra info for the caller */ 4696 return del_filter_wr(sc, t->idx); 4697 } 4698 4699 return (0); 4700 } 4701 4702 static void 4703 clear_filter(struct filter_entry *f) 4704 { 4705 if (f->l2t) 4706 t4_l2t_release(f->l2t); 4707 4708 bzero(f, sizeof (*f)); 4709 } 4710 4711 static int 4712 set_filter_wr(struct adapter *sc, int fidx) 4713 { 4714 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4715 struct mbuf *m; 4716 struct fw_filter_wr *fwr; 4717 unsigned int ftid; 4718 4719 ADAPTER_LOCK_ASSERT_OWNED(sc); 4720 4721 if (f->fs.newdmac || f->fs.newvlan) { 4722 /* This filter needs an L2T entry; allocate one. */ 4723 f->l2t = t4_l2t_alloc_switching(sc->l2t); 4724 if (f->l2t == NULL) 4725 return (EAGAIN); 4726 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 4727 f->fs.dmac)) { 4728 t4_l2t_release(f->l2t); 4729 f->l2t = NULL; 4730 return (ENOMEM); 4731 } 4732 } 4733 4734 ftid = sc->tids.ftid_base + fidx; 4735 4736 m = m_gethdr(M_NOWAIT, MT_DATA); 4737 if (m == NULL) 4738 return (ENOMEM); 4739 4740 fwr = mtod(m, struct fw_filter_wr *); 4741 m->m_len = m->m_pkthdr.len = sizeof(*fwr); 4742 bzero(fwr, sizeof (*fwr)); 4743 4744 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 4745 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 4746 fwr->tid_to_iq = 4747 htobe32(V_FW_FILTER_WR_TID(ftid) | 4748 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 4749 V_FW_FILTER_WR_NOREPLY(0) | 4750 V_FW_FILTER_WR_IQ(f->fs.iq)); 4751 fwr->del_filter_to_l2tix = 4752 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 4753 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 4754 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 4755 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 4756 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 4757 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 4758 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 4759 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 4760 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 4761 f->fs.newvlan == VLAN_REWRITE) | 4762 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 4763 f->fs.newvlan == VLAN_REWRITE) | 4764 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 4765 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 4766 V_FW_FILTER_WR_PRIO(f->fs.prio) | 4767 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 4768 fwr->ethtype = htobe16(f->fs.val.ethtype); 4769 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 4770 fwr->frag_to_ovlan_vldm = 4771 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 4772 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 4773 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 4774 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 4775 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 4776 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 4777 fwr->smac_sel = 0; 4778 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 4779 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 4780 fwr->maci_to_matchtypem = 4781 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 4782 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 4783 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 4784 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 4785 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 4786 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 4787 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 4788 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 4789 fwr->ptcl = f->fs.val.proto; 4790 fwr->ptclm = f->fs.mask.proto; 4791 fwr->ttyp = f->fs.val.tos; 4792 fwr->ttypm = f->fs.mask.tos; 4793 fwr->ivlan = htobe16(f->fs.val.vlan); 4794 fwr->ivlanm = htobe16(f->fs.mask.vlan); 4795 fwr->ovlan = htobe16(f->fs.val.vnic); 4796 fwr->ovlanm = htobe16(f->fs.mask.vnic); 4797 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 4798 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 4799 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 4800 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 4801 fwr->lp = htobe16(f->fs.val.dport); 4802 fwr->lpm = htobe16(f->fs.mask.dport); 4803 fwr->fp = htobe16(f->fs.val.sport); 4804 fwr->fpm = htobe16(f->fs.mask.sport); 4805 if (f->fs.newsmac) 4806 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 4807 4808 f->pending = 1; 4809 sc->tids.ftids_in_use++; 4810 4811 t4_mgmt_tx(sc, m); 4812 return (0); 4813 } 4814 4815 static int 4816 del_filter_wr(struct adapter *sc, int fidx) 4817 { 4818 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4819 struct mbuf *m; 4820 struct fw_filter_wr *fwr; 4821 unsigned int ftid; 4822 4823 ADAPTER_LOCK_ASSERT_OWNED(sc); 4824 4825 ftid = sc->tids.ftid_base + fidx; 4826 4827 m = m_gethdr(M_NOWAIT, MT_DATA); 4828 if (m == NULL) 4829 return (ENOMEM); 4830 4831 fwr = mtod(m, struct fw_filter_wr *); 4832 m->m_len = m->m_pkthdr.len = sizeof(*fwr); 4833 bzero(fwr, sizeof (*fwr)); 4834 4835 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 4836 4837 f->pending = 1; 4838 t4_mgmt_tx(sc, m); 4839 return (0); 4840 } 4841 4842 static int 4843 filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4844 { 4845 struct adapter *sc = iq->adapter; 4846 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 4847 unsigned int idx = GET_TID(rpl); 4848 4849 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 4850 rss->opcode)); 4851 4852 if (idx >= sc->tids.ftid_base && 4853 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 4854 unsigned int rc = G_COOKIE(rpl->cookie); 4855 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 4856 4857 if (rc == FW_FILTER_WR_FLT_ADDED) { 4858 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 4859 f->pending = 0; /* asynchronous setup completed */ 4860 f->valid = 1; 4861 return (0); 4862 } 4863 4864 if (rc != FW_FILTER_WR_FLT_DELETED) { 4865 /* Add or delete failed, need to display an error */ 4866 device_printf(sc->dev, 4867 "filter %u setup failed with error %u\n", idx, rc); 4868 } 4869 4870 clear_filter(f); 4871 ADAPTER_LOCK(sc); 4872 sc->tids.ftids_in_use--; 4873 ADAPTER_UNLOCK(sc); 4874 } 4875 4876 return (0); 4877 } 4878 4879 static int 4880 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 4881 { 4882 int rc = EINVAL; 4883 4884 if (cntxt->cid > M_CTXTQID) 4885 return (rc); 4886 4887 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 4888 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 4889 return (rc); 4890 4891 if (sc->flags & FW_OK) { 4892 ADAPTER_LOCK(sc); /* Avoid parallel t4_wr_mbox */ 4893 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 4894 &cntxt->data[0]); 4895 ADAPTER_UNLOCK(sc); 4896 } 4897 4898 if (rc != 0) { 4899 /* Read via firmware failed or wasn't even attempted */ 4900 4901 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, 4902 &cntxt->data[0]); 4903 } 4904 4905 return (rc); 4906 } 4907 4908 static int 4909 read_card_mem(struct adapter *sc, struct t4_mem_range *mr) 4910 { 4911 uint32_t base, size, lo, hi, win, off, remaining, i, n; 4912 uint32_t *buf, *b; 4913 int rc; 4914 4915 /* reads are in multiples of 32 bits */ 4916 if (mr->addr & 3 || mr->len & 3 || mr->len == 0) 4917 return (EINVAL); 4918 4919 /* 4920 * We don't want to deal with potential holes so we mandate that the 4921 * requested region must lie entirely within one of the 3 memories. 4922 */ 4923 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 4924 if (lo & F_EDRAM0_ENABLE) { 4925 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 4926 base = G_EDRAM0_BASE(hi) << 20; 4927 size = G_EDRAM0_SIZE(hi) << 20; 4928 if (size > 0 && 4929 mr->addr >= base && mr->addr < base + size && 4930 mr->addr + mr->len <= base + size) 4931 goto proceed; 4932 } 4933 if (lo & F_EDRAM1_ENABLE) { 4934 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 4935 base = G_EDRAM1_BASE(hi) << 20; 4936 size = G_EDRAM1_SIZE(hi) << 20; 4937 if (size > 0 && 4938 mr->addr >= base && mr->addr < base + size && 4939 mr->addr + mr->len <= base + size) 4940 goto proceed; 4941 } 4942 if (lo & F_EXT_MEM_ENABLE) { 4943 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 4944 base = G_EXT_MEM_BASE(hi) << 20; 4945 size = G_EXT_MEM_SIZE(hi) << 20; 4946 if (size > 0 && 4947 mr->addr >= base && mr->addr < base + size && 4948 mr->addr + mr->len <= base + size) 4949 goto proceed; 4950 } 4951 return (ENXIO); 4952 4953 proceed: 4954 buf = b = malloc(mr->len, M_CXGBE, M_WAITOK); 4955 4956 /* 4957 * Position the PCIe window (we use memwin2) to the 16B aligned area 4958 * just at/before the requested region. 4959 */ 4960 win = mr->addr & ~0xf; 4961 off = mr->addr - win; /* offset of the requested region in the win */ 4962 remaining = mr->len; 4963 4964 while (remaining) { 4965 t4_write_reg(sc, 4966 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 4967 t4_read_reg(sc, 4968 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 4969 4970 /* number of bytes that we'll copy in the inner loop */ 4971 n = min(remaining, MEMWIN2_APERTURE - off); 4972 4973 for (i = 0; i < n; i += 4, remaining -= 4) 4974 *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i); 4975 4976 win += MEMWIN2_APERTURE; 4977 off = 0; 4978 } 4979 4980 rc = copyout(buf, mr->data, mr->len); 4981 free(buf, M_CXGBE); 4982 4983 return (rc); 4984 } 4985 4986 int 4987 t4_os_find_pci_capability(struct adapter *sc, int cap) 4988 { 4989 int i; 4990 4991 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 4992 } 4993 4994 int 4995 t4_os_pci_save_state(struct adapter *sc) 4996 { 4997 device_t dev; 4998 struct pci_devinfo *dinfo; 4999 5000 dev = sc->dev; 5001 dinfo = device_get_ivars(dev); 5002 5003 pci_cfg_save(dev, dinfo, 0); 5004 return (0); 5005 } 5006 5007 int 5008 t4_os_pci_restore_state(struct adapter *sc) 5009 { 5010 device_t dev; 5011 struct pci_devinfo *dinfo; 5012 5013 dev = sc->dev; 5014 dinfo = device_get_ivars(dev); 5015 5016 pci_cfg_restore(dev, dinfo); 5017 return (0); 5018 } 5019 5020 void 5021 t4_os_portmod_changed(const struct adapter *sc, int idx) 5022 { 5023 struct port_info *pi = sc->port[idx]; 5024 static const char *mod_str[] = { 5025 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 5026 }; 5027 5028 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 5029 if_printf(pi->ifp, "transceiver unplugged.\n"); 5030 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 5031 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 5032 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 5033 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 5034 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) { 5035 if_printf(pi->ifp, "%s transceiver inserted.\n", 5036 mod_str[pi->mod_type]); 5037 } else { 5038 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 5039 pi->mod_type); 5040 } 5041 } 5042 5043 void 5044 t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 5045 { 5046 struct port_info *pi = sc->port[idx]; 5047 struct ifnet *ifp = pi->ifp; 5048 5049 if (link_stat) { 5050 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 5051 if_link_state_change(ifp, LINK_STATE_UP); 5052 } else 5053 if_link_state_change(ifp, LINK_STATE_DOWN); 5054 } 5055 5056 void 5057 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 5058 { 5059 struct adapter *sc; 5060 5061 mtx_lock(&t4_list_lock); 5062 SLIST_FOREACH(sc, &t4_list, link) { 5063 /* 5064 * func should not make any assumptions about what state sc is 5065 * in - the only guarantee is that sc->sc_lock is a valid lock. 5066 */ 5067 func(sc, arg); 5068 } 5069 mtx_unlock(&t4_list_lock); 5070 } 5071 5072 static int 5073 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 5074 { 5075 return (0); 5076 } 5077 5078 static int 5079 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 5080 { 5081 return (0); 5082 } 5083 5084 static int 5085 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 5086 struct thread *td) 5087 { 5088 int rc; 5089 struct adapter *sc = dev->si_drv1; 5090 5091 rc = priv_check(td, PRIV_DRIVER); 5092 if (rc != 0) 5093 return (rc); 5094 5095 switch (cmd) { 5096 case CHELSIO_T4_GETREG: { 5097 struct t4_reg *edata = (struct t4_reg *)data; 5098 5099 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5100 return (EFAULT); 5101 5102 if (edata->size == 4) 5103 edata->val = t4_read_reg(sc, edata->addr); 5104 else if (edata->size == 8) 5105 edata->val = t4_read_reg64(sc, edata->addr); 5106 else 5107 return (EINVAL); 5108 5109 break; 5110 } 5111 case CHELSIO_T4_SETREG: { 5112 struct t4_reg *edata = (struct t4_reg *)data; 5113 5114 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5115 return (EFAULT); 5116 5117 if (edata->size == 4) { 5118 if (edata->val & 0xffffffff00000000) 5119 return (EINVAL); 5120 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 5121 } else if (edata->size == 8) 5122 t4_write_reg64(sc, edata->addr, edata->val); 5123 else 5124 return (EINVAL); 5125 break; 5126 } 5127 case CHELSIO_T4_REGDUMP: { 5128 struct t4_regdump *regs = (struct t4_regdump *)data; 5129 int reglen = T4_REGDUMP_SIZE; 5130 uint8_t *buf; 5131 5132 if (regs->len < reglen) { 5133 regs->len = reglen; /* hint to the caller */ 5134 return (ENOBUFS); 5135 } 5136 5137 regs->len = reglen; 5138 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 5139 t4_get_regs(sc, regs, buf); 5140 rc = copyout(buf, regs->data, reglen); 5141 free(buf, M_CXGBE); 5142 break; 5143 } 5144 case CHELSIO_T4_GET_FILTER_MODE: 5145 rc = get_filter_mode(sc, (uint32_t *)data); 5146 break; 5147 case CHELSIO_T4_SET_FILTER_MODE: 5148 rc = set_filter_mode(sc, *(uint32_t *)data); 5149 break; 5150 case CHELSIO_T4_GET_FILTER: 5151 ADAPTER_LOCK(sc); 5152 rc = get_filter(sc, (struct t4_filter *)data); 5153 ADAPTER_UNLOCK(sc); 5154 break; 5155 case CHELSIO_T4_SET_FILTER: 5156 ADAPTER_LOCK(sc); 5157 rc = set_filter(sc, (struct t4_filter *)data); 5158 ADAPTER_UNLOCK(sc); 5159 break; 5160 case CHELSIO_T4_DEL_FILTER: 5161 ADAPTER_LOCK(sc); 5162 rc = del_filter(sc, (struct t4_filter *)data); 5163 ADAPTER_UNLOCK(sc); 5164 break; 5165 case CHELSIO_T4_GET_SGE_CONTEXT: 5166 rc = get_sge_context(sc, (struct t4_sge_context *)data); 5167 break; 5168 case CHELSIO_T4_LOAD_FW: { 5169 struct t4_data *fw = (struct t4_data *)data; 5170 uint8_t *fw_data; 5171 5172 if (sc->flags & FULL_INIT_DONE) 5173 return (EBUSY); 5174 5175 fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT); 5176 if (fw_data == NULL) 5177 return (ENOMEM); 5178 5179 rc = copyin(fw->data, fw_data, fw->len); 5180 if (rc == 0) 5181 rc = -t4_load_fw(sc, fw_data, fw->len); 5182 5183 free(fw_data, M_CXGBE); 5184 break; 5185 } 5186 case CHELSIO_T4_GET_MEM: 5187 rc = read_card_mem(sc, (struct t4_mem_range *)data); 5188 break; 5189 default: 5190 rc = EINVAL; 5191 } 5192 5193 return (rc); 5194 } 5195 5196 #ifndef TCP_OFFLOAD_DISABLE 5197 static int 5198 toe_capability(struct port_info *pi, int enable) 5199 { 5200 int rc; 5201 struct adapter *sc = pi->adapter; 5202 5203 ADAPTER_LOCK_ASSERT_OWNED(sc); 5204 5205 if (!is_offload(sc)) 5206 return (ENODEV); 5207 5208 if (enable) { 5209 if (isset(&sc->offload_map, pi->port_id)) 5210 return (0); 5211 5212 if (sc->offload_map == 0) { 5213 rc = activate_uld(sc, ULD_TOM, &sc->tom); 5214 if (rc != 0) 5215 return (rc); 5216 } 5217 5218 setbit(&sc->offload_map, pi->port_id); 5219 } else { 5220 if (!isset(&sc->offload_map, pi->port_id)) 5221 return (0); 5222 5223 clrbit(&sc->offload_map, pi->port_id); 5224 5225 if (sc->offload_map == 0) { 5226 rc = deactivate_uld(&sc->tom); 5227 if (rc != 0) { 5228 setbit(&sc->offload_map, pi->port_id); 5229 return (rc); 5230 } 5231 } 5232 } 5233 5234 return (0); 5235 } 5236 5237 /* 5238 * Add an upper layer driver to the global list. 5239 */ 5240 int 5241 t4_register_uld(struct uld_info *ui) 5242 { 5243 int rc = 0; 5244 struct uld_info *u; 5245 5246 mtx_lock(&t4_uld_list_lock); 5247 SLIST_FOREACH(u, &t4_uld_list, link) { 5248 if (u->uld_id == ui->uld_id) { 5249 rc = EEXIST; 5250 goto done; 5251 } 5252 } 5253 5254 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 5255 ui->refcount = 0; 5256 done: 5257 mtx_unlock(&t4_uld_list_lock); 5258 return (rc); 5259 } 5260 5261 int 5262 t4_unregister_uld(struct uld_info *ui) 5263 { 5264 int rc = EINVAL; 5265 struct uld_info *u; 5266 5267 mtx_lock(&t4_uld_list_lock); 5268 5269 SLIST_FOREACH(u, &t4_uld_list, link) { 5270 if (u == ui) { 5271 if (ui->refcount > 0) { 5272 rc = EBUSY; 5273 goto done; 5274 } 5275 5276 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 5277 rc = 0; 5278 goto done; 5279 } 5280 } 5281 done: 5282 mtx_unlock(&t4_uld_list_lock); 5283 return (rc); 5284 } 5285 5286 static int 5287 activate_uld(struct adapter *sc, int id, struct uld_softc *usc) 5288 { 5289 int rc = EAGAIN; 5290 struct uld_info *ui; 5291 5292 mtx_lock(&t4_uld_list_lock); 5293 5294 SLIST_FOREACH(ui, &t4_uld_list, link) { 5295 if (ui->uld_id == id) { 5296 rc = ui->attach(sc, &usc->softc); 5297 if (rc == 0) { 5298 KASSERT(usc->softc != NULL, 5299 ("%s: ULD %d has no state", __func__, id)); 5300 ui->refcount++; 5301 usc->uld = ui; 5302 } 5303 goto done; 5304 } 5305 } 5306 done: 5307 mtx_unlock(&t4_uld_list_lock); 5308 5309 return (rc); 5310 } 5311 5312 static int 5313 deactivate_uld(struct uld_softc *usc) 5314 { 5315 int rc; 5316 5317 mtx_lock(&t4_uld_list_lock); 5318 5319 if (usc->uld == NULL || usc->softc == NULL) { 5320 rc = EINVAL; 5321 goto done; 5322 } 5323 5324 rc = usc->uld->detach(usc->softc); 5325 if (rc == 0) { 5326 KASSERT(usc->uld->refcount > 0, 5327 ("%s: ULD has bad refcount", __func__)); 5328 usc->uld->refcount--; 5329 usc->uld = NULL; 5330 usc->softc = NULL; 5331 } 5332 done: 5333 mtx_unlock(&t4_uld_list_lock); 5334 5335 return (rc); 5336 } 5337 #endif 5338 5339 /* 5340 * Come up with reasonable defaults for some of the tunables, provided they're 5341 * not set by the user (in which case we'll use the values as is). 5342 */ 5343 static void 5344 tweak_tunables(void) 5345 { 5346 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 5347 5348 if (t4_ntxq10g < 1) 5349 t4_ntxq10g = min(nc, NTXQ_10G); 5350 5351 if (t4_ntxq1g < 1) 5352 t4_ntxq1g = min(nc, NTXQ_1G); 5353 5354 if (t4_nrxq10g < 1) 5355 t4_nrxq10g = min(nc, NRXQ_10G); 5356 5357 if (t4_nrxq1g < 1) 5358 t4_nrxq1g = min(nc, NRXQ_1G); 5359 5360 #ifndef TCP_OFFLOAD_DISABLE 5361 if (t4_nofldtxq10g < 1) 5362 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 5363 5364 if (t4_nofldtxq1g < 1) 5365 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 5366 5367 if (t4_nofldrxq10g < 1) 5368 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 5369 5370 if (t4_nofldrxq1g < 1) 5371 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 5372 #endif 5373 5374 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 5375 t4_tmr_idx_10g = TMR_IDX_10G; 5376 5377 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 5378 t4_pktc_idx_10g = PKTC_IDX_10G; 5379 5380 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 5381 t4_tmr_idx_1g = TMR_IDX_1G; 5382 5383 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 5384 t4_pktc_idx_1g = PKTC_IDX_1G; 5385 5386 if (t4_qsize_txq < 128) 5387 t4_qsize_txq = 128; 5388 5389 if (t4_qsize_rxq < 128) 5390 t4_qsize_rxq = 128; 5391 while (t4_qsize_rxq & 7) 5392 t4_qsize_rxq++; 5393 5394 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 5395 } 5396 5397 static int 5398 t4_mod_event(module_t mod, int cmd, void *arg) 5399 { 5400 int rc = 0; 5401 5402 switch (cmd) { 5403 case MOD_LOAD: 5404 t4_sge_modload(); 5405 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 5406 SLIST_INIT(&t4_list); 5407 #ifndef TCP_OFFLOAD_DISABLE 5408 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 5409 SLIST_INIT(&t4_uld_list); 5410 #endif 5411 tweak_tunables(); 5412 break; 5413 5414 case MOD_UNLOAD: 5415 #ifndef TCP_OFFLOAD_DISABLE 5416 mtx_lock(&t4_uld_list_lock); 5417 if (!SLIST_EMPTY(&t4_uld_list)) { 5418 rc = EBUSY; 5419 mtx_unlock(&t4_uld_list_lock); 5420 break; 5421 } 5422 mtx_unlock(&t4_uld_list_lock); 5423 mtx_destroy(&t4_uld_list_lock); 5424 #endif 5425 mtx_lock(&t4_list_lock); 5426 if (!SLIST_EMPTY(&t4_list)) { 5427 rc = EBUSY; 5428 mtx_unlock(&t4_list_lock); 5429 break; 5430 } 5431 mtx_unlock(&t4_list_lock); 5432 mtx_destroy(&t4_list_lock); 5433 break; 5434 } 5435 5436 return (rc); 5437 } 5438 5439 static devclass_t t4_devclass; 5440 static devclass_t cxgbe_devclass; 5441 5442 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0); 5443 MODULE_VERSION(t4nex, 1); 5444 5445 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 5446 MODULE_VERSION(cxgbe, 1); 5447