1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 59 #include "common/common.h" 60 #include "common/t4_msg.h" 61 #include "common/t4_regs.h" 62 #include "common/t4_regs_values.h" 63 #include "t4_ioctl.h" 64 #include "t4_l2t.h" 65 66 /* T4 bus driver interface */ 67 static int t4_probe(device_t); 68 static int t4_attach(device_t); 69 static int t4_detach(device_t); 70 static device_method_t t4_methods[] = { 71 DEVMETHOD(device_probe, t4_probe), 72 DEVMETHOD(device_attach, t4_attach), 73 DEVMETHOD(device_detach, t4_detach), 74 75 DEVMETHOD_END 76 }; 77 static driver_t t4_driver = { 78 "t4nex", 79 t4_methods, 80 sizeof(struct adapter) 81 }; 82 83 84 /* T4 port (cxgbe) interface */ 85 static int cxgbe_probe(device_t); 86 static int cxgbe_attach(device_t); 87 static int cxgbe_detach(device_t); 88 static device_method_t cxgbe_methods[] = { 89 DEVMETHOD(device_probe, cxgbe_probe), 90 DEVMETHOD(device_attach, cxgbe_attach), 91 DEVMETHOD(device_detach, cxgbe_detach), 92 { 0, 0 } 93 }; 94 static driver_t cxgbe_driver = { 95 "cxgbe", 96 cxgbe_methods, 97 sizeof(struct port_info) 98 }; 99 100 static d_ioctl_t t4_ioctl; 101 static d_open_t t4_open; 102 static d_close_t t4_close; 103 104 static struct cdevsw t4_cdevsw = { 105 .d_version = D_VERSION, 106 .d_flags = 0, 107 .d_open = t4_open, 108 .d_close = t4_close, 109 .d_ioctl = t4_ioctl, 110 .d_name = "t4nex", 111 }; 112 113 /* ifnet + media interface */ 114 static void cxgbe_init(void *); 115 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 116 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 117 static void cxgbe_qflush(struct ifnet *); 118 static int cxgbe_media_change(struct ifnet *); 119 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 120 121 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services"); 122 123 /* 124 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 125 * then ADAPTER_LOCK, then t4_uld_list_lock. 126 */ 127 static struct mtx t4_list_lock; 128 static SLIST_HEAD(, adapter) t4_list; 129 #ifdef TCP_OFFLOAD 130 static struct mtx t4_uld_list_lock; 131 static SLIST_HEAD(, uld_info) t4_uld_list; 132 #endif 133 134 /* 135 * Tunables. See tweak_tunables() too. 136 */ 137 138 /* 139 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 140 */ 141 #define NTXQ_10G 16 142 static int t4_ntxq10g = -1; 143 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 144 145 #define NRXQ_10G 8 146 static int t4_nrxq10g = -1; 147 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 148 149 #define NTXQ_1G 4 150 static int t4_ntxq1g = -1; 151 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 152 153 #define NRXQ_1G 2 154 static int t4_nrxq1g = -1; 155 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 156 157 #ifdef TCP_OFFLOAD 158 #define NOFLDTXQ_10G 8 159 static int t4_nofldtxq10g = -1; 160 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 161 162 #define NOFLDRXQ_10G 2 163 static int t4_nofldrxq10g = -1; 164 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 165 166 #define NOFLDTXQ_1G 2 167 static int t4_nofldtxq1g = -1; 168 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 169 170 #define NOFLDRXQ_1G 1 171 static int t4_nofldrxq1g = -1; 172 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 173 #endif 174 175 /* 176 * Holdoff parameters for 10G and 1G ports. 177 */ 178 #define TMR_IDX_10G 1 179 static int t4_tmr_idx_10g = TMR_IDX_10G; 180 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 181 182 #define PKTC_IDX_10G (-1) 183 static int t4_pktc_idx_10g = PKTC_IDX_10G; 184 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 185 186 #define TMR_IDX_1G 1 187 static int t4_tmr_idx_1g = TMR_IDX_1G; 188 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 189 190 #define PKTC_IDX_1G (-1) 191 static int t4_pktc_idx_1g = PKTC_IDX_1G; 192 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 193 194 /* 195 * Size (# of entries) of each tx and rx queue. 196 */ 197 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 198 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 199 200 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 201 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 202 203 /* 204 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 205 */ 206 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 207 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 208 209 /* 210 * Configuration file. 211 */ 212 static char t4_cfg_file[32] = "default"; 213 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 214 215 /* 216 * ASIC features that will be used. Disable the ones you don't want so that the 217 * chip resources aren't wasted on features that will not be used. 218 */ 219 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 220 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 221 222 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 223 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 224 225 static int t4_toecaps_allowed = -1; 226 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 227 228 static int t4_rdmacaps_allowed = 0; 229 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 230 231 static int t4_iscsicaps_allowed = 0; 232 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 233 234 static int t4_fcoecaps_allowed = 0; 235 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 236 237 struct intrs_and_queues { 238 int intr_type; /* INTx, MSI, or MSI-X */ 239 int nirq; /* Number of vectors */ 240 int intr_flags; 241 int ntxq10g; /* # of NIC txq's for each 10G port */ 242 int nrxq10g; /* # of NIC rxq's for each 10G port */ 243 int ntxq1g; /* # of NIC txq's for each 1G port */ 244 int nrxq1g; /* # of NIC rxq's for each 1G port */ 245 #ifdef TCP_OFFLOAD 246 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 247 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 248 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 249 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 250 #endif 251 }; 252 253 struct filter_entry { 254 uint32_t valid:1; /* filter allocated and valid */ 255 uint32_t locked:1; /* filter is administratively locked */ 256 uint32_t pending:1; /* filter action is pending firmware reply */ 257 uint32_t smtidx:8; /* Source MAC Table index for smac */ 258 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 259 260 struct t4_filter_specification fs; 261 }; 262 263 enum { 264 XGMAC_MTU = (1 << 0), 265 XGMAC_PROMISC = (1 << 1), 266 XGMAC_ALLMULTI = (1 << 2), 267 XGMAC_VLANEX = (1 << 3), 268 XGMAC_UCADDR = (1 << 4), 269 XGMAC_MCADDRS = (1 << 5), 270 271 XGMAC_ALL = 0xffff 272 }; 273 274 static int map_bars(struct adapter *); 275 static void setup_memwin(struct adapter *); 276 static int cfg_itype_and_nqueues(struct adapter *, int, int, 277 struct intrs_and_queues *); 278 static int prep_firmware(struct adapter *); 279 static int upload_config_file(struct adapter *, const struct firmware *, 280 uint32_t *, uint32_t *); 281 static int partition_resources(struct adapter *, const struct firmware *); 282 static int get_params__pre_init(struct adapter *); 283 static int get_params__post_init(struct adapter *); 284 static void t4_set_desc(struct adapter *); 285 static void build_medialist(struct port_info *); 286 static int update_mac_settings(struct port_info *, int); 287 static int cxgbe_init_locked(struct port_info *); 288 static int cxgbe_init_synchronized(struct port_info *); 289 static int cxgbe_uninit_locked(struct port_info *); 290 static int cxgbe_uninit_synchronized(struct port_info *); 291 static int setup_intr_handlers(struct adapter *); 292 static int adapter_full_init(struct adapter *); 293 static int adapter_full_uninit(struct adapter *); 294 static int port_full_init(struct port_info *); 295 static int port_full_uninit(struct port_info *); 296 static void quiesce_eq(struct adapter *, struct sge_eq *); 297 static void quiesce_iq(struct adapter *, struct sge_iq *); 298 static void quiesce_fl(struct adapter *, struct sge_fl *); 299 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 300 driver_intr_t *, void *, char *); 301 static int t4_free_irq(struct adapter *, struct irq *); 302 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 303 unsigned int); 304 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 305 static void cxgbe_tick(void *); 306 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 307 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 308 struct mbuf *); 309 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 310 static int fw_msg_not_handled(struct adapter *, const __be64 *); 311 static int t4_sysctls(struct adapter *); 312 static int cxgbe_sysctls(struct port_info *); 313 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 314 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 315 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 316 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 317 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 318 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 319 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 320 #ifdef SBUF_DRAIN 321 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 322 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 323 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 324 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 325 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 326 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 327 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 328 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 329 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 330 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 331 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 332 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 333 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 334 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 335 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 336 #endif 337 static inline void txq_start(struct ifnet *, struct sge_txq *); 338 static uint32_t fconf_to_mode(uint32_t); 339 static uint32_t mode_to_fconf(uint32_t); 340 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 341 static int get_filter_mode(struct adapter *, uint32_t *); 342 static int set_filter_mode(struct adapter *, uint32_t); 343 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 344 static int get_filter(struct adapter *, struct t4_filter *); 345 static int set_filter(struct adapter *, struct t4_filter *); 346 static int del_filter(struct adapter *, struct t4_filter *); 347 static void clear_filter(struct filter_entry *); 348 static int set_filter_wr(struct adapter *, int); 349 static int del_filter_wr(struct adapter *, int); 350 static int get_sge_context(struct adapter *, struct t4_sge_context *); 351 static int read_card_mem(struct adapter *, struct t4_mem_range *); 352 static int read_i2c(struct adapter *, struct t4_i2c_data *); 353 #ifdef TCP_OFFLOAD 354 static int toe_capability(struct port_info *, int); 355 #endif 356 static int t4_mod_event(module_t, int, void *); 357 358 struct t4_pciids { 359 uint16_t device; 360 char *desc; 361 } t4_pciids[] = { 362 {0xa000, "Chelsio Terminator 4 FPGA"}, 363 {0x4400, "Chelsio T440-dbg"}, 364 {0x4401, "Chelsio T420-CR"}, 365 {0x4402, "Chelsio T422-CR"}, 366 {0x4403, "Chelsio T440-CR"}, 367 {0x4404, "Chelsio T420-BCH"}, 368 {0x4405, "Chelsio T440-BCH"}, 369 {0x4406, "Chelsio T440-CH"}, 370 {0x4407, "Chelsio T420-SO"}, 371 {0x4408, "Chelsio T420-CX"}, 372 {0x4409, "Chelsio T420-BT"}, 373 {0x440a, "Chelsio T404-BT"}, 374 }; 375 376 #ifdef TCP_OFFLOAD 377 /* 378 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 379 * exactly the same for both rxq and ofld_rxq. 380 */ 381 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 382 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 383 #endif 384 385 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 386 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 387 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 388 389 static int 390 t4_probe(device_t dev) 391 { 392 int i; 393 uint16_t v = pci_get_vendor(dev); 394 uint16_t d = pci_get_device(dev); 395 uint8_t f = pci_get_function(dev); 396 397 if (v != PCI_VENDOR_ID_CHELSIO) 398 return (ENXIO); 399 400 /* Attach only to PF0 of the FPGA */ 401 if (d == 0xa000 && f != 0) 402 return (ENXIO); 403 404 for (i = 0; i < nitems(t4_pciids); i++) { 405 if (d == t4_pciids[i].device) { 406 device_set_desc(dev, t4_pciids[i].desc); 407 return (BUS_PROBE_DEFAULT); 408 } 409 } 410 411 return (ENXIO); 412 } 413 414 static int 415 t4_attach(device_t dev) 416 { 417 struct adapter *sc; 418 int rc = 0, i, n10g, n1g, rqidx, tqidx; 419 struct intrs_and_queues iaq; 420 struct sge *s; 421 #ifdef TCP_OFFLOAD 422 int ofld_rqidx, ofld_tqidx; 423 #endif 424 425 sc = device_get_softc(dev); 426 sc->dev = dev; 427 428 pci_enable_busmaster(dev); 429 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 430 uint32_t v; 431 432 pci_set_max_read_req(dev, 4096); 433 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 434 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 435 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 436 } 437 438 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 439 device_get_nameunit(dev)); 440 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 441 mtx_lock(&t4_list_lock); 442 SLIST_INSERT_HEAD(&t4_list, sc, link); 443 mtx_unlock(&t4_list_lock); 444 445 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 446 TAILQ_INIT(&sc->sfl); 447 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 448 449 rc = map_bars(sc); 450 if (rc != 0) 451 goto done; /* error message displayed already */ 452 453 /* 454 * This is the real PF# to which we're attaching. Works from within PCI 455 * passthrough environments too, where pci_get_function() could return a 456 * different PF# depending on the passthrough configuration. We need to 457 * use the real PF# in all our communication with the firmware. 458 */ 459 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 460 sc->mbox = sc->pf; 461 462 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 463 sc->an_handler = an_not_handled; 464 for (i = 0; i < nitems(sc->cpl_handler); i++) 465 sc->cpl_handler[i] = cpl_not_handled; 466 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 467 sc->fw_msg_handler[i] = fw_msg_not_handled; 468 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 469 470 /* Prepare the adapter for operation */ 471 rc = -t4_prep_adapter(sc); 472 if (rc != 0) { 473 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 474 goto done; 475 } 476 477 /* 478 * Do this really early, with the memory windows set up even before the 479 * character device. The userland tool's register i/o and mem read 480 * will work even in "recovery mode". 481 */ 482 setup_memwin(sc); 483 sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT, 484 GID_WHEEL, 0600, "%s", device_get_nameunit(dev)); 485 sc->cdev->si_drv1 = sc; 486 487 /* Go no further if recovery mode has been requested. */ 488 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 489 device_printf(dev, "recovery mode.\n"); 490 goto done; 491 } 492 493 /* Prepare the firmware for operation */ 494 rc = prep_firmware(sc); 495 if (rc != 0) 496 goto done; /* error message displayed already */ 497 498 rc = get_params__pre_init(sc); 499 if (rc != 0) 500 goto done; /* error message displayed already */ 501 502 rc = t4_sge_init(sc); 503 if (rc != 0) 504 goto done; /* error message displayed already */ 505 506 if (sc->flags & MASTER_PF) { 507 /* get basic stuff going */ 508 rc = -t4_fw_initialize(sc, sc->mbox); 509 if (rc != 0) { 510 device_printf(dev, "early init failed: %d.\n", rc); 511 goto done; 512 } 513 } 514 515 rc = get_params__post_init(sc); 516 if (rc != 0) 517 goto done; /* error message displayed already */ 518 519 if (sc->flags & MASTER_PF) { 520 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 521 522 /* final tweaks to some settings */ 523 524 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, 525 sc->params.b_wnd); 526 /* 4K, 16K, 64K, 256K DDP "page sizes" */ 527 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(0) | V_HPZ1(2) | 528 V_HPZ2(4) | V_HPZ3(6)); 529 t4_set_reg_field(sc, A_ULP_RX_CTL, F_TDDPTAGTCB, F_TDDPTAGTCB); 530 t4_set_reg_field(sc, A_TP_PARA_REG5, 531 V_INDICATESIZE(M_INDICATESIZE) | 532 F_REARMDDPOFFSET | F_RESETDDPOFFSET, 533 V_INDICATESIZE(indsz) | 534 F_REARMDDPOFFSET | F_RESETDDPOFFSET); 535 } else { 536 /* 537 * XXX: Verify that we can live with whatever the master driver 538 * has done so far, and hope that it doesn't change any global 539 * setting from underneath us in the future. 540 */ 541 } 542 543 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 544 A_TP_VLAN_PRI_MAP); 545 546 for (i = 0; i < NCHAN; i++) 547 sc->params.tp.tx_modq[i] = i; 548 549 rc = t4_create_dma_tag(sc); 550 if (rc != 0) 551 goto done; /* error message displayed already */ 552 553 /* 554 * First pass over all the ports - allocate VIs and initialize some 555 * basic parameters like mac address, port type, etc. We also figure 556 * out whether a port is 10G or 1G and use that information when 557 * calculating how many interrupts to attempt to allocate. 558 */ 559 n10g = n1g = 0; 560 for_each_port(sc, i) { 561 struct port_info *pi; 562 563 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 564 sc->port[i] = pi; 565 566 /* These must be set before t4_port_init */ 567 pi->adapter = sc; 568 pi->port_id = i; 569 570 /* Allocate the vi and initialize parameters like mac addr */ 571 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 572 if (rc != 0) { 573 device_printf(dev, "unable to initialize port %d: %d\n", 574 i, rc); 575 free(pi, M_CXGBE); 576 sc->port[i] = NULL; 577 goto done; 578 } 579 580 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 581 device_get_nameunit(dev), i); 582 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 583 584 if (is_10G_port(pi)) { 585 n10g++; 586 pi->tmr_idx = t4_tmr_idx_10g; 587 pi->pktc_idx = t4_pktc_idx_10g; 588 } else { 589 n1g++; 590 pi->tmr_idx = t4_tmr_idx_1g; 591 pi->pktc_idx = t4_pktc_idx_1g; 592 } 593 594 pi->xact_addr_filt = -1; 595 596 pi->qsize_rxq = t4_qsize_rxq; 597 pi->qsize_txq = t4_qsize_txq; 598 599 pi->dev = device_add_child(dev, "cxgbe", -1); 600 if (pi->dev == NULL) { 601 device_printf(dev, 602 "failed to add device for port %d.\n", i); 603 rc = ENXIO; 604 goto done; 605 } 606 device_set_softc(pi->dev, pi); 607 } 608 609 /* 610 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 611 */ 612 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 613 if (rc != 0) 614 goto done; /* error message displayed already */ 615 616 sc->intr_type = iaq.intr_type; 617 sc->intr_count = iaq.nirq; 618 sc->flags |= iaq.intr_flags; 619 620 s = &sc->sge; 621 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 622 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 623 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 624 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 625 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 626 627 #ifdef TCP_OFFLOAD 628 if (is_offload(sc)) { 629 630 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 631 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 632 s->neq += s->nofldtxq + s->nofldrxq; 633 s->niq += s->nofldrxq; 634 635 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 636 M_CXGBE, M_ZERO | M_WAITOK); 637 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 638 M_CXGBE, M_ZERO | M_WAITOK); 639 } 640 #endif 641 642 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 643 M_ZERO | M_WAITOK); 644 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 645 M_ZERO | M_WAITOK); 646 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 647 M_ZERO | M_WAITOK); 648 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 649 M_ZERO | M_WAITOK); 650 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 651 M_ZERO | M_WAITOK); 652 653 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 654 M_ZERO | M_WAITOK); 655 656 t4_init_l2t(sc, M_WAITOK); 657 658 /* 659 * Second pass over the ports. This time we know the number of rx and 660 * tx queues that each port should get. 661 */ 662 rqidx = tqidx = 0; 663 #ifdef TCP_OFFLOAD 664 ofld_rqidx = ofld_tqidx = 0; 665 #endif 666 for_each_port(sc, i) { 667 struct port_info *pi = sc->port[i]; 668 669 if (pi == NULL) 670 continue; 671 672 pi->first_rxq = rqidx; 673 pi->first_txq = tqidx; 674 if (is_10G_port(pi)) { 675 pi->nrxq = iaq.nrxq10g; 676 pi->ntxq = iaq.ntxq10g; 677 } else { 678 pi->nrxq = iaq.nrxq1g; 679 pi->ntxq = iaq.ntxq1g; 680 } 681 682 rqidx += pi->nrxq; 683 tqidx += pi->ntxq; 684 685 #ifdef TCP_OFFLOAD 686 if (is_offload(sc)) { 687 pi->first_ofld_rxq = ofld_rqidx; 688 pi->first_ofld_txq = ofld_tqidx; 689 if (is_10G_port(pi)) { 690 pi->nofldrxq = iaq.nofldrxq10g; 691 pi->nofldtxq = iaq.nofldtxq10g; 692 } else { 693 pi->nofldrxq = iaq.nofldrxq1g; 694 pi->nofldtxq = iaq.nofldtxq1g; 695 } 696 ofld_rqidx += pi->nofldrxq; 697 ofld_tqidx += pi->nofldtxq; 698 } 699 #endif 700 } 701 702 rc = setup_intr_handlers(sc); 703 if (rc != 0) { 704 device_printf(dev, 705 "failed to setup interrupt handlers: %d\n", rc); 706 goto done; 707 } 708 709 rc = bus_generic_attach(dev); 710 if (rc != 0) { 711 device_printf(dev, 712 "failed to attach all child ports: %d\n", rc); 713 goto done; 714 } 715 716 device_printf(dev, 717 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 718 sc->params.pci.width, sc->params.nports, sc->intr_count, 719 sc->intr_type == INTR_MSIX ? "MSI-X" : 720 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 721 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 722 723 t4_set_desc(sc); 724 725 done: 726 if (rc != 0 && sc->cdev) { 727 /* cdev was created and so cxgbetool works; recover that way. */ 728 device_printf(dev, 729 "error during attach, adapter is now in recovery mode.\n"); 730 rc = 0; 731 } 732 733 if (rc != 0) 734 t4_detach(dev); 735 else 736 t4_sysctls(sc); 737 738 return (rc); 739 } 740 741 /* 742 * Idempotent 743 */ 744 static int 745 t4_detach(device_t dev) 746 { 747 struct adapter *sc; 748 struct port_info *pi; 749 int i, rc; 750 751 sc = device_get_softc(dev); 752 753 if (sc->flags & FULL_INIT_DONE) 754 t4_intr_disable(sc); 755 756 if (sc->cdev) { 757 destroy_dev(sc->cdev); 758 sc->cdev = NULL; 759 } 760 761 rc = bus_generic_detach(dev); 762 if (rc) { 763 device_printf(dev, 764 "failed to detach child devices: %d\n", rc); 765 return (rc); 766 } 767 768 for (i = 0; i < sc->intr_count; i++) 769 t4_free_irq(sc, &sc->irq[i]); 770 771 for (i = 0; i < MAX_NPORTS; i++) { 772 pi = sc->port[i]; 773 if (pi) { 774 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 775 if (pi->dev) 776 device_delete_child(dev, pi->dev); 777 778 mtx_destroy(&pi->pi_lock); 779 free(pi, M_CXGBE); 780 } 781 } 782 783 if (sc->flags & FULL_INIT_DONE) 784 adapter_full_uninit(sc); 785 786 if (sc->flags & FW_OK) 787 t4_fw_bye(sc, sc->mbox); 788 789 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 790 pci_release_msi(dev); 791 792 if (sc->regs_res) 793 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 794 sc->regs_res); 795 796 if (sc->msix_res) 797 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 798 sc->msix_res); 799 800 if (sc->l2t) 801 t4_free_l2t(sc->l2t); 802 803 #ifdef TCP_OFFLOAD 804 free(sc->sge.ofld_rxq, M_CXGBE); 805 free(sc->sge.ofld_txq, M_CXGBE); 806 #endif 807 free(sc->irq, M_CXGBE); 808 free(sc->sge.rxq, M_CXGBE); 809 free(sc->sge.txq, M_CXGBE); 810 free(sc->sge.ctrlq, M_CXGBE); 811 free(sc->sge.iqmap, M_CXGBE); 812 free(sc->sge.eqmap, M_CXGBE); 813 free(sc->tids.ftid_tab, M_CXGBE); 814 t4_destroy_dma_tag(sc); 815 if (mtx_initialized(&sc->sc_lock)) { 816 mtx_lock(&t4_list_lock); 817 SLIST_REMOVE(&t4_list, sc, adapter, link); 818 mtx_unlock(&t4_list_lock); 819 mtx_destroy(&sc->sc_lock); 820 } 821 822 if (mtx_initialized(&sc->sfl_lock)) 823 mtx_destroy(&sc->sfl_lock); 824 825 bzero(sc, sizeof(*sc)); 826 827 return (0); 828 } 829 830 831 static int 832 cxgbe_probe(device_t dev) 833 { 834 char buf[128]; 835 struct port_info *pi = device_get_softc(dev); 836 837 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 838 device_set_desc_copy(dev, buf); 839 840 return (BUS_PROBE_DEFAULT); 841 } 842 843 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 844 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 845 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 846 #define T4_CAP_ENABLE (T4_CAP) 847 848 static int 849 cxgbe_attach(device_t dev) 850 { 851 struct port_info *pi = device_get_softc(dev); 852 struct ifnet *ifp; 853 854 /* Allocate an ifnet and set it up */ 855 ifp = if_alloc(IFT_ETHER); 856 if (ifp == NULL) { 857 device_printf(dev, "Cannot allocate ifnet\n"); 858 return (ENOMEM); 859 } 860 pi->ifp = ifp; 861 ifp->if_softc = pi; 862 863 callout_init(&pi->tick, CALLOUT_MPSAFE); 864 865 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 866 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 867 868 ifp->if_init = cxgbe_init; 869 ifp->if_ioctl = cxgbe_ioctl; 870 ifp->if_transmit = cxgbe_transmit; 871 ifp->if_qflush = cxgbe_qflush; 872 873 ifp->if_capabilities = T4_CAP; 874 #ifdef TCP_OFFLOAD 875 if (is_offload(pi->adapter)) 876 ifp->if_capabilities |= IFCAP_TOE4; 877 #endif 878 ifp->if_capenable = T4_CAP_ENABLE; 879 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 880 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 881 882 /* Initialize ifmedia for this port */ 883 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 884 cxgbe_media_status); 885 build_medialist(pi); 886 887 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 888 EVENTHANDLER_PRI_ANY); 889 890 ether_ifattach(ifp, pi->hw_addr); 891 892 #ifdef TCP_OFFLOAD 893 if (is_offload(pi->adapter)) { 894 device_printf(dev, 895 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 896 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 897 } else 898 #endif 899 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 900 901 cxgbe_sysctls(pi); 902 903 return (0); 904 } 905 906 static int 907 cxgbe_detach(device_t dev) 908 { 909 struct port_info *pi = device_get_softc(dev); 910 struct adapter *sc = pi->adapter; 911 struct ifnet *ifp = pi->ifp; 912 913 /* Tell if_ioctl and if_init that the port is going away */ 914 ADAPTER_LOCK(sc); 915 SET_DOOMED(pi); 916 wakeup(&sc->flags); 917 while (IS_BUSY(sc)) 918 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 919 SET_BUSY(sc); 920 ADAPTER_UNLOCK(sc); 921 922 if (pi->vlan_c) 923 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 924 925 PORT_LOCK(pi); 926 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 927 callout_stop(&pi->tick); 928 PORT_UNLOCK(pi); 929 callout_drain(&pi->tick); 930 931 /* Let detach proceed even if these fail. */ 932 cxgbe_uninit_synchronized(pi); 933 port_full_uninit(pi); 934 935 ifmedia_removeall(&pi->media); 936 ether_ifdetach(pi->ifp); 937 if_free(pi->ifp); 938 939 ADAPTER_LOCK(sc); 940 CLR_BUSY(sc); 941 wakeup_one(&sc->flags); 942 ADAPTER_UNLOCK(sc); 943 944 return (0); 945 } 946 947 static void 948 cxgbe_init(void *arg) 949 { 950 struct port_info *pi = arg; 951 struct adapter *sc = pi->adapter; 952 953 ADAPTER_LOCK(sc); 954 cxgbe_init_locked(pi); /* releases adapter lock */ 955 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 956 } 957 958 static int 959 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 960 { 961 int rc = 0, mtu, flags; 962 struct port_info *pi = ifp->if_softc; 963 struct adapter *sc = pi->adapter; 964 struct ifreq *ifr = (struct ifreq *)data; 965 uint32_t mask; 966 967 switch (cmd) { 968 case SIOCSIFMTU: 969 ADAPTER_LOCK(sc); 970 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 971 if (rc) { 972 fail: 973 ADAPTER_UNLOCK(sc); 974 return (rc); 975 } 976 977 mtu = ifr->ifr_mtu; 978 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 979 rc = EINVAL; 980 } else { 981 ifp->if_mtu = mtu; 982 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 983 t4_update_fl_bufsize(ifp); 984 PORT_LOCK(pi); 985 rc = update_mac_settings(pi, XGMAC_MTU); 986 PORT_UNLOCK(pi); 987 } 988 } 989 ADAPTER_UNLOCK(sc); 990 break; 991 992 case SIOCSIFFLAGS: 993 ADAPTER_LOCK(sc); 994 if (IS_DOOMED(pi)) { 995 rc = ENXIO; 996 goto fail; 997 } 998 if (ifp->if_flags & IFF_UP) { 999 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1000 flags = pi->if_flags; 1001 if ((ifp->if_flags ^ flags) & 1002 (IFF_PROMISC | IFF_ALLMULTI)) { 1003 if (IS_BUSY(sc)) { 1004 rc = EBUSY; 1005 goto fail; 1006 } 1007 PORT_LOCK(pi); 1008 rc = update_mac_settings(pi, 1009 XGMAC_PROMISC | XGMAC_ALLMULTI); 1010 PORT_UNLOCK(pi); 1011 } 1012 ADAPTER_UNLOCK(sc); 1013 } else 1014 rc = cxgbe_init_locked(pi); 1015 pi->if_flags = ifp->if_flags; 1016 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1017 rc = cxgbe_uninit_locked(pi); 1018 else 1019 ADAPTER_UNLOCK(sc); 1020 1021 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1022 break; 1023 1024 case SIOCADDMULTI: 1025 case SIOCDELMULTI: /* these two can be called with a mutex held :-( */ 1026 ADAPTER_LOCK(sc); 1027 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1028 if (rc) 1029 goto fail; 1030 1031 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1032 PORT_LOCK(pi); 1033 rc = update_mac_settings(pi, XGMAC_MCADDRS); 1034 PORT_UNLOCK(pi); 1035 } 1036 ADAPTER_UNLOCK(sc); 1037 break; 1038 1039 case SIOCSIFCAP: 1040 ADAPTER_LOCK(sc); 1041 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1042 if (rc) 1043 goto fail; 1044 1045 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1046 if (mask & IFCAP_TXCSUM) { 1047 ifp->if_capenable ^= IFCAP_TXCSUM; 1048 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1049 1050 if (IFCAP_TSO4 & ifp->if_capenable && 1051 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1052 ifp->if_capenable &= ~IFCAP_TSO4; 1053 if_printf(ifp, 1054 "tso4 disabled due to -txcsum.\n"); 1055 } 1056 } 1057 if (mask & IFCAP_TXCSUM_IPV6) { 1058 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1059 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1060 1061 if (IFCAP_TSO6 & ifp->if_capenable && 1062 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1063 ifp->if_capenable &= ~IFCAP_TSO6; 1064 if_printf(ifp, 1065 "tso6 disabled due to -txcsum6.\n"); 1066 } 1067 } 1068 if (mask & IFCAP_RXCSUM) 1069 ifp->if_capenable ^= IFCAP_RXCSUM; 1070 if (mask & IFCAP_RXCSUM_IPV6) 1071 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1072 1073 /* 1074 * Note that we leave CSUM_TSO alone (it is always set). The 1075 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1076 * sending a TSO request our way, so it's sufficient to toggle 1077 * IFCAP_TSOx only. 1078 */ 1079 if (mask & IFCAP_TSO4) { 1080 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1081 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1082 if_printf(ifp, "enable txcsum first.\n"); 1083 rc = EAGAIN; 1084 goto fail; 1085 } 1086 ifp->if_capenable ^= IFCAP_TSO4; 1087 } 1088 if (mask & IFCAP_TSO6) { 1089 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1090 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1091 if_printf(ifp, "enable txcsum6 first.\n"); 1092 rc = EAGAIN; 1093 goto fail; 1094 } 1095 ifp->if_capenable ^= IFCAP_TSO6; 1096 } 1097 if (mask & IFCAP_LRO) { 1098 #if defined(INET) || defined(INET6) 1099 int i; 1100 struct sge_rxq *rxq; 1101 1102 ifp->if_capenable ^= IFCAP_LRO; 1103 for_each_rxq(pi, i, rxq) { 1104 if (ifp->if_capenable & IFCAP_LRO) 1105 rxq->iq.flags |= IQ_LRO_ENABLED; 1106 else 1107 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1108 } 1109 #endif 1110 } 1111 #ifdef TCP_OFFLOAD 1112 if (mask & IFCAP_TOE) { 1113 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1114 1115 rc = toe_capability(pi, enable); 1116 if (rc != 0) 1117 goto fail; 1118 1119 ifp->if_capenable ^= mask; 1120 } 1121 #endif 1122 if (mask & IFCAP_VLAN_HWTAGGING) { 1123 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1124 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1125 PORT_LOCK(pi); 1126 rc = update_mac_settings(pi, XGMAC_VLANEX); 1127 PORT_UNLOCK(pi); 1128 } 1129 } 1130 if (mask & IFCAP_VLAN_MTU) { 1131 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1132 1133 /* Need to find out how to disable auto-mtu-inflation */ 1134 } 1135 if (mask & IFCAP_VLAN_HWTSO) 1136 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1137 if (mask & IFCAP_VLAN_HWCSUM) 1138 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1139 1140 #ifdef VLAN_CAPABILITIES 1141 VLAN_CAPABILITIES(ifp); 1142 #endif 1143 ADAPTER_UNLOCK(sc); 1144 break; 1145 1146 case SIOCSIFMEDIA: 1147 case SIOCGIFMEDIA: 1148 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1149 break; 1150 1151 default: 1152 rc = ether_ioctl(ifp, cmd, data); 1153 } 1154 1155 return (rc); 1156 } 1157 1158 static int 1159 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1160 { 1161 struct port_info *pi = ifp->if_softc; 1162 struct adapter *sc = pi->adapter; 1163 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1164 struct buf_ring *br; 1165 int rc; 1166 1167 M_ASSERTPKTHDR(m); 1168 1169 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1170 m_freem(m); 1171 return (ENETDOWN); 1172 } 1173 1174 if (m->m_flags & M_FLOWID) 1175 txq += (m->m_pkthdr.flowid % pi->ntxq); 1176 br = txq->br; 1177 1178 if (TXQ_TRYLOCK(txq) == 0) { 1179 struct sge_eq *eq = &txq->eq; 1180 1181 /* 1182 * It is possible that t4_eth_tx finishes up and releases the 1183 * lock between the TRYLOCK above and the drbr_enqueue here. We 1184 * need to make sure that this mbuf doesn't just sit there in 1185 * the drbr. 1186 */ 1187 1188 rc = drbr_enqueue(ifp, br, m); 1189 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1190 !(eq->flags & EQ_DOOMED)) 1191 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1192 return (rc); 1193 } 1194 1195 /* 1196 * txq->m is the mbuf that is held up due to a temporary shortage of 1197 * resources and it should be put on the wire first. Then what's in 1198 * drbr and finally the mbuf that was just passed in to us. 1199 * 1200 * Return code should indicate the fate of the mbuf that was passed in 1201 * this time. 1202 */ 1203 1204 TXQ_LOCK_ASSERT_OWNED(txq); 1205 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1206 1207 /* Queued for transmission. */ 1208 1209 rc = drbr_enqueue(ifp, br, m); 1210 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1211 (void) t4_eth_tx(ifp, txq, m); 1212 TXQ_UNLOCK(txq); 1213 return (rc); 1214 } 1215 1216 /* Direct transmission. */ 1217 rc = t4_eth_tx(ifp, txq, m); 1218 if (rc != 0 && txq->m) 1219 rc = 0; /* held, will be transmitted soon (hopefully) */ 1220 1221 TXQ_UNLOCK(txq); 1222 return (rc); 1223 } 1224 1225 static void 1226 cxgbe_qflush(struct ifnet *ifp) 1227 { 1228 struct port_info *pi = ifp->if_softc; 1229 struct sge_txq *txq; 1230 int i; 1231 struct mbuf *m; 1232 1233 /* queues do not exist if !PORT_INIT_DONE. */ 1234 if (pi->flags & PORT_INIT_DONE) { 1235 for_each_txq(pi, i, txq) { 1236 TXQ_LOCK(txq); 1237 m_freem(txq->m); 1238 txq->m = NULL; 1239 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1240 m_freem(m); 1241 TXQ_UNLOCK(txq); 1242 } 1243 } 1244 if_qflush(ifp); 1245 } 1246 1247 static int 1248 cxgbe_media_change(struct ifnet *ifp) 1249 { 1250 struct port_info *pi = ifp->if_softc; 1251 1252 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1253 1254 return (EOPNOTSUPP); 1255 } 1256 1257 static void 1258 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1259 { 1260 struct port_info *pi = ifp->if_softc; 1261 struct ifmedia_entry *cur = pi->media.ifm_cur; 1262 int speed = pi->link_cfg.speed; 1263 int data = (pi->port_type << 8) | pi->mod_type; 1264 1265 if (cur->ifm_data != data) { 1266 build_medialist(pi); 1267 cur = pi->media.ifm_cur; 1268 } 1269 1270 ifmr->ifm_status = IFM_AVALID; 1271 if (!pi->link_cfg.link_ok) 1272 return; 1273 1274 ifmr->ifm_status |= IFM_ACTIVE; 1275 1276 /* active and current will differ iff current media is autoselect. */ 1277 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1278 return; 1279 1280 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1281 if (speed == SPEED_10000) 1282 ifmr->ifm_active |= IFM_10G_T; 1283 else if (speed == SPEED_1000) 1284 ifmr->ifm_active |= IFM_1000_T; 1285 else if (speed == SPEED_100) 1286 ifmr->ifm_active |= IFM_100_TX; 1287 else if (speed == SPEED_10) 1288 ifmr->ifm_active |= IFM_10_T; 1289 else 1290 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1291 speed)); 1292 } 1293 1294 void 1295 t4_fatal_err(struct adapter *sc) 1296 { 1297 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1298 t4_intr_disable(sc); 1299 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1300 device_get_nameunit(sc->dev)); 1301 } 1302 1303 static int 1304 map_bars(struct adapter *sc) 1305 { 1306 sc->regs_rid = PCIR_BAR(0); 1307 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1308 &sc->regs_rid, RF_ACTIVE); 1309 if (sc->regs_res == NULL) { 1310 device_printf(sc->dev, "cannot map registers.\n"); 1311 return (ENXIO); 1312 } 1313 sc->bt = rman_get_bustag(sc->regs_res); 1314 sc->bh = rman_get_bushandle(sc->regs_res); 1315 sc->mmio_len = rman_get_size(sc->regs_res); 1316 1317 sc->msix_rid = PCIR_BAR(4); 1318 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1319 &sc->msix_rid, RF_ACTIVE); 1320 if (sc->msix_res == NULL) { 1321 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1322 return (ENXIO); 1323 } 1324 1325 return (0); 1326 } 1327 1328 static void 1329 setup_memwin(struct adapter *sc) 1330 { 1331 uint32_t bar0; 1332 1333 /* 1334 * Read low 32b of bar0 indirectly via the hardware backdoor mechanism. 1335 * Works from within PCI passthrough environments too, where 1336 * rman_get_start() can return a different value. We need to program 1337 * the memory window decoders with the actual addresses that will be 1338 * coming across the PCIe link. 1339 */ 1340 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1341 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1342 1343 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1344 (bar0 + MEMWIN0_BASE) | V_BIR(0) | 1345 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1346 1347 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1348 (bar0 + MEMWIN1_BASE) | V_BIR(0) | 1349 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1350 1351 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1352 (bar0 + MEMWIN2_BASE) | V_BIR(0) | 1353 V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10)); 1354 1355 /* flush */ 1356 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1357 } 1358 1359 static int 1360 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1361 struct intrs_and_queues *iaq) 1362 { 1363 int rc, itype, navail, nrxq10g, nrxq1g, n; 1364 int nofldrxq10g = 0, nofldrxq1g = 0; 1365 1366 bzero(iaq, sizeof(*iaq)); 1367 1368 iaq->ntxq10g = t4_ntxq10g; 1369 iaq->ntxq1g = t4_ntxq1g; 1370 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1371 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1372 #ifdef TCP_OFFLOAD 1373 if (is_offload(sc)) { 1374 iaq->nofldtxq10g = t4_nofldtxq10g; 1375 iaq->nofldtxq1g = t4_nofldtxq1g; 1376 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1377 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1378 } 1379 #endif 1380 1381 for (itype = INTR_MSIX; itype; itype >>= 1) { 1382 1383 if ((itype & t4_intr_types) == 0) 1384 continue; /* not allowed */ 1385 1386 if (itype == INTR_MSIX) 1387 navail = pci_msix_count(sc->dev); 1388 else if (itype == INTR_MSI) 1389 navail = pci_msi_count(sc->dev); 1390 else 1391 navail = 1; 1392 restart: 1393 if (navail == 0) 1394 continue; 1395 1396 iaq->intr_type = itype; 1397 iaq->intr_flags = 0; 1398 1399 /* 1400 * Best option: an interrupt vector for errors, one for the 1401 * firmware event queue, and one each for each rxq (NIC as well 1402 * as offload). 1403 */ 1404 iaq->nirq = T4_EXTRA_INTR; 1405 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1406 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1407 if (iaq->nirq <= navail && 1408 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1409 iaq->intr_flags |= INTR_DIRECT; 1410 goto allocate; 1411 } 1412 1413 /* 1414 * Second best option: an interrupt vector for errors, one for 1415 * the firmware event queue, and one each for either NIC or 1416 * offload rxq's. 1417 */ 1418 iaq->nirq = T4_EXTRA_INTR; 1419 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1420 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1421 if (iaq->nirq <= navail && 1422 (itype != INTR_MSI || powerof2(iaq->nirq))) 1423 goto allocate; 1424 1425 /* 1426 * Next best option: an interrupt vector for errors, one for the 1427 * firmware event queue, and at least one per port. At this 1428 * point we know we'll have to downsize nrxq or nofldrxq to fit 1429 * what's available to us. 1430 */ 1431 iaq->nirq = T4_EXTRA_INTR; 1432 iaq->nirq += n10g + n1g; 1433 if (iaq->nirq <= navail) { 1434 int leftover = navail - iaq->nirq; 1435 1436 if (n10g > 0) { 1437 int target = max(nrxq10g, nofldrxq10g); 1438 1439 n = 1; 1440 while (n < target && leftover >= n10g) { 1441 leftover -= n10g; 1442 iaq->nirq += n10g; 1443 n++; 1444 } 1445 iaq->nrxq10g = min(n, nrxq10g); 1446 #ifdef TCP_OFFLOAD 1447 if (is_offload(sc)) 1448 iaq->nofldrxq10g = min(n, nofldrxq10g); 1449 #endif 1450 } 1451 1452 if (n1g > 0) { 1453 int target = max(nrxq1g, nofldrxq1g); 1454 1455 n = 1; 1456 while (n < target && leftover >= n1g) { 1457 leftover -= n1g; 1458 iaq->nirq += n1g; 1459 n++; 1460 } 1461 iaq->nrxq1g = min(n, nrxq1g); 1462 #ifdef TCP_OFFLOAD 1463 if (is_offload(sc)) 1464 iaq->nofldrxq1g = min(n, nofldrxq1g); 1465 #endif 1466 } 1467 1468 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1469 goto allocate; 1470 } 1471 1472 /* 1473 * Least desirable option: one interrupt vector for everything. 1474 */ 1475 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1476 #ifdef TCP_OFFLOAD 1477 if (is_offload(sc)) 1478 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1479 #endif 1480 1481 allocate: 1482 navail = iaq->nirq; 1483 rc = 0; 1484 if (itype == INTR_MSIX) 1485 rc = pci_alloc_msix(sc->dev, &navail); 1486 else if (itype == INTR_MSI) 1487 rc = pci_alloc_msi(sc->dev, &navail); 1488 1489 if (rc == 0) { 1490 if (navail == iaq->nirq) 1491 return (0); 1492 1493 /* 1494 * Didn't get the number requested. Use whatever number 1495 * the kernel is willing to allocate (it's in navail). 1496 */ 1497 device_printf(sc->dev, "fewer vectors than requested, " 1498 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1499 itype, iaq->nirq, navail); 1500 pci_release_msi(sc->dev); 1501 goto restart; 1502 } 1503 1504 device_printf(sc->dev, 1505 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1506 itype, rc, iaq->nirq, navail); 1507 } 1508 1509 device_printf(sc->dev, 1510 "failed to find a usable interrupt type. " 1511 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1512 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1513 1514 return (ENXIO); 1515 } 1516 1517 /* 1518 * Install a compatible firmware (if required), establish contact with it (by 1519 * saying hello), and reset the device. If we end up as the master driver, 1520 * partition adapter resources by providing a configuration file to the 1521 * firmware. 1522 */ 1523 static int 1524 prep_firmware(struct adapter *sc) 1525 { 1526 const struct firmware *fw = NULL, *cfg = NULL, *default_cfg; 1527 int rc; 1528 enum dev_state state; 1529 1530 default_cfg = firmware_get(T4_CFGNAME); 1531 1532 /* Check firmware version and install a different one if necessary */ 1533 rc = t4_check_fw_version(sc); 1534 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 1535 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1536 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1537 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1538 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1539 if (rc != 0) { 1540 uint32_t v = 0; 1541 1542 fw = firmware_get(T4_FWNAME); 1543 if (fw != NULL) { 1544 const struct fw_hdr *hdr = (const void *)fw->data; 1545 1546 v = ntohl(hdr->fw_ver); 1547 1548 /* 1549 * The firmware module will not be used if it isn't the 1550 * same major version as what the driver was compiled 1551 * with. 1552 */ 1553 if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) { 1554 device_printf(sc->dev, 1555 "Found firmware image but version %d " 1556 "can not be used with this driver (%d)\n", 1557 G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR); 1558 1559 firmware_put(fw, FIRMWARE_UNLOAD); 1560 fw = NULL; 1561 } 1562 } 1563 1564 if (fw == NULL && rc < 0) { 1565 device_printf(sc->dev, "No usable firmware. " 1566 "card has %d.%d.%d, driver compiled with %d.%d.%d", 1567 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1568 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1569 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1570 FW_VERSION_MAJOR, FW_VERSION_MINOR, 1571 FW_VERSION_MICRO); 1572 rc = EAGAIN; 1573 goto done; 1574 } 1575 1576 /* 1577 * Always upgrade, even for minor/micro/build mismatches. 1578 * Downgrade only for a major version mismatch or if 1579 * force_firmware_install was specified. 1580 */ 1581 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) { 1582 device_printf(sc->dev, 1583 "installing firmware %d.%d.%d.%d on card.\n", 1584 G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v), 1585 G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v)); 1586 1587 rc = -t4_load_fw(sc, fw->data, fw->datasize); 1588 if (rc != 0) { 1589 device_printf(sc->dev, 1590 "failed to install firmware: %d\n", rc); 1591 goto done; 1592 } else { 1593 /* refresh */ 1594 (void) t4_check_fw_version(sc); 1595 snprintf(sc->fw_version, 1596 sizeof(sc->fw_version), "%u.%u.%u.%u", 1597 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1598 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1599 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1600 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1601 } 1602 } 1603 } 1604 1605 /* Contact firmware. */ 1606 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1607 if (rc < 0) { 1608 rc = -rc; 1609 device_printf(sc->dev, 1610 "failed to connect to the firmware: %d.\n", rc); 1611 goto done; 1612 } 1613 if (rc == sc->mbox) 1614 sc->flags |= MASTER_PF; 1615 1616 /* Reset device */ 1617 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1618 if (rc != 0) { 1619 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 1620 if (rc != ETIMEDOUT && rc != EIO) 1621 t4_fw_bye(sc, sc->mbox); 1622 goto done; 1623 } 1624 1625 /* Partition adapter resources as specified in the config file. */ 1626 if (sc->flags & MASTER_PF) { 1627 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) { 1628 char s[32]; 1629 1630 snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file); 1631 cfg = firmware_get(s); 1632 if (cfg == NULL) { 1633 device_printf(sc->dev, 1634 "unable to locate %s module, " 1635 "will use default config file.\n", s); 1636 } 1637 } 1638 1639 rc = partition_resources(sc, cfg ? cfg : default_cfg); 1640 if (rc != 0) 1641 goto done; /* error message displayed already */ 1642 } 1643 1644 sc->flags |= FW_OK; 1645 1646 done: 1647 if (fw != NULL) 1648 firmware_put(fw, FIRMWARE_UNLOAD); 1649 if (cfg != NULL) 1650 firmware_put(cfg, FIRMWARE_UNLOAD); 1651 if (default_cfg != NULL) 1652 firmware_put(default_cfg, FIRMWARE_UNLOAD); 1653 1654 return (rc); 1655 } 1656 1657 #define FW_PARAM_DEV(param) \ 1658 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1659 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1660 #define FW_PARAM_PFVF(param) \ 1661 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1662 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1663 1664 /* 1665 * Upload configuration file to card's memory. 1666 */ 1667 static int 1668 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt, 1669 uint32_t *ma) 1670 { 1671 int rc, i; 1672 uint32_t param, val, mtype, maddr, bar, off, win, remaining; 1673 const uint32_t *b; 1674 1675 /* Figure out where the firmware wants us to upload it. */ 1676 param = FW_PARAM_DEV(CF); 1677 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1678 if (rc != 0) { 1679 /* Firmwares without config file support will fail this way */ 1680 device_printf(sc->dev, 1681 "failed to query config file location: %d.\n", rc); 1682 return (rc); 1683 } 1684 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1685 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1686 1687 if (maddr & 3) { 1688 device_printf(sc->dev, 1689 "cannot upload config file (type %u, addr %x).\n", 1690 mtype, maddr); 1691 return (EFAULT); 1692 } 1693 1694 /* Translate mtype/maddr to an address suitable for the PCIe window */ 1695 val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1696 val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE; 1697 switch (mtype) { 1698 case FW_MEMTYPE_CF_EDC0: 1699 if (!(val & F_EDRAM0_ENABLE)) 1700 goto err; 1701 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1702 maddr += G_EDRAM0_BASE(bar) << 20; 1703 break; 1704 1705 case FW_MEMTYPE_CF_EDC1: 1706 if (!(val & F_EDRAM1_ENABLE)) 1707 goto err; 1708 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1709 maddr += G_EDRAM1_BASE(bar) << 20; 1710 break; 1711 1712 case FW_MEMTYPE_CF_EXTMEM: 1713 if (!(val & F_EXT_MEM_ENABLE)) 1714 goto err; 1715 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1716 maddr += G_EXT_MEM_BASE(bar) << 20; 1717 break; 1718 1719 default: 1720 err: 1721 device_printf(sc->dev, 1722 "cannot upload config file (type %u, enabled %u).\n", 1723 mtype, val); 1724 return (EFAULT); 1725 } 1726 1727 /* 1728 * Position the PCIe window (we use memwin2) to the 16B aligned area 1729 * just at/before the upload location. 1730 */ 1731 win = maddr & ~0xf; 1732 off = maddr - win; /* offset from the start of the window. */ 1733 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 1734 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 1735 1736 remaining = fw->datasize; 1737 if (remaining > FLASH_CFG_MAX_SIZE || 1738 remaining > MEMWIN2_APERTURE - off) { 1739 device_printf(sc->dev, "cannot upload config file all at once " 1740 "(size %u, max %u, room %u).\n", 1741 remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off); 1742 return (EFBIG); 1743 } 1744 1745 /* 1746 * XXX: sheer laziness. We deliberately added 4 bytes of useless 1747 * stuffing/comments at the end of the config file so it's ok to simply 1748 * throw away the last remaining bytes when the config file is not an 1749 * exact multiple of 4. 1750 */ 1751 b = fw->data; 1752 for (i = 0; remaining >= 4; i += 4, remaining -= 4) 1753 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++); 1754 1755 return (rc); 1756 } 1757 1758 /* 1759 * Partition chip resources for use between various PFs, VFs, etc. This is done 1760 * by uploading the firmware configuration file to the adapter and instructing 1761 * the firmware to process it. 1762 */ 1763 static int 1764 partition_resources(struct adapter *sc, const struct firmware *cfg) 1765 { 1766 int rc; 1767 struct fw_caps_config_cmd caps; 1768 uint32_t mtype, maddr, finicsum, cfcsum; 1769 1770 rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT; 1771 if (rc != 0) { 1772 mtype = FW_MEMTYPE_CF_FLASH; 1773 maddr = t4_flash_cfg_addr(sc); 1774 } 1775 1776 bzero(&caps, sizeof(caps)); 1777 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1778 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1779 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1780 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1781 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1782 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1783 if (rc != 0) { 1784 device_printf(sc->dev, 1785 "failed to pre-process config file: %d.\n", rc); 1786 return (rc); 1787 } 1788 1789 finicsum = be32toh(caps.finicsum); 1790 cfcsum = be32toh(caps.cfcsum); 1791 if (finicsum != cfcsum) { 1792 device_printf(sc->dev, 1793 "WARNING: config file checksum mismatch: %08x %08x\n", 1794 finicsum, cfcsum); 1795 } 1796 sc->cfcsum = cfcsum; 1797 1798 #define LIMIT_CAPS(x) do { \ 1799 caps.x &= htobe16(t4_##x##_allowed); \ 1800 sc->x = htobe16(caps.x); \ 1801 } while (0) 1802 1803 /* 1804 * Let the firmware know what features will (not) be used so it can tune 1805 * things accordingly. 1806 */ 1807 LIMIT_CAPS(linkcaps); 1808 LIMIT_CAPS(niccaps); 1809 LIMIT_CAPS(toecaps); 1810 LIMIT_CAPS(rdmacaps); 1811 LIMIT_CAPS(iscsicaps); 1812 LIMIT_CAPS(fcoecaps); 1813 #undef LIMIT_CAPS 1814 1815 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1816 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1817 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1818 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 1819 if (rc != 0) { 1820 device_printf(sc->dev, 1821 "failed to process config file: %d.\n", rc); 1822 return (rc); 1823 } 1824 1825 return (0); 1826 } 1827 1828 /* 1829 * Retrieve parameters that are needed (or nice to have) prior to calling 1830 * t4_sge_init and t4_fw_initialize. 1831 */ 1832 static int 1833 get_params__pre_init(struct adapter *sc) 1834 { 1835 int rc; 1836 uint32_t param[2], val[2]; 1837 struct fw_devlog_cmd cmd; 1838 struct devlog_params *dlog = &sc->params.devlog; 1839 1840 param[0] = FW_PARAM_DEV(PORTVEC); 1841 param[1] = FW_PARAM_DEV(CCLK); 1842 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1843 if (rc != 0) { 1844 device_printf(sc->dev, 1845 "failed to query parameters (pre_init): %d.\n", rc); 1846 return (rc); 1847 } 1848 1849 sc->params.portvec = val[0]; 1850 sc->params.nports = bitcount32(val[0]); 1851 sc->params.vpd.cclk = val[1]; 1852 1853 /* Read device log parameters. */ 1854 bzero(&cmd, sizeof(cmd)); 1855 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1856 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1857 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 1858 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 1859 if (rc != 0) { 1860 device_printf(sc->dev, 1861 "failed to get devlog parameters: %d.\n", rc); 1862 bzero(dlog, sizeof (*dlog)); 1863 rc = 0; /* devlog isn't critical for device operation */ 1864 } else { 1865 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 1866 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1867 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1868 dlog->size = be32toh(cmd.memsize_devlog); 1869 } 1870 1871 return (rc); 1872 } 1873 1874 /* 1875 * Retrieve various parameters that are of interest to the driver. The device 1876 * has been initialized by the firmware at this point. 1877 */ 1878 static int 1879 get_params__post_init(struct adapter *sc) 1880 { 1881 int rc; 1882 uint32_t param[7], val[7]; 1883 struct fw_caps_config_cmd caps; 1884 1885 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1886 param[1] = FW_PARAM_PFVF(EQ_START); 1887 param[2] = FW_PARAM_PFVF(FILTER_START); 1888 param[3] = FW_PARAM_PFVF(FILTER_END); 1889 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); 1890 if (rc != 0) { 1891 device_printf(sc->dev, 1892 "failed to query parameters (post_init): %d.\n", rc); 1893 return (rc); 1894 } 1895 1896 sc->sge.iq_start = val[0]; 1897 sc->sge.eq_start = val[1]; 1898 sc->tids.ftid_base = val[2]; 1899 sc->tids.nftids = val[3] - val[2] + 1; 1900 1901 /* get capabilites */ 1902 bzero(&caps, sizeof(caps)); 1903 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1904 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1905 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1906 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1907 if (rc != 0) { 1908 device_printf(sc->dev, 1909 "failed to get card capabilities: %d.\n", rc); 1910 return (rc); 1911 } 1912 1913 if (caps.toecaps) { 1914 /* query offload-related parameters */ 1915 param[0] = FW_PARAM_DEV(NTID); 1916 param[1] = FW_PARAM_PFVF(SERVER_START); 1917 param[2] = FW_PARAM_PFVF(SERVER_END); 1918 param[3] = FW_PARAM_PFVF(TDDP_START); 1919 param[4] = FW_PARAM_PFVF(TDDP_END); 1920 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1921 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1922 if (rc != 0) { 1923 device_printf(sc->dev, 1924 "failed to query TOE parameters: %d.\n", rc); 1925 return (rc); 1926 } 1927 sc->tids.ntids = val[0]; 1928 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1929 sc->tids.stid_base = val[1]; 1930 sc->tids.nstids = val[2] - val[1] + 1; 1931 sc->vres.ddp.start = val[3]; 1932 sc->vres.ddp.size = val[4] - val[3] + 1; 1933 sc->params.ofldq_wr_cred = val[5]; 1934 sc->params.offload = 1; 1935 } 1936 if (caps.rdmacaps) { 1937 param[0] = FW_PARAM_PFVF(STAG_START); 1938 param[1] = FW_PARAM_PFVF(STAG_END); 1939 param[2] = FW_PARAM_PFVF(RQ_START); 1940 param[3] = FW_PARAM_PFVF(RQ_END); 1941 param[4] = FW_PARAM_PFVF(PBL_START); 1942 param[5] = FW_PARAM_PFVF(PBL_END); 1943 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1944 if (rc != 0) { 1945 device_printf(sc->dev, 1946 "failed to query RDMA parameters(1): %d.\n", rc); 1947 return (rc); 1948 } 1949 sc->vres.stag.start = val[0]; 1950 sc->vres.stag.size = val[1] - val[0] + 1; 1951 sc->vres.rq.start = val[2]; 1952 sc->vres.rq.size = val[3] - val[2] + 1; 1953 sc->vres.pbl.start = val[4]; 1954 sc->vres.pbl.size = val[5] - val[4] + 1; 1955 1956 param[0] = FW_PARAM_PFVF(SQRQ_START); 1957 param[1] = FW_PARAM_PFVF(SQRQ_END); 1958 param[2] = FW_PARAM_PFVF(CQ_START); 1959 param[3] = FW_PARAM_PFVF(CQ_END); 1960 param[4] = FW_PARAM_PFVF(OCQ_START); 1961 param[5] = FW_PARAM_PFVF(OCQ_END); 1962 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 1963 if (rc != 0) { 1964 device_printf(sc->dev, 1965 "failed to query RDMA parameters(2): %d.\n", rc); 1966 return (rc); 1967 } 1968 sc->vres.qp.start = val[0]; 1969 sc->vres.qp.size = val[1] - val[0] + 1; 1970 sc->vres.cq.start = val[2]; 1971 sc->vres.cq.size = val[3] - val[2] + 1; 1972 sc->vres.ocq.start = val[4]; 1973 sc->vres.ocq.size = val[5] - val[4] + 1; 1974 } 1975 if (caps.iscsicaps) { 1976 param[0] = FW_PARAM_PFVF(ISCSI_START); 1977 param[1] = FW_PARAM_PFVF(ISCSI_END); 1978 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1979 if (rc != 0) { 1980 device_printf(sc->dev, 1981 "failed to query iSCSI parameters: %d.\n", rc); 1982 return (rc); 1983 } 1984 sc->vres.iscsi.start = val[0]; 1985 sc->vres.iscsi.size = val[1] - val[0] + 1; 1986 } 1987 1988 /* These are finalized by FW initialization, load their values now */ 1989 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1990 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1991 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1992 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1993 1994 return (rc); 1995 } 1996 1997 #undef FW_PARAM_PFVF 1998 #undef FW_PARAM_DEV 1999 2000 static void 2001 t4_set_desc(struct adapter *sc) 2002 { 2003 char buf[128]; 2004 struct adapter_params *p = &sc->params; 2005 2006 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 2007 p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec); 2008 2009 device_set_desc_copy(sc->dev, buf); 2010 } 2011 2012 static void 2013 build_medialist(struct port_info *pi) 2014 { 2015 struct ifmedia *media = &pi->media; 2016 int data, m; 2017 2018 PORT_LOCK(pi); 2019 2020 ifmedia_removeall(media); 2021 2022 m = IFM_ETHER | IFM_FDX; 2023 data = (pi->port_type << 8) | pi->mod_type; 2024 2025 switch(pi->port_type) { 2026 case FW_PORT_TYPE_BT_XFI: 2027 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2028 break; 2029 2030 case FW_PORT_TYPE_BT_XAUI: 2031 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2032 /* fall through */ 2033 2034 case FW_PORT_TYPE_BT_SGMII: 2035 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2036 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2037 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2038 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2039 break; 2040 2041 case FW_PORT_TYPE_CX4: 2042 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2043 ifmedia_set(media, m | IFM_10G_CX4); 2044 break; 2045 2046 case FW_PORT_TYPE_SFP: 2047 case FW_PORT_TYPE_FIBER_XFI: 2048 case FW_PORT_TYPE_FIBER_XAUI: 2049 switch (pi->mod_type) { 2050 2051 case FW_PORT_MOD_TYPE_LR: 2052 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2053 ifmedia_set(media, m | IFM_10G_LR); 2054 break; 2055 2056 case FW_PORT_MOD_TYPE_SR: 2057 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2058 ifmedia_set(media, m | IFM_10G_SR); 2059 break; 2060 2061 case FW_PORT_MOD_TYPE_LRM: 2062 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2063 ifmedia_set(media, m | IFM_10G_LRM); 2064 break; 2065 2066 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2067 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2068 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2069 ifmedia_set(media, m | IFM_10G_TWINAX); 2070 break; 2071 2072 case FW_PORT_MOD_TYPE_NONE: 2073 m &= ~IFM_FDX; 2074 ifmedia_add(media, m | IFM_NONE, data, NULL); 2075 ifmedia_set(media, m | IFM_NONE); 2076 break; 2077 2078 case FW_PORT_MOD_TYPE_NA: 2079 case FW_PORT_MOD_TYPE_ER: 2080 default: 2081 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2082 ifmedia_set(media, m | IFM_UNKNOWN); 2083 break; 2084 } 2085 break; 2086 2087 case FW_PORT_TYPE_KX4: 2088 case FW_PORT_TYPE_KX: 2089 case FW_PORT_TYPE_KR: 2090 default: 2091 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2092 ifmedia_set(media, m | IFM_UNKNOWN); 2093 break; 2094 } 2095 2096 PORT_UNLOCK(pi); 2097 } 2098 2099 #define FW_MAC_EXACT_CHUNK 7 2100 2101 /* 2102 * Program the port's XGMAC based on parameters in ifnet. The caller also 2103 * indicates which parameters should be programmed (the rest are left alone). 2104 */ 2105 static int 2106 update_mac_settings(struct port_info *pi, int flags) 2107 { 2108 int rc; 2109 struct ifnet *ifp = pi->ifp; 2110 struct adapter *sc = pi->adapter; 2111 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2112 2113 PORT_LOCK_ASSERT_OWNED(pi); 2114 KASSERT(flags, ("%s: not told what to update.", __func__)); 2115 2116 if (flags & XGMAC_MTU) 2117 mtu = ifp->if_mtu; 2118 2119 if (flags & XGMAC_PROMISC) 2120 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2121 2122 if (flags & XGMAC_ALLMULTI) 2123 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2124 2125 if (flags & XGMAC_VLANEX) 2126 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2127 2128 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2129 vlanex, false); 2130 if (rc) { 2131 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2132 return (rc); 2133 } 2134 2135 if (flags & XGMAC_UCADDR) { 2136 uint8_t ucaddr[ETHER_ADDR_LEN]; 2137 2138 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2139 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2140 ucaddr, true, true); 2141 if (rc < 0) { 2142 rc = -rc; 2143 if_printf(ifp, "change_mac failed: %d\n", rc); 2144 return (rc); 2145 } else { 2146 pi->xact_addr_filt = rc; 2147 rc = 0; 2148 } 2149 } 2150 2151 if (flags & XGMAC_MCADDRS) { 2152 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2153 int del = 1; 2154 uint64_t hash = 0; 2155 struct ifmultiaddr *ifma; 2156 int i = 0, j; 2157 2158 if_maddr_rlock(ifp); 2159 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2160 if (ifma->ifma_addr->sa_family != AF_LINK) 2161 continue; 2162 mcaddr[i++] = 2163 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2164 2165 if (i == FW_MAC_EXACT_CHUNK) { 2166 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2167 del, i, mcaddr, NULL, &hash, 0); 2168 if (rc < 0) { 2169 rc = -rc; 2170 for (j = 0; j < i; j++) { 2171 if_printf(ifp, 2172 "failed to add mc address" 2173 " %02x:%02x:%02x:" 2174 "%02x:%02x:%02x rc=%d\n", 2175 mcaddr[j][0], mcaddr[j][1], 2176 mcaddr[j][2], mcaddr[j][3], 2177 mcaddr[j][4], mcaddr[j][5], 2178 rc); 2179 } 2180 goto mcfail; 2181 } 2182 del = 0; 2183 i = 0; 2184 } 2185 } 2186 if (i > 0) { 2187 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2188 del, i, mcaddr, NULL, &hash, 0); 2189 if (rc < 0) { 2190 rc = -rc; 2191 for (j = 0; j < i; j++) { 2192 if_printf(ifp, 2193 "failed to add mc address" 2194 " %02x:%02x:%02x:" 2195 "%02x:%02x:%02x rc=%d\n", 2196 mcaddr[j][0], mcaddr[j][1], 2197 mcaddr[j][2], mcaddr[j][3], 2198 mcaddr[j][4], mcaddr[j][5], 2199 rc); 2200 } 2201 goto mcfail; 2202 } 2203 } 2204 2205 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2206 if (rc != 0) 2207 if_printf(ifp, "failed to set mc address hash: %d", rc); 2208 mcfail: 2209 if_maddr_runlock(ifp); 2210 } 2211 2212 return (rc); 2213 } 2214 2215 static int 2216 cxgbe_init_locked(struct port_info *pi) 2217 { 2218 struct adapter *sc = pi->adapter; 2219 int rc = 0; 2220 2221 ADAPTER_LOCK_ASSERT_OWNED(sc); 2222 2223 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2224 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) { 2225 rc = EINTR; 2226 goto done; 2227 } 2228 } 2229 if (IS_DOOMED(pi)) { 2230 rc = ENXIO; 2231 goto done; 2232 } 2233 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2234 2235 /* Give up the adapter lock, port init code can sleep. */ 2236 SET_BUSY(sc); 2237 ADAPTER_UNLOCK(sc); 2238 2239 rc = cxgbe_init_synchronized(pi); 2240 2241 done: 2242 ADAPTER_LOCK(sc); 2243 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2244 CLR_BUSY(sc); 2245 wakeup_one(&sc->flags); 2246 ADAPTER_UNLOCK(sc); 2247 return (rc); 2248 } 2249 2250 static int 2251 cxgbe_init_synchronized(struct port_info *pi) 2252 { 2253 struct adapter *sc = pi->adapter; 2254 struct ifnet *ifp = pi->ifp; 2255 int rc = 0; 2256 2257 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2258 2259 if (isset(&sc->open_device_map, pi->port_id)) { 2260 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2261 ("mismatch between open_device_map and if_drv_flags")); 2262 return (0); /* already running */ 2263 } 2264 2265 if (!(sc->flags & FULL_INIT_DONE) && 2266 ((rc = adapter_full_init(sc)) != 0)) 2267 return (rc); /* error message displayed already */ 2268 2269 if (!(pi->flags & PORT_INIT_DONE) && 2270 ((rc = port_full_init(pi)) != 0)) 2271 return (rc); /* error message displayed already */ 2272 2273 PORT_LOCK(pi); 2274 rc = update_mac_settings(pi, XGMAC_ALL); 2275 PORT_UNLOCK(pi); 2276 if (rc) 2277 goto done; /* error message displayed already */ 2278 2279 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2280 if (rc != 0) { 2281 if_printf(ifp, "start_link failed: %d\n", rc); 2282 goto done; 2283 } 2284 2285 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2286 if (rc != 0) { 2287 if_printf(ifp, "enable_vi failed: %d\n", rc); 2288 goto done; 2289 } 2290 2291 /* all ok */ 2292 setbit(&sc->open_device_map, pi->port_id); 2293 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2294 2295 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2296 done: 2297 if (rc != 0) 2298 cxgbe_uninit_synchronized(pi); 2299 2300 return (rc); 2301 } 2302 2303 static int 2304 cxgbe_uninit_locked(struct port_info *pi) 2305 { 2306 struct adapter *sc = pi->adapter; 2307 int rc; 2308 2309 ADAPTER_LOCK_ASSERT_OWNED(sc); 2310 2311 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2312 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) { 2313 rc = EINTR; 2314 goto done; 2315 } 2316 } 2317 if (IS_DOOMED(pi)) { 2318 rc = ENXIO; 2319 goto done; 2320 } 2321 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2322 SET_BUSY(sc); 2323 ADAPTER_UNLOCK(sc); 2324 2325 rc = cxgbe_uninit_synchronized(pi); 2326 2327 ADAPTER_LOCK(sc); 2328 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2329 CLR_BUSY(sc); 2330 wakeup_one(&sc->flags); 2331 done: 2332 ADAPTER_UNLOCK(sc); 2333 return (rc); 2334 } 2335 2336 /* 2337 * Idempotent. 2338 */ 2339 static int 2340 cxgbe_uninit_synchronized(struct port_info *pi) 2341 { 2342 struct adapter *sc = pi->adapter; 2343 struct ifnet *ifp = pi->ifp; 2344 int rc; 2345 2346 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2347 2348 /* 2349 * Disable the VI so that all its data in either direction is discarded 2350 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2351 * tick) intact as the TP can deliver negative advice or data that it's 2352 * holding in its RAM (for an offloaded connection) even after the VI is 2353 * disabled. 2354 */ 2355 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2356 if (rc) { 2357 if_printf(ifp, "disable_vi failed: %d\n", rc); 2358 return (rc); 2359 } 2360 2361 clrbit(&sc->open_device_map, pi->port_id); 2362 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2363 2364 pi->link_cfg.link_ok = 0; 2365 pi->link_cfg.speed = 0; 2366 t4_os_link_changed(sc, pi->port_id, 0); 2367 2368 return (0); 2369 } 2370 2371 /* 2372 * It is ok for this function to fail midway and return right away. t4_detach 2373 * will walk the entire sc->irq list and clean up whatever is valid. 2374 */ 2375 static int 2376 setup_intr_handlers(struct adapter *sc) 2377 { 2378 int rc, rid, p, q; 2379 char s[8]; 2380 struct irq *irq; 2381 struct port_info *pi; 2382 struct sge_rxq *rxq; 2383 #ifdef TCP_OFFLOAD 2384 struct sge_ofld_rxq *ofld_rxq; 2385 #endif 2386 2387 /* 2388 * Setup interrupts. 2389 */ 2390 irq = &sc->irq[0]; 2391 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2392 if (sc->intr_count == 1) { 2393 KASSERT(!(sc->flags & INTR_DIRECT), 2394 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2395 2396 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 2397 if (rc != 0) 2398 return (rc); 2399 } else { 2400 /* Multiple interrupts. */ 2401 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2402 ("%s: too few intr.", __func__)); 2403 2404 /* The first one is always error intr */ 2405 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 2406 if (rc != 0) 2407 return (rc); 2408 irq++; 2409 rid++; 2410 2411 /* The second one is always the firmware event queue */ 2412 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 2413 "evt"); 2414 if (rc != 0) 2415 return (rc); 2416 irq++; 2417 rid++; 2418 2419 /* 2420 * Note that if INTR_DIRECT is not set then either the NIC rx 2421 * queues or (exclusive or) the TOE rx queueus will be taking 2422 * direct interrupts. 2423 * 2424 * There is no need to check for is_offload(sc) as nofldrxq 2425 * will be 0 if offload is disabled. 2426 */ 2427 for_each_port(sc, p) { 2428 pi = sc->port[p]; 2429 2430 #ifdef TCP_OFFLOAD 2431 /* 2432 * Skip over the NIC queues if they aren't taking direct 2433 * interrupts. 2434 */ 2435 if (!(sc->flags & INTR_DIRECT) && 2436 pi->nofldrxq > pi->nrxq) 2437 goto ofld_queues; 2438 #endif 2439 rxq = &sc->sge.rxq[pi->first_rxq]; 2440 for (q = 0; q < pi->nrxq; q++, rxq++) { 2441 snprintf(s, sizeof(s), "%d.%d", p, q); 2442 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 2443 s); 2444 if (rc != 0) 2445 return (rc); 2446 irq++; 2447 rid++; 2448 } 2449 2450 #ifdef TCP_OFFLOAD 2451 /* 2452 * Skip over the offload queues if they aren't taking 2453 * direct interrupts. 2454 */ 2455 if (!(sc->flags & INTR_DIRECT)) 2456 continue; 2457 ofld_queues: 2458 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2459 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2460 snprintf(s, sizeof(s), "%d,%d", p, q); 2461 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 2462 ofld_rxq, s); 2463 if (rc != 0) 2464 return (rc); 2465 irq++; 2466 rid++; 2467 } 2468 #endif 2469 } 2470 } 2471 2472 return (0); 2473 } 2474 2475 static int 2476 adapter_full_init(struct adapter *sc) 2477 { 2478 int rc, i; 2479 2480 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2481 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2482 ("%s: FULL_INIT_DONE already", __func__)); 2483 2484 /* 2485 * queues that belong to the adapter (not any particular port). 2486 */ 2487 rc = t4_setup_adapter_queues(sc); 2488 if (rc != 0) 2489 goto done; 2490 2491 for (i = 0; i < nitems(sc->tq); i++) { 2492 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 2493 taskqueue_thread_enqueue, &sc->tq[i]); 2494 if (sc->tq[i] == NULL) { 2495 device_printf(sc->dev, 2496 "failed to allocate task queue %d\n", i); 2497 rc = ENOMEM; 2498 goto done; 2499 } 2500 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 2501 device_get_nameunit(sc->dev), i); 2502 } 2503 2504 t4_intr_enable(sc); 2505 sc->flags |= FULL_INIT_DONE; 2506 done: 2507 if (rc != 0) 2508 adapter_full_uninit(sc); 2509 2510 return (rc); 2511 } 2512 2513 static int 2514 adapter_full_uninit(struct adapter *sc) 2515 { 2516 int i; 2517 2518 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2519 2520 t4_teardown_adapter_queues(sc); 2521 2522 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 2523 taskqueue_free(sc->tq[i]); 2524 sc->tq[i] = NULL; 2525 } 2526 2527 sc->flags &= ~FULL_INIT_DONE; 2528 2529 return (0); 2530 } 2531 2532 static int 2533 port_full_init(struct port_info *pi) 2534 { 2535 struct adapter *sc = pi->adapter; 2536 struct ifnet *ifp = pi->ifp; 2537 uint16_t *rss; 2538 struct sge_rxq *rxq; 2539 int rc, i; 2540 2541 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2542 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 2543 ("%s: PORT_INIT_DONE already", __func__)); 2544 2545 sysctl_ctx_init(&pi->ctx); 2546 pi->flags |= PORT_SYSCTL_CTX; 2547 2548 /* 2549 * Allocate tx/rx/fl queues for this port. 2550 */ 2551 rc = t4_setup_port_queues(pi); 2552 if (rc != 0) 2553 goto done; /* error message displayed already */ 2554 2555 /* 2556 * Setup RSS for this port. 2557 */ 2558 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 2559 M_ZERO | M_WAITOK); 2560 for_each_rxq(pi, i, rxq) { 2561 rss[i] = rxq->iq.abs_id; 2562 } 2563 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2564 pi->rss_size, rss, pi->nrxq); 2565 free(rss, M_CXGBE); 2566 if (rc != 0) { 2567 if_printf(ifp, "rss_config failed: %d\n", rc); 2568 goto done; 2569 } 2570 2571 pi->flags |= PORT_INIT_DONE; 2572 done: 2573 if (rc != 0) 2574 port_full_uninit(pi); 2575 2576 return (rc); 2577 } 2578 2579 /* 2580 * Idempotent. 2581 */ 2582 static int 2583 port_full_uninit(struct port_info *pi) 2584 { 2585 struct adapter *sc = pi->adapter; 2586 int i; 2587 struct sge_rxq *rxq; 2588 struct sge_txq *txq; 2589 #ifdef TCP_OFFLOAD 2590 struct sge_ofld_rxq *ofld_rxq; 2591 struct sge_wrq *ofld_txq; 2592 #endif 2593 2594 if (pi->flags & PORT_INIT_DONE) { 2595 2596 /* Need to quiesce queues. XXX: ctrl queues? */ 2597 2598 for_each_txq(pi, i, txq) { 2599 quiesce_eq(sc, &txq->eq); 2600 } 2601 2602 #ifdef TCP_OFFLOAD 2603 for_each_ofld_txq(pi, i, ofld_txq) { 2604 quiesce_eq(sc, &ofld_txq->eq); 2605 } 2606 #endif 2607 2608 for_each_rxq(pi, i, rxq) { 2609 quiesce_iq(sc, &rxq->iq); 2610 quiesce_fl(sc, &rxq->fl); 2611 } 2612 2613 #ifdef TCP_OFFLOAD 2614 for_each_ofld_rxq(pi, i, ofld_rxq) { 2615 quiesce_iq(sc, &ofld_rxq->iq); 2616 quiesce_fl(sc, &ofld_rxq->fl); 2617 } 2618 #endif 2619 } 2620 2621 t4_teardown_port_queues(pi); 2622 pi->flags &= ~PORT_INIT_DONE; 2623 2624 return (0); 2625 } 2626 2627 static void 2628 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 2629 { 2630 EQ_LOCK(eq); 2631 eq->flags |= EQ_DOOMED; 2632 2633 /* 2634 * Wait for the response to a credit flush if one's 2635 * pending. 2636 */ 2637 while (eq->flags & EQ_CRFLUSHED) 2638 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 2639 EQ_UNLOCK(eq); 2640 2641 callout_drain(&eq->tx_callout); /* XXX: iffy */ 2642 pause("callout", 10); /* Still iffy */ 2643 2644 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 2645 } 2646 2647 static void 2648 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 2649 { 2650 (void) sc; /* unused */ 2651 2652 /* Synchronize with the interrupt handler */ 2653 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 2654 pause("iqfree", 1); 2655 } 2656 2657 static void 2658 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 2659 { 2660 mtx_lock(&sc->sfl_lock); 2661 FL_LOCK(fl); 2662 fl->flags |= FL_DOOMED; 2663 FL_UNLOCK(fl); 2664 mtx_unlock(&sc->sfl_lock); 2665 2666 callout_drain(&sc->sfl_callout); 2667 KASSERT((fl->flags & FL_STARVING) == 0, 2668 ("%s: still starving", __func__)); 2669 } 2670 2671 static int 2672 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 2673 driver_intr_t *handler, void *arg, char *name) 2674 { 2675 int rc; 2676 2677 irq->rid = rid; 2678 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 2679 RF_SHAREABLE | RF_ACTIVE); 2680 if (irq->res == NULL) { 2681 device_printf(sc->dev, 2682 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 2683 return (ENOMEM); 2684 } 2685 2686 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 2687 NULL, handler, arg, &irq->tag); 2688 if (rc != 0) { 2689 device_printf(sc->dev, 2690 "failed to setup interrupt for rid %d, name %s: %d\n", 2691 rid, name, rc); 2692 } else if (name) 2693 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 2694 2695 return (rc); 2696 } 2697 2698 static int 2699 t4_free_irq(struct adapter *sc, struct irq *irq) 2700 { 2701 if (irq->tag) 2702 bus_teardown_intr(sc->dev, irq->res, irq->tag); 2703 if (irq->res) 2704 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 2705 2706 bzero(irq, sizeof(*irq)); 2707 2708 return (0); 2709 } 2710 2711 static void 2712 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 2713 unsigned int end) 2714 { 2715 uint32_t *p = (uint32_t *)(buf + start); 2716 2717 for ( ; start <= end; start += sizeof(uint32_t)) 2718 *p++ = t4_read_reg(sc, start); 2719 } 2720 2721 static void 2722 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 2723 { 2724 int i; 2725 static const unsigned int reg_ranges[] = { 2726 0x1008, 0x1108, 2727 0x1180, 0x11b4, 2728 0x11fc, 0x123c, 2729 0x1300, 0x173c, 2730 0x1800, 0x18fc, 2731 0x3000, 0x30d8, 2732 0x30e0, 0x5924, 2733 0x5960, 0x59d4, 2734 0x5a00, 0x5af8, 2735 0x6000, 0x6098, 2736 0x6100, 0x6150, 2737 0x6200, 0x6208, 2738 0x6240, 0x6248, 2739 0x6280, 0x6338, 2740 0x6370, 0x638c, 2741 0x6400, 0x643c, 2742 0x6500, 0x6524, 2743 0x6a00, 0x6a38, 2744 0x6a60, 0x6a78, 2745 0x6b00, 0x6b84, 2746 0x6bf0, 0x6c84, 2747 0x6cf0, 0x6d84, 2748 0x6df0, 0x6e84, 2749 0x6ef0, 0x6f84, 2750 0x6ff0, 0x7084, 2751 0x70f0, 0x7184, 2752 0x71f0, 0x7284, 2753 0x72f0, 0x7384, 2754 0x73f0, 0x7450, 2755 0x7500, 0x7530, 2756 0x7600, 0x761c, 2757 0x7680, 0x76cc, 2758 0x7700, 0x7798, 2759 0x77c0, 0x77fc, 2760 0x7900, 0x79fc, 2761 0x7b00, 0x7c38, 2762 0x7d00, 0x7efc, 2763 0x8dc0, 0x8e1c, 2764 0x8e30, 0x8e78, 2765 0x8ea0, 0x8f6c, 2766 0x8fc0, 0x9074, 2767 0x90fc, 0x90fc, 2768 0x9400, 0x9458, 2769 0x9600, 0x96bc, 2770 0x9800, 0x9808, 2771 0x9820, 0x983c, 2772 0x9850, 0x9864, 2773 0x9c00, 0x9c6c, 2774 0x9c80, 0x9cec, 2775 0x9d00, 0x9d6c, 2776 0x9d80, 0x9dec, 2777 0x9e00, 0x9e6c, 2778 0x9e80, 0x9eec, 2779 0x9f00, 0x9f6c, 2780 0x9f80, 0x9fec, 2781 0xd004, 0xd03c, 2782 0xdfc0, 0xdfe0, 2783 0xe000, 0xea7c, 2784 0xf000, 0x11190, 2785 0x19040, 0x1906c, 2786 0x19078, 0x19080, 2787 0x1908c, 0x19124, 2788 0x19150, 0x191b0, 2789 0x191d0, 0x191e8, 2790 0x19238, 0x1924c, 2791 0x193f8, 0x19474, 2792 0x19490, 0x194f8, 2793 0x19800, 0x19f30, 2794 0x1a000, 0x1a06c, 2795 0x1a0b0, 0x1a120, 2796 0x1a128, 0x1a138, 2797 0x1a190, 0x1a1c4, 2798 0x1a1fc, 0x1a1fc, 2799 0x1e040, 0x1e04c, 2800 0x1e284, 0x1e28c, 2801 0x1e2c0, 0x1e2c0, 2802 0x1e2e0, 0x1e2e0, 2803 0x1e300, 0x1e384, 2804 0x1e3c0, 0x1e3c8, 2805 0x1e440, 0x1e44c, 2806 0x1e684, 0x1e68c, 2807 0x1e6c0, 0x1e6c0, 2808 0x1e6e0, 0x1e6e0, 2809 0x1e700, 0x1e784, 2810 0x1e7c0, 0x1e7c8, 2811 0x1e840, 0x1e84c, 2812 0x1ea84, 0x1ea8c, 2813 0x1eac0, 0x1eac0, 2814 0x1eae0, 0x1eae0, 2815 0x1eb00, 0x1eb84, 2816 0x1ebc0, 0x1ebc8, 2817 0x1ec40, 0x1ec4c, 2818 0x1ee84, 0x1ee8c, 2819 0x1eec0, 0x1eec0, 2820 0x1eee0, 0x1eee0, 2821 0x1ef00, 0x1ef84, 2822 0x1efc0, 0x1efc8, 2823 0x1f040, 0x1f04c, 2824 0x1f284, 0x1f28c, 2825 0x1f2c0, 0x1f2c0, 2826 0x1f2e0, 0x1f2e0, 2827 0x1f300, 0x1f384, 2828 0x1f3c0, 0x1f3c8, 2829 0x1f440, 0x1f44c, 2830 0x1f684, 0x1f68c, 2831 0x1f6c0, 0x1f6c0, 2832 0x1f6e0, 0x1f6e0, 2833 0x1f700, 0x1f784, 2834 0x1f7c0, 0x1f7c8, 2835 0x1f840, 0x1f84c, 2836 0x1fa84, 0x1fa8c, 2837 0x1fac0, 0x1fac0, 2838 0x1fae0, 0x1fae0, 2839 0x1fb00, 0x1fb84, 2840 0x1fbc0, 0x1fbc8, 2841 0x1fc40, 0x1fc4c, 2842 0x1fe84, 0x1fe8c, 2843 0x1fec0, 0x1fec0, 2844 0x1fee0, 0x1fee0, 2845 0x1ff00, 0x1ff84, 2846 0x1ffc0, 0x1ffc8, 2847 0x20000, 0x2002c, 2848 0x20100, 0x2013c, 2849 0x20190, 0x201c8, 2850 0x20200, 0x20318, 2851 0x20400, 0x20528, 2852 0x20540, 0x20614, 2853 0x21000, 0x21040, 2854 0x2104c, 0x21060, 2855 0x210c0, 0x210ec, 2856 0x21200, 0x21268, 2857 0x21270, 0x21284, 2858 0x212fc, 0x21388, 2859 0x21400, 0x21404, 2860 0x21500, 0x21518, 2861 0x2152c, 0x2153c, 2862 0x21550, 0x21554, 2863 0x21600, 0x21600, 2864 0x21608, 0x21628, 2865 0x21630, 0x2163c, 2866 0x21700, 0x2171c, 2867 0x21780, 0x2178c, 2868 0x21800, 0x21c38, 2869 0x21c80, 0x21d7c, 2870 0x21e00, 0x21e04, 2871 0x22000, 0x2202c, 2872 0x22100, 0x2213c, 2873 0x22190, 0x221c8, 2874 0x22200, 0x22318, 2875 0x22400, 0x22528, 2876 0x22540, 0x22614, 2877 0x23000, 0x23040, 2878 0x2304c, 0x23060, 2879 0x230c0, 0x230ec, 2880 0x23200, 0x23268, 2881 0x23270, 0x23284, 2882 0x232fc, 0x23388, 2883 0x23400, 0x23404, 2884 0x23500, 0x23518, 2885 0x2352c, 0x2353c, 2886 0x23550, 0x23554, 2887 0x23600, 0x23600, 2888 0x23608, 0x23628, 2889 0x23630, 0x2363c, 2890 0x23700, 0x2371c, 2891 0x23780, 0x2378c, 2892 0x23800, 0x23c38, 2893 0x23c80, 0x23d7c, 2894 0x23e00, 0x23e04, 2895 0x24000, 0x2402c, 2896 0x24100, 0x2413c, 2897 0x24190, 0x241c8, 2898 0x24200, 0x24318, 2899 0x24400, 0x24528, 2900 0x24540, 0x24614, 2901 0x25000, 0x25040, 2902 0x2504c, 0x25060, 2903 0x250c0, 0x250ec, 2904 0x25200, 0x25268, 2905 0x25270, 0x25284, 2906 0x252fc, 0x25388, 2907 0x25400, 0x25404, 2908 0x25500, 0x25518, 2909 0x2552c, 0x2553c, 2910 0x25550, 0x25554, 2911 0x25600, 0x25600, 2912 0x25608, 0x25628, 2913 0x25630, 0x2563c, 2914 0x25700, 0x2571c, 2915 0x25780, 0x2578c, 2916 0x25800, 0x25c38, 2917 0x25c80, 0x25d7c, 2918 0x25e00, 0x25e04, 2919 0x26000, 0x2602c, 2920 0x26100, 0x2613c, 2921 0x26190, 0x261c8, 2922 0x26200, 0x26318, 2923 0x26400, 0x26528, 2924 0x26540, 0x26614, 2925 0x27000, 0x27040, 2926 0x2704c, 0x27060, 2927 0x270c0, 0x270ec, 2928 0x27200, 0x27268, 2929 0x27270, 0x27284, 2930 0x272fc, 0x27388, 2931 0x27400, 0x27404, 2932 0x27500, 0x27518, 2933 0x2752c, 0x2753c, 2934 0x27550, 0x27554, 2935 0x27600, 0x27600, 2936 0x27608, 0x27628, 2937 0x27630, 0x2763c, 2938 0x27700, 0x2771c, 2939 0x27780, 0x2778c, 2940 0x27800, 0x27c38, 2941 0x27c80, 0x27d7c, 2942 0x27e00, 0x27e04 2943 }; 2944 2945 regs->version = 4 | (sc->params.rev << 10); 2946 for (i = 0; i < nitems(reg_ranges); i += 2) 2947 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 2948 } 2949 2950 static void 2951 cxgbe_tick(void *arg) 2952 { 2953 struct port_info *pi = arg; 2954 struct ifnet *ifp = pi->ifp; 2955 struct sge_txq *txq; 2956 int i, drops; 2957 struct port_stats *s = &pi->stats; 2958 2959 PORT_LOCK(pi); 2960 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2961 PORT_UNLOCK(pi); 2962 return; /* without scheduling another callout */ 2963 } 2964 2965 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 2966 2967 ifp->if_opackets = s->tx_frames - s->tx_pause; 2968 ifp->if_ipackets = s->rx_frames - s->rx_pause; 2969 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 2970 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 2971 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 2972 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 2973 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 2974 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 2975 s->rx_trunc3; 2976 2977 drops = s->tx_drop; 2978 for_each_txq(pi, i, txq) 2979 drops += txq->br->br_drops; 2980 ifp->if_snd.ifq_drops = drops; 2981 2982 ifp->if_oerrors = s->tx_error_frames; 2983 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 2984 s->rx_fcs_err + s->rx_len_err; 2985 2986 callout_schedule(&pi->tick, hz); 2987 PORT_UNLOCK(pi); 2988 } 2989 2990 static void 2991 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 2992 { 2993 struct ifnet *vlan; 2994 2995 if (arg != ifp || ifp->if_type != IFT_ETHER) 2996 return; 2997 2998 vlan = VLAN_DEVAT(ifp, vid); 2999 VLAN_SETCOOKIE(vlan, ifp); 3000 } 3001 3002 static int 3003 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 3004 { 3005 3006 #ifdef INVARIANTS 3007 panic("%s: opcode 0x%02x on iq %p with payload %p", 3008 __func__, rss->opcode, iq, m); 3009 #else 3010 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 3011 __func__, rss->opcode, iq, m); 3012 m_freem(m); 3013 #endif 3014 return (EDOOFUS); 3015 } 3016 3017 int 3018 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3019 { 3020 uintptr_t *loc, new; 3021 3022 if (opcode >= nitems(sc->cpl_handler)) 3023 return (EINVAL); 3024 3025 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3026 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3027 atomic_store_rel_ptr(loc, new); 3028 3029 return (0); 3030 } 3031 3032 static int 3033 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 3034 { 3035 3036 #ifdef INVARIANTS 3037 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 3038 #else 3039 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 3040 __func__, iq, ctrl); 3041 #endif 3042 return (EDOOFUS); 3043 } 3044 3045 int 3046 t4_register_an_handler(struct adapter *sc, an_handler_t h) 3047 { 3048 uintptr_t *loc, new; 3049 3050 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 3051 loc = (uintptr_t *) &sc->an_handler; 3052 atomic_store_rel_ptr(loc, new); 3053 3054 return (0); 3055 } 3056 3057 static int 3058 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 3059 { 3060 const struct cpl_fw6_msg *cpl = 3061 __containerof(rpl, struct cpl_fw6_msg, data[0]); 3062 3063 #ifdef INVARIANTS 3064 panic("%s: fw_msg type %d", __func__, cpl->type); 3065 #else 3066 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 3067 #endif 3068 return (EDOOFUS); 3069 } 3070 3071 int 3072 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 3073 { 3074 uintptr_t *loc, new; 3075 3076 if (type >= nitems(sc->fw_msg_handler)) 3077 return (EINVAL); 3078 3079 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 3080 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 3081 atomic_store_rel_ptr(loc, new); 3082 3083 return (0); 3084 } 3085 3086 static int 3087 t4_sysctls(struct adapter *sc) 3088 { 3089 struct sysctl_ctx_list *ctx; 3090 struct sysctl_oid *oid; 3091 struct sysctl_oid_list *children, *c0; 3092 static char *caps[] = { 3093 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 3094 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 3095 "\20\1TOE", /* caps[2] toecaps */ 3096 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 3097 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 3098 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 3099 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 3100 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 3101 }; 3102 3103 ctx = device_get_sysctl_ctx(sc->dev); 3104 3105 /* 3106 * dev.t4nex.X. 3107 */ 3108 oid = device_get_sysctl_tree(sc->dev); 3109 c0 = children = SYSCTL_CHILDREN(oid); 3110 3111 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, 3112 &sc->params.nports, 0, "# of ports"); 3113 3114 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 3115 &sc->params.rev, 0, "chip hardware revision"); 3116 3117 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 3118 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 3119 3120 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 3121 CTLFLAG_RD, &t4_cfg_file, 0, "configuration file"); 3122 3123 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, 3124 &sc->cfcsum, 0, "config file checksum"); 3125 3126 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 3127 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 3128 sysctl_bitfield, "A", "available link capabilities"); 3129 3130 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 3131 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 3132 sysctl_bitfield, "A", "available NIC capabilities"); 3133 3134 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 3135 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 3136 sysctl_bitfield, "A", "available TCP offload capabilities"); 3137 3138 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 3139 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 3140 sysctl_bitfield, "A", "available RDMA capabilities"); 3141 3142 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 3143 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 3144 sysctl_bitfield, "A", "available iSCSI capabilities"); 3145 3146 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 3147 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 3148 sysctl_bitfield, "A", "available FCoE capabilities"); 3149 3150 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, 3151 &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)"); 3152 3153 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 3154 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 3155 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 3156 "interrupt holdoff timer values (us)"); 3157 3158 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 3159 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 3160 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 3161 "interrupt holdoff packet counter values"); 3162 3163 #ifdef SBUF_DRAIN 3164 /* 3165 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 3166 */ 3167 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 3168 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 3169 "logs and miscellaneous information"); 3170 children = SYSCTL_CHILDREN(oid); 3171 3172 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 3173 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3174 sysctl_cctrl, "A", "congestion control"); 3175 3176 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 3177 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3178 sysctl_cpl_stats, "A", "CPL statistics"); 3179 3180 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 3181 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3182 sysctl_ddp_stats, "A", "DDP statistics"); 3183 3184 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 3185 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3186 sysctl_devlog, "A", "firmware's device log"); 3187 3188 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 3189 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3190 sysctl_fcoe_stats, "A", "FCoE statistics"); 3191 3192 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 3193 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3194 sysctl_hw_sched, "A", "hardware scheduler "); 3195 3196 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 3197 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3198 sysctl_l2t, "A", "hardware L2 table"); 3199 3200 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 3201 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3202 sysctl_lb_stats, "A", "loopback statistics"); 3203 3204 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 3205 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3206 sysctl_meminfo, "A", "memory regions"); 3207 3208 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 3209 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3210 sysctl_path_mtus, "A", "path MTUs"); 3211 3212 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 3213 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3214 sysctl_pm_stats, "A", "PM statistics"); 3215 3216 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 3217 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3218 sysctl_rdma_stats, "A", "RDMA statistics"); 3219 3220 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 3221 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3222 sysctl_tcp_stats, "A", "TCP statistics"); 3223 3224 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 3225 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3226 sysctl_tids, "A", "TID information"); 3227 3228 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 3229 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3230 sysctl_tp_err_stats, "A", "TP error statistics"); 3231 3232 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 3233 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3234 sysctl_tx_rate, "A", "Tx rate"); 3235 #endif 3236 3237 #ifdef TCP_OFFLOAD 3238 if (is_offload(sc)) { 3239 /* 3240 * dev.t4nex.X.toe. 3241 */ 3242 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 3243 NULL, "TOE parameters"); 3244 children = SYSCTL_CHILDREN(oid); 3245 3246 sc->tt.sndbuf = 256 * 1024; 3247 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 3248 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 3249 3250 sc->tt.ddp = 0; 3251 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 3252 &sc->tt.ddp, 0, "DDP allowed"); 3253 3254 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 3255 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 3256 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 3257 3258 sc->tt.ddp_thres = 3259 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 3260 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 3261 &sc->tt.ddp_thres, 0, "DDP threshold"); 3262 } 3263 #endif 3264 3265 3266 return (0); 3267 } 3268 3269 static int 3270 cxgbe_sysctls(struct port_info *pi) 3271 { 3272 struct sysctl_ctx_list *ctx; 3273 struct sysctl_oid *oid; 3274 struct sysctl_oid_list *children; 3275 3276 ctx = device_get_sysctl_ctx(pi->dev); 3277 3278 /* 3279 * dev.cxgbe.X. 3280 */ 3281 oid = device_get_sysctl_tree(pi->dev); 3282 children = SYSCTL_CHILDREN(oid); 3283 3284 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 3285 &pi->nrxq, 0, "# of rx queues"); 3286 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 3287 &pi->ntxq, 0, "# of tx queues"); 3288 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 3289 &pi->first_rxq, 0, "index of first rx queue"); 3290 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 3291 &pi->first_txq, 0, "index of first tx queue"); 3292 3293 #ifdef TCP_OFFLOAD 3294 if (is_offload(pi->adapter)) { 3295 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 3296 &pi->nofldrxq, 0, 3297 "# of rx queues for offloaded TCP connections"); 3298 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 3299 &pi->nofldtxq, 0, 3300 "# of tx queues for offloaded TCP connections"); 3301 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 3302 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 3303 "index of first TOE rx queue"); 3304 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 3305 CTLFLAG_RD, &pi->first_ofld_txq, 0, 3306 "index of first TOE tx queue"); 3307 } 3308 #endif 3309 3310 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 3311 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 3312 "holdoff timer index"); 3313 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 3314 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 3315 "holdoff packet counter index"); 3316 3317 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 3318 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 3319 "rx queue size"); 3320 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 3321 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 3322 "tx queue size"); 3323 3324 /* 3325 * dev.cxgbe.X.stats. 3326 */ 3327 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 3328 NULL, "port statistics"); 3329 children = SYSCTL_CHILDREN(oid); 3330 3331 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 3332 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 3333 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 3334 sysctl_handle_t4_reg64, "QU", desc) 3335 3336 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 3337 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 3338 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 3339 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 3340 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 3341 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 3342 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 3343 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 3344 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 3345 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 3346 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 3347 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 3348 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 3349 "# of tx frames in this range", 3350 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 3351 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 3352 "# of tx frames in this range", 3353 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 3354 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 3355 "# of tx frames in this range", 3356 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 3357 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 3358 "# of tx frames in this range", 3359 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 3360 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 3361 "# of tx frames in this range", 3362 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 3363 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 3364 "# of tx frames in this range", 3365 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 3366 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 3367 "# of tx frames in this range", 3368 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 3369 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 3370 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 3371 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 3372 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 3373 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 3374 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 3375 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 3376 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 3377 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 3378 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 3379 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 3380 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 3381 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 3382 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 3383 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 3384 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 3385 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 3386 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 3387 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 3388 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 3389 3390 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 3391 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 3392 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 3393 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 3394 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 3395 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 3396 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 3397 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 3398 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 3399 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 3400 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 3401 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 3402 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 3403 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 3404 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 3405 "# of frames received with bad FCS", 3406 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 3407 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 3408 "# of frames received with length error", 3409 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 3410 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 3411 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 3412 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 3413 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 3414 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 3415 "# of rx frames in this range", 3416 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 3417 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 3418 "# of rx frames in this range", 3419 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 3420 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 3421 "# of rx frames in this range", 3422 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 3423 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 3424 "# of rx frames in this range", 3425 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 3426 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 3427 "# of rx frames in this range", 3428 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 3429 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 3430 "# of rx frames in this range", 3431 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 3432 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 3433 "# of rx frames in this range", 3434 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 3435 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 3436 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 3437 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 3438 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 3439 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 3440 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 3441 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 3442 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 3443 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 3444 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 3445 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 3446 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 3447 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 3448 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 3449 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 3450 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 3451 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 3452 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 3453 3454 #undef SYSCTL_ADD_T4_REG64 3455 3456 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 3457 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 3458 &pi->stats.name, desc) 3459 3460 /* We get these from port_stats and they may be stale by upto 1s */ 3461 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 3462 "# drops due to buffer-group 0 overflows"); 3463 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 3464 "# drops due to buffer-group 1 overflows"); 3465 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 3466 "# drops due to buffer-group 2 overflows"); 3467 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 3468 "# drops due to buffer-group 3 overflows"); 3469 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 3470 "# of buffer-group 0 truncated packets"); 3471 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 3472 "# of buffer-group 1 truncated packets"); 3473 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 3474 "# of buffer-group 2 truncated packets"); 3475 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 3476 "# of buffer-group 3 truncated packets"); 3477 3478 #undef SYSCTL_ADD_T4_PORTSTAT 3479 3480 return (0); 3481 } 3482 3483 static int 3484 sysctl_int_array(SYSCTL_HANDLER_ARGS) 3485 { 3486 int rc, *i; 3487 struct sbuf sb; 3488 3489 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 3490 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 3491 sbuf_printf(&sb, "%d ", *i); 3492 sbuf_trim(&sb); 3493 sbuf_finish(&sb); 3494 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 3495 sbuf_delete(&sb); 3496 return (rc); 3497 } 3498 3499 static int 3500 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 3501 { 3502 int rc; 3503 struct sbuf *sb; 3504 3505 rc = sysctl_wire_old_buffer(req, 0); 3506 if (rc != 0) 3507 return(rc); 3508 3509 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 3510 if (sb == NULL) 3511 return (ENOMEM); 3512 3513 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 3514 rc = sbuf_finish(sb); 3515 sbuf_delete(sb); 3516 3517 return (rc); 3518 } 3519 3520 static int 3521 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 3522 { 3523 struct port_info *pi = arg1; 3524 struct adapter *sc = pi->adapter; 3525 int idx, rc, i; 3526 3527 idx = pi->tmr_idx; 3528 3529 rc = sysctl_handle_int(oidp, &idx, 0, req); 3530 if (rc != 0 || req->newptr == NULL) 3531 return (rc); 3532 3533 if (idx < 0 || idx >= SGE_NTIMERS) 3534 return (EINVAL); 3535 3536 ADAPTER_LOCK(sc); 3537 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3538 if (rc == 0) { 3539 struct sge_rxq *rxq; 3540 uint8_t v; 3541 3542 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 3543 for_each_rxq(pi, i, rxq) { 3544 #ifdef atomic_store_rel_8 3545 atomic_store_rel_8(&rxq->iq.intr_params, v); 3546 #else 3547 rxq->iq.intr_params = v; 3548 #endif 3549 } 3550 pi->tmr_idx = idx; 3551 } 3552 3553 ADAPTER_UNLOCK(sc); 3554 return (rc); 3555 } 3556 3557 static int 3558 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 3559 { 3560 struct port_info *pi = arg1; 3561 struct adapter *sc = pi->adapter; 3562 int idx, rc; 3563 3564 idx = pi->pktc_idx; 3565 3566 rc = sysctl_handle_int(oidp, &idx, 0, req); 3567 if (rc != 0 || req->newptr == NULL) 3568 return (rc); 3569 3570 if (idx < -1 || idx >= SGE_NCOUNTERS) 3571 return (EINVAL); 3572 3573 ADAPTER_LOCK(sc); 3574 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3575 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3576 rc = EBUSY; /* cannot be changed once the queues are created */ 3577 3578 if (rc == 0) 3579 pi->pktc_idx = idx; 3580 3581 ADAPTER_UNLOCK(sc); 3582 return (rc); 3583 } 3584 3585 static int 3586 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 3587 { 3588 struct port_info *pi = arg1; 3589 struct adapter *sc = pi->adapter; 3590 int qsize, rc; 3591 3592 qsize = pi->qsize_rxq; 3593 3594 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3595 if (rc != 0 || req->newptr == NULL) 3596 return (rc); 3597 3598 if (qsize < 128 || (qsize & 7)) 3599 return (EINVAL); 3600 3601 ADAPTER_LOCK(sc); 3602 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3603 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3604 rc = EBUSY; /* cannot be changed once the queues are created */ 3605 3606 if (rc == 0) 3607 pi->qsize_rxq = qsize; 3608 3609 ADAPTER_UNLOCK(sc); 3610 return (rc); 3611 } 3612 3613 static int 3614 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 3615 { 3616 struct port_info *pi = arg1; 3617 struct adapter *sc = pi->adapter; 3618 int qsize, rc; 3619 3620 qsize = pi->qsize_txq; 3621 3622 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3623 if (rc != 0 || req->newptr == NULL) 3624 return (rc); 3625 3626 if (qsize < 128) 3627 return (EINVAL); 3628 3629 ADAPTER_LOCK(sc); 3630 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3631 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3632 rc = EBUSY; /* cannot be changed once the queues are created */ 3633 3634 if (rc == 0) 3635 pi->qsize_txq = qsize; 3636 3637 ADAPTER_UNLOCK(sc); 3638 return (rc); 3639 } 3640 3641 static int 3642 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 3643 { 3644 struct adapter *sc = arg1; 3645 int reg = arg2; 3646 uint64_t val; 3647 3648 val = t4_read_reg64(sc, reg); 3649 3650 return (sysctl_handle_64(oidp, &val, 0, req)); 3651 } 3652 3653 #ifdef SBUF_DRAIN 3654 static int 3655 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 3656 { 3657 struct adapter *sc = arg1; 3658 struct sbuf *sb; 3659 int rc, i; 3660 uint16_t incr[NMTUS][NCCTRL_WIN]; 3661 static const char *dec_fac[] = { 3662 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 3663 "0.9375" 3664 }; 3665 3666 rc = sysctl_wire_old_buffer(req, 0); 3667 if (rc != 0) 3668 return (rc); 3669 3670 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3671 if (sb == NULL) 3672 return (ENOMEM); 3673 3674 t4_read_cong_tbl(sc, incr); 3675 3676 for (i = 0; i < NCCTRL_WIN; ++i) { 3677 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 3678 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 3679 incr[5][i], incr[6][i], incr[7][i]); 3680 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 3681 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 3682 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 3683 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 3684 } 3685 3686 rc = sbuf_finish(sb); 3687 sbuf_delete(sb); 3688 3689 return (rc); 3690 } 3691 3692 static int 3693 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 3694 { 3695 struct adapter *sc = arg1; 3696 struct sbuf *sb; 3697 int rc; 3698 struct tp_cpl_stats stats; 3699 3700 rc = sysctl_wire_old_buffer(req, 0); 3701 if (rc != 0) 3702 return (rc); 3703 3704 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3705 if (sb == NULL) 3706 return (ENOMEM); 3707 3708 t4_tp_get_cpl_stats(sc, &stats); 3709 3710 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 3711 "channel 3\n"); 3712 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 3713 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 3714 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 3715 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 3716 3717 rc = sbuf_finish(sb); 3718 sbuf_delete(sb); 3719 3720 return (rc); 3721 } 3722 3723 static int 3724 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 3725 { 3726 struct adapter *sc = arg1; 3727 struct sbuf *sb; 3728 int rc; 3729 struct tp_usm_stats stats; 3730 3731 rc = sysctl_wire_old_buffer(req, 0); 3732 if (rc != 0) 3733 return(rc); 3734 3735 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3736 if (sb == NULL) 3737 return (ENOMEM); 3738 3739 t4_get_usm_stats(sc, &stats); 3740 3741 sbuf_printf(sb, "Frames: %u\n", stats.frames); 3742 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 3743 sbuf_printf(sb, "Drops: %u", stats.drops); 3744 3745 rc = sbuf_finish(sb); 3746 sbuf_delete(sb); 3747 3748 return (rc); 3749 } 3750 3751 const char *devlog_level_strings[] = { 3752 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 3753 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 3754 [FW_DEVLOG_LEVEL_ERR] = "ERR", 3755 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 3756 [FW_DEVLOG_LEVEL_INFO] = "INFO", 3757 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 3758 }; 3759 3760 const char *devlog_facility_strings[] = { 3761 [FW_DEVLOG_FACILITY_CORE] = "CORE", 3762 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 3763 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 3764 [FW_DEVLOG_FACILITY_RES] = "RES", 3765 [FW_DEVLOG_FACILITY_HW] = "HW", 3766 [FW_DEVLOG_FACILITY_FLR] = "FLR", 3767 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 3768 [FW_DEVLOG_FACILITY_PHY] = "PHY", 3769 [FW_DEVLOG_FACILITY_MAC] = "MAC", 3770 [FW_DEVLOG_FACILITY_PORT] = "PORT", 3771 [FW_DEVLOG_FACILITY_VI] = "VI", 3772 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 3773 [FW_DEVLOG_FACILITY_ACL] = "ACL", 3774 [FW_DEVLOG_FACILITY_TM] = "TM", 3775 [FW_DEVLOG_FACILITY_QFC] = "QFC", 3776 [FW_DEVLOG_FACILITY_DCB] = "DCB", 3777 [FW_DEVLOG_FACILITY_ETH] = "ETH", 3778 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 3779 [FW_DEVLOG_FACILITY_RI] = "RI", 3780 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 3781 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 3782 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 3783 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 3784 }; 3785 3786 static int 3787 sysctl_devlog(SYSCTL_HANDLER_ARGS) 3788 { 3789 struct adapter *sc = arg1; 3790 struct devlog_params *dparams = &sc->params.devlog; 3791 struct fw_devlog_e *buf, *e; 3792 int i, j, rc, nentries, first = 0; 3793 struct sbuf *sb; 3794 uint64_t ftstamp = UINT64_MAX; 3795 3796 if (dparams->start == 0) 3797 return (ENXIO); 3798 3799 nentries = dparams->size / sizeof(struct fw_devlog_e); 3800 3801 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 3802 if (buf == NULL) 3803 return (ENOMEM); 3804 3805 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 3806 (void *)buf); 3807 if (rc != 0) 3808 goto done; 3809 3810 for (i = 0; i < nentries; i++) { 3811 e = &buf[i]; 3812 3813 if (e->timestamp == 0) 3814 break; /* end */ 3815 3816 e->timestamp = be64toh(e->timestamp); 3817 e->seqno = be32toh(e->seqno); 3818 for (j = 0; j < 8; j++) 3819 e->params[j] = be32toh(e->params[j]); 3820 3821 if (e->timestamp < ftstamp) { 3822 ftstamp = e->timestamp; 3823 first = i; 3824 } 3825 } 3826 3827 if (buf[first].timestamp == 0) 3828 goto done; /* nothing in the log */ 3829 3830 rc = sysctl_wire_old_buffer(req, 0); 3831 if (rc != 0) 3832 goto done; 3833 3834 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3835 if (sb == NULL) { 3836 rc = ENOMEM; 3837 goto done; 3838 } 3839 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 3840 "Seq#", "Tstamp", "Level", "Facility", "Message"); 3841 3842 i = first; 3843 do { 3844 e = &buf[i]; 3845 if (e->timestamp == 0) 3846 break; /* end */ 3847 3848 sbuf_printf(sb, "%10d %15ju %8s %8s ", 3849 e->seqno, e->timestamp, 3850 (e->level < nitems(devlog_level_strings) ? 3851 devlog_level_strings[e->level] : "UNKNOWN"), 3852 (e->facility < nitems(devlog_facility_strings) ? 3853 devlog_facility_strings[e->facility] : "UNKNOWN")); 3854 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 3855 e->params[2], e->params[3], e->params[4], 3856 e->params[5], e->params[6], e->params[7]); 3857 3858 if (++i == nentries) 3859 i = 0; 3860 } while (i != first); 3861 3862 rc = sbuf_finish(sb); 3863 sbuf_delete(sb); 3864 done: 3865 free(buf, M_CXGBE); 3866 return (rc); 3867 } 3868 3869 static int 3870 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 3871 { 3872 struct adapter *sc = arg1; 3873 struct sbuf *sb; 3874 int rc; 3875 struct tp_fcoe_stats stats[4]; 3876 3877 rc = sysctl_wire_old_buffer(req, 0); 3878 if (rc != 0) 3879 return (rc); 3880 3881 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3882 if (sb == NULL) 3883 return (ENOMEM); 3884 3885 t4_get_fcoe_stats(sc, 0, &stats[0]); 3886 t4_get_fcoe_stats(sc, 1, &stats[1]); 3887 t4_get_fcoe_stats(sc, 2, &stats[2]); 3888 t4_get_fcoe_stats(sc, 3, &stats[3]); 3889 3890 sbuf_printf(sb, " channel 0 channel 1 " 3891 "channel 2 channel 3\n"); 3892 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 3893 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 3894 stats[3].octetsDDP); 3895 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 3896 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 3897 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 3898 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 3899 stats[3].framesDrop); 3900 3901 rc = sbuf_finish(sb); 3902 sbuf_delete(sb); 3903 3904 return (rc); 3905 } 3906 3907 static int 3908 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 3909 { 3910 struct adapter *sc = arg1; 3911 struct sbuf *sb; 3912 int rc, i; 3913 unsigned int map, kbps, ipg, mode; 3914 unsigned int pace_tab[NTX_SCHED]; 3915 3916 rc = sysctl_wire_old_buffer(req, 0); 3917 if (rc != 0) 3918 return (rc); 3919 3920 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3921 if (sb == NULL) 3922 return (ENOMEM); 3923 3924 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 3925 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 3926 t4_read_pace_tbl(sc, pace_tab); 3927 3928 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 3929 "Class IPG (0.1 ns) Flow IPG (us)"); 3930 3931 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 3932 t4_get_tx_sched(sc, i, &kbps, &ipg); 3933 sbuf_printf(sb, "\n %u %-5s %u ", i, 3934 (mode & (1 << i)) ? "flow" : "class", map & 3); 3935 if (kbps) 3936 sbuf_printf(sb, "%9u ", kbps); 3937 else 3938 sbuf_printf(sb, " disabled "); 3939 3940 if (ipg) 3941 sbuf_printf(sb, "%13u ", ipg); 3942 else 3943 sbuf_printf(sb, " disabled "); 3944 3945 if (pace_tab[i]) 3946 sbuf_printf(sb, "%10u", pace_tab[i]); 3947 else 3948 sbuf_printf(sb, " disabled"); 3949 } 3950 3951 rc = sbuf_finish(sb); 3952 sbuf_delete(sb); 3953 3954 return (rc); 3955 } 3956 3957 static int 3958 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 3959 { 3960 struct adapter *sc = arg1; 3961 struct sbuf *sb; 3962 int rc, i, j; 3963 uint64_t *p0, *p1; 3964 struct lb_port_stats s[2]; 3965 static const char *stat_name[] = { 3966 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 3967 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 3968 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 3969 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 3970 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 3971 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 3972 "BG2FramesTrunc:", "BG3FramesTrunc:" 3973 }; 3974 3975 rc = sysctl_wire_old_buffer(req, 0); 3976 if (rc != 0) 3977 return (rc); 3978 3979 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3980 if (sb == NULL) 3981 return (ENOMEM); 3982 3983 memset(s, 0, sizeof(s)); 3984 3985 for (i = 0; i < 4; i += 2) { 3986 t4_get_lb_stats(sc, i, &s[0]); 3987 t4_get_lb_stats(sc, i + 1, &s[1]); 3988 3989 p0 = &s[0].octets; 3990 p1 = &s[1].octets; 3991 sbuf_printf(sb, "%s Loopback %u" 3992 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 3993 3994 for (j = 0; j < nitems(stat_name); j++) 3995 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 3996 *p0++, *p1++); 3997 } 3998 3999 rc = sbuf_finish(sb); 4000 sbuf_delete(sb); 4001 4002 return (rc); 4003 } 4004 4005 struct mem_desc { 4006 unsigned int base; 4007 unsigned int limit; 4008 unsigned int idx; 4009 }; 4010 4011 static int 4012 mem_desc_cmp(const void *a, const void *b) 4013 { 4014 return ((const struct mem_desc *)a)->base - 4015 ((const struct mem_desc *)b)->base; 4016 } 4017 4018 static void 4019 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 4020 unsigned int to) 4021 { 4022 unsigned int size; 4023 4024 size = to - from + 1; 4025 if (size == 0) 4026 return; 4027 4028 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 4029 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 4030 } 4031 4032 static int 4033 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 4034 { 4035 struct adapter *sc = arg1; 4036 struct sbuf *sb; 4037 int rc, i, n; 4038 uint32_t lo, hi; 4039 static const char *memory[] = { "EDC0:", "EDC1:", "MC:" }; 4040 static const char *region[] = { 4041 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 4042 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 4043 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 4044 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 4045 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:", 4046 "ULPTX state:", "On-chip queues:" 4047 }; 4048 struct mem_desc avail[3]; 4049 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 4050 struct mem_desc *md = mem; 4051 4052 rc = sysctl_wire_old_buffer(req, 0); 4053 if (rc != 0) 4054 return (rc); 4055 4056 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4057 if (sb == NULL) 4058 return (ENOMEM); 4059 4060 for (i = 0; i < nitems(mem); i++) { 4061 mem[i].limit = 0; 4062 mem[i].idx = i; 4063 } 4064 4065 /* Find and sort the populated memory ranges */ 4066 i = 0; 4067 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 4068 if (lo & F_EDRAM0_ENABLE) { 4069 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 4070 avail[i].base = G_EDRAM0_BASE(hi) << 20; 4071 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 4072 avail[i].idx = 0; 4073 i++; 4074 } 4075 if (lo & F_EDRAM1_ENABLE) { 4076 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 4077 avail[i].base = G_EDRAM1_BASE(hi) << 20; 4078 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 4079 avail[i].idx = 1; 4080 i++; 4081 } 4082 if (lo & F_EXT_MEM_ENABLE) { 4083 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 4084 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 4085 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20); 4086 avail[i].idx = 2; 4087 i++; 4088 } 4089 if (!i) /* no memory available */ 4090 return 0; 4091 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 4092 4093 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 4094 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 4095 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 4096 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4097 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 4098 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 4099 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 4100 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 4101 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 4102 4103 /* the next few have explicit upper bounds */ 4104 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 4105 md->limit = md->base - 1 + 4106 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 4107 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 4108 md++; 4109 4110 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 4111 md->limit = md->base - 1 + 4112 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 4113 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 4114 md++; 4115 4116 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4117 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 4118 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 4119 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 4120 } else { 4121 md->base = 0; 4122 md->idx = nitems(region); /* hide it */ 4123 } 4124 md++; 4125 4126 #define ulp_region(reg) \ 4127 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 4128 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 4129 4130 ulp_region(RX_ISCSI); 4131 ulp_region(RX_TDDP); 4132 ulp_region(TX_TPT); 4133 ulp_region(RX_STAG); 4134 ulp_region(RX_RQ); 4135 ulp_region(RX_RQUDP); 4136 ulp_region(RX_PBL); 4137 ulp_region(TX_PBL); 4138 #undef ulp_region 4139 4140 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 4141 md->limit = md->base + sc->tids.ntids - 1; 4142 md++; 4143 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 4144 md->limit = md->base + sc->tids.ntids - 1; 4145 md++; 4146 4147 md->base = sc->vres.ocq.start; 4148 if (sc->vres.ocq.size) 4149 md->limit = md->base + sc->vres.ocq.size - 1; 4150 else 4151 md->idx = nitems(region); /* hide it */ 4152 md++; 4153 4154 /* add any address-space holes, there can be up to 3 */ 4155 for (n = 0; n < i - 1; n++) 4156 if (avail[n].limit < avail[n + 1].base) 4157 (md++)->base = avail[n].limit; 4158 if (avail[n].limit) 4159 (md++)->base = avail[n].limit; 4160 4161 n = md - mem; 4162 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 4163 4164 for (lo = 0; lo < i; lo++) 4165 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 4166 avail[lo].limit - 1); 4167 4168 sbuf_printf(sb, "\n"); 4169 for (i = 0; i < n; i++) { 4170 if (mem[i].idx >= nitems(region)) 4171 continue; /* skip holes */ 4172 if (!mem[i].limit) 4173 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 4174 mem_region_show(sb, region[mem[i].idx], mem[i].base, 4175 mem[i].limit); 4176 } 4177 4178 sbuf_printf(sb, "\n"); 4179 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 4180 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 4181 mem_region_show(sb, "uP RAM:", lo, hi); 4182 4183 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 4184 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 4185 mem_region_show(sb, "uP Extmem2:", lo, hi); 4186 4187 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 4188 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 4189 G_PMRXMAXPAGE(lo), 4190 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 4191 (lo & F_PMRXNUMCHN) ? 2 : 1); 4192 4193 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 4194 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 4195 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 4196 G_PMTXMAXPAGE(lo), 4197 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 4198 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 4199 sbuf_printf(sb, "%u p-structs\n", 4200 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 4201 4202 for (i = 0; i < 4; i++) { 4203 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 4204 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 4205 i, G_USED(lo), G_ALLOC(lo)); 4206 } 4207 for (i = 0; i < 4; i++) { 4208 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 4209 sbuf_printf(sb, 4210 "\nLoopback %d using %u pages out of %u allocated", 4211 i, G_USED(lo), G_ALLOC(lo)); 4212 } 4213 4214 rc = sbuf_finish(sb); 4215 sbuf_delete(sb); 4216 4217 return (rc); 4218 } 4219 4220 static int 4221 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 4222 { 4223 struct adapter *sc = arg1; 4224 struct sbuf *sb; 4225 int rc; 4226 uint16_t mtus[NMTUS]; 4227 4228 rc = sysctl_wire_old_buffer(req, 0); 4229 if (rc != 0) 4230 return (rc); 4231 4232 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4233 if (sb == NULL) 4234 return (ENOMEM); 4235 4236 t4_read_mtu_tbl(sc, mtus, NULL); 4237 4238 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 4239 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 4240 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 4241 mtus[14], mtus[15]); 4242 4243 rc = sbuf_finish(sb); 4244 sbuf_delete(sb); 4245 4246 return (rc); 4247 } 4248 4249 static int 4250 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 4251 { 4252 struct adapter *sc = arg1; 4253 struct sbuf *sb; 4254 int rc, i; 4255 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 4256 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 4257 static const char *pm_stats[] = { 4258 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 4259 }; 4260 4261 rc = sysctl_wire_old_buffer(req, 0); 4262 if (rc != 0) 4263 return (rc); 4264 4265 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4266 if (sb == NULL) 4267 return (ENOMEM); 4268 4269 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 4270 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 4271 4272 sbuf_printf(sb, " Tx count Tx cycles " 4273 "Rx count Rx cycles"); 4274 for (i = 0; i < PM_NSTATS; i++) 4275 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 4276 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 4277 4278 rc = sbuf_finish(sb); 4279 sbuf_delete(sb); 4280 4281 return (rc); 4282 } 4283 4284 static int 4285 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 4286 { 4287 struct adapter *sc = arg1; 4288 struct sbuf *sb; 4289 int rc; 4290 struct tp_rdma_stats stats; 4291 4292 rc = sysctl_wire_old_buffer(req, 0); 4293 if (rc != 0) 4294 return (rc); 4295 4296 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4297 if (sb == NULL) 4298 return (ENOMEM); 4299 4300 t4_tp_get_rdma_stats(sc, &stats); 4301 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 4302 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 4303 4304 rc = sbuf_finish(sb); 4305 sbuf_delete(sb); 4306 4307 return (rc); 4308 } 4309 4310 static int 4311 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 4312 { 4313 struct adapter *sc = arg1; 4314 struct sbuf *sb; 4315 int rc; 4316 struct tp_tcp_stats v4, v6; 4317 4318 rc = sysctl_wire_old_buffer(req, 0); 4319 if (rc != 0) 4320 return (rc); 4321 4322 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4323 if (sb == NULL) 4324 return (ENOMEM); 4325 4326 t4_tp_get_tcp_stats(sc, &v4, &v6); 4327 sbuf_printf(sb, 4328 " IP IPv6\n"); 4329 sbuf_printf(sb, "OutRsts: %20u %20u\n", 4330 v4.tcpOutRsts, v6.tcpOutRsts); 4331 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 4332 v4.tcpInSegs, v6.tcpInSegs); 4333 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 4334 v4.tcpOutSegs, v6.tcpOutSegs); 4335 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 4336 v4.tcpRetransSegs, v6.tcpRetransSegs); 4337 4338 rc = sbuf_finish(sb); 4339 sbuf_delete(sb); 4340 4341 return (rc); 4342 } 4343 4344 static int 4345 sysctl_tids(SYSCTL_HANDLER_ARGS) 4346 { 4347 struct adapter *sc = arg1; 4348 struct sbuf *sb; 4349 int rc; 4350 struct tid_info *t = &sc->tids; 4351 4352 rc = sysctl_wire_old_buffer(req, 0); 4353 if (rc != 0) 4354 return (rc); 4355 4356 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4357 if (sb == NULL) 4358 return (ENOMEM); 4359 4360 if (t->natids) { 4361 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 4362 t->atids_in_use); 4363 } 4364 4365 if (t->ntids) { 4366 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4367 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 4368 4369 if (b) { 4370 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 4371 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4372 t->ntids - 1); 4373 } else { 4374 sbuf_printf(sb, "TID range: %u-%u", 4375 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4376 t->ntids - 1); 4377 } 4378 } else 4379 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 4380 sbuf_printf(sb, ", in use: %u\n", 4381 atomic_load_acq_int(&t->tids_in_use)); 4382 } 4383 4384 if (t->nstids) { 4385 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 4386 t->stid_base + t->nstids - 1, t->stids_in_use); 4387 } 4388 4389 if (t->nftids) { 4390 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 4391 t->ftid_base + t->nftids - 1); 4392 } 4393 4394 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 4395 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 4396 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 4397 4398 rc = sbuf_finish(sb); 4399 sbuf_delete(sb); 4400 4401 return (rc); 4402 } 4403 4404 static int 4405 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 4406 { 4407 struct adapter *sc = arg1; 4408 struct sbuf *sb; 4409 int rc; 4410 struct tp_err_stats stats; 4411 4412 rc = sysctl_wire_old_buffer(req, 0); 4413 if (rc != 0) 4414 return (rc); 4415 4416 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4417 if (sb == NULL) 4418 return (ENOMEM); 4419 4420 t4_tp_get_err_stats(sc, &stats); 4421 4422 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4423 "channel 3\n"); 4424 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 4425 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 4426 stats.macInErrs[3]); 4427 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 4428 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 4429 stats.hdrInErrs[3]); 4430 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 4431 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 4432 stats.tcpInErrs[3]); 4433 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 4434 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 4435 stats.tcp6InErrs[3]); 4436 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 4437 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 4438 stats.tnlCongDrops[3]); 4439 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 4440 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 4441 stats.tnlTxDrops[3]); 4442 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 4443 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 4444 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 4445 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 4446 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 4447 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 4448 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 4449 stats.ofldNoNeigh, stats.ofldCongDefer); 4450 4451 rc = sbuf_finish(sb); 4452 sbuf_delete(sb); 4453 4454 return (rc); 4455 } 4456 4457 static int 4458 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 4459 { 4460 struct adapter *sc = arg1; 4461 struct sbuf *sb; 4462 int rc; 4463 u64 nrate[NCHAN], orate[NCHAN]; 4464 4465 rc = sysctl_wire_old_buffer(req, 0); 4466 if (rc != 0) 4467 return (rc); 4468 4469 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4470 if (sb == NULL) 4471 return (ENOMEM); 4472 4473 t4_get_chan_txrate(sc, nrate, orate); 4474 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4475 "channel 3\n"); 4476 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 4477 nrate[0], nrate[1], nrate[2], nrate[3]); 4478 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 4479 orate[0], orate[1], orate[2], orate[3]); 4480 4481 rc = sbuf_finish(sb); 4482 sbuf_delete(sb); 4483 4484 return (rc); 4485 } 4486 #endif 4487 4488 static inline void 4489 txq_start(struct ifnet *ifp, struct sge_txq *txq) 4490 { 4491 struct buf_ring *br; 4492 struct mbuf *m; 4493 4494 TXQ_LOCK_ASSERT_OWNED(txq); 4495 4496 br = txq->br; 4497 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 4498 if (m) 4499 t4_eth_tx(ifp, txq, m); 4500 } 4501 4502 void 4503 t4_tx_callout(void *arg) 4504 { 4505 struct sge_eq *eq = arg; 4506 struct adapter *sc; 4507 4508 if (EQ_TRYLOCK(eq) == 0) 4509 goto reschedule; 4510 4511 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 4512 EQ_UNLOCK(eq); 4513 reschedule: 4514 if (__predict_true(!(eq->flags && EQ_DOOMED))) 4515 callout_schedule(&eq->tx_callout, 1); 4516 return; 4517 } 4518 4519 EQ_LOCK_ASSERT_OWNED(eq); 4520 4521 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 4522 4523 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4524 struct sge_txq *txq = arg; 4525 struct port_info *pi = txq->ifp->if_softc; 4526 4527 sc = pi->adapter; 4528 } else { 4529 struct sge_wrq *wrq = arg; 4530 4531 sc = wrq->adapter; 4532 } 4533 4534 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 4535 } 4536 4537 EQ_UNLOCK(eq); 4538 } 4539 4540 void 4541 t4_tx_task(void *arg, int count) 4542 { 4543 struct sge_eq *eq = arg; 4544 4545 EQ_LOCK(eq); 4546 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4547 struct sge_txq *txq = arg; 4548 txq_start(txq->ifp, txq); 4549 } else { 4550 struct sge_wrq *wrq = arg; 4551 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 4552 } 4553 EQ_UNLOCK(eq); 4554 } 4555 4556 static uint32_t 4557 fconf_to_mode(uint32_t fconf) 4558 { 4559 uint32_t mode; 4560 4561 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 4562 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 4563 4564 if (fconf & F_FRAGMENTATION) 4565 mode |= T4_FILTER_IP_FRAGMENT; 4566 4567 if (fconf & F_MPSHITTYPE) 4568 mode |= T4_FILTER_MPS_HIT_TYPE; 4569 4570 if (fconf & F_MACMATCH) 4571 mode |= T4_FILTER_MAC_IDX; 4572 4573 if (fconf & F_ETHERTYPE) 4574 mode |= T4_FILTER_ETH_TYPE; 4575 4576 if (fconf & F_PROTOCOL) 4577 mode |= T4_FILTER_IP_PROTO; 4578 4579 if (fconf & F_TOS) 4580 mode |= T4_FILTER_IP_TOS; 4581 4582 if (fconf & F_VLAN) 4583 mode |= T4_FILTER_VLAN; 4584 4585 if (fconf & F_VNIC_ID) 4586 mode |= T4_FILTER_VNIC; 4587 4588 if (fconf & F_PORT) 4589 mode |= T4_FILTER_PORT; 4590 4591 if (fconf & F_FCOE) 4592 mode |= T4_FILTER_FCoE; 4593 4594 return (mode); 4595 } 4596 4597 static uint32_t 4598 mode_to_fconf(uint32_t mode) 4599 { 4600 uint32_t fconf = 0; 4601 4602 if (mode & T4_FILTER_IP_FRAGMENT) 4603 fconf |= F_FRAGMENTATION; 4604 4605 if (mode & T4_FILTER_MPS_HIT_TYPE) 4606 fconf |= F_MPSHITTYPE; 4607 4608 if (mode & T4_FILTER_MAC_IDX) 4609 fconf |= F_MACMATCH; 4610 4611 if (mode & T4_FILTER_ETH_TYPE) 4612 fconf |= F_ETHERTYPE; 4613 4614 if (mode & T4_FILTER_IP_PROTO) 4615 fconf |= F_PROTOCOL; 4616 4617 if (mode & T4_FILTER_IP_TOS) 4618 fconf |= F_TOS; 4619 4620 if (mode & T4_FILTER_VLAN) 4621 fconf |= F_VLAN; 4622 4623 if (mode & T4_FILTER_VNIC) 4624 fconf |= F_VNIC_ID; 4625 4626 if (mode & T4_FILTER_PORT) 4627 fconf |= F_PORT; 4628 4629 if (mode & T4_FILTER_FCoE) 4630 fconf |= F_FCOE; 4631 4632 return (fconf); 4633 } 4634 4635 static uint32_t 4636 fspec_to_fconf(struct t4_filter_specification *fs) 4637 { 4638 uint32_t fconf = 0; 4639 4640 if (fs->val.frag || fs->mask.frag) 4641 fconf |= F_FRAGMENTATION; 4642 4643 if (fs->val.matchtype || fs->mask.matchtype) 4644 fconf |= F_MPSHITTYPE; 4645 4646 if (fs->val.macidx || fs->mask.macidx) 4647 fconf |= F_MACMATCH; 4648 4649 if (fs->val.ethtype || fs->mask.ethtype) 4650 fconf |= F_ETHERTYPE; 4651 4652 if (fs->val.proto || fs->mask.proto) 4653 fconf |= F_PROTOCOL; 4654 4655 if (fs->val.tos || fs->mask.tos) 4656 fconf |= F_TOS; 4657 4658 if (fs->val.vlan_vld || fs->mask.vlan_vld) 4659 fconf |= F_VLAN; 4660 4661 if (fs->val.vnic_vld || fs->mask.vnic_vld) 4662 fconf |= F_VNIC_ID; 4663 4664 if (fs->val.iport || fs->mask.iport) 4665 fconf |= F_PORT; 4666 4667 if (fs->val.fcoe || fs->mask.fcoe) 4668 fconf |= F_FCOE; 4669 4670 return (fconf); 4671 } 4672 4673 static int 4674 get_filter_mode(struct adapter *sc, uint32_t *mode) 4675 { 4676 uint32_t fconf; 4677 4678 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 4679 A_TP_VLAN_PRI_MAP); 4680 4681 if (sc->filter_mode != fconf) { 4682 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 4683 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 4684 sc->filter_mode = fconf; 4685 } 4686 4687 *mode = fconf_to_mode(sc->filter_mode); 4688 4689 return (0); 4690 } 4691 4692 static int 4693 set_filter_mode(struct adapter *sc, uint32_t mode) 4694 { 4695 uint32_t fconf; 4696 int rc; 4697 4698 fconf = mode_to_fconf(mode); 4699 4700 ADAPTER_LOCK(sc); 4701 if (IS_BUSY(sc)) { 4702 rc = EAGAIN; 4703 goto done; 4704 } 4705 4706 if (sc->tids.ftids_in_use > 0) { 4707 rc = EBUSY; 4708 goto done; 4709 } 4710 4711 #ifdef TCP_OFFLOAD 4712 if (sc->offload_map) { 4713 rc = EBUSY; 4714 goto done; 4715 } 4716 #endif 4717 4718 #ifdef notyet 4719 rc = -t4_set_filter_mode(sc, fconf); 4720 if (rc == 0) 4721 sc->filter_mode = fconf; 4722 #else 4723 rc = ENOTSUP; 4724 #endif 4725 4726 done: 4727 ADAPTER_UNLOCK(sc); 4728 return (rc); 4729 } 4730 4731 static inline uint64_t 4732 get_filter_hits(struct adapter *sc, uint32_t fid) 4733 { 4734 uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4735 uint64_t hits; 4736 4737 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0), 4738 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 4739 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0)); 4740 hits = t4_read_reg64(sc, MEMWIN0_BASE + 16); 4741 4742 return (be64toh(hits)); 4743 } 4744 4745 static int 4746 get_filter(struct adapter *sc, struct t4_filter *t) 4747 { 4748 int i, nfilters = sc->tids.nftids; 4749 struct filter_entry *f; 4750 4751 ADAPTER_LOCK_ASSERT_OWNED(sc); 4752 4753 if (IS_BUSY(sc)) 4754 return (EAGAIN); 4755 4756 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 4757 t->idx >= nfilters) { 4758 t->idx = 0xffffffff; 4759 return (0); 4760 } 4761 4762 f = &sc->tids.ftid_tab[t->idx]; 4763 for (i = t->idx; i < nfilters; i++, f++) { 4764 if (f->valid) { 4765 t->idx = i; 4766 t->l2tidx = f->l2t ? f->l2t->idx : 0; 4767 t->smtidx = f->smtidx; 4768 if (f->fs.hitcnts) 4769 t->hits = get_filter_hits(sc, t->idx); 4770 else 4771 t->hits = UINT64_MAX; 4772 t->fs = f->fs; 4773 4774 return (0); 4775 } 4776 } 4777 4778 t->idx = 0xffffffff; 4779 return (0); 4780 } 4781 4782 static int 4783 set_filter(struct adapter *sc, struct t4_filter *t) 4784 { 4785 unsigned int nfilters, nports; 4786 struct filter_entry *f; 4787 int i; 4788 4789 ADAPTER_LOCK_ASSERT_OWNED(sc); 4790 4791 nfilters = sc->tids.nftids; 4792 nports = sc->params.nports; 4793 4794 if (nfilters == 0) 4795 return (ENOTSUP); 4796 4797 if (!(sc->flags & FULL_INIT_DONE)) 4798 return (EAGAIN); 4799 4800 if (t->idx >= nfilters) 4801 return (EINVAL); 4802 4803 /* Validate against the global filter mode */ 4804 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) 4805 return (E2BIG); 4806 4807 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) 4808 return (EINVAL); 4809 4810 if (t->fs.val.iport >= nports) 4811 return (EINVAL); 4812 4813 /* Can't specify an iq if not steering to it */ 4814 if (!t->fs.dirsteer && t->fs.iq) 4815 return (EINVAL); 4816 4817 /* IPv6 filter idx must be 4 aligned */ 4818 if (t->fs.type == 1 && 4819 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) 4820 return (EINVAL); 4821 4822 if (sc->tids.ftid_tab == NULL) { 4823 KASSERT(sc->tids.ftids_in_use == 0, 4824 ("%s: no memory allocated but filters_in_use > 0", 4825 __func__)); 4826 4827 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 4828 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 4829 if (sc->tids.ftid_tab == NULL) 4830 return (ENOMEM); 4831 } 4832 4833 for (i = 0; i < 4; i++) { 4834 f = &sc->tids.ftid_tab[t->idx + i]; 4835 4836 if (f->pending || f->valid) 4837 return (EBUSY); 4838 if (f->locked) 4839 return (EPERM); 4840 4841 if (t->fs.type == 0) 4842 break; 4843 } 4844 4845 f = &sc->tids.ftid_tab[t->idx]; 4846 f->fs = t->fs; 4847 4848 return set_filter_wr(sc, t->idx); 4849 } 4850 4851 static int 4852 del_filter(struct adapter *sc, struct t4_filter *t) 4853 { 4854 unsigned int nfilters; 4855 struct filter_entry *f; 4856 4857 ADAPTER_LOCK_ASSERT_OWNED(sc); 4858 4859 if (IS_BUSY(sc)) 4860 return (EAGAIN); 4861 4862 nfilters = sc->tids.nftids; 4863 4864 if (nfilters == 0) 4865 return (ENOTSUP); 4866 4867 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 4868 t->idx >= nfilters) 4869 return (EINVAL); 4870 4871 if (!(sc->flags & FULL_INIT_DONE)) 4872 return (EAGAIN); 4873 4874 f = &sc->tids.ftid_tab[t->idx]; 4875 4876 if (f->pending) 4877 return (EBUSY); 4878 if (f->locked) 4879 return (EPERM); 4880 4881 if (f->valid) { 4882 t->fs = f->fs; /* extra info for the caller */ 4883 return del_filter_wr(sc, t->idx); 4884 } 4885 4886 return (0); 4887 } 4888 4889 static void 4890 clear_filter(struct filter_entry *f) 4891 { 4892 if (f->l2t) 4893 t4_l2t_release(f->l2t); 4894 4895 bzero(f, sizeof (*f)); 4896 } 4897 4898 static int 4899 set_filter_wr(struct adapter *sc, int fidx) 4900 { 4901 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4902 struct wrqe *wr; 4903 struct fw_filter_wr *fwr; 4904 unsigned int ftid; 4905 4906 ADAPTER_LOCK_ASSERT_OWNED(sc); 4907 4908 if (f->fs.newdmac || f->fs.newvlan) { 4909 /* This filter needs an L2T entry; allocate one. */ 4910 f->l2t = t4_l2t_alloc_switching(sc->l2t); 4911 if (f->l2t == NULL) 4912 return (EAGAIN); 4913 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 4914 f->fs.dmac)) { 4915 t4_l2t_release(f->l2t); 4916 f->l2t = NULL; 4917 return (ENOMEM); 4918 } 4919 } 4920 4921 ftid = sc->tids.ftid_base + fidx; 4922 4923 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 4924 if (wr == NULL) 4925 return (ENOMEM); 4926 4927 fwr = wrtod(wr); 4928 bzero(fwr, sizeof (*fwr)); 4929 4930 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 4931 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 4932 fwr->tid_to_iq = 4933 htobe32(V_FW_FILTER_WR_TID(ftid) | 4934 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 4935 V_FW_FILTER_WR_NOREPLY(0) | 4936 V_FW_FILTER_WR_IQ(f->fs.iq)); 4937 fwr->del_filter_to_l2tix = 4938 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 4939 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 4940 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 4941 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 4942 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 4943 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 4944 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 4945 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 4946 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 4947 f->fs.newvlan == VLAN_REWRITE) | 4948 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 4949 f->fs.newvlan == VLAN_REWRITE) | 4950 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 4951 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 4952 V_FW_FILTER_WR_PRIO(f->fs.prio) | 4953 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 4954 fwr->ethtype = htobe16(f->fs.val.ethtype); 4955 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 4956 fwr->frag_to_ovlan_vldm = 4957 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 4958 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 4959 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 4960 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 4961 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 4962 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 4963 fwr->smac_sel = 0; 4964 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 4965 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 4966 fwr->maci_to_matchtypem = 4967 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 4968 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 4969 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 4970 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 4971 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 4972 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 4973 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 4974 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 4975 fwr->ptcl = f->fs.val.proto; 4976 fwr->ptclm = f->fs.mask.proto; 4977 fwr->ttyp = f->fs.val.tos; 4978 fwr->ttypm = f->fs.mask.tos; 4979 fwr->ivlan = htobe16(f->fs.val.vlan); 4980 fwr->ivlanm = htobe16(f->fs.mask.vlan); 4981 fwr->ovlan = htobe16(f->fs.val.vnic); 4982 fwr->ovlanm = htobe16(f->fs.mask.vnic); 4983 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 4984 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 4985 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 4986 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 4987 fwr->lp = htobe16(f->fs.val.dport); 4988 fwr->lpm = htobe16(f->fs.mask.dport); 4989 fwr->fp = htobe16(f->fs.val.sport); 4990 fwr->fpm = htobe16(f->fs.mask.sport); 4991 if (f->fs.newsmac) 4992 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 4993 4994 f->pending = 1; 4995 sc->tids.ftids_in_use++; 4996 4997 t4_wrq_tx(sc, wr); 4998 return (0); 4999 } 5000 5001 static int 5002 del_filter_wr(struct adapter *sc, int fidx) 5003 { 5004 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 5005 struct wrqe *wr; 5006 struct fw_filter_wr *fwr; 5007 unsigned int ftid; 5008 5009 ADAPTER_LOCK_ASSERT_OWNED(sc); 5010 5011 ftid = sc->tids.ftid_base + fidx; 5012 5013 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 5014 if (wr == NULL) 5015 return (ENOMEM); 5016 fwr = wrtod(wr); 5017 bzero(fwr, sizeof (*fwr)); 5018 5019 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 5020 5021 f->pending = 1; 5022 t4_wrq_tx(sc, wr); 5023 return (0); 5024 } 5025 5026 int 5027 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5028 { 5029 struct adapter *sc = iq->adapter; 5030 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 5031 unsigned int idx = GET_TID(rpl); 5032 5033 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 5034 rss->opcode)); 5035 5036 if (idx >= sc->tids.ftid_base && 5037 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 5038 unsigned int rc = G_COOKIE(rpl->cookie); 5039 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 5040 5041 ADAPTER_LOCK(sc); 5042 if (rc == FW_FILTER_WR_FLT_ADDED) { 5043 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 5044 f->pending = 0; /* asynchronous setup completed */ 5045 f->valid = 1; 5046 } else { 5047 if (rc != FW_FILTER_WR_FLT_DELETED) { 5048 /* Add or delete failed, display an error */ 5049 log(LOG_ERR, 5050 "filter %u setup failed with error %u\n", 5051 idx, rc); 5052 } 5053 5054 clear_filter(f); 5055 sc->tids.ftids_in_use--; 5056 } 5057 ADAPTER_UNLOCK(sc); 5058 } 5059 5060 return (0); 5061 } 5062 5063 static int 5064 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 5065 { 5066 int rc = EINVAL; 5067 5068 if (cntxt->cid > M_CTXTQID) 5069 return (rc); 5070 5071 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 5072 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 5073 return (rc); 5074 5075 if (sc->flags & FW_OK) { 5076 ADAPTER_LOCK(sc); /* Avoid parallel t4_wr_mbox */ 5077 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 5078 &cntxt->data[0]); 5079 ADAPTER_UNLOCK(sc); 5080 } 5081 5082 if (rc != 0) { 5083 /* Read via firmware failed or wasn't even attempted */ 5084 5085 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, 5086 &cntxt->data[0]); 5087 } 5088 5089 return (rc); 5090 } 5091 5092 static int 5093 read_card_mem(struct adapter *sc, struct t4_mem_range *mr) 5094 { 5095 uint32_t base, size, lo, hi, win, off, remaining, i, n; 5096 uint32_t *buf, *b; 5097 int rc; 5098 5099 /* reads are in multiples of 32 bits */ 5100 if (mr->addr & 3 || mr->len & 3 || mr->len == 0) 5101 return (EINVAL); 5102 5103 /* 5104 * We don't want to deal with potential holes so we mandate that the 5105 * requested region must lie entirely within one of the 3 memories. 5106 */ 5107 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5108 if (lo & F_EDRAM0_ENABLE) { 5109 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5110 base = G_EDRAM0_BASE(hi) << 20; 5111 size = G_EDRAM0_SIZE(hi) << 20; 5112 if (size > 0 && 5113 mr->addr >= base && mr->addr < base + size && 5114 mr->addr + mr->len <= base + size) 5115 goto proceed; 5116 } 5117 if (lo & F_EDRAM1_ENABLE) { 5118 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5119 base = G_EDRAM1_BASE(hi) << 20; 5120 size = G_EDRAM1_SIZE(hi) << 20; 5121 if (size > 0 && 5122 mr->addr >= base && mr->addr < base + size && 5123 mr->addr + mr->len <= base + size) 5124 goto proceed; 5125 } 5126 if (lo & F_EXT_MEM_ENABLE) { 5127 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5128 base = G_EXT_MEM_BASE(hi) << 20; 5129 size = G_EXT_MEM_SIZE(hi) << 20; 5130 if (size > 0 && 5131 mr->addr >= base && mr->addr < base + size && 5132 mr->addr + mr->len <= base + size) 5133 goto proceed; 5134 } 5135 return (ENXIO); 5136 5137 proceed: 5138 buf = b = malloc(mr->len, M_CXGBE, M_WAITOK); 5139 5140 /* 5141 * Position the PCIe window (we use memwin2) to the 16B aligned area 5142 * just at/before the requested region. 5143 */ 5144 win = mr->addr & ~0xf; 5145 off = mr->addr - win; /* offset of the requested region in the win */ 5146 remaining = mr->len; 5147 5148 while (remaining) { 5149 t4_write_reg(sc, 5150 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 5151 t4_read_reg(sc, 5152 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 5153 5154 /* number of bytes that we'll copy in the inner loop */ 5155 n = min(remaining, MEMWIN2_APERTURE - off); 5156 5157 for (i = 0; i < n; i += 4, remaining -= 4) 5158 *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i); 5159 5160 win += MEMWIN2_APERTURE; 5161 off = 0; 5162 } 5163 5164 rc = copyout(buf, mr->data, mr->len); 5165 free(buf, M_CXGBE); 5166 5167 return (rc); 5168 } 5169 5170 static int 5171 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 5172 { 5173 int rc; 5174 5175 ADAPTER_LOCK_ASSERT_OWNED(sc); /* for mbox */ 5176 5177 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 5178 return (EINVAL); 5179 5180 if (i2cd->len > 1) { 5181 /* XXX: need fw support for longer reads in one go */ 5182 return (ENOTSUP); 5183 } 5184 5185 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 5186 i2cd->offset, &i2cd->data[0]); 5187 5188 return (rc); 5189 } 5190 5191 int 5192 t4_os_find_pci_capability(struct adapter *sc, int cap) 5193 { 5194 int i; 5195 5196 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 5197 } 5198 5199 int 5200 t4_os_pci_save_state(struct adapter *sc) 5201 { 5202 device_t dev; 5203 struct pci_devinfo *dinfo; 5204 5205 dev = sc->dev; 5206 dinfo = device_get_ivars(dev); 5207 5208 pci_cfg_save(dev, dinfo, 0); 5209 return (0); 5210 } 5211 5212 int 5213 t4_os_pci_restore_state(struct adapter *sc) 5214 { 5215 device_t dev; 5216 struct pci_devinfo *dinfo; 5217 5218 dev = sc->dev; 5219 dinfo = device_get_ivars(dev); 5220 5221 pci_cfg_restore(dev, dinfo); 5222 return (0); 5223 } 5224 5225 void 5226 t4_os_portmod_changed(const struct adapter *sc, int idx) 5227 { 5228 struct port_info *pi = sc->port[idx]; 5229 static const char *mod_str[] = { 5230 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 5231 }; 5232 5233 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 5234 if_printf(pi->ifp, "transceiver unplugged.\n"); 5235 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 5236 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 5237 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 5238 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 5239 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 5240 if_printf(pi->ifp, "%s transceiver inserted.\n", 5241 mod_str[pi->mod_type]); 5242 } else { 5243 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 5244 pi->mod_type); 5245 } 5246 } 5247 5248 void 5249 t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 5250 { 5251 struct port_info *pi = sc->port[idx]; 5252 struct ifnet *ifp = pi->ifp; 5253 5254 if (link_stat) { 5255 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 5256 if_link_state_change(ifp, LINK_STATE_UP); 5257 } else 5258 if_link_state_change(ifp, LINK_STATE_DOWN); 5259 } 5260 5261 void 5262 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 5263 { 5264 struct adapter *sc; 5265 5266 mtx_lock(&t4_list_lock); 5267 SLIST_FOREACH(sc, &t4_list, link) { 5268 /* 5269 * func should not make any assumptions about what state sc is 5270 * in - the only guarantee is that sc->sc_lock is a valid lock. 5271 */ 5272 func(sc, arg); 5273 } 5274 mtx_unlock(&t4_list_lock); 5275 } 5276 5277 static int 5278 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 5279 { 5280 return (0); 5281 } 5282 5283 static int 5284 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 5285 { 5286 return (0); 5287 } 5288 5289 static int 5290 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 5291 struct thread *td) 5292 { 5293 int rc; 5294 struct adapter *sc = dev->si_drv1; 5295 5296 rc = priv_check(td, PRIV_DRIVER); 5297 if (rc != 0) 5298 return (rc); 5299 5300 switch (cmd) { 5301 case CHELSIO_T4_GETREG: { 5302 struct t4_reg *edata = (struct t4_reg *)data; 5303 5304 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5305 return (EFAULT); 5306 5307 if (edata->size == 4) 5308 edata->val = t4_read_reg(sc, edata->addr); 5309 else if (edata->size == 8) 5310 edata->val = t4_read_reg64(sc, edata->addr); 5311 else 5312 return (EINVAL); 5313 5314 break; 5315 } 5316 case CHELSIO_T4_SETREG: { 5317 struct t4_reg *edata = (struct t4_reg *)data; 5318 5319 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5320 return (EFAULT); 5321 5322 if (edata->size == 4) { 5323 if (edata->val & 0xffffffff00000000) 5324 return (EINVAL); 5325 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 5326 } else if (edata->size == 8) 5327 t4_write_reg64(sc, edata->addr, edata->val); 5328 else 5329 return (EINVAL); 5330 break; 5331 } 5332 case CHELSIO_T4_REGDUMP: { 5333 struct t4_regdump *regs = (struct t4_regdump *)data; 5334 int reglen = T4_REGDUMP_SIZE; 5335 uint8_t *buf; 5336 5337 if (regs->len < reglen) { 5338 regs->len = reglen; /* hint to the caller */ 5339 return (ENOBUFS); 5340 } 5341 5342 regs->len = reglen; 5343 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 5344 t4_get_regs(sc, regs, buf); 5345 rc = copyout(buf, regs->data, reglen); 5346 free(buf, M_CXGBE); 5347 break; 5348 } 5349 case CHELSIO_T4_GET_FILTER_MODE: 5350 rc = get_filter_mode(sc, (uint32_t *)data); 5351 break; 5352 case CHELSIO_T4_SET_FILTER_MODE: 5353 rc = set_filter_mode(sc, *(uint32_t *)data); 5354 break; 5355 case CHELSIO_T4_GET_FILTER: 5356 ADAPTER_LOCK(sc); 5357 rc = get_filter(sc, (struct t4_filter *)data); 5358 ADAPTER_UNLOCK(sc); 5359 break; 5360 case CHELSIO_T4_SET_FILTER: 5361 ADAPTER_LOCK(sc); 5362 rc = set_filter(sc, (struct t4_filter *)data); 5363 ADAPTER_UNLOCK(sc); 5364 break; 5365 case CHELSIO_T4_DEL_FILTER: 5366 ADAPTER_LOCK(sc); 5367 rc = del_filter(sc, (struct t4_filter *)data); 5368 ADAPTER_UNLOCK(sc); 5369 break; 5370 case CHELSIO_T4_GET_SGE_CONTEXT: 5371 rc = get_sge_context(sc, (struct t4_sge_context *)data); 5372 break; 5373 case CHELSIO_T4_LOAD_FW: { 5374 struct t4_data *fw = (struct t4_data *)data; 5375 uint8_t *fw_data; 5376 5377 if (sc->flags & FULL_INIT_DONE) 5378 return (EBUSY); 5379 5380 fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT); 5381 if (fw_data == NULL) 5382 return (ENOMEM); 5383 5384 rc = copyin(fw->data, fw_data, fw->len); 5385 if (rc == 0) 5386 rc = -t4_load_fw(sc, fw_data, fw->len); 5387 5388 free(fw_data, M_CXGBE); 5389 break; 5390 } 5391 case CHELSIO_T4_GET_MEM: 5392 rc = read_card_mem(sc, (struct t4_mem_range *)data); 5393 break; 5394 case CHELSIO_T4_GET_I2C: 5395 ADAPTER_LOCK(sc); 5396 rc = read_i2c(sc, (struct t4_i2c_data *)data); 5397 ADAPTER_UNLOCK(sc); 5398 break; 5399 case CHELSIO_T4_CLEAR_STATS: { 5400 u_int port_id = *(uint32_t *)data; 5401 5402 if (port_id >= sc->params.nports) 5403 return (EINVAL); 5404 5405 t4_clr_port_stats(sc, port_id); 5406 break; 5407 } 5408 default: 5409 rc = EINVAL; 5410 } 5411 5412 return (rc); 5413 } 5414 5415 #ifdef TCP_OFFLOAD 5416 static int 5417 toe_capability(struct port_info *pi, int enable) 5418 { 5419 int rc; 5420 struct adapter *sc = pi->adapter; 5421 5422 ADAPTER_LOCK_ASSERT_OWNED(sc); 5423 5424 if (!is_offload(sc)) 5425 return (ENODEV); 5426 5427 if (enable) { 5428 if (!(sc->flags & FULL_INIT_DONE)) { 5429 log(LOG_WARNING, 5430 "You must enable a cxgbe interface first\n"); 5431 return (EAGAIN); 5432 } 5433 5434 if (isset(&sc->offload_map, pi->port_id)) 5435 return (0); 5436 5437 if (!(sc->flags & TOM_INIT_DONE)) { 5438 rc = t4_activate_uld(sc, ULD_TOM); 5439 if (rc == EAGAIN) { 5440 log(LOG_WARNING, 5441 "You must kldload t4_tom.ko before trying " 5442 "to enable TOE on a cxgbe interface.\n"); 5443 } 5444 if (rc != 0) 5445 return (rc); 5446 KASSERT(sc->tom_softc != NULL, 5447 ("%s: TOM activated but softc NULL", __func__)); 5448 KASSERT(sc->flags & TOM_INIT_DONE, 5449 ("%s: TOM activated but flag not set", __func__)); 5450 } 5451 5452 setbit(&sc->offload_map, pi->port_id); 5453 } else { 5454 if (!isset(&sc->offload_map, pi->port_id)) 5455 return (0); 5456 5457 KASSERT(sc->flags & TOM_INIT_DONE, 5458 ("%s: TOM never initialized?", __func__)); 5459 clrbit(&sc->offload_map, pi->port_id); 5460 } 5461 5462 return (0); 5463 } 5464 5465 /* 5466 * Add an upper layer driver to the global list. 5467 */ 5468 int 5469 t4_register_uld(struct uld_info *ui) 5470 { 5471 int rc = 0; 5472 struct uld_info *u; 5473 5474 mtx_lock(&t4_uld_list_lock); 5475 SLIST_FOREACH(u, &t4_uld_list, link) { 5476 if (u->uld_id == ui->uld_id) { 5477 rc = EEXIST; 5478 goto done; 5479 } 5480 } 5481 5482 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 5483 ui->refcount = 0; 5484 done: 5485 mtx_unlock(&t4_uld_list_lock); 5486 return (rc); 5487 } 5488 5489 int 5490 t4_unregister_uld(struct uld_info *ui) 5491 { 5492 int rc = EINVAL; 5493 struct uld_info *u; 5494 5495 mtx_lock(&t4_uld_list_lock); 5496 5497 SLIST_FOREACH(u, &t4_uld_list, link) { 5498 if (u == ui) { 5499 if (ui->refcount > 0) { 5500 rc = EBUSY; 5501 goto done; 5502 } 5503 5504 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 5505 rc = 0; 5506 goto done; 5507 } 5508 } 5509 done: 5510 mtx_unlock(&t4_uld_list_lock); 5511 return (rc); 5512 } 5513 5514 int 5515 t4_activate_uld(struct adapter *sc, int id) 5516 { 5517 int rc = EAGAIN; 5518 struct uld_info *ui; 5519 5520 mtx_lock(&t4_uld_list_lock); 5521 5522 SLIST_FOREACH(ui, &t4_uld_list, link) { 5523 if (ui->uld_id == id) { 5524 rc = ui->activate(sc); 5525 if (rc == 0) 5526 ui->refcount++; 5527 goto done; 5528 } 5529 } 5530 done: 5531 mtx_unlock(&t4_uld_list_lock); 5532 5533 return (rc); 5534 } 5535 5536 int 5537 t4_deactivate_uld(struct adapter *sc, int id) 5538 { 5539 int rc = EINVAL; 5540 struct uld_info *ui; 5541 5542 mtx_lock(&t4_uld_list_lock); 5543 5544 SLIST_FOREACH(ui, &t4_uld_list, link) { 5545 if (ui->uld_id == id) { 5546 rc = ui->deactivate(sc); 5547 if (rc == 0) 5548 ui->refcount--; 5549 goto done; 5550 } 5551 } 5552 done: 5553 mtx_unlock(&t4_uld_list_lock); 5554 5555 return (rc); 5556 } 5557 #endif 5558 5559 /* 5560 * Come up with reasonable defaults for some of the tunables, provided they're 5561 * not set by the user (in which case we'll use the values as is). 5562 */ 5563 static void 5564 tweak_tunables(void) 5565 { 5566 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 5567 5568 if (t4_ntxq10g < 1) 5569 t4_ntxq10g = min(nc, NTXQ_10G); 5570 5571 if (t4_ntxq1g < 1) 5572 t4_ntxq1g = min(nc, NTXQ_1G); 5573 5574 if (t4_nrxq10g < 1) 5575 t4_nrxq10g = min(nc, NRXQ_10G); 5576 5577 if (t4_nrxq1g < 1) 5578 t4_nrxq1g = min(nc, NRXQ_1G); 5579 5580 #ifdef TCP_OFFLOAD 5581 if (t4_nofldtxq10g < 1) 5582 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 5583 5584 if (t4_nofldtxq1g < 1) 5585 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 5586 5587 if (t4_nofldrxq10g < 1) 5588 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 5589 5590 if (t4_nofldrxq1g < 1) 5591 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 5592 5593 if (t4_toecaps_allowed == -1) 5594 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 5595 #else 5596 if (t4_toecaps_allowed == -1) 5597 t4_toecaps_allowed = 0; 5598 #endif 5599 5600 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 5601 t4_tmr_idx_10g = TMR_IDX_10G; 5602 5603 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 5604 t4_pktc_idx_10g = PKTC_IDX_10G; 5605 5606 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 5607 t4_tmr_idx_1g = TMR_IDX_1G; 5608 5609 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 5610 t4_pktc_idx_1g = PKTC_IDX_1G; 5611 5612 if (t4_qsize_txq < 128) 5613 t4_qsize_txq = 128; 5614 5615 if (t4_qsize_rxq < 128) 5616 t4_qsize_rxq = 128; 5617 while (t4_qsize_rxq & 7) 5618 t4_qsize_rxq++; 5619 5620 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 5621 } 5622 5623 static int 5624 t4_mod_event(module_t mod, int cmd, void *arg) 5625 { 5626 int rc = 0; 5627 5628 switch (cmd) { 5629 case MOD_LOAD: 5630 t4_sge_modload(); 5631 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 5632 SLIST_INIT(&t4_list); 5633 #ifdef TCP_OFFLOAD 5634 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 5635 SLIST_INIT(&t4_uld_list); 5636 #endif 5637 tweak_tunables(); 5638 break; 5639 5640 case MOD_UNLOAD: 5641 #ifdef TCP_OFFLOAD 5642 mtx_lock(&t4_uld_list_lock); 5643 if (!SLIST_EMPTY(&t4_uld_list)) { 5644 rc = EBUSY; 5645 mtx_unlock(&t4_uld_list_lock); 5646 break; 5647 } 5648 mtx_unlock(&t4_uld_list_lock); 5649 mtx_destroy(&t4_uld_list_lock); 5650 #endif 5651 mtx_lock(&t4_list_lock); 5652 if (!SLIST_EMPTY(&t4_list)) { 5653 rc = EBUSY; 5654 mtx_unlock(&t4_list_lock); 5655 break; 5656 } 5657 mtx_unlock(&t4_list_lock); 5658 mtx_destroy(&t4_list_lock); 5659 break; 5660 } 5661 5662 return (rc); 5663 } 5664 5665 static devclass_t t4_devclass; 5666 static devclass_t cxgbe_devclass; 5667 5668 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0); 5669 MODULE_VERSION(t4nex, 1); 5670 5671 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 5672 MODULE_VERSION(cxgbe, 1); 5673