1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 59 #include "common/common.h" 60 #include "common/t4_msg.h" 61 #include "common/t4_regs.h" 62 #include "common/t4_regs_values.h" 63 #include "t4_ioctl.h" 64 #include "t4_l2t.h" 65 66 /* T4 bus driver interface */ 67 static int t4_probe(device_t); 68 static int t4_attach(device_t); 69 static int t4_detach(device_t); 70 static device_method_t t4_methods[] = { 71 DEVMETHOD(device_probe, t4_probe), 72 DEVMETHOD(device_attach, t4_attach), 73 DEVMETHOD(device_detach, t4_detach), 74 75 DEVMETHOD_END 76 }; 77 static driver_t t4_driver = { 78 "t4nex", 79 t4_methods, 80 sizeof(struct adapter) 81 }; 82 83 84 /* T4 port (cxgbe) interface */ 85 static int cxgbe_probe(device_t); 86 static int cxgbe_attach(device_t); 87 static int cxgbe_detach(device_t); 88 static device_method_t cxgbe_methods[] = { 89 DEVMETHOD(device_probe, cxgbe_probe), 90 DEVMETHOD(device_attach, cxgbe_attach), 91 DEVMETHOD(device_detach, cxgbe_detach), 92 { 0, 0 } 93 }; 94 static driver_t cxgbe_driver = { 95 "cxgbe", 96 cxgbe_methods, 97 sizeof(struct port_info) 98 }; 99 100 static d_ioctl_t t4_ioctl; 101 static d_open_t t4_open; 102 static d_close_t t4_close; 103 104 static struct cdevsw t4_cdevsw = { 105 .d_version = D_VERSION, 106 .d_flags = 0, 107 .d_open = t4_open, 108 .d_close = t4_close, 109 .d_ioctl = t4_ioctl, 110 .d_name = "t4nex", 111 }; 112 113 /* ifnet + media interface */ 114 static void cxgbe_init(void *); 115 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 116 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 117 static void cxgbe_qflush(struct ifnet *); 118 static int cxgbe_media_change(struct ifnet *); 119 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 120 121 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services"); 122 123 /* 124 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 125 * then ADAPTER_LOCK, then t4_uld_list_lock. 126 */ 127 static struct mtx t4_list_lock; 128 static SLIST_HEAD(, adapter) t4_list; 129 #ifdef TCP_OFFLOAD 130 static struct mtx t4_uld_list_lock; 131 static SLIST_HEAD(, uld_info) t4_uld_list; 132 #endif 133 134 /* 135 * Tunables. See tweak_tunables() too. 136 */ 137 138 /* 139 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 140 */ 141 #define NTXQ_10G 16 142 static int t4_ntxq10g = -1; 143 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 144 145 #define NRXQ_10G 8 146 static int t4_nrxq10g = -1; 147 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 148 149 #define NTXQ_1G 4 150 static int t4_ntxq1g = -1; 151 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 152 153 #define NRXQ_1G 2 154 static int t4_nrxq1g = -1; 155 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 156 157 #ifdef TCP_OFFLOAD 158 #define NOFLDTXQ_10G 8 159 static int t4_nofldtxq10g = -1; 160 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 161 162 #define NOFLDRXQ_10G 2 163 static int t4_nofldrxq10g = -1; 164 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 165 166 #define NOFLDTXQ_1G 2 167 static int t4_nofldtxq1g = -1; 168 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 169 170 #define NOFLDRXQ_1G 1 171 static int t4_nofldrxq1g = -1; 172 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 173 #endif 174 175 /* 176 * Holdoff parameters for 10G and 1G ports. 177 */ 178 #define TMR_IDX_10G 1 179 static int t4_tmr_idx_10g = TMR_IDX_10G; 180 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 181 182 #define PKTC_IDX_10G (-1) 183 static int t4_pktc_idx_10g = PKTC_IDX_10G; 184 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 185 186 #define TMR_IDX_1G 1 187 static int t4_tmr_idx_1g = TMR_IDX_1G; 188 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 189 190 #define PKTC_IDX_1G (-1) 191 static int t4_pktc_idx_1g = PKTC_IDX_1G; 192 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 193 194 /* 195 * Size (# of entries) of each tx and rx queue. 196 */ 197 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 198 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 199 200 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 201 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 202 203 /* 204 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 205 */ 206 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 207 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 208 209 /* 210 * Configuration file. 211 */ 212 static char t4_cfg_file[32] = "default"; 213 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 214 215 /* 216 * ASIC features that will be used. Disable the ones you don't want so that the 217 * chip resources aren't wasted on features that will not be used. 218 */ 219 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 220 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 221 222 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 223 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 224 225 static int t4_toecaps_allowed = -1; 226 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 227 228 static int t4_rdmacaps_allowed = 0; 229 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 230 231 static int t4_iscsicaps_allowed = 0; 232 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 233 234 static int t4_fcoecaps_allowed = 0; 235 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 236 237 struct intrs_and_queues { 238 int intr_type; /* INTx, MSI, or MSI-X */ 239 int nirq; /* Number of vectors */ 240 int intr_flags; 241 int ntxq10g; /* # of NIC txq's for each 10G port */ 242 int nrxq10g; /* # of NIC rxq's for each 10G port */ 243 int ntxq1g; /* # of NIC txq's for each 1G port */ 244 int nrxq1g; /* # of NIC rxq's for each 1G port */ 245 #ifdef TCP_OFFLOAD 246 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 247 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 248 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 249 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 250 #endif 251 }; 252 253 struct filter_entry { 254 uint32_t valid:1; /* filter allocated and valid */ 255 uint32_t locked:1; /* filter is administratively locked */ 256 uint32_t pending:1; /* filter action is pending firmware reply */ 257 uint32_t smtidx:8; /* Source MAC Table index for smac */ 258 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 259 260 struct t4_filter_specification fs; 261 }; 262 263 enum { 264 XGMAC_MTU = (1 << 0), 265 XGMAC_PROMISC = (1 << 1), 266 XGMAC_ALLMULTI = (1 << 2), 267 XGMAC_VLANEX = (1 << 3), 268 XGMAC_UCADDR = (1 << 4), 269 XGMAC_MCADDRS = (1 << 5), 270 271 XGMAC_ALL = 0xffff 272 }; 273 274 static int map_bars(struct adapter *); 275 static void setup_memwin(struct adapter *); 276 static int cfg_itype_and_nqueues(struct adapter *, int, int, 277 struct intrs_and_queues *); 278 static int prep_firmware(struct adapter *); 279 static int upload_config_file(struct adapter *, const struct firmware *, 280 uint32_t *, uint32_t *); 281 static int partition_resources(struct adapter *, const struct firmware *); 282 static int get_params__pre_init(struct adapter *); 283 static int get_params__post_init(struct adapter *); 284 static void t4_set_desc(struct adapter *); 285 static void build_medialist(struct port_info *); 286 static int update_mac_settings(struct port_info *, int); 287 static int cxgbe_init_locked(struct port_info *); 288 static int cxgbe_init_synchronized(struct port_info *); 289 static int cxgbe_uninit_locked(struct port_info *); 290 static int cxgbe_uninit_synchronized(struct port_info *); 291 static int setup_intr_handlers(struct adapter *); 292 static int adapter_full_init(struct adapter *); 293 static int adapter_full_uninit(struct adapter *); 294 static int port_full_init(struct port_info *); 295 static int port_full_uninit(struct port_info *); 296 static void quiesce_eq(struct adapter *, struct sge_eq *); 297 static void quiesce_iq(struct adapter *, struct sge_iq *); 298 static void quiesce_fl(struct adapter *, struct sge_fl *); 299 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 300 driver_intr_t *, void *, char *); 301 static int t4_free_irq(struct adapter *, struct irq *); 302 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 303 unsigned int); 304 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 305 static void cxgbe_tick(void *); 306 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 307 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 308 struct mbuf *); 309 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 310 static int fw_msg_not_handled(struct adapter *, const __be64 *); 311 static int t4_sysctls(struct adapter *); 312 static int cxgbe_sysctls(struct port_info *); 313 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 314 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 315 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 316 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 317 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 318 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 319 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 320 #ifdef SBUF_DRAIN 321 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 322 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 323 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 324 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 325 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 326 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 327 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 328 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 329 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 330 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 331 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 332 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 333 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 334 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 335 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 336 #endif 337 static inline void txq_start(struct ifnet *, struct sge_txq *); 338 static uint32_t fconf_to_mode(uint32_t); 339 static uint32_t mode_to_fconf(uint32_t); 340 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 341 static int get_filter_mode(struct adapter *, uint32_t *); 342 static int set_filter_mode(struct adapter *, uint32_t); 343 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 344 static int get_filter(struct adapter *, struct t4_filter *); 345 static int set_filter(struct adapter *, struct t4_filter *); 346 static int del_filter(struct adapter *, struct t4_filter *); 347 static void clear_filter(struct filter_entry *); 348 static int set_filter_wr(struct adapter *, int); 349 static int del_filter_wr(struct adapter *, int); 350 static int get_sge_context(struct adapter *, struct t4_sge_context *); 351 static int read_card_mem(struct adapter *, struct t4_mem_range *); 352 static int read_i2c(struct adapter *, struct t4_i2c_data *); 353 #ifdef TCP_OFFLOAD 354 static int toe_capability(struct port_info *, int); 355 #endif 356 static int t4_mod_event(module_t, int, void *); 357 358 struct t4_pciids { 359 uint16_t device; 360 char *desc; 361 } t4_pciids[] = { 362 {0xa000, "Chelsio Terminator 4 FPGA"}, 363 {0x4400, "Chelsio T440-dbg"}, 364 {0x4401, "Chelsio T420-CR"}, 365 {0x4402, "Chelsio T422-CR"}, 366 {0x4403, "Chelsio T440-CR"}, 367 {0x4404, "Chelsio T420-BCH"}, 368 {0x4405, "Chelsio T440-BCH"}, 369 {0x4406, "Chelsio T440-CH"}, 370 {0x4407, "Chelsio T420-SO"}, 371 {0x4408, "Chelsio T420-CX"}, 372 {0x4409, "Chelsio T420-BT"}, 373 {0x440a, "Chelsio T404-BT"}, 374 {0x440e, "Chelsio T440-LP-CR"}, 375 }; 376 377 #ifdef TCP_OFFLOAD 378 /* 379 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 380 * exactly the same for both rxq and ofld_rxq. 381 */ 382 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 383 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 384 #endif 385 386 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 387 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 388 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 389 390 static int 391 t4_probe(device_t dev) 392 { 393 int i; 394 uint16_t v = pci_get_vendor(dev); 395 uint16_t d = pci_get_device(dev); 396 uint8_t f = pci_get_function(dev); 397 398 if (v != PCI_VENDOR_ID_CHELSIO) 399 return (ENXIO); 400 401 /* Attach only to PF0 of the FPGA */ 402 if (d == 0xa000 && f != 0) 403 return (ENXIO); 404 405 for (i = 0; i < nitems(t4_pciids); i++) { 406 if (d == t4_pciids[i].device) { 407 device_set_desc(dev, t4_pciids[i].desc); 408 return (BUS_PROBE_DEFAULT); 409 } 410 } 411 412 return (ENXIO); 413 } 414 415 static int 416 t4_attach(device_t dev) 417 { 418 struct adapter *sc; 419 int rc = 0, i, n10g, n1g, rqidx, tqidx; 420 struct intrs_and_queues iaq; 421 struct sge *s; 422 #ifdef TCP_OFFLOAD 423 int ofld_rqidx, ofld_tqidx; 424 #endif 425 426 sc = device_get_softc(dev); 427 sc->dev = dev; 428 429 pci_enable_busmaster(dev); 430 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 431 uint32_t v; 432 433 pci_set_max_read_req(dev, 4096); 434 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 435 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 436 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 437 } 438 439 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 440 device_get_nameunit(dev)); 441 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 442 mtx_lock(&t4_list_lock); 443 SLIST_INSERT_HEAD(&t4_list, sc, link); 444 mtx_unlock(&t4_list_lock); 445 446 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 447 TAILQ_INIT(&sc->sfl); 448 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 449 450 rc = map_bars(sc); 451 if (rc != 0) 452 goto done; /* error message displayed already */ 453 454 /* 455 * This is the real PF# to which we're attaching. Works from within PCI 456 * passthrough environments too, where pci_get_function() could return a 457 * different PF# depending on the passthrough configuration. We need to 458 * use the real PF# in all our communication with the firmware. 459 */ 460 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 461 sc->mbox = sc->pf; 462 463 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 464 sc->an_handler = an_not_handled; 465 for (i = 0; i < nitems(sc->cpl_handler); i++) 466 sc->cpl_handler[i] = cpl_not_handled; 467 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 468 sc->fw_msg_handler[i] = fw_msg_not_handled; 469 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 470 471 /* Prepare the adapter for operation */ 472 rc = -t4_prep_adapter(sc); 473 if (rc != 0) { 474 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 475 goto done; 476 } 477 478 /* 479 * Do this really early, with the memory windows set up even before the 480 * character device. The userland tool's register i/o and mem read 481 * will work even in "recovery mode". 482 */ 483 setup_memwin(sc); 484 sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT, 485 GID_WHEEL, 0600, "%s", device_get_nameunit(dev)); 486 sc->cdev->si_drv1 = sc; 487 488 /* Go no further if recovery mode has been requested. */ 489 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 490 device_printf(dev, "recovery mode.\n"); 491 goto done; 492 } 493 494 /* Prepare the firmware for operation */ 495 rc = prep_firmware(sc); 496 if (rc != 0) 497 goto done; /* error message displayed already */ 498 499 rc = get_params__pre_init(sc); 500 if (rc != 0) 501 goto done; /* error message displayed already */ 502 503 rc = t4_sge_init(sc); 504 if (rc != 0) 505 goto done; /* error message displayed already */ 506 507 if (sc->flags & MASTER_PF) { 508 /* get basic stuff going */ 509 rc = -t4_fw_initialize(sc, sc->mbox); 510 if (rc != 0) { 511 device_printf(dev, "early init failed: %d.\n", rc); 512 goto done; 513 } 514 } 515 516 rc = get_params__post_init(sc); 517 if (rc != 0) 518 goto done; /* error message displayed already */ 519 520 if (sc->flags & MASTER_PF) { 521 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 522 523 /* final tweaks to some settings */ 524 525 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, 526 sc->params.b_wnd); 527 /* 4K, 16K, 64K, 256K DDP "page sizes" */ 528 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(0) | V_HPZ1(2) | 529 V_HPZ2(4) | V_HPZ3(6)); 530 t4_set_reg_field(sc, A_ULP_RX_CTL, F_TDDPTAGTCB, F_TDDPTAGTCB); 531 t4_set_reg_field(sc, A_TP_PARA_REG5, 532 V_INDICATESIZE(M_INDICATESIZE) | 533 F_REARMDDPOFFSET | F_RESETDDPOFFSET, 534 V_INDICATESIZE(indsz) | 535 F_REARMDDPOFFSET | F_RESETDDPOFFSET); 536 } else { 537 /* 538 * XXX: Verify that we can live with whatever the master driver 539 * has done so far, and hope that it doesn't change any global 540 * setting from underneath us in the future. 541 */ 542 } 543 544 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 545 A_TP_VLAN_PRI_MAP); 546 547 for (i = 0; i < NCHAN; i++) 548 sc->params.tp.tx_modq[i] = i; 549 550 rc = t4_create_dma_tag(sc); 551 if (rc != 0) 552 goto done; /* error message displayed already */ 553 554 /* 555 * First pass over all the ports - allocate VIs and initialize some 556 * basic parameters like mac address, port type, etc. We also figure 557 * out whether a port is 10G or 1G and use that information when 558 * calculating how many interrupts to attempt to allocate. 559 */ 560 n10g = n1g = 0; 561 for_each_port(sc, i) { 562 struct port_info *pi; 563 564 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 565 sc->port[i] = pi; 566 567 /* These must be set before t4_port_init */ 568 pi->adapter = sc; 569 pi->port_id = i; 570 571 /* Allocate the vi and initialize parameters like mac addr */ 572 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 573 if (rc != 0) { 574 device_printf(dev, "unable to initialize port %d: %d\n", 575 i, rc); 576 free(pi, M_CXGBE); 577 sc->port[i] = NULL; 578 goto done; 579 } 580 581 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 582 device_get_nameunit(dev), i); 583 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 584 585 if (is_10G_port(pi)) { 586 n10g++; 587 pi->tmr_idx = t4_tmr_idx_10g; 588 pi->pktc_idx = t4_pktc_idx_10g; 589 } else { 590 n1g++; 591 pi->tmr_idx = t4_tmr_idx_1g; 592 pi->pktc_idx = t4_pktc_idx_1g; 593 } 594 595 pi->xact_addr_filt = -1; 596 597 pi->qsize_rxq = t4_qsize_rxq; 598 pi->qsize_txq = t4_qsize_txq; 599 600 pi->dev = device_add_child(dev, "cxgbe", -1); 601 if (pi->dev == NULL) { 602 device_printf(dev, 603 "failed to add device for port %d.\n", i); 604 rc = ENXIO; 605 goto done; 606 } 607 device_set_softc(pi->dev, pi); 608 } 609 610 /* 611 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 612 */ 613 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 614 if (rc != 0) 615 goto done; /* error message displayed already */ 616 617 sc->intr_type = iaq.intr_type; 618 sc->intr_count = iaq.nirq; 619 sc->flags |= iaq.intr_flags; 620 621 s = &sc->sge; 622 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 623 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 624 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 625 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 626 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 627 628 #ifdef TCP_OFFLOAD 629 if (is_offload(sc)) { 630 631 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 632 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 633 s->neq += s->nofldtxq + s->nofldrxq; 634 s->niq += s->nofldrxq; 635 636 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 637 M_CXGBE, M_ZERO | M_WAITOK); 638 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 639 M_CXGBE, M_ZERO | M_WAITOK); 640 } 641 #endif 642 643 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 644 M_ZERO | M_WAITOK); 645 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 646 M_ZERO | M_WAITOK); 647 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 648 M_ZERO | M_WAITOK); 649 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 650 M_ZERO | M_WAITOK); 651 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 652 M_ZERO | M_WAITOK); 653 654 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 655 M_ZERO | M_WAITOK); 656 657 t4_init_l2t(sc, M_WAITOK); 658 659 /* 660 * Second pass over the ports. This time we know the number of rx and 661 * tx queues that each port should get. 662 */ 663 rqidx = tqidx = 0; 664 #ifdef TCP_OFFLOAD 665 ofld_rqidx = ofld_tqidx = 0; 666 #endif 667 for_each_port(sc, i) { 668 struct port_info *pi = sc->port[i]; 669 670 if (pi == NULL) 671 continue; 672 673 pi->first_rxq = rqidx; 674 pi->first_txq = tqidx; 675 if (is_10G_port(pi)) { 676 pi->nrxq = iaq.nrxq10g; 677 pi->ntxq = iaq.ntxq10g; 678 } else { 679 pi->nrxq = iaq.nrxq1g; 680 pi->ntxq = iaq.ntxq1g; 681 } 682 683 rqidx += pi->nrxq; 684 tqidx += pi->ntxq; 685 686 #ifdef TCP_OFFLOAD 687 if (is_offload(sc)) { 688 pi->first_ofld_rxq = ofld_rqidx; 689 pi->first_ofld_txq = ofld_tqidx; 690 if (is_10G_port(pi)) { 691 pi->nofldrxq = iaq.nofldrxq10g; 692 pi->nofldtxq = iaq.nofldtxq10g; 693 } else { 694 pi->nofldrxq = iaq.nofldrxq1g; 695 pi->nofldtxq = iaq.nofldtxq1g; 696 } 697 ofld_rqidx += pi->nofldrxq; 698 ofld_tqidx += pi->nofldtxq; 699 } 700 #endif 701 } 702 703 rc = setup_intr_handlers(sc); 704 if (rc != 0) { 705 device_printf(dev, 706 "failed to setup interrupt handlers: %d\n", rc); 707 goto done; 708 } 709 710 rc = bus_generic_attach(dev); 711 if (rc != 0) { 712 device_printf(dev, 713 "failed to attach all child ports: %d\n", rc); 714 goto done; 715 } 716 717 device_printf(dev, 718 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 719 sc->params.pci.width, sc->params.nports, sc->intr_count, 720 sc->intr_type == INTR_MSIX ? "MSI-X" : 721 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 722 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 723 724 t4_set_desc(sc); 725 726 done: 727 if (rc != 0 && sc->cdev) { 728 /* cdev was created and so cxgbetool works; recover that way. */ 729 device_printf(dev, 730 "error during attach, adapter is now in recovery mode.\n"); 731 rc = 0; 732 } 733 734 if (rc != 0) 735 t4_detach(dev); 736 else 737 t4_sysctls(sc); 738 739 return (rc); 740 } 741 742 /* 743 * Idempotent 744 */ 745 static int 746 t4_detach(device_t dev) 747 { 748 struct adapter *sc; 749 struct port_info *pi; 750 int i, rc; 751 752 sc = device_get_softc(dev); 753 754 if (sc->flags & FULL_INIT_DONE) 755 t4_intr_disable(sc); 756 757 if (sc->cdev) { 758 destroy_dev(sc->cdev); 759 sc->cdev = NULL; 760 } 761 762 rc = bus_generic_detach(dev); 763 if (rc) { 764 device_printf(dev, 765 "failed to detach child devices: %d\n", rc); 766 return (rc); 767 } 768 769 for (i = 0; i < sc->intr_count; i++) 770 t4_free_irq(sc, &sc->irq[i]); 771 772 for (i = 0; i < MAX_NPORTS; i++) { 773 pi = sc->port[i]; 774 if (pi) { 775 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 776 if (pi->dev) 777 device_delete_child(dev, pi->dev); 778 779 mtx_destroy(&pi->pi_lock); 780 free(pi, M_CXGBE); 781 } 782 } 783 784 if (sc->flags & FULL_INIT_DONE) 785 adapter_full_uninit(sc); 786 787 if (sc->flags & FW_OK) 788 t4_fw_bye(sc, sc->mbox); 789 790 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 791 pci_release_msi(dev); 792 793 if (sc->regs_res) 794 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 795 sc->regs_res); 796 797 if (sc->msix_res) 798 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 799 sc->msix_res); 800 801 if (sc->l2t) 802 t4_free_l2t(sc->l2t); 803 804 #ifdef TCP_OFFLOAD 805 free(sc->sge.ofld_rxq, M_CXGBE); 806 free(sc->sge.ofld_txq, M_CXGBE); 807 #endif 808 free(sc->irq, M_CXGBE); 809 free(sc->sge.rxq, M_CXGBE); 810 free(sc->sge.txq, M_CXGBE); 811 free(sc->sge.ctrlq, M_CXGBE); 812 free(sc->sge.iqmap, M_CXGBE); 813 free(sc->sge.eqmap, M_CXGBE); 814 free(sc->tids.ftid_tab, M_CXGBE); 815 t4_destroy_dma_tag(sc); 816 if (mtx_initialized(&sc->sc_lock)) { 817 mtx_lock(&t4_list_lock); 818 SLIST_REMOVE(&t4_list, sc, adapter, link); 819 mtx_unlock(&t4_list_lock); 820 mtx_destroy(&sc->sc_lock); 821 } 822 823 if (mtx_initialized(&sc->sfl_lock)) 824 mtx_destroy(&sc->sfl_lock); 825 826 bzero(sc, sizeof(*sc)); 827 828 return (0); 829 } 830 831 832 static int 833 cxgbe_probe(device_t dev) 834 { 835 char buf[128]; 836 struct port_info *pi = device_get_softc(dev); 837 838 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 839 device_set_desc_copy(dev, buf); 840 841 return (BUS_PROBE_DEFAULT); 842 } 843 844 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 845 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 846 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 847 #define T4_CAP_ENABLE (T4_CAP) 848 849 static int 850 cxgbe_attach(device_t dev) 851 { 852 struct port_info *pi = device_get_softc(dev); 853 struct ifnet *ifp; 854 855 /* Allocate an ifnet and set it up */ 856 ifp = if_alloc(IFT_ETHER); 857 if (ifp == NULL) { 858 device_printf(dev, "Cannot allocate ifnet\n"); 859 return (ENOMEM); 860 } 861 pi->ifp = ifp; 862 ifp->if_softc = pi; 863 864 callout_init(&pi->tick, CALLOUT_MPSAFE); 865 866 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 867 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 868 869 ifp->if_init = cxgbe_init; 870 ifp->if_ioctl = cxgbe_ioctl; 871 ifp->if_transmit = cxgbe_transmit; 872 ifp->if_qflush = cxgbe_qflush; 873 874 ifp->if_capabilities = T4_CAP; 875 #ifdef TCP_OFFLOAD 876 if (is_offload(pi->adapter)) 877 ifp->if_capabilities |= IFCAP_TOE4; 878 #endif 879 ifp->if_capenable = T4_CAP_ENABLE; 880 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 881 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 882 883 /* Initialize ifmedia for this port */ 884 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 885 cxgbe_media_status); 886 build_medialist(pi); 887 888 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 889 EVENTHANDLER_PRI_ANY); 890 891 ether_ifattach(ifp, pi->hw_addr); 892 893 #ifdef TCP_OFFLOAD 894 if (is_offload(pi->adapter)) { 895 device_printf(dev, 896 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 897 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 898 } else 899 #endif 900 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 901 902 cxgbe_sysctls(pi); 903 904 return (0); 905 } 906 907 static int 908 cxgbe_detach(device_t dev) 909 { 910 struct port_info *pi = device_get_softc(dev); 911 struct adapter *sc = pi->adapter; 912 struct ifnet *ifp = pi->ifp; 913 914 /* Tell if_ioctl and if_init that the port is going away */ 915 ADAPTER_LOCK(sc); 916 SET_DOOMED(pi); 917 wakeup(&sc->flags); 918 while (IS_BUSY(sc)) 919 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 920 SET_BUSY(sc); 921 ADAPTER_UNLOCK(sc); 922 923 if (pi->vlan_c) 924 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 925 926 PORT_LOCK(pi); 927 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 928 callout_stop(&pi->tick); 929 PORT_UNLOCK(pi); 930 callout_drain(&pi->tick); 931 932 /* Let detach proceed even if these fail. */ 933 cxgbe_uninit_synchronized(pi); 934 port_full_uninit(pi); 935 936 ifmedia_removeall(&pi->media); 937 ether_ifdetach(pi->ifp); 938 if_free(pi->ifp); 939 940 ADAPTER_LOCK(sc); 941 CLR_BUSY(sc); 942 wakeup_one(&sc->flags); 943 ADAPTER_UNLOCK(sc); 944 945 return (0); 946 } 947 948 static void 949 cxgbe_init(void *arg) 950 { 951 struct port_info *pi = arg; 952 struct adapter *sc = pi->adapter; 953 954 ADAPTER_LOCK(sc); 955 cxgbe_init_locked(pi); /* releases adapter lock */ 956 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 957 } 958 959 static int 960 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 961 { 962 int rc = 0, mtu, flags; 963 struct port_info *pi = ifp->if_softc; 964 struct adapter *sc = pi->adapter; 965 struct ifreq *ifr = (struct ifreq *)data; 966 uint32_t mask; 967 968 switch (cmd) { 969 case SIOCSIFMTU: 970 ADAPTER_LOCK(sc); 971 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 972 if (rc) { 973 fail: 974 ADAPTER_UNLOCK(sc); 975 return (rc); 976 } 977 978 mtu = ifr->ifr_mtu; 979 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 980 rc = EINVAL; 981 } else { 982 ifp->if_mtu = mtu; 983 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 984 t4_update_fl_bufsize(ifp); 985 PORT_LOCK(pi); 986 rc = update_mac_settings(pi, XGMAC_MTU); 987 PORT_UNLOCK(pi); 988 } 989 } 990 ADAPTER_UNLOCK(sc); 991 break; 992 993 case SIOCSIFFLAGS: 994 ADAPTER_LOCK(sc); 995 if (IS_DOOMED(pi)) { 996 rc = ENXIO; 997 goto fail; 998 } 999 if (ifp->if_flags & IFF_UP) { 1000 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1001 flags = pi->if_flags; 1002 if ((ifp->if_flags ^ flags) & 1003 (IFF_PROMISC | IFF_ALLMULTI)) { 1004 if (IS_BUSY(sc)) { 1005 rc = EBUSY; 1006 goto fail; 1007 } 1008 PORT_LOCK(pi); 1009 rc = update_mac_settings(pi, 1010 XGMAC_PROMISC | XGMAC_ALLMULTI); 1011 PORT_UNLOCK(pi); 1012 } 1013 ADAPTER_UNLOCK(sc); 1014 } else 1015 rc = cxgbe_init_locked(pi); 1016 pi->if_flags = ifp->if_flags; 1017 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1018 rc = cxgbe_uninit_locked(pi); 1019 else 1020 ADAPTER_UNLOCK(sc); 1021 1022 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1023 break; 1024 1025 case SIOCADDMULTI: 1026 case SIOCDELMULTI: /* these two can be called with a mutex held :-( */ 1027 ADAPTER_LOCK(sc); 1028 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1029 if (rc) 1030 goto fail; 1031 1032 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1033 PORT_LOCK(pi); 1034 rc = update_mac_settings(pi, XGMAC_MCADDRS); 1035 PORT_UNLOCK(pi); 1036 } 1037 ADAPTER_UNLOCK(sc); 1038 break; 1039 1040 case SIOCSIFCAP: 1041 ADAPTER_LOCK(sc); 1042 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1043 if (rc) 1044 goto fail; 1045 1046 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1047 if (mask & IFCAP_TXCSUM) { 1048 ifp->if_capenable ^= IFCAP_TXCSUM; 1049 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1050 1051 if (IFCAP_TSO4 & ifp->if_capenable && 1052 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1053 ifp->if_capenable &= ~IFCAP_TSO4; 1054 if_printf(ifp, 1055 "tso4 disabled due to -txcsum.\n"); 1056 } 1057 } 1058 if (mask & IFCAP_TXCSUM_IPV6) { 1059 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1060 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1061 1062 if (IFCAP_TSO6 & ifp->if_capenable && 1063 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1064 ifp->if_capenable &= ~IFCAP_TSO6; 1065 if_printf(ifp, 1066 "tso6 disabled due to -txcsum6.\n"); 1067 } 1068 } 1069 if (mask & IFCAP_RXCSUM) 1070 ifp->if_capenable ^= IFCAP_RXCSUM; 1071 if (mask & IFCAP_RXCSUM_IPV6) 1072 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1073 1074 /* 1075 * Note that we leave CSUM_TSO alone (it is always set). The 1076 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1077 * sending a TSO request our way, so it's sufficient to toggle 1078 * IFCAP_TSOx only. 1079 */ 1080 if (mask & IFCAP_TSO4) { 1081 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1082 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1083 if_printf(ifp, "enable txcsum first.\n"); 1084 rc = EAGAIN; 1085 goto fail; 1086 } 1087 ifp->if_capenable ^= IFCAP_TSO4; 1088 } 1089 if (mask & IFCAP_TSO6) { 1090 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1091 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1092 if_printf(ifp, "enable txcsum6 first.\n"); 1093 rc = EAGAIN; 1094 goto fail; 1095 } 1096 ifp->if_capenable ^= IFCAP_TSO6; 1097 } 1098 if (mask & IFCAP_LRO) { 1099 #if defined(INET) || defined(INET6) 1100 int i; 1101 struct sge_rxq *rxq; 1102 1103 ifp->if_capenable ^= IFCAP_LRO; 1104 for_each_rxq(pi, i, rxq) { 1105 if (ifp->if_capenable & IFCAP_LRO) 1106 rxq->iq.flags |= IQ_LRO_ENABLED; 1107 else 1108 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1109 } 1110 #endif 1111 } 1112 #ifdef TCP_OFFLOAD 1113 if (mask & IFCAP_TOE) { 1114 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1115 1116 rc = toe_capability(pi, enable); 1117 if (rc != 0) 1118 goto fail; 1119 1120 ifp->if_capenable ^= mask; 1121 } 1122 #endif 1123 if (mask & IFCAP_VLAN_HWTAGGING) { 1124 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1125 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1126 PORT_LOCK(pi); 1127 rc = update_mac_settings(pi, XGMAC_VLANEX); 1128 PORT_UNLOCK(pi); 1129 } 1130 } 1131 if (mask & IFCAP_VLAN_MTU) { 1132 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1133 1134 /* Need to find out how to disable auto-mtu-inflation */ 1135 } 1136 if (mask & IFCAP_VLAN_HWTSO) 1137 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1138 if (mask & IFCAP_VLAN_HWCSUM) 1139 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1140 1141 #ifdef VLAN_CAPABILITIES 1142 VLAN_CAPABILITIES(ifp); 1143 #endif 1144 ADAPTER_UNLOCK(sc); 1145 break; 1146 1147 case SIOCSIFMEDIA: 1148 case SIOCGIFMEDIA: 1149 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1150 break; 1151 1152 default: 1153 rc = ether_ioctl(ifp, cmd, data); 1154 } 1155 1156 return (rc); 1157 } 1158 1159 static int 1160 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1161 { 1162 struct port_info *pi = ifp->if_softc; 1163 struct adapter *sc = pi->adapter; 1164 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1165 struct buf_ring *br; 1166 int rc; 1167 1168 M_ASSERTPKTHDR(m); 1169 1170 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1171 m_freem(m); 1172 return (ENETDOWN); 1173 } 1174 1175 if (m->m_flags & M_FLOWID) 1176 txq += (m->m_pkthdr.flowid % pi->ntxq); 1177 br = txq->br; 1178 1179 if (TXQ_TRYLOCK(txq) == 0) { 1180 struct sge_eq *eq = &txq->eq; 1181 1182 /* 1183 * It is possible that t4_eth_tx finishes up and releases the 1184 * lock between the TRYLOCK above and the drbr_enqueue here. We 1185 * need to make sure that this mbuf doesn't just sit there in 1186 * the drbr. 1187 */ 1188 1189 rc = drbr_enqueue(ifp, br, m); 1190 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1191 !(eq->flags & EQ_DOOMED)) 1192 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1193 return (rc); 1194 } 1195 1196 /* 1197 * txq->m is the mbuf that is held up due to a temporary shortage of 1198 * resources and it should be put on the wire first. Then what's in 1199 * drbr and finally the mbuf that was just passed in to us. 1200 * 1201 * Return code should indicate the fate of the mbuf that was passed in 1202 * this time. 1203 */ 1204 1205 TXQ_LOCK_ASSERT_OWNED(txq); 1206 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1207 1208 /* Queued for transmission. */ 1209 1210 rc = drbr_enqueue(ifp, br, m); 1211 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1212 (void) t4_eth_tx(ifp, txq, m); 1213 TXQ_UNLOCK(txq); 1214 return (rc); 1215 } 1216 1217 /* Direct transmission. */ 1218 rc = t4_eth_tx(ifp, txq, m); 1219 if (rc != 0 && txq->m) 1220 rc = 0; /* held, will be transmitted soon (hopefully) */ 1221 1222 TXQ_UNLOCK(txq); 1223 return (rc); 1224 } 1225 1226 static void 1227 cxgbe_qflush(struct ifnet *ifp) 1228 { 1229 struct port_info *pi = ifp->if_softc; 1230 struct sge_txq *txq; 1231 int i; 1232 struct mbuf *m; 1233 1234 /* queues do not exist if !PORT_INIT_DONE. */ 1235 if (pi->flags & PORT_INIT_DONE) { 1236 for_each_txq(pi, i, txq) { 1237 TXQ_LOCK(txq); 1238 m_freem(txq->m); 1239 txq->m = NULL; 1240 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1241 m_freem(m); 1242 TXQ_UNLOCK(txq); 1243 } 1244 } 1245 if_qflush(ifp); 1246 } 1247 1248 static int 1249 cxgbe_media_change(struct ifnet *ifp) 1250 { 1251 struct port_info *pi = ifp->if_softc; 1252 1253 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1254 1255 return (EOPNOTSUPP); 1256 } 1257 1258 static void 1259 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1260 { 1261 struct port_info *pi = ifp->if_softc; 1262 struct ifmedia_entry *cur = pi->media.ifm_cur; 1263 int speed = pi->link_cfg.speed; 1264 int data = (pi->port_type << 8) | pi->mod_type; 1265 1266 if (cur->ifm_data != data) { 1267 build_medialist(pi); 1268 cur = pi->media.ifm_cur; 1269 } 1270 1271 ifmr->ifm_status = IFM_AVALID; 1272 if (!pi->link_cfg.link_ok) 1273 return; 1274 1275 ifmr->ifm_status |= IFM_ACTIVE; 1276 1277 /* active and current will differ iff current media is autoselect. */ 1278 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1279 return; 1280 1281 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1282 if (speed == SPEED_10000) 1283 ifmr->ifm_active |= IFM_10G_T; 1284 else if (speed == SPEED_1000) 1285 ifmr->ifm_active |= IFM_1000_T; 1286 else if (speed == SPEED_100) 1287 ifmr->ifm_active |= IFM_100_TX; 1288 else if (speed == SPEED_10) 1289 ifmr->ifm_active |= IFM_10_T; 1290 else 1291 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1292 speed)); 1293 } 1294 1295 void 1296 t4_fatal_err(struct adapter *sc) 1297 { 1298 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1299 t4_intr_disable(sc); 1300 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1301 device_get_nameunit(sc->dev)); 1302 } 1303 1304 static int 1305 map_bars(struct adapter *sc) 1306 { 1307 sc->regs_rid = PCIR_BAR(0); 1308 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1309 &sc->regs_rid, RF_ACTIVE); 1310 if (sc->regs_res == NULL) { 1311 device_printf(sc->dev, "cannot map registers.\n"); 1312 return (ENXIO); 1313 } 1314 sc->bt = rman_get_bustag(sc->regs_res); 1315 sc->bh = rman_get_bushandle(sc->regs_res); 1316 sc->mmio_len = rman_get_size(sc->regs_res); 1317 1318 sc->msix_rid = PCIR_BAR(4); 1319 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1320 &sc->msix_rid, RF_ACTIVE); 1321 if (sc->msix_res == NULL) { 1322 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1323 return (ENXIO); 1324 } 1325 1326 return (0); 1327 } 1328 1329 static void 1330 setup_memwin(struct adapter *sc) 1331 { 1332 uint32_t bar0; 1333 1334 /* 1335 * Read low 32b of bar0 indirectly via the hardware backdoor mechanism. 1336 * Works from within PCI passthrough environments too, where 1337 * rman_get_start() can return a different value. We need to program 1338 * the memory window decoders with the actual addresses that will be 1339 * coming across the PCIe link. 1340 */ 1341 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1342 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1343 1344 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1345 (bar0 + MEMWIN0_BASE) | V_BIR(0) | 1346 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1347 1348 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1349 (bar0 + MEMWIN1_BASE) | V_BIR(0) | 1350 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1351 1352 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1353 (bar0 + MEMWIN2_BASE) | V_BIR(0) | 1354 V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10)); 1355 1356 /* flush */ 1357 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1358 } 1359 1360 static int 1361 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1362 struct intrs_and_queues *iaq) 1363 { 1364 int rc, itype, navail, nrxq10g, nrxq1g, n; 1365 int nofldrxq10g = 0, nofldrxq1g = 0; 1366 1367 bzero(iaq, sizeof(*iaq)); 1368 1369 iaq->ntxq10g = t4_ntxq10g; 1370 iaq->ntxq1g = t4_ntxq1g; 1371 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1372 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1373 #ifdef TCP_OFFLOAD 1374 if (is_offload(sc)) { 1375 iaq->nofldtxq10g = t4_nofldtxq10g; 1376 iaq->nofldtxq1g = t4_nofldtxq1g; 1377 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1378 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1379 } 1380 #endif 1381 1382 for (itype = INTR_MSIX; itype; itype >>= 1) { 1383 1384 if ((itype & t4_intr_types) == 0) 1385 continue; /* not allowed */ 1386 1387 if (itype == INTR_MSIX) 1388 navail = pci_msix_count(sc->dev); 1389 else if (itype == INTR_MSI) 1390 navail = pci_msi_count(sc->dev); 1391 else 1392 navail = 1; 1393 restart: 1394 if (navail == 0) 1395 continue; 1396 1397 iaq->intr_type = itype; 1398 iaq->intr_flags = 0; 1399 1400 /* 1401 * Best option: an interrupt vector for errors, one for the 1402 * firmware event queue, and one each for each rxq (NIC as well 1403 * as offload). 1404 */ 1405 iaq->nirq = T4_EXTRA_INTR; 1406 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1407 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1408 if (iaq->nirq <= navail && 1409 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1410 iaq->intr_flags |= INTR_DIRECT; 1411 goto allocate; 1412 } 1413 1414 /* 1415 * Second best option: an interrupt vector for errors, one for 1416 * the firmware event queue, and one each for either NIC or 1417 * offload rxq's. 1418 */ 1419 iaq->nirq = T4_EXTRA_INTR; 1420 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1421 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1422 if (iaq->nirq <= navail && 1423 (itype != INTR_MSI || powerof2(iaq->nirq))) 1424 goto allocate; 1425 1426 /* 1427 * Next best option: an interrupt vector for errors, one for the 1428 * firmware event queue, and at least one per port. At this 1429 * point we know we'll have to downsize nrxq or nofldrxq to fit 1430 * what's available to us. 1431 */ 1432 iaq->nirq = T4_EXTRA_INTR; 1433 iaq->nirq += n10g + n1g; 1434 if (iaq->nirq <= navail) { 1435 int leftover = navail - iaq->nirq; 1436 1437 if (n10g > 0) { 1438 int target = max(nrxq10g, nofldrxq10g); 1439 1440 n = 1; 1441 while (n < target && leftover >= n10g) { 1442 leftover -= n10g; 1443 iaq->nirq += n10g; 1444 n++; 1445 } 1446 iaq->nrxq10g = min(n, nrxq10g); 1447 #ifdef TCP_OFFLOAD 1448 if (is_offload(sc)) 1449 iaq->nofldrxq10g = min(n, nofldrxq10g); 1450 #endif 1451 } 1452 1453 if (n1g > 0) { 1454 int target = max(nrxq1g, nofldrxq1g); 1455 1456 n = 1; 1457 while (n < target && leftover >= n1g) { 1458 leftover -= n1g; 1459 iaq->nirq += n1g; 1460 n++; 1461 } 1462 iaq->nrxq1g = min(n, nrxq1g); 1463 #ifdef TCP_OFFLOAD 1464 if (is_offload(sc)) 1465 iaq->nofldrxq1g = min(n, nofldrxq1g); 1466 #endif 1467 } 1468 1469 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1470 goto allocate; 1471 } 1472 1473 /* 1474 * Least desirable option: one interrupt vector for everything. 1475 */ 1476 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1477 #ifdef TCP_OFFLOAD 1478 if (is_offload(sc)) 1479 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1480 #endif 1481 1482 allocate: 1483 navail = iaq->nirq; 1484 rc = 0; 1485 if (itype == INTR_MSIX) 1486 rc = pci_alloc_msix(sc->dev, &navail); 1487 else if (itype == INTR_MSI) 1488 rc = pci_alloc_msi(sc->dev, &navail); 1489 1490 if (rc == 0) { 1491 if (navail == iaq->nirq) 1492 return (0); 1493 1494 /* 1495 * Didn't get the number requested. Use whatever number 1496 * the kernel is willing to allocate (it's in navail). 1497 */ 1498 device_printf(sc->dev, "fewer vectors than requested, " 1499 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1500 itype, iaq->nirq, navail); 1501 pci_release_msi(sc->dev); 1502 goto restart; 1503 } 1504 1505 device_printf(sc->dev, 1506 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1507 itype, rc, iaq->nirq, navail); 1508 } 1509 1510 device_printf(sc->dev, 1511 "failed to find a usable interrupt type. " 1512 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1513 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1514 1515 return (ENXIO); 1516 } 1517 1518 /* 1519 * Install a compatible firmware (if required), establish contact with it (by 1520 * saying hello), and reset the device. If we end up as the master driver, 1521 * partition adapter resources by providing a configuration file to the 1522 * firmware. 1523 */ 1524 static int 1525 prep_firmware(struct adapter *sc) 1526 { 1527 const struct firmware *fw = NULL, *cfg = NULL, *default_cfg; 1528 int rc; 1529 enum dev_state state; 1530 1531 default_cfg = firmware_get(T4_CFGNAME); 1532 1533 /* Check firmware version and install a different one if necessary */ 1534 rc = t4_check_fw_version(sc); 1535 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 1536 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1537 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1538 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1539 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1540 if (rc != 0) { 1541 uint32_t v = 0; 1542 1543 fw = firmware_get(T4_FWNAME); 1544 if (fw != NULL) { 1545 const struct fw_hdr *hdr = (const void *)fw->data; 1546 1547 v = ntohl(hdr->fw_ver); 1548 1549 /* 1550 * The firmware module will not be used if it isn't the 1551 * same major version as what the driver was compiled 1552 * with. 1553 */ 1554 if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) { 1555 device_printf(sc->dev, 1556 "Found firmware image but version %d " 1557 "can not be used with this driver (%d)\n", 1558 G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR); 1559 1560 firmware_put(fw, FIRMWARE_UNLOAD); 1561 fw = NULL; 1562 } 1563 } 1564 1565 if (fw == NULL && rc < 0) { 1566 device_printf(sc->dev, "No usable firmware. " 1567 "card has %d.%d.%d, driver compiled with %d.%d.%d", 1568 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1569 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1570 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1571 FW_VERSION_MAJOR, FW_VERSION_MINOR, 1572 FW_VERSION_MICRO); 1573 rc = EAGAIN; 1574 goto done; 1575 } 1576 1577 /* 1578 * Always upgrade, even for minor/micro/build mismatches. 1579 * Downgrade only for a major version mismatch or if 1580 * force_firmware_install was specified. 1581 */ 1582 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) { 1583 device_printf(sc->dev, 1584 "installing firmware %d.%d.%d.%d on card.\n", 1585 G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v), 1586 G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v)); 1587 1588 rc = -t4_load_fw(sc, fw->data, fw->datasize); 1589 if (rc != 0) { 1590 device_printf(sc->dev, 1591 "failed to install firmware: %d\n", rc); 1592 goto done; 1593 } else { 1594 /* refresh */ 1595 (void) t4_check_fw_version(sc); 1596 snprintf(sc->fw_version, 1597 sizeof(sc->fw_version), "%u.%u.%u.%u", 1598 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1599 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1600 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1601 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1602 } 1603 } 1604 } 1605 1606 /* Contact firmware. */ 1607 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1608 if (rc < 0) { 1609 rc = -rc; 1610 device_printf(sc->dev, 1611 "failed to connect to the firmware: %d.\n", rc); 1612 goto done; 1613 } 1614 if (rc == sc->mbox) 1615 sc->flags |= MASTER_PF; 1616 1617 /* Reset device */ 1618 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1619 if (rc != 0) { 1620 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 1621 if (rc != ETIMEDOUT && rc != EIO) 1622 t4_fw_bye(sc, sc->mbox); 1623 goto done; 1624 } 1625 1626 /* Partition adapter resources as specified in the config file. */ 1627 if (sc->flags & MASTER_PF) { 1628 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) { 1629 char s[32]; 1630 1631 snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file); 1632 cfg = firmware_get(s); 1633 if (cfg == NULL) { 1634 device_printf(sc->dev, 1635 "unable to locate %s module, " 1636 "will use default config file.\n", s); 1637 } 1638 } 1639 1640 rc = partition_resources(sc, cfg ? cfg : default_cfg); 1641 if (rc != 0) 1642 goto done; /* error message displayed already */ 1643 } 1644 1645 sc->flags |= FW_OK; 1646 1647 done: 1648 if (fw != NULL) 1649 firmware_put(fw, FIRMWARE_UNLOAD); 1650 if (cfg != NULL) 1651 firmware_put(cfg, FIRMWARE_UNLOAD); 1652 if (default_cfg != NULL) 1653 firmware_put(default_cfg, FIRMWARE_UNLOAD); 1654 1655 return (rc); 1656 } 1657 1658 #define FW_PARAM_DEV(param) \ 1659 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1660 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1661 #define FW_PARAM_PFVF(param) \ 1662 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1663 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1664 1665 /* 1666 * Upload configuration file to card's memory. 1667 */ 1668 static int 1669 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt, 1670 uint32_t *ma) 1671 { 1672 int rc, i; 1673 uint32_t param, val, mtype, maddr, bar, off, win, remaining; 1674 const uint32_t *b; 1675 1676 /* Figure out where the firmware wants us to upload it. */ 1677 param = FW_PARAM_DEV(CF); 1678 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1679 if (rc != 0) { 1680 /* Firmwares without config file support will fail this way */ 1681 device_printf(sc->dev, 1682 "failed to query config file location: %d.\n", rc); 1683 return (rc); 1684 } 1685 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1686 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1687 1688 if (maddr & 3) { 1689 device_printf(sc->dev, 1690 "cannot upload config file (type %u, addr %x).\n", 1691 mtype, maddr); 1692 return (EFAULT); 1693 } 1694 1695 /* Translate mtype/maddr to an address suitable for the PCIe window */ 1696 val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1697 val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE; 1698 switch (mtype) { 1699 case FW_MEMTYPE_CF_EDC0: 1700 if (!(val & F_EDRAM0_ENABLE)) 1701 goto err; 1702 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1703 maddr += G_EDRAM0_BASE(bar) << 20; 1704 break; 1705 1706 case FW_MEMTYPE_CF_EDC1: 1707 if (!(val & F_EDRAM1_ENABLE)) 1708 goto err; 1709 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1710 maddr += G_EDRAM1_BASE(bar) << 20; 1711 break; 1712 1713 case FW_MEMTYPE_CF_EXTMEM: 1714 if (!(val & F_EXT_MEM_ENABLE)) 1715 goto err; 1716 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1717 maddr += G_EXT_MEM_BASE(bar) << 20; 1718 break; 1719 1720 default: 1721 err: 1722 device_printf(sc->dev, 1723 "cannot upload config file (type %u, enabled %u).\n", 1724 mtype, val); 1725 return (EFAULT); 1726 } 1727 1728 /* 1729 * Position the PCIe window (we use memwin2) to the 16B aligned area 1730 * just at/before the upload location. 1731 */ 1732 win = maddr & ~0xf; 1733 off = maddr - win; /* offset from the start of the window. */ 1734 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 1735 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 1736 1737 remaining = fw->datasize; 1738 if (remaining > FLASH_CFG_MAX_SIZE || 1739 remaining > MEMWIN2_APERTURE - off) { 1740 device_printf(sc->dev, "cannot upload config file all at once " 1741 "(size %u, max %u, room %u).\n", 1742 remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off); 1743 return (EFBIG); 1744 } 1745 1746 /* 1747 * XXX: sheer laziness. We deliberately added 4 bytes of useless 1748 * stuffing/comments at the end of the config file so it's ok to simply 1749 * throw away the last remaining bytes when the config file is not an 1750 * exact multiple of 4. 1751 */ 1752 b = fw->data; 1753 for (i = 0; remaining >= 4; i += 4, remaining -= 4) 1754 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++); 1755 1756 return (rc); 1757 } 1758 1759 /* 1760 * Partition chip resources for use between various PFs, VFs, etc. This is done 1761 * by uploading the firmware configuration file to the adapter and instructing 1762 * the firmware to process it. 1763 */ 1764 static int 1765 partition_resources(struct adapter *sc, const struct firmware *cfg) 1766 { 1767 int rc; 1768 struct fw_caps_config_cmd caps; 1769 uint32_t mtype, maddr, finicsum, cfcsum; 1770 1771 rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT; 1772 if (rc != 0) { 1773 mtype = FW_MEMTYPE_CF_FLASH; 1774 maddr = t4_flash_cfg_addr(sc); 1775 } 1776 1777 bzero(&caps, sizeof(caps)); 1778 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1779 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1780 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1781 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1782 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1783 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1784 if (rc != 0) { 1785 device_printf(sc->dev, 1786 "failed to pre-process config file: %d.\n", rc); 1787 return (rc); 1788 } 1789 1790 finicsum = be32toh(caps.finicsum); 1791 cfcsum = be32toh(caps.cfcsum); 1792 if (finicsum != cfcsum) { 1793 device_printf(sc->dev, 1794 "WARNING: config file checksum mismatch: %08x %08x\n", 1795 finicsum, cfcsum); 1796 } 1797 sc->cfcsum = cfcsum; 1798 1799 #define LIMIT_CAPS(x) do { \ 1800 caps.x &= htobe16(t4_##x##_allowed); \ 1801 sc->x = htobe16(caps.x); \ 1802 } while (0) 1803 1804 /* 1805 * Let the firmware know what features will (not) be used so it can tune 1806 * things accordingly. 1807 */ 1808 LIMIT_CAPS(linkcaps); 1809 LIMIT_CAPS(niccaps); 1810 LIMIT_CAPS(toecaps); 1811 LIMIT_CAPS(rdmacaps); 1812 LIMIT_CAPS(iscsicaps); 1813 LIMIT_CAPS(fcoecaps); 1814 #undef LIMIT_CAPS 1815 1816 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1817 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1818 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1819 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 1820 if (rc != 0) { 1821 device_printf(sc->dev, 1822 "failed to process config file: %d.\n", rc); 1823 return (rc); 1824 } 1825 1826 return (0); 1827 } 1828 1829 /* 1830 * Retrieve parameters that are needed (or nice to have) prior to calling 1831 * t4_sge_init and t4_fw_initialize. 1832 */ 1833 static int 1834 get_params__pre_init(struct adapter *sc) 1835 { 1836 int rc; 1837 uint32_t param[2], val[2]; 1838 struct fw_devlog_cmd cmd; 1839 struct devlog_params *dlog = &sc->params.devlog; 1840 1841 param[0] = FW_PARAM_DEV(PORTVEC); 1842 param[1] = FW_PARAM_DEV(CCLK); 1843 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1844 if (rc != 0) { 1845 device_printf(sc->dev, 1846 "failed to query parameters (pre_init): %d.\n", rc); 1847 return (rc); 1848 } 1849 1850 sc->params.portvec = val[0]; 1851 sc->params.nports = bitcount32(val[0]); 1852 sc->params.vpd.cclk = val[1]; 1853 1854 /* Read device log parameters. */ 1855 bzero(&cmd, sizeof(cmd)); 1856 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1857 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1858 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 1859 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 1860 if (rc != 0) { 1861 device_printf(sc->dev, 1862 "failed to get devlog parameters: %d.\n", rc); 1863 bzero(dlog, sizeof (*dlog)); 1864 rc = 0; /* devlog isn't critical for device operation */ 1865 } else { 1866 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 1867 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1868 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1869 dlog->size = be32toh(cmd.memsize_devlog); 1870 } 1871 1872 return (rc); 1873 } 1874 1875 /* 1876 * Retrieve various parameters that are of interest to the driver. The device 1877 * has been initialized by the firmware at this point. 1878 */ 1879 static int 1880 get_params__post_init(struct adapter *sc) 1881 { 1882 int rc; 1883 uint32_t param[7], val[7]; 1884 struct fw_caps_config_cmd caps; 1885 1886 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1887 param[1] = FW_PARAM_PFVF(EQ_START); 1888 param[2] = FW_PARAM_PFVF(FILTER_START); 1889 param[3] = FW_PARAM_PFVF(FILTER_END); 1890 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); 1891 if (rc != 0) { 1892 device_printf(sc->dev, 1893 "failed to query parameters (post_init): %d.\n", rc); 1894 return (rc); 1895 } 1896 1897 sc->sge.iq_start = val[0]; 1898 sc->sge.eq_start = val[1]; 1899 sc->tids.ftid_base = val[2]; 1900 sc->tids.nftids = val[3] - val[2] + 1; 1901 1902 /* get capabilites */ 1903 bzero(&caps, sizeof(caps)); 1904 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1905 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1906 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1907 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1908 if (rc != 0) { 1909 device_printf(sc->dev, 1910 "failed to get card capabilities: %d.\n", rc); 1911 return (rc); 1912 } 1913 1914 if (caps.toecaps) { 1915 /* query offload-related parameters */ 1916 param[0] = FW_PARAM_DEV(NTID); 1917 param[1] = FW_PARAM_PFVF(SERVER_START); 1918 param[2] = FW_PARAM_PFVF(SERVER_END); 1919 param[3] = FW_PARAM_PFVF(TDDP_START); 1920 param[4] = FW_PARAM_PFVF(TDDP_END); 1921 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1922 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1923 if (rc != 0) { 1924 device_printf(sc->dev, 1925 "failed to query TOE parameters: %d.\n", rc); 1926 return (rc); 1927 } 1928 sc->tids.ntids = val[0]; 1929 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1930 sc->tids.stid_base = val[1]; 1931 sc->tids.nstids = val[2] - val[1] + 1; 1932 sc->vres.ddp.start = val[3]; 1933 sc->vres.ddp.size = val[4] - val[3] + 1; 1934 sc->params.ofldq_wr_cred = val[5]; 1935 sc->params.offload = 1; 1936 } 1937 if (caps.rdmacaps) { 1938 param[0] = FW_PARAM_PFVF(STAG_START); 1939 param[1] = FW_PARAM_PFVF(STAG_END); 1940 param[2] = FW_PARAM_PFVF(RQ_START); 1941 param[3] = FW_PARAM_PFVF(RQ_END); 1942 param[4] = FW_PARAM_PFVF(PBL_START); 1943 param[5] = FW_PARAM_PFVF(PBL_END); 1944 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1945 if (rc != 0) { 1946 device_printf(sc->dev, 1947 "failed to query RDMA parameters(1): %d.\n", rc); 1948 return (rc); 1949 } 1950 sc->vres.stag.start = val[0]; 1951 sc->vres.stag.size = val[1] - val[0] + 1; 1952 sc->vres.rq.start = val[2]; 1953 sc->vres.rq.size = val[3] - val[2] + 1; 1954 sc->vres.pbl.start = val[4]; 1955 sc->vres.pbl.size = val[5] - val[4] + 1; 1956 1957 param[0] = FW_PARAM_PFVF(SQRQ_START); 1958 param[1] = FW_PARAM_PFVF(SQRQ_END); 1959 param[2] = FW_PARAM_PFVF(CQ_START); 1960 param[3] = FW_PARAM_PFVF(CQ_END); 1961 param[4] = FW_PARAM_PFVF(OCQ_START); 1962 param[5] = FW_PARAM_PFVF(OCQ_END); 1963 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 1964 if (rc != 0) { 1965 device_printf(sc->dev, 1966 "failed to query RDMA parameters(2): %d.\n", rc); 1967 return (rc); 1968 } 1969 sc->vres.qp.start = val[0]; 1970 sc->vres.qp.size = val[1] - val[0] + 1; 1971 sc->vres.cq.start = val[2]; 1972 sc->vres.cq.size = val[3] - val[2] + 1; 1973 sc->vres.ocq.start = val[4]; 1974 sc->vres.ocq.size = val[5] - val[4] + 1; 1975 } 1976 if (caps.iscsicaps) { 1977 param[0] = FW_PARAM_PFVF(ISCSI_START); 1978 param[1] = FW_PARAM_PFVF(ISCSI_END); 1979 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1980 if (rc != 0) { 1981 device_printf(sc->dev, 1982 "failed to query iSCSI parameters: %d.\n", rc); 1983 return (rc); 1984 } 1985 sc->vres.iscsi.start = val[0]; 1986 sc->vres.iscsi.size = val[1] - val[0] + 1; 1987 } 1988 1989 /* These are finalized by FW initialization, load their values now */ 1990 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1991 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1992 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1993 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1994 1995 return (rc); 1996 } 1997 1998 #undef FW_PARAM_PFVF 1999 #undef FW_PARAM_DEV 2000 2001 static void 2002 t4_set_desc(struct adapter *sc) 2003 { 2004 char buf[128]; 2005 struct adapter_params *p = &sc->params; 2006 2007 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 2008 p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec); 2009 2010 device_set_desc_copy(sc->dev, buf); 2011 } 2012 2013 static void 2014 build_medialist(struct port_info *pi) 2015 { 2016 struct ifmedia *media = &pi->media; 2017 int data, m; 2018 2019 PORT_LOCK(pi); 2020 2021 ifmedia_removeall(media); 2022 2023 m = IFM_ETHER | IFM_FDX; 2024 data = (pi->port_type << 8) | pi->mod_type; 2025 2026 switch(pi->port_type) { 2027 case FW_PORT_TYPE_BT_XFI: 2028 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2029 break; 2030 2031 case FW_PORT_TYPE_BT_XAUI: 2032 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2033 /* fall through */ 2034 2035 case FW_PORT_TYPE_BT_SGMII: 2036 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2037 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2038 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2039 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2040 break; 2041 2042 case FW_PORT_TYPE_CX4: 2043 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2044 ifmedia_set(media, m | IFM_10G_CX4); 2045 break; 2046 2047 case FW_PORT_TYPE_SFP: 2048 case FW_PORT_TYPE_FIBER_XFI: 2049 case FW_PORT_TYPE_FIBER_XAUI: 2050 switch (pi->mod_type) { 2051 2052 case FW_PORT_MOD_TYPE_LR: 2053 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2054 ifmedia_set(media, m | IFM_10G_LR); 2055 break; 2056 2057 case FW_PORT_MOD_TYPE_SR: 2058 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2059 ifmedia_set(media, m | IFM_10G_SR); 2060 break; 2061 2062 case FW_PORT_MOD_TYPE_LRM: 2063 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2064 ifmedia_set(media, m | IFM_10G_LRM); 2065 break; 2066 2067 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2068 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2069 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2070 ifmedia_set(media, m | IFM_10G_TWINAX); 2071 break; 2072 2073 case FW_PORT_MOD_TYPE_NONE: 2074 m &= ~IFM_FDX; 2075 ifmedia_add(media, m | IFM_NONE, data, NULL); 2076 ifmedia_set(media, m | IFM_NONE); 2077 break; 2078 2079 case FW_PORT_MOD_TYPE_NA: 2080 case FW_PORT_MOD_TYPE_ER: 2081 default: 2082 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2083 ifmedia_set(media, m | IFM_UNKNOWN); 2084 break; 2085 } 2086 break; 2087 2088 case FW_PORT_TYPE_KX4: 2089 case FW_PORT_TYPE_KX: 2090 case FW_PORT_TYPE_KR: 2091 default: 2092 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2093 ifmedia_set(media, m | IFM_UNKNOWN); 2094 break; 2095 } 2096 2097 PORT_UNLOCK(pi); 2098 } 2099 2100 #define FW_MAC_EXACT_CHUNK 7 2101 2102 /* 2103 * Program the port's XGMAC based on parameters in ifnet. The caller also 2104 * indicates which parameters should be programmed (the rest are left alone). 2105 */ 2106 static int 2107 update_mac_settings(struct port_info *pi, int flags) 2108 { 2109 int rc; 2110 struct ifnet *ifp = pi->ifp; 2111 struct adapter *sc = pi->adapter; 2112 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2113 2114 PORT_LOCK_ASSERT_OWNED(pi); 2115 KASSERT(flags, ("%s: not told what to update.", __func__)); 2116 2117 if (flags & XGMAC_MTU) 2118 mtu = ifp->if_mtu; 2119 2120 if (flags & XGMAC_PROMISC) 2121 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2122 2123 if (flags & XGMAC_ALLMULTI) 2124 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2125 2126 if (flags & XGMAC_VLANEX) 2127 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2128 2129 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2130 vlanex, false); 2131 if (rc) { 2132 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2133 return (rc); 2134 } 2135 2136 if (flags & XGMAC_UCADDR) { 2137 uint8_t ucaddr[ETHER_ADDR_LEN]; 2138 2139 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2140 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2141 ucaddr, true, true); 2142 if (rc < 0) { 2143 rc = -rc; 2144 if_printf(ifp, "change_mac failed: %d\n", rc); 2145 return (rc); 2146 } else { 2147 pi->xact_addr_filt = rc; 2148 rc = 0; 2149 } 2150 } 2151 2152 if (flags & XGMAC_MCADDRS) { 2153 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2154 int del = 1; 2155 uint64_t hash = 0; 2156 struct ifmultiaddr *ifma; 2157 int i = 0, j; 2158 2159 if_maddr_rlock(ifp); 2160 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2161 if (ifma->ifma_addr->sa_family != AF_LINK) 2162 continue; 2163 mcaddr[i++] = 2164 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2165 2166 if (i == FW_MAC_EXACT_CHUNK) { 2167 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2168 del, i, mcaddr, NULL, &hash, 0); 2169 if (rc < 0) { 2170 rc = -rc; 2171 for (j = 0; j < i; j++) { 2172 if_printf(ifp, 2173 "failed to add mc address" 2174 " %02x:%02x:%02x:" 2175 "%02x:%02x:%02x rc=%d\n", 2176 mcaddr[j][0], mcaddr[j][1], 2177 mcaddr[j][2], mcaddr[j][3], 2178 mcaddr[j][4], mcaddr[j][5], 2179 rc); 2180 } 2181 goto mcfail; 2182 } 2183 del = 0; 2184 i = 0; 2185 } 2186 } 2187 if (i > 0) { 2188 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2189 del, i, mcaddr, NULL, &hash, 0); 2190 if (rc < 0) { 2191 rc = -rc; 2192 for (j = 0; j < i; j++) { 2193 if_printf(ifp, 2194 "failed to add mc address" 2195 " %02x:%02x:%02x:" 2196 "%02x:%02x:%02x rc=%d\n", 2197 mcaddr[j][0], mcaddr[j][1], 2198 mcaddr[j][2], mcaddr[j][3], 2199 mcaddr[j][4], mcaddr[j][5], 2200 rc); 2201 } 2202 goto mcfail; 2203 } 2204 } 2205 2206 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2207 if (rc != 0) 2208 if_printf(ifp, "failed to set mc address hash: %d", rc); 2209 mcfail: 2210 if_maddr_runlock(ifp); 2211 } 2212 2213 return (rc); 2214 } 2215 2216 static int 2217 cxgbe_init_locked(struct port_info *pi) 2218 { 2219 struct adapter *sc = pi->adapter; 2220 int rc = 0; 2221 2222 ADAPTER_LOCK_ASSERT_OWNED(sc); 2223 2224 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2225 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) { 2226 rc = EINTR; 2227 goto done; 2228 } 2229 } 2230 if (IS_DOOMED(pi)) { 2231 rc = ENXIO; 2232 goto done; 2233 } 2234 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2235 2236 /* Give up the adapter lock, port init code can sleep. */ 2237 SET_BUSY(sc); 2238 ADAPTER_UNLOCK(sc); 2239 2240 rc = cxgbe_init_synchronized(pi); 2241 2242 done: 2243 ADAPTER_LOCK(sc); 2244 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2245 CLR_BUSY(sc); 2246 wakeup_one(&sc->flags); 2247 ADAPTER_UNLOCK(sc); 2248 return (rc); 2249 } 2250 2251 static int 2252 cxgbe_init_synchronized(struct port_info *pi) 2253 { 2254 struct adapter *sc = pi->adapter; 2255 struct ifnet *ifp = pi->ifp; 2256 int rc = 0; 2257 2258 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2259 2260 if (isset(&sc->open_device_map, pi->port_id)) { 2261 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2262 ("mismatch between open_device_map and if_drv_flags")); 2263 return (0); /* already running */ 2264 } 2265 2266 if (!(sc->flags & FULL_INIT_DONE) && 2267 ((rc = adapter_full_init(sc)) != 0)) 2268 return (rc); /* error message displayed already */ 2269 2270 if (!(pi->flags & PORT_INIT_DONE) && 2271 ((rc = port_full_init(pi)) != 0)) 2272 return (rc); /* error message displayed already */ 2273 2274 PORT_LOCK(pi); 2275 rc = update_mac_settings(pi, XGMAC_ALL); 2276 PORT_UNLOCK(pi); 2277 if (rc) 2278 goto done; /* error message displayed already */ 2279 2280 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2281 if (rc != 0) { 2282 if_printf(ifp, "start_link failed: %d\n", rc); 2283 goto done; 2284 } 2285 2286 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2287 if (rc != 0) { 2288 if_printf(ifp, "enable_vi failed: %d\n", rc); 2289 goto done; 2290 } 2291 2292 /* all ok */ 2293 setbit(&sc->open_device_map, pi->port_id); 2294 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2295 2296 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2297 done: 2298 if (rc != 0) 2299 cxgbe_uninit_synchronized(pi); 2300 2301 return (rc); 2302 } 2303 2304 static int 2305 cxgbe_uninit_locked(struct port_info *pi) 2306 { 2307 struct adapter *sc = pi->adapter; 2308 int rc; 2309 2310 ADAPTER_LOCK_ASSERT_OWNED(sc); 2311 2312 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2313 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) { 2314 rc = EINTR; 2315 goto done; 2316 } 2317 } 2318 if (IS_DOOMED(pi)) { 2319 rc = ENXIO; 2320 goto done; 2321 } 2322 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2323 SET_BUSY(sc); 2324 ADAPTER_UNLOCK(sc); 2325 2326 rc = cxgbe_uninit_synchronized(pi); 2327 2328 ADAPTER_LOCK(sc); 2329 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2330 CLR_BUSY(sc); 2331 wakeup_one(&sc->flags); 2332 done: 2333 ADAPTER_UNLOCK(sc); 2334 return (rc); 2335 } 2336 2337 /* 2338 * Idempotent. 2339 */ 2340 static int 2341 cxgbe_uninit_synchronized(struct port_info *pi) 2342 { 2343 struct adapter *sc = pi->adapter; 2344 struct ifnet *ifp = pi->ifp; 2345 int rc; 2346 2347 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2348 2349 /* 2350 * Disable the VI so that all its data in either direction is discarded 2351 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2352 * tick) intact as the TP can deliver negative advice or data that it's 2353 * holding in its RAM (for an offloaded connection) even after the VI is 2354 * disabled. 2355 */ 2356 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2357 if (rc) { 2358 if_printf(ifp, "disable_vi failed: %d\n", rc); 2359 return (rc); 2360 } 2361 2362 clrbit(&sc->open_device_map, pi->port_id); 2363 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2364 2365 pi->link_cfg.link_ok = 0; 2366 pi->link_cfg.speed = 0; 2367 t4_os_link_changed(sc, pi->port_id, 0); 2368 2369 return (0); 2370 } 2371 2372 /* 2373 * It is ok for this function to fail midway and return right away. t4_detach 2374 * will walk the entire sc->irq list and clean up whatever is valid. 2375 */ 2376 static int 2377 setup_intr_handlers(struct adapter *sc) 2378 { 2379 int rc, rid, p, q; 2380 char s[8]; 2381 struct irq *irq; 2382 struct port_info *pi; 2383 struct sge_rxq *rxq; 2384 #ifdef TCP_OFFLOAD 2385 struct sge_ofld_rxq *ofld_rxq; 2386 #endif 2387 2388 /* 2389 * Setup interrupts. 2390 */ 2391 irq = &sc->irq[0]; 2392 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2393 if (sc->intr_count == 1) { 2394 KASSERT(!(sc->flags & INTR_DIRECT), 2395 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2396 2397 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 2398 if (rc != 0) 2399 return (rc); 2400 } else { 2401 /* Multiple interrupts. */ 2402 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2403 ("%s: too few intr.", __func__)); 2404 2405 /* The first one is always error intr */ 2406 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 2407 if (rc != 0) 2408 return (rc); 2409 irq++; 2410 rid++; 2411 2412 /* The second one is always the firmware event queue */ 2413 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 2414 "evt"); 2415 if (rc != 0) 2416 return (rc); 2417 irq++; 2418 rid++; 2419 2420 /* 2421 * Note that if INTR_DIRECT is not set then either the NIC rx 2422 * queues or (exclusive or) the TOE rx queueus will be taking 2423 * direct interrupts. 2424 * 2425 * There is no need to check for is_offload(sc) as nofldrxq 2426 * will be 0 if offload is disabled. 2427 */ 2428 for_each_port(sc, p) { 2429 pi = sc->port[p]; 2430 2431 #ifdef TCP_OFFLOAD 2432 /* 2433 * Skip over the NIC queues if they aren't taking direct 2434 * interrupts. 2435 */ 2436 if (!(sc->flags & INTR_DIRECT) && 2437 pi->nofldrxq > pi->nrxq) 2438 goto ofld_queues; 2439 #endif 2440 rxq = &sc->sge.rxq[pi->first_rxq]; 2441 for (q = 0; q < pi->nrxq; q++, rxq++) { 2442 snprintf(s, sizeof(s), "%d.%d", p, q); 2443 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 2444 s); 2445 if (rc != 0) 2446 return (rc); 2447 irq++; 2448 rid++; 2449 } 2450 2451 #ifdef TCP_OFFLOAD 2452 /* 2453 * Skip over the offload queues if they aren't taking 2454 * direct interrupts. 2455 */ 2456 if (!(sc->flags & INTR_DIRECT)) 2457 continue; 2458 ofld_queues: 2459 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2460 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2461 snprintf(s, sizeof(s), "%d,%d", p, q); 2462 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 2463 ofld_rxq, s); 2464 if (rc != 0) 2465 return (rc); 2466 irq++; 2467 rid++; 2468 } 2469 #endif 2470 } 2471 } 2472 2473 return (0); 2474 } 2475 2476 static int 2477 adapter_full_init(struct adapter *sc) 2478 { 2479 int rc, i; 2480 2481 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2482 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2483 ("%s: FULL_INIT_DONE already", __func__)); 2484 2485 /* 2486 * queues that belong to the adapter (not any particular port). 2487 */ 2488 rc = t4_setup_adapter_queues(sc); 2489 if (rc != 0) 2490 goto done; 2491 2492 for (i = 0; i < nitems(sc->tq); i++) { 2493 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 2494 taskqueue_thread_enqueue, &sc->tq[i]); 2495 if (sc->tq[i] == NULL) { 2496 device_printf(sc->dev, 2497 "failed to allocate task queue %d\n", i); 2498 rc = ENOMEM; 2499 goto done; 2500 } 2501 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 2502 device_get_nameunit(sc->dev), i); 2503 } 2504 2505 t4_intr_enable(sc); 2506 sc->flags |= FULL_INIT_DONE; 2507 done: 2508 if (rc != 0) 2509 adapter_full_uninit(sc); 2510 2511 return (rc); 2512 } 2513 2514 static int 2515 adapter_full_uninit(struct adapter *sc) 2516 { 2517 int i; 2518 2519 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2520 2521 t4_teardown_adapter_queues(sc); 2522 2523 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 2524 taskqueue_free(sc->tq[i]); 2525 sc->tq[i] = NULL; 2526 } 2527 2528 sc->flags &= ~FULL_INIT_DONE; 2529 2530 return (0); 2531 } 2532 2533 static int 2534 port_full_init(struct port_info *pi) 2535 { 2536 struct adapter *sc = pi->adapter; 2537 struct ifnet *ifp = pi->ifp; 2538 uint16_t *rss; 2539 struct sge_rxq *rxq; 2540 int rc, i; 2541 2542 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2543 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 2544 ("%s: PORT_INIT_DONE already", __func__)); 2545 2546 sysctl_ctx_init(&pi->ctx); 2547 pi->flags |= PORT_SYSCTL_CTX; 2548 2549 /* 2550 * Allocate tx/rx/fl queues for this port. 2551 */ 2552 rc = t4_setup_port_queues(pi); 2553 if (rc != 0) 2554 goto done; /* error message displayed already */ 2555 2556 /* 2557 * Setup RSS for this port. 2558 */ 2559 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 2560 M_ZERO | M_WAITOK); 2561 for_each_rxq(pi, i, rxq) { 2562 rss[i] = rxq->iq.abs_id; 2563 } 2564 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2565 pi->rss_size, rss, pi->nrxq); 2566 free(rss, M_CXGBE); 2567 if (rc != 0) { 2568 if_printf(ifp, "rss_config failed: %d\n", rc); 2569 goto done; 2570 } 2571 2572 pi->flags |= PORT_INIT_DONE; 2573 done: 2574 if (rc != 0) 2575 port_full_uninit(pi); 2576 2577 return (rc); 2578 } 2579 2580 /* 2581 * Idempotent. 2582 */ 2583 static int 2584 port_full_uninit(struct port_info *pi) 2585 { 2586 struct adapter *sc = pi->adapter; 2587 int i; 2588 struct sge_rxq *rxq; 2589 struct sge_txq *txq; 2590 #ifdef TCP_OFFLOAD 2591 struct sge_ofld_rxq *ofld_rxq; 2592 struct sge_wrq *ofld_txq; 2593 #endif 2594 2595 if (pi->flags & PORT_INIT_DONE) { 2596 2597 /* Need to quiesce queues. XXX: ctrl queues? */ 2598 2599 for_each_txq(pi, i, txq) { 2600 quiesce_eq(sc, &txq->eq); 2601 } 2602 2603 #ifdef TCP_OFFLOAD 2604 for_each_ofld_txq(pi, i, ofld_txq) { 2605 quiesce_eq(sc, &ofld_txq->eq); 2606 } 2607 #endif 2608 2609 for_each_rxq(pi, i, rxq) { 2610 quiesce_iq(sc, &rxq->iq); 2611 quiesce_fl(sc, &rxq->fl); 2612 } 2613 2614 #ifdef TCP_OFFLOAD 2615 for_each_ofld_rxq(pi, i, ofld_rxq) { 2616 quiesce_iq(sc, &ofld_rxq->iq); 2617 quiesce_fl(sc, &ofld_rxq->fl); 2618 } 2619 #endif 2620 } 2621 2622 t4_teardown_port_queues(pi); 2623 pi->flags &= ~PORT_INIT_DONE; 2624 2625 return (0); 2626 } 2627 2628 static void 2629 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 2630 { 2631 EQ_LOCK(eq); 2632 eq->flags |= EQ_DOOMED; 2633 2634 /* 2635 * Wait for the response to a credit flush if one's 2636 * pending. 2637 */ 2638 while (eq->flags & EQ_CRFLUSHED) 2639 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 2640 EQ_UNLOCK(eq); 2641 2642 callout_drain(&eq->tx_callout); /* XXX: iffy */ 2643 pause("callout", 10); /* Still iffy */ 2644 2645 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 2646 } 2647 2648 static void 2649 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 2650 { 2651 (void) sc; /* unused */ 2652 2653 /* Synchronize with the interrupt handler */ 2654 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 2655 pause("iqfree", 1); 2656 } 2657 2658 static void 2659 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 2660 { 2661 mtx_lock(&sc->sfl_lock); 2662 FL_LOCK(fl); 2663 fl->flags |= FL_DOOMED; 2664 FL_UNLOCK(fl); 2665 mtx_unlock(&sc->sfl_lock); 2666 2667 callout_drain(&sc->sfl_callout); 2668 KASSERT((fl->flags & FL_STARVING) == 0, 2669 ("%s: still starving", __func__)); 2670 } 2671 2672 static int 2673 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 2674 driver_intr_t *handler, void *arg, char *name) 2675 { 2676 int rc; 2677 2678 irq->rid = rid; 2679 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 2680 RF_SHAREABLE | RF_ACTIVE); 2681 if (irq->res == NULL) { 2682 device_printf(sc->dev, 2683 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 2684 return (ENOMEM); 2685 } 2686 2687 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 2688 NULL, handler, arg, &irq->tag); 2689 if (rc != 0) { 2690 device_printf(sc->dev, 2691 "failed to setup interrupt for rid %d, name %s: %d\n", 2692 rid, name, rc); 2693 } else if (name) 2694 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 2695 2696 return (rc); 2697 } 2698 2699 static int 2700 t4_free_irq(struct adapter *sc, struct irq *irq) 2701 { 2702 if (irq->tag) 2703 bus_teardown_intr(sc->dev, irq->res, irq->tag); 2704 if (irq->res) 2705 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 2706 2707 bzero(irq, sizeof(*irq)); 2708 2709 return (0); 2710 } 2711 2712 static void 2713 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 2714 unsigned int end) 2715 { 2716 uint32_t *p = (uint32_t *)(buf + start); 2717 2718 for ( ; start <= end; start += sizeof(uint32_t)) 2719 *p++ = t4_read_reg(sc, start); 2720 } 2721 2722 static void 2723 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 2724 { 2725 int i; 2726 static const unsigned int reg_ranges[] = { 2727 0x1008, 0x1108, 2728 0x1180, 0x11b4, 2729 0x11fc, 0x123c, 2730 0x1300, 0x173c, 2731 0x1800, 0x18fc, 2732 0x3000, 0x30d8, 2733 0x30e0, 0x5924, 2734 0x5960, 0x59d4, 2735 0x5a00, 0x5af8, 2736 0x6000, 0x6098, 2737 0x6100, 0x6150, 2738 0x6200, 0x6208, 2739 0x6240, 0x6248, 2740 0x6280, 0x6338, 2741 0x6370, 0x638c, 2742 0x6400, 0x643c, 2743 0x6500, 0x6524, 2744 0x6a00, 0x6a38, 2745 0x6a60, 0x6a78, 2746 0x6b00, 0x6b84, 2747 0x6bf0, 0x6c84, 2748 0x6cf0, 0x6d84, 2749 0x6df0, 0x6e84, 2750 0x6ef0, 0x6f84, 2751 0x6ff0, 0x7084, 2752 0x70f0, 0x7184, 2753 0x71f0, 0x7284, 2754 0x72f0, 0x7384, 2755 0x73f0, 0x7450, 2756 0x7500, 0x7530, 2757 0x7600, 0x761c, 2758 0x7680, 0x76cc, 2759 0x7700, 0x7798, 2760 0x77c0, 0x77fc, 2761 0x7900, 0x79fc, 2762 0x7b00, 0x7c38, 2763 0x7d00, 0x7efc, 2764 0x8dc0, 0x8e1c, 2765 0x8e30, 0x8e78, 2766 0x8ea0, 0x8f6c, 2767 0x8fc0, 0x9074, 2768 0x90fc, 0x90fc, 2769 0x9400, 0x9458, 2770 0x9600, 0x96bc, 2771 0x9800, 0x9808, 2772 0x9820, 0x983c, 2773 0x9850, 0x9864, 2774 0x9c00, 0x9c6c, 2775 0x9c80, 0x9cec, 2776 0x9d00, 0x9d6c, 2777 0x9d80, 0x9dec, 2778 0x9e00, 0x9e6c, 2779 0x9e80, 0x9eec, 2780 0x9f00, 0x9f6c, 2781 0x9f80, 0x9fec, 2782 0xd004, 0xd03c, 2783 0xdfc0, 0xdfe0, 2784 0xe000, 0xea7c, 2785 0xf000, 0x11190, 2786 0x19040, 0x1906c, 2787 0x19078, 0x19080, 2788 0x1908c, 0x19124, 2789 0x19150, 0x191b0, 2790 0x191d0, 0x191e8, 2791 0x19238, 0x1924c, 2792 0x193f8, 0x19474, 2793 0x19490, 0x194f8, 2794 0x19800, 0x19f30, 2795 0x1a000, 0x1a06c, 2796 0x1a0b0, 0x1a120, 2797 0x1a128, 0x1a138, 2798 0x1a190, 0x1a1c4, 2799 0x1a1fc, 0x1a1fc, 2800 0x1e040, 0x1e04c, 2801 0x1e284, 0x1e28c, 2802 0x1e2c0, 0x1e2c0, 2803 0x1e2e0, 0x1e2e0, 2804 0x1e300, 0x1e384, 2805 0x1e3c0, 0x1e3c8, 2806 0x1e440, 0x1e44c, 2807 0x1e684, 0x1e68c, 2808 0x1e6c0, 0x1e6c0, 2809 0x1e6e0, 0x1e6e0, 2810 0x1e700, 0x1e784, 2811 0x1e7c0, 0x1e7c8, 2812 0x1e840, 0x1e84c, 2813 0x1ea84, 0x1ea8c, 2814 0x1eac0, 0x1eac0, 2815 0x1eae0, 0x1eae0, 2816 0x1eb00, 0x1eb84, 2817 0x1ebc0, 0x1ebc8, 2818 0x1ec40, 0x1ec4c, 2819 0x1ee84, 0x1ee8c, 2820 0x1eec0, 0x1eec0, 2821 0x1eee0, 0x1eee0, 2822 0x1ef00, 0x1ef84, 2823 0x1efc0, 0x1efc8, 2824 0x1f040, 0x1f04c, 2825 0x1f284, 0x1f28c, 2826 0x1f2c0, 0x1f2c0, 2827 0x1f2e0, 0x1f2e0, 2828 0x1f300, 0x1f384, 2829 0x1f3c0, 0x1f3c8, 2830 0x1f440, 0x1f44c, 2831 0x1f684, 0x1f68c, 2832 0x1f6c0, 0x1f6c0, 2833 0x1f6e0, 0x1f6e0, 2834 0x1f700, 0x1f784, 2835 0x1f7c0, 0x1f7c8, 2836 0x1f840, 0x1f84c, 2837 0x1fa84, 0x1fa8c, 2838 0x1fac0, 0x1fac0, 2839 0x1fae0, 0x1fae0, 2840 0x1fb00, 0x1fb84, 2841 0x1fbc0, 0x1fbc8, 2842 0x1fc40, 0x1fc4c, 2843 0x1fe84, 0x1fe8c, 2844 0x1fec0, 0x1fec0, 2845 0x1fee0, 0x1fee0, 2846 0x1ff00, 0x1ff84, 2847 0x1ffc0, 0x1ffc8, 2848 0x20000, 0x2002c, 2849 0x20100, 0x2013c, 2850 0x20190, 0x201c8, 2851 0x20200, 0x20318, 2852 0x20400, 0x20528, 2853 0x20540, 0x20614, 2854 0x21000, 0x21040, 2855 0x2104c, 0x21060, 2856 0x210c0, 0x210ec, 2857 0x21200, 0x21268, 2858 0x21270, 0x21284, 2859 0x212fc, 0x21388, 2860 0x21400, 0x21404, 2861 0x21500, 0x21518, 2862 0x2152c, 0x2153c, 2863 0x21550, 0x21554, 2864 0x21600, 0x21600, 2865 0x21608, 0x21628, 2866 0x21630, 0x2163c, 2867 0x21700, 0x2171c, 2868 0x21780, 0x2178c, 2869 0x21800, 0x21c38, 2870 0x21c80, 0x21d7c, 2871 0x21e00, 0x21e04, 2872 0x22000, 0x2202c, 2873 0x22100, 0x2213c, 2874 0x22190, 0x221c8, 2875 0x22200, 0x22318, 2876 0x22400, 0x22528, 2877 0x22540, 0x22614, 2878 0x23000, 0x23040, 2879 0x2304c, 0x23060, 2880 0x230c0, 0x230ec, 2881 0x23200, 0x23268, 2882 0x23270, 0x23284, 2883 0x232fc, 0x23388, 2884 0x23400, 0x23404, 2885 0x23500, 0x23518, 2886 0x2352c, 0x2353c, 2887 0x23550, 0x23554, 2888 0x23600, 0x23600, 2889 0x23608, 0x23628, 2890 0x23630, 0x2363c, 2891 0x23700, 0x2371c, 2892 0x23780, 0x2378c, 2893 0x23800, 0x23c38, 2894 0x23c80, 0x23d7c, 2895 0x23e00, 0x23e04, 2896 0x24000, 0x2402c, 2897 0x24100, 0x2413c, 2898 0x24190, 0x241c8, 2899 0x24200, 0x24318, 2900 0x24400, 0x24528, 2901 0x24540, 0x24614, 2902 0x25000, 0x25040, 2903 0x2504c, 0x25060, 2904 0x250c0, 0x250ec, 2905 0x25200, 0x25268, 2906 0x25270, 0x25284, 2907 0x252fc, 0x25388, 2908 0x25400, 0x25404, 2909 0x25500, 0x25518, 2910 0x2552c, 0x2553c, 2911 0x25550, 0x25554, 2912 0x25600, 0x25600, 2913 0x25608, 0x25628, 2914 0x25630, 0x2563c, 2915 0x25700, 0x2571c, 2916 0x25780, 0x2578c, 2917 0x25800, 0x25c38, 2918 0x25c80, 0x25d7c, 2919 0x25e00, 0x25e04, 2920 0x26000, 0x2602c, 2921 0x26100, 0x2613c, 2922 0x26190, 0x261c8, 2923 0x26200, 0x26318, 2924 0x26400, 0x26528, 2925 0x26540, 0x26614, 2926 0x27000, 0x27040, 2927 0x2704c, 0x27060, 2928 0x270c0, 0x270ec, 2929 0x27200, 0x27268, 2930 0x27270, 0x27284, 2931 0x272fc, 0x27388, 2932 0x27400, 0x27404, 2933 0x27500, 0x27518, 2934 0x2752c, 0x2753c, 2935 0x27550, 0x27554, 2936 0x27600, 0x27600, 2937 0x27608, 0x27628, 2938 0x27630, 0x2763c, 2939 0x27700, 0x2771c, 2940 0x27780, 0x2778c, 2941 0x27800, 0x27c38, 2942 0x27c80, 0x27d7c, 2943 0x27e00, 0x27e04 2944 }; 2945 2946 regs->version = 4 | (sc->params.rev << 10); 2947 for (i = 0; i < nitems(reg_ranges); i += 2) 2948 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 2949 } 2950 2951 static void 2952 cxgbe_tick(void *arg) 2953 { 2954 struct port_info *pi = arg; 2955 struct ifnet *ifp = pi->ifp; 2956 struct sge_txq *txq; 2957 int i, drops; 2958 struct port_stats *s = &pi->stats; 2959 2960 PORT_LOCK(pi); 2961 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2962 PORT_UNLOCK(pi); 2963 return; /* without scheduling another callout */ 2964 } 2965 2966 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 2967 2968 ifp->if_opackets = s->tx_frames - s->tx_pause; 2969 ifp->if_ipackets = s->rx_frames - s->rx_pause; 2970 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 2971 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 2972 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 2973 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 2974 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 2975 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 2976 s->rx_trunc3; 2977 2978 drops = s->tx_drop; 2979 for_each_txq(pi, i, txq) 2980 drops += txq->br->br_drops; 2981 ifp->if_snd.ifq_drops = drops; 2982 2983 ifp->if_oerrors = s->tx_error_frames; 2984 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 2985 s->rx_fcs_err + s->rx_len_err; 2986 2987 callout_schedule(&pi->tick, hz); 2988 PORT_UNLOCK(pi); 2989 } 2990 2991 static void 2992 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 2993 { 2994 struct ifnet *vlan; 2995 2996 if (arg != ifp || ifp->if_type != IFT_ETHER) 2997 return; 2998 2999 vlan = VLAN_DEVAT(ifp, vid); 3000 VLAN_SETCOOKIE(vlan, ifp); 3001 } 3002 3003 static int 3004 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 3005 { 3006 3007 #ifdef INVARIANTS 3008 panic("%s: opcode 0x%02x on iq %p with payload %p", 3009 __func__, rss->opcode, iq, m); 3010 #else 3011 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 3012 __func__, rss->opcode, iq, m); 3013 m_freem(m); 3014 #endif 3015 return (EDOOFUS); 3016 } 3017 3018 int 3019 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3020 { 3021 uintptr_t *loc, new; 3022 3023 if (opcode >= nitems(sc->cpl_handler)) 3024 return (EINVAL); 3025 3026 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3027 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3028 atomic_store_rel_ptr(loc, new); 3029 3030 return (0); 3031 } 3032 3033 static int 3034 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 3035 { 3036 3037 #ifdef INVARIANTS 3038 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 3039 #else 3040 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 3041 __func__, iq, ctrl); 3042 #endif 3043 return (EDOOFUS); 3044 } 3045 3046 int 3047 t4_register_an_handler(struct adapter *sc, an_handler_t h) 3048 { 3049 uintptr_t *loc, new; 3050 3051 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 3052 loc = (uintptr_t *) &sc->an_handler; 3053 atomic_store_rel_ptr(loc, new); 3054 3055 return (0); 3056 } 3057 3058 static int 3059 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 3060 { 3061 const struct cpl_fw6_msg *cpl = 3062 __containerof(rpl, struct cpl_fw6_msg, data[0]); 3063 3064 #ifdef INVARIANTS 3065 panic("%s: fw_msg type %d", __func__, cpl->type); 3066 #else 3067 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 3068 #endif 3069 return (EDOOFUS); 3070 } 3071 3072 int 3073 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 3074 { 3075 uintptr_t *loc, new; 3076 3077 if (type >= nitems(sc->fw_msg_handler)) 3078 return (EINVAL); 3079 3080 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 3081 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 3082 atomic_store_rel_ptr(loc, new); 3083 3084 return (0); 3085 } 3086 3087 static int 3088 t4_sysctls(struct adapter *sc) 3089 { 3090 struct sysctl_ctx_list *ctx; 3091 struct sysctl_oid *oid; 3092 struct sysctl_oid_list *children, *c0; 3093 static char *caps[] = { 3094 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 3095 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 3096 "\20\1TOE", /* caps[2] toecaps */ 3097 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 3098 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 3099 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 3100 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 3101 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 3102 }; 3103 3104 ctx = device_get_sysctl_ctx(sc->dev); 3105 3106 /* 3107 * dev.t4nex.X. 3108 */ 3109 oid = device_get_sysctl_tree(sc->dev); 3110 c0 = children = SYSCTL_CHILDREN(oid); 3111 3112 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, 3113 &sc->params.nports, 0, "# of ports"); 3114 3115 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 3116 &sc->params.rev, 0, "chip hardware revision"); 3117 3118 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 3119 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 3120 3121 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 3122 CTLFLAG_RD, &t4_cfg_file, 0, "configuration file"); 3123 3124 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, 3125 &sc->cfcsum, 0, "config file checksum"); 3126 3127 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 3128 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 3129 sysctl_bitfield, "A", "available link capabilities"); 3130 3131 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 3132 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 3133 sysctl_bitfield, "A", "available NIC capabilities"); 3134 3135 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 3136 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 3137 sysctl_bitfield, "A", "available TCP offload capabilities"); 3138 3139 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 3140 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 3141 sysctl_bitfield, "A", "available RDMA capabilities"); 3142 3143 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 3144 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 3145 sysctl_bitfield, "A", "available iSCSI capabilities"); 3146 3147 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 3148 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 3149 sysctl_bitfield, "A", "available FCoE capabilities"); 3150 3151 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, 3152 &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)"); 3153 3154 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 3155 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 3156 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 3157 "interrupt holdoff timer values (us)"); 3158 3159 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 3160 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 3161 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 3162 "interrupt holdoff packet counter values"); 3163 3164 #ifdef SBUF_DRAIN 3165 /* 3166 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 3167 */ 3168 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 3169 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 3170 "logs and miscellaneous information"); 3171 children = SYSCTL_CHILDREN(oid); 3172 3173 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 3174 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3175 sysctl_cctrl, "A", "congestion control"); 3176 3177 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 3178 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3179 sysctl_cpl_stats, "A", "CPL statistics"); 3180 3181 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 3182 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3183 sysctl_ddp_stats, "A", "DDP statistics"); 3184 3185 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 3186 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3187 sysctl_devlog, "A", "firmware's device log"); 3188 3189 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 3190 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3191 sysctl_fcoe_stats, "A", "FCoE statistics"); 3192 3193 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 3194 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3195 sysctl_hw_sched, "A", "hardware scheduler "); 3196 3197 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 3198 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3199 sysctl_l2t, "A", "hardware L2 table"); 3200 3201 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 3202 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3203 sysctl_lb_stats, "A", "loopback statistics"); 3204 3205 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 3206 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3207 sysctl_meminfo, "A", "memory regions"); 3208 3209 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 3210 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3211 sysctl_path_mtus, "A", "path MTUs"); 3212 3213 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 3214 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3215 sysctl_pm_stats, "A", "PM statistics"); 3216 3217 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 3218 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3219 sysctl_rdma_stats, "A", "RDMA statistics"); 3220 3221 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 3222 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3223 sysctl_tcp_stats, "A", "TCP statistics"); 3224 3225 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 3226 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3227 sysctl_tids, "A", "TID information"); 3228 3229 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 3230 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3231 sysctl_tp_err_stats, "A", "TP error statistics"); 3232 3233 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 3234 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3235 sysctl_tx_rate, "A", "Tx rate"); 3236 #endif 3237 3238 #ifdef TCP_OFFLOAD 3239 if (is_offload(sc)) { 3240 /* 3241 * dev.t4nex.X.toe. 3242 */ 3243 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 3244 NULL, "TOE parameters"); 3245 children = SYSCTL_CHILDREN(oid); 3246 3247 sc->tt.sndbuf = 256 * 1024; 3248 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 3249 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 3250 3251 sc->tt.ddp = 0; 3252 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 3253 &sc->tt.ddp, 0, "DDP allowed"); 3254 3255 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 3256 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 3257 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 3258 3259 sc->tt.ddp_thres = 3260 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 3261 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 3262 &sc->tt.ddp_thres, 0, "DDP threshold"); 3263 } 3264 #endif 3265 3266 3267 return (0); 3268 } 3269 3270 static int 3271 cxgbe_sysctls(struct port_info *pi) 3272 { 3273 struct sysctl_ctx_list *ctx; 3274 struct sysctl_oid *oid; 3275 struct sysctl_oid_list *children; 3276 3277 ctx = device_get_sysctl_ctx(pi->dev); 3278 3279 /* 3280 * dev.cxgbe.X. 3281 */ 3282 oid = device_get_sysctl_tree(pi->dev); 3283 children = SYSCTL_CHILDREN(oid); 3284 3285 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 3286 &pi->nrxq, 0, "# of rx queues"); 3287 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 3288 &pi->ntxq, 0, "# of tx queues"); 3289 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 3290 &pi->first_rxq, 0, "index of first rx queue"); 3291 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 3292 &pi->first_txq, 0, "index of first tx queue"); 3293 3294 #ifdef TCP_OFFLOAD 3295 if (is_offload(pi->adapter)) { 3296 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 3297 &pi->nofldrxq, 0, 3298 "# of rx queues for offloaded TCP connections"); 3299 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 3300 &pi->nofldtxq, 0, 3301 "# of tx queues for offloaded TCP connections"); 3302 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 3303 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 3304 "index of first TOE rx queue"); 3305 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 3306 CTLFLAG_RD, &pi->first_ofld_txq, 0, 3307 "index of first TOE tx queue"); 3308 } 3309 #endif 3310 3311 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 3312 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 3313 "holdoff timer index"); 3314 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 3315 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 3316 "holdoff packet counter index"); 3317 3318 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 3319 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 3320 "rx queue size"); 3321 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 3322 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 3323 "tx queue size"); 3324 3325 /* 3326 * dev.cxgbe.X.stats. 3327 */ 3328 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 3329 NULL, "port statistics"); 3330 children = SYSCTL_CHILDREN(oid); 3331 3332 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 3333 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 3334 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 3335 sysctl_handle_t4_reg64, "QU", desc) 3336 3337 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 3338 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 3339 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 3340 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 3341 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 3342 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 3343 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 3344 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 3345 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 3346 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 3347 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 3348 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 3349 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 3350 "# of tx frames in this range", 3351 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 3352 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 3353 "# of tx frames in this range", 3354 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 3355 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 3356 "# of tx frames in this range", 3357 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 3358 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 3359 "# of tx frames in this range", 3360 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 3361 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 3362 "# of tx frames in this range", 3363 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 3364 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 3365 "# of tx frames in this range", 3366 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 3367 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 3368 "# of tx frames in this range", 3369 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 3370 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 3371 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 3372 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 3373 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 3374 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 3375 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 3376 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 3377 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 3378 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 3379 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 3380 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 3381 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 3382 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 3383 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 3384 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 3385 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 3386 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 3387 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 3388 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 3389 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 3390 3391 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 3392 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 3393 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 3394 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 3395 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 3396 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 3397 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 3398 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 3399 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 3400 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 3401 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 3402 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 3403 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 3404 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 3405 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 3406 "# of frames received with bad FCS", 3407 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 3408 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 3409 "# of frames received with length error", 3410 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 3411 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 3412 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 3413 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 3414 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 3415 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 3416 "# of rx frames in this range", 3417 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 3418 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 3419 "# of rx frames in this range", 3420 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 3421 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 3422 "# of rx frames in this range", 3423 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 3424 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 3425 "# of rx frames in this range", 3426 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 3427 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 3428 "# of rx frames in this range", 3429 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 3430 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 3431 "# of rx frames in this range", 3432 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 3433 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 3434 "# of rx frames in this range", 3435 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 3436 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 3437 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 3438 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 3439 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 3440 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 3441 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 3442 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 3443 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 3444 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 3445 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 3446 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 3447 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 3448 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 3449 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 3450 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 3451 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 3452 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 3453 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 3454 3455 #undef SYSCTL_ADD_T4_REG64 3456 3457 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 3458 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 3459 &pi->stats.name, desc) 3460 3461 /* We get these from port_stats and they may be stale by upto 1s */ 3462 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 3463 "# drops due to buffer-group 0 overflows"); 3464 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 3465 "# drops due to buffer-group 1 overflows"); 3466 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 3467 "# drops due to buffer-group 2 overflows"); 3468 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 3469 "# drops due to buffer-group 3 overflows"); 3470 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 3471 "# of buffer-group 0 truncated packets"); 3472 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 3473 "# of buffer-group 1 truncated packets"); 3474 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 3475 "# of buffer-group 2 truncated packets"); 3476 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 3477 "# of buffer-group 3 truncated packets"); 3478 3479 #undef SYSCTL_ADD_T4_PORTSTAT 3480 3481 return (0); 3482 } 3483 3484 static int 3485 sysctl_int_array(SYSCTL_HANDLER_ARGS) 3486 { 3487 int rc, *i; 3488 struct sbuf sb; 3489 3490 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 3491 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 3492 sbuf_printf(&sb, "%d ", *i); 3493 sbuf_trim(&sb); 3494 sbuf_finish(&sb); 3495 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 3496 sbuf_delete(&sb); 3497 return (rc); 3498 } 3499 3500 static int 3501 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 3502 { 3503 int rc; 3504 struct sbuf *sb; 3505 3506 rc = sysctl_wire_old_buffer(req, 0); 3507 if (rc != 0) 3508 return(rc); 3509 3510 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 3511 if (sb == NULL) 3512 return (ENOMEM); 3513 3514 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 3515 rc = sbuf_finish(sb); 3516 sbuf_delete(sb); 3517 3518 return (rc); 3519 } 3520 3521 static int 3522 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 3523 { 3524 struct port_info *pi = arg1; 3525 struct adapter *sc = pi->adapter; 3526 int idx, rc, i; 3527 3528 idx = pi->tmr_idx; 3529 3530 rc = sysctl_handle_int(oidp, &idx, 0, req); 3531 if (rc != 0 || req->newptr == NULL) 3532 return (rc); 3533 3534 if (idx < 0 || idx >= SGE_NTIMERS) 3535 return (EINVAL); 3536 3537 ADAPTER_LOCK(sc); 3538 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3539 if (rc == 0) { 3540 struct sge_rxq *rxq; 3541 uint8_t v; 3542 3543 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 3544 for_each_rxq(pi, i, rxq) { 3545 #ifdef atomic_store_rel_8 3546 atomic_store_rel_8(&rxq->iq.intr_params, v); 3547 #else 3548 rxq->iq.intr_params = v; 3549 #endif 3550 } 3551 pi->tmr_idx = idx; 3552 } 3553 3554 ADAPTER_UNLOCK(sc); 3555 return (rc); 3556 } 3557 3558 static int 3559 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 3560 { 3561 struct port_info *pi = arg1; 3562 struct adapter *sc = pi->adapter; 3563 int idx, rc; 3564 3565 idx = pi->pktc_idx; 3566 3567 rc = sysctl_handle_int(oidp, &idx, 0, req); 3568 if (rc != 0 || req->newptr == NULL) 3569 return (rc); 3570 3571 if (idx < -1 || idx >= SGE_NCOUNTERS) 3572 return (EINVAL); 3573 3574 ADAPTER_LOCK(sc); 3575 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3576 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3577 rc = EBUSY; /* cannot be changed once the queues are created */ 3578 3579 if (rc == 0) 3580 pi->pktc_idx = idx; 3581 3582 ADAPTER_UNLOCK(sc); 3583 return (rc); 3584 } 3585 3586 static int 3587 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 3588 { 3589 struct port_info *pi = arg1; 3590 struct adapter *sc = pi->adapter; 3591 int qsize, rc; 3592 3593 qsize = pi->qsize_rxq; 3594 3595 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3596 if (rc != 0 || req->newptr == NULL) 3597 return (rc); 3598 3599 if (qsize < 128 || (qsize & 7)) 3600 return (EINVAL); 3601 3602 ADAPTER_LOCK(sc); 3603 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3604 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3605 rc = EBUSY; /* cannot be changed once the queues are created */ 3606 3607 if (rc == 0) 3608 pi->qsize_rxq = qsize; 3609 3610 ADAPTER_UNLOCK(sc); 3611 return (rc); 3612 } 3613 3614 static int 3615 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 3616 { 3617 struct port_info *pi = arg1; 3618 struct adapter *sc = pi->adapter; 3619 int qsize, rc; 3620 3621 qsize = pi->qsize_txq; 3622 3623 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3624 if (rc != 0 || req->newptr == NULL) 3625 return (rc); 3626 3627 if (qsize < 128) 3628 return (EINVAL); 3629 3630 ADAPTER_LOCK(sc); 3631 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3632 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3633 rc = EBUSY; /* cannot be changed once the queues are created */ 3634 3635 if (rc == 0) 3636 pi->qsize_txq = qsize; 3637 3638 ADAPTER_UNLOCK(sc); 3639 return (rc); 3640 } 3641 3642 static int 3643 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 3644 { 3645 struct adapter *sc = arg1; 3646 int reg = arg2; 3647 uint64_t val; 3648 3649 val = t4_read_reg64(sc, reg); 3650 3651 return (sysctl_handle_64(oidp, &val, 0, req)); 3652 } 3653 3654 #ifdef SBUF_DRAIN 3655 static int 3656 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 3657 { 3658 struct adapter *sc = arg1; 3659 struct sbuf *sb; 3660 int rc, i; 3661 uint16_t incr[NMTUS][NCCTRL_WIN]; 3662 static const char *dec_fac[] = { 3663 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 3664 "0.9375" 3665 }; 3666 3667 rc = sysctl_wire_old_buffer(req, 0); 3668 if (rc != 0) 3669 return (rc); 3670 3671 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3672 if (sb == NULL) 3673 return (ENOMEM); 3674 3675 t4_read_cong_tbl(sc, incr); 3676 3677 for (i = 0; i < NCCTRL_WIN; ++i) { 3678 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 3679 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 3680 incr[5][i], incr[6][i], incr[7][i]); 3681 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 3682 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 3683 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 3684 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 3685 } 3686 3687 rc = sbuf_finish(sb); 3688 sbuf_delete(sb); 3689 3690 return (rc); 3691 } 3692 3693 static int 3694 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 3695 { 3696 struct adapter *sc = arg1; 3697 struct sbuf *sb; 3698 int rc; 3699 struct tp_cpl_stats stats; 3700 3701 rc = sysctl_wire_old_buffer(req, 0); 3702 if (rc != 0) 3703 return (rc); 3704 3705 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3706 if (sb == NULL) 3707 return (ENOMEM); 3708 3709 t4_tp_get_cpl_stats(sc, &stats); 3710 3711 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 3712 "channel 3\n"); 3713 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 3714 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 3715 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 3716 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 3717 3718 rc = sbuf_finish(sb); 3719 sbuf_delete(sb); 3720 3721 return (rc); 3722 } 3723 3724 static int 3725 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 3726 { 3727 struct adapter *sc = arg1; 3728 struct sbuf *sb; 3729 int rc; 3730 struct tp_usm_stats stats; 3731 3732 rc = sysctl_wire_old_buffer(req, 0); 3733 if (rc != 0) 3734 return(rc); 3735 3736 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3737 if (sb == NULL) 3738 return (ENOMEM); 3739 3740 t4_get_usm_stats(sc, &stats); 3741 3742 sbuf_printf(sb, "Frames: %u\n", stats.frames); 3743 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 3744 sbuf_printf(sb, "Drops: %u", stats.drops); 3745 3746 rc = sbuf_finish(sb); 3747 sbuf_delete(sb); 3748 3749 return (rc); 3750 } 3751 3752 const char *devlog_level_strings[] = { 3753 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 3754 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 3755 [FW_DEVLOG_LEVEL_ERR] = "ERR", 3756 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 3757 [FW_DEVLOG_LEVEL_INFO] = "INFO", 3758 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 3759 }; 3760 3761 const char *devlog_facility_strings[] = { 3762 [FW_DEVLOG_FACILITY_CORE] = "CORE", 3763 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 3764 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 3765 [FW_DEVLOG_FACILITY_RES] = "RES", 3766 [FW_DEVLOG_FACILITY_HW] = "HW", 3767 [FW_DEVLOG_FACILITY_FLR] = "FLR", 3768 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 3769 [FW_DEVLOG_FACILITY_PHY] = "PHY", 3770 [FW_DEVLOG_FACILITY_MAC] = "MAC", 3771 [FW_DEVLOG_FACILITY_PORT] = "PORT", 3772 [FW_DEVLOG_FACILITY_VI] = "VI", 3773 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 3774 [FW_DEVLOG_FACILITY_ACL] = "ACL", 3775 [FW_DEVLOG_FACILITY_TM] = "TM", 3776 [FW_DEVLOG_FACILITY_QFC] = "QFC", 3777 [FW_DEVLOG_FACILITY_DCB] = "DCB", 3778 [FW_DEVLOG_FACILITY_ETH] = "ETH", 3779 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 3780 [FW_DEVLOG_FACILITY_RI] = "RI", 3781 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 3782 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 3783 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 3784 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 3785 }; 3786 3787 static int 3788 sysctl_devlog(SYSCTL_HANDLER_ARGS) 3789 { 3790 struct adapter *sc = arg1; 3791 struct devlog_params *dparams = &sc->params.devlog; 3792 struct fw_devlog_e *buf, *e; 3793 int i, j, rc, nentries, first = 0; 3794 struct sbuf *sb; 3795 uint64_t ftstamp = UINT64_MAX; 3796 3797 if (dparams->start == 0) 3798 return (ENXIO); 3799 3800 nentries = dparams->size / sizeof(struct fw_devlog_e); 3801 3802 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 3803 if (buf == NULL) 3804 return (ENOMEM); 3805 3806 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 3807 (void *)buf); 3808 if (rc != 0) 3809 goto done; 3810 3811 for (i = 0; i < nentries; i++) { 3812 e = &buf[i]; 3813 3814 if (e->timestamp == 0) 3815 break; /* end */ 3816 3817 e->timestamp = be64toh(e->timestamp); 3818 e->seqno = be32toh(e->seqno); 3819 for (j = 0; j < 8; j++) 3820 e->params[j] = be32toh(e->params[j]); 3821 3822 if (e->timestamp < ftstamp) { 3823 ftstamp = e->timestamp; 3824 first = i; 3825 } 3826 } 3827 3828 if (buf[first].timestamp == 0) 3829 goto done; /* nothing in the log */ 3830 3831 rc = sysctl_wire_old_buffer(req, 0); 3832 if (rc != 0) 3833 goto done; 3834 3835 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3836 if (sb == NULL) { 3837 rc = ENOMEM; 3838 goto done; 3839 } 3840 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 3841 "Seq#", "Tstamp", "Level", "Facility", "Message"); 3842 3843 i = first; 3844 do { 3845 e = &buf[i]; 3846 if (e->timestamp == 0) 3847 break; /* end */ 3848 3849 sbuf_printf(sb, "%10d %15ju %8s %8s ", 3850 e->seqno, e->timestamp, 3851 (e->level < nitems(devlog_level_strings) ? 3852 devlog_level_strings[e->level] : "UNKNOWN"), 3853 (e->facility < nitems(devlog_facility_strings) ? 3854 devlog_facility_strings[e->facility] : "UNKNOWN")); 3855 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 3856 e->params[2], e->params[3], e->params[4], 3857 e->params[5], e->params[6], e->params[7]); 3858 3859 if (++i == nentries) 3860 i = 0; 3861 } while (i != first); 3862 3863 rc = sbuf_finish(sb); 3864 sbuf_delete(sb); 3865 done: 3866 free(buf, M_CXGBE); 3867 return (rc); 3868 } 3869 3870 static int 3871 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 3872 { 3873 struct adapter *sc = arg1; 3874 struct sbuf *sb; 3875 int rc; 3876 struct tp_fcoe_stats stats[4]; 3877 3878 rc = sysctl_wire_old_buffer(req, 0); 3879 if (rc != 0) 3880 return (rc); 3881 3882 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3883 if (sb == NULL) 3884 return (ENOMEM); 3885 3886 t4_get_fcoe_stats(sc, 0, &stats[0]); 3887 t4_get_fcoe_stats(sc, 1, &stats[1]); 3888 t4_get_fcoe_stats(sc, 2, &stats[2]); 3889 t4_get_fcoe_stats(sc, 3, &stats[3]); 3890 3891 sbuf_printf(sb, " channel 0 channel 1 " 3892 "channel 2 channel 3\n"); 3893 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 3894 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 3895 stats[3].octetsDDP); 3896 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 3897 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 3898 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 3899 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 3900 stats[3].framesDrop); 3901 3902 rc = sbuf_finish(sb); 3903 sbuf_delete(sb); 3904 3905 return (rc); 3906 } 3907 3908 static int 3909 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 3910 { 3911 struct adapter *sc = arg1; 3912 struct sbuf *sb; 3913 int rc, i; 3914 unsigned int map, kbps, ipg, mode; 3915 unsigned int pace_tab[NTX_SCHED]; 3916 3917 rc = sysctl_wire_old_buffer(req, 0); 3918 if (rc != 0) 3919 return (rc); 3920 3921 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3922 if (sb == NULL) 3923 return (ENOMEM); 3924 3925 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 3926 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 3927 t4_read_pace_tbl(sc, pace_tab); 3928 3929 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 3930 "Class IPG (0.1 ns) Flow IPG (us)"); 3931 3932 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 3933 t4_get_tx_sched(sc, i, &kbps, &ipg); 3934 sbuf_printf(sb, "\n %u %-5s %u ", i, 3935 (mode & (1 << i)) ? "flow" : "class", map & 3); 3936 if (kbps) 3937 sbuf_printf(sb, "%9u ", kbps); 3938 else 3939 sbuf_printf(sb, " disabled "); 3940 3941 if (ipg) 3942 sbuf_printf(sb, "%13u ", ipg); 3943 else 3944 sbuf_printf(sb, " disabled "); 3945 3946 if (pace_tab[i]) 3947 sbuf_printf(sb, "%10u", pace_tab[i]); 3948 else 3949 sbuf_printf(sb, " disabled"); 3950 } 3951 3952 rc = sbuf_finish(sb); 3953 sbuf_delete(sb); 3954 3955 return (rc); 3956 } 3957 3958 static int 3959 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 3960 { 3961 struct adapter *sc = arg1; 3962 struct sbuf *sb; 3963 int rc, i, j; 3964 uint64_t *p0, *p1; 3965 struct lb_port_stats s[2]; 3966 static const char *stat_name[] = { 3967 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 3968 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 3969 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 3970 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 3971 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 3972 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 3973 "BG2FramesTrunc:", "BG3FramesTrunc:" 3974 }; 3975 3976 rc = sysctl_wire_old_buffer(req, 0); 3977 if (rc != 0) 3978 return (rc); 3979 3980 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3981 if (sb == NULL) 3982 return (ENOMEM); 3983 3984 memset(s, 0, sizeof(s)); 3985 3986 for (i = 0; i < 4; i += 2) { 3987 t4_get_lb_stats(sc, i, &s[0]); 3988 t4_get_lb_stats(sc, i + 1, &s[1]); 3989 3990 p0 = &s[0].octets; 3991 p1 = &s[1].octets; 3992 sbuf_printf(sb, "%s Loopback %u" 3993 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 3994 3995 for (j = 0; j < nitems(stat_name); j++) 3996 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 3997 *p0++, *p1++); 3998 } 3999 4000 rc = sbuf_finish(sb); 4001 sbuf_delete(sb); 4002 4003 return (rc); 4004 } 4005 4006 struct mem_desc { 4007 unsigned int base; 4008 unsigned int limit; 4009 unsigned int idx; 4010 }; 4011 4012 static int 4013 mem_desc_cmp(const void *a, const void *b) 4014 { 4015 return ((const struct mem_desc *)a)->base - 4016 ((const struct mem_desc *)b)->base; 4017 } 4018 4019 static void 4020 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 4021 unsigned int to) 4022 { 4023 unsigned int size; 4024 4025 size = to - from + 1; 4026 if (size == 0) 4027 return; 4028 4029 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 4030 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 4031 } 4032 4033 static int 4034 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 4035 { 4036 struct adapter *sc = arg1; 4037 struct sbuf *sb; 4038 int rc, i, n; 4039 uint32_t lo, hi; 4040 static const char *memory[] = { "EDC0:", "EDC1:", "MC:" }; 4041 static const char *region[] = { 4042 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 4043 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 4044 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 4045 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 4046 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:", 4047 "ULPTX state:", "On-chip queues:" 4048 }; 4049 struct mem_desc avail[3]; 4050 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 4051 struct mem_desc *md = mem; 4052 4053 rc = sysctl_wire_old_buffer(req, 0); 4054 if (rc != 0) 4055 return (rc); 4056 4057 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4058 if (sb == NULL) 4059 return (ENOMEM); 4060 4061 for (i = 0; i < nitems(mem); i++) { 4062 mem[i].limit = 0; 4063 mem[i].idx = i; 4064 } 4065 4066 /* Find and sort the populated memory ranges */ 4067 i = 0; 4068 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 4069 if (lo & F_EDRAM0_ENABLE) { 4070 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 4071 avail[i].base = G_EDRAM0_BASE(hi) << 20; 4072 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 4073 avail[i].idx = 0; 4074 i++; 4075 } 4076 if (lo & F_EDRAM1_ENABLE) { 4077 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 4078 avail[i].base = G_EDRAM1_BASE(hi) << 20; 4079 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 4080 avail[i].idx = 1; 4081 i++; 4082 } 4083 if (lo & F_EXT_MEM_ENABLE) { 4084 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 4085 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 4086 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20); 4087 avail[i].idx = 2; 4088 i++; 4089 } 4090 if (!i) /* no memory available */ 4091 return 0; 4092 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 4093 4094 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 4095 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 4096 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 4097 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4098 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 4099 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 4100 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 4101 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 4102 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 4103 4104 /* the next few have explicit upper bounds */ 4105 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 4106 md->limit = md->base - 1 + 4107 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 4108 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 4109 md++; 4110 4111 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 4112 md->limit = md->base - 1 + 4113 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 4114 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 4115 md++; 4116 4117 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4118 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 4119 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 4120 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 4121 } else { 4122 md->base = 0; 4123 md->idx = nitems(region); /* hide it */ 4124 } 4125 md++; 4126 4127 #define ulp_region(reg) \ 4128 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 4129 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 4130 4131 ulp_region(RX_ISCSI); 4132 ulp_region(RX_TDDP); 4133 ulp_region(TX_TPT); 4134 ulp_region(RX_STAG); 4135 ulp_region(RX_RQ); 4136 ulp_region(RX_RQUDP); 4137 ulp_region(RX_PBL); 4138 ulp_region(TX_PBL); 4139 #undef ulp_region 4140 4141 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 4142 md->limit = md->base + sc->tids.ntids - 1; 4143 md++; 4144 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 4145 md->limit = md->base + sc->tids.ntids - 1; 4146 md++; 4147 4148 md->base = sc->vres.ocq.start; 4149 if (sc->vres.ocq.size) 4150 md->limit = md->base + sc->vres.ocq.size - 1; 4151 else 4152 md->idx = nitems(region); /* hide it */ 4153 md++; 4154 4155 /* add any address-space holes, there can be up to 3 */ 4156 for (n = 0; n < i - 1; n++) 4157 if (avail[n].limit < avail[n + 1].base) 4158 (md++)->base = avail[n].limit; 4159 if (avail[n].limit) 4160 (md++)->base = avail[n].limit; 4161 4162 n = md - mem; 4163 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 4164 4165 for (lo = 0; lo < i; lo++) 4166 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 4167 avail[lo].limit - 1); 4168 4169 sbuf_printf(sb, "\n"); 4170 for (i = 0; i < n; i++) { 4171 if (mem[i].idx >= nitems(region)) 4172 continue; /* skip holes */ 4173 if (!mem[i].limit) 4174 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 4175 mem_region_show(sb, region[mem[i].idx], mem[i].base, 4176 mem[i].limit); 4177 } 4178 4179 sbuf_printf(sb, "\n"); 4180 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 4181 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 4182 mem_region_show(sb, "uP RAM:", lo, hi); 4183 4184 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 4185 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 4186 mem_region_show(sb, "uP Extmem2:", lo, hi); 4187 4188 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 4189 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 4190 G_PMRXMAXPAGE(lo), 4191 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 4192 (lo & F_PMRXNUMCHN) ? 2 : 1); 4193 4194 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 4195 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 4196 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 4197 G_PMTXMAXPAGE(lo), 4198 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 4199 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 4200 sbuf_printf(sb, "%u p-structs\n", 4201 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 4202 4203 for (i = 0; i < 4; i++) { 4204 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 4205 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 4206 i, G_USED(lo), G_ALLOC(lo)); 4207 } 4208 for (i = 0; i < 4; i++) { 4209 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 4210 sbuf_printf(sb, 4211 "\nLoopback %d using %u pages out of %u allocated", 4212 i, G_USED(lo), G_ALLOC(lo)); 4213 } 4214 4215 rc = sbuf_finish(sb); 4216 sbuf_delete(sb); 4217 4218 return (rc); 4219 } 4220 4221 static int 4222 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 4223 { 4224 struct adapter *sc = arg1; 4225 struct sbuf *sb; 4226 int rc; 4227 uint16_t mtus[NMTUS]; 4228 4229 rc = sysctl_wire_old_buffer(req, 0); 4230 if (rc != 0) 4231 return (rc); 4232 4233 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4234 if (sb == NULL) 4235 return (ENOMEM); 4236 4237 t4_read_mtu_tbl(sc, mtus, NULL); 4238 4239 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 4240 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 4241 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 4242 mtus[14], mtus[15]); 4243 4244 rc = sbuf_finish(sb); 4245 sbuf_delete(sb); 4246 4247 return (rc); 4248 } 4249 4250 static int 4251 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 4252 { 4253 struct adapter *sc = arg1; 4254 struct sbuf *sb; 4255 int rc, i; 4256 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 4257 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 4258 static const char *pm_stats[] = { 4259 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 4260 }; 4261 4262 rc = sysctl_wire_old_buffer(req, 0); 4263 if (rc != 0) 4264 return (rc); 4265 4266 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4267 if (sb == NULL) 4268 return (ENOMEM); 4269 4270 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 4271 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 4272 4273 sbuf_printf(sb, " Tx count Tx cycles " 4274 "Rx count Rx cycles"); 4275 for (i = 0; i < PM_NSTATS; i++) 4276 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 4277 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 4278 4279 rc = sbuf_finish(sb); 4280 sbuf_delete(sb); 4281 4282 return (rc); 4283 } 4284 4285 static int 4286 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 4287 { 4288 struct adapter *sc = arg1; 4289 struct sbuf *sb; 4290 int rc; 4291 struct tp_rdma_stats stats; 4292 4293 rc = sysctl_wire_old_buffer(req, 0); 4294 if (rc != 0) 4295 return (rc); 4296 4297 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4298 if (sb == NULL) 4299 return (ENOMEM); 4300 4301 t4_tp_get_rdma_stats(sc, &stats); 4302 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 4303 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 4304 4305 rc = sbuf_finish(sb); 4306 sbuf_delete(sb); 4307 4308 return (rc); 4309 } 4310 4311 static int 4312 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 4313 { 4314 struct adapter *sc = arg1; 4315 struct sbuf *sb; 4316 int rc; 4317 struct tp_tcp_stats v4, v6; 4318 4319 rc = sysctl_wire_old_buffer(req, 0); 4320 if (rc != 0) 4321 return (rc); 4322 4323 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4324 if (sb == NULL) 4325 return (ENOMEM); 4326 4327 t4_tp_get_tcp_stats(sc, &v4, &v6); 4328 sbuf_printf(sb, 4329 " IP IPv6\n"); 4330 sbuf_printf(sb, "OutRsts: %20u %20u\n", 4331 v4.tcpOutRsts, v6.tcpOutRsts); 4332 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 4333 v4.tcpInSegs, v6.tcpInSegs); 4334 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 4335 v4.tcpOutSegs, v6.tcpOutSegs); 4336 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 4337 v4.tcpRetransSegs, v6.tcpRetransSegs); 4338 4339 rc = sbuf_finish(sb); 4340 sbuf_delete(sb); 4341 4342 return (rc); 4343 } 4344 4345 static int 4346 sysctl_tids(SYSCTL_HANDLER_ARGS) 4347 { 4348 struct adapter *sc = arg1; 4349 struct sbuf *sb; 4350 int rc; 4351 struct tid_info *t = &sc->tids; 4352 4353 rc = sysctl_wire_old_buffer(req, 0); 4354 if (rc != 0) 4355 return (rc); 4356 4357 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4358 if (sb == NULL) 4359 return (ENOMEM); 4360 4361 if (t->natids) { 4362 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 4363 t->atids_in_use); 4364 } 4365 4366 if (t->ntids) { 4367 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4368 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 4369 4370 if (b) { 4371 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 4372 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4373 t->ntids - 1); 4374 } else { 4375 sbuf_printf(sb, "TID range: %u-%u", 4376 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4377 t->ntids - 1); 4378 } 4379 } else 4380 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 4381 sbuf_printf(sb, ", in use: %u\n", 4382 atomic_load_acq_int(&t->tids_in_use)); 4383 } 4384 4385 if (t->nstids) { 4386 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 4387 t->stid_base + t->nstids - 1, t->stids_in_use); 4388 } 4389 4390 if (t->nftids) { 4391 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 4392 t->ftid_base + t->nftids - 1); 4393 } 4394 4395 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 4396 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 4397 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 4398 4399 rc = sbuf_finish(sb); 4400 sbuf_delete(sb); 4401 4402 return (rc); 4403 } 4404 4405 static int 4406 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 4407 { 4408 struct adapter *sc = arg1; 4409 struct sbuf *sb; 4410 int rc; 4411 struct tp_err_stats stats; 4412 4413 rc = sysctl_wire_old_buffer(req, 0); 4414 if (rc != 0) 4415 return (rc); 4416 4417 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4418 if (sb == NULL) 4419 return (ENOMEM); 4420 4421 t4_tp_get_err_stats(sc, &stats); 4422 4423 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4424 "channel 3\n"); 4425 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 4426 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 4427 stats.macInErrs[3]); 4428 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 4429 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 4430 stats.hdrInErrs[3]); 4431 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 4432 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 4433 stats.tcpInErrs[3]); 4434 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 4435 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 4436 stats.tcp6InErrs[3]); 4437 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 4438 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 4439 stats.tnlCongDrops[3]); 4440 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 4441 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 4442 stats.tnlTxDrops[3]); 4443 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 4444 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 4445 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 4446 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 4447 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 4448 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 4449 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 4450 stats.ofldNoNeigh, stats.ofldCongDefer); 4451 4452 rc = sbuf_finish(sb); 4453 sbuf_delete(sb); 4454 4455 return (rc); 4456 } 4457 4458 static int 4459 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 4460 { 4461 struct adapter *sc = arg1; 4462 struct sbuf *sb; 4463 int rc; 4464 u64 nrate[NCHAN], orate[NCHAN]; 4465 4466 rc = sysctl_wire_old_buffer(req, 0); 4467 if (rc != 0) 4468 return (rc); 4469 4470 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4471 if (sb == NULL) 4472 return (ENOMEM); 4473 4474 t4_get_chan_txrate(sc, nrate, orate); 4475 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4476 "channel 3\n"); 4477 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 4478 nrate[0], nrate[1], nrate[2], nrate[3]); 4479 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 4480 orate[0], orate[1], orate[2], orate[3]); 4481 4482 rc = sbuf_finish(sb); 4483 sbuf_delete(sb); 4484 4485 return (rc); 4486 } 4487 #endif 4488 4489 static inline void 4490 txq_start(struct ifnet *ifp, struct sge_txq *txq) 4491 { 4492 struct buf_ring *br; 4493 struct mbuf *m; 4494 4495 TXQ_LOCK_ASSERT_OWNED(txq); 4496 4497 br = txq->br; 4498 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 4499 if (m) 4500 t4_eth_tx(ifp, txq, m); 4501 } 4502 4503 void 4504 t4_tx_callout(void *arg) 4505 { 4506 struct sge_eq *eq = arg; 4507 struct adapter *sc; 4508 4509 if (EQ_TRYLOCK(eq) == 0) 4510 goto reschedule; 4511 4512 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 4513 EQ_UNLOCK(eq); 4514 reschedule: 4515 if (__predict_true(!(eq->flags && EQ_DOOMED))) 4516 callout_schedule(&eq->tx_callout, 1); 4517 return; 4518 } 4519 4520 EQ_LOCK_ASSERT_OWNED(eq); 4521 4522 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 4523 4524 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4525 struct sge_txq *txq = arg; 4526 struct port_info *pi = txq->ifp->if_softc; 4527 4528 sc = pi->adapter; 4529 } else { 4530 struct sge_wrq *wrq = arg; 4531 4532 sc = wrq->adapter; 4533 } 4534 4535 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 4536 } 4537 4538 EQ_UNLOCK(eq); 4539 } 4540 4541 void 4542 t4_tx_task(void *arg, int count) 4543 { 4544 struct sge_eq *eq = arg; 4545 4546 EQ_LOCK(eq); 4547 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4548 struct sge_txq *txq = arg; 4549 txq_start(txq->ifp, txq); 4550 } else { 4551 struct sge_wrq *wrq = arg; 4552 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 4553 } 4554 EQ_UNLOCK(eq); 4555 } 4556 4557 static uint32_t 4558 fconf_to_mode(uint32_t fconf) 4559 { 4560 uint32_t mode; 4561 4562 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 4563 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 4564 4565 if (fconf & F_FRAGMENTATION) 4566 mode |= T4_FILTER_IP_FRAGMENT; 4567 4568 if (fconf & F_MPSHITTYPE) 4569 mode |= T4_FILTER_MPS_HIT_TYPE; 4570 4571 if (fconf & F_MACMATCH) 4572 mode |= T4_FILTER_MAC_IDX; 4573 4574 if (fconf & F_ETHERTYPE) 4575 mode |= T4_FILTER_ETH_TYPE; 4576 4577 if (fconf & F_PROTOCOL) 4578 mode |= T4_FILTER_IP_PROTO; 4579 4580 if (fconf & F_TOS) 4581 mode |= T4_FILTER_IP_TOS; 4582 4583 if (fconf & F_VLAN) 4584 mode |= T4_FILTER_VLAN; 4585 4586 if (fconf & F_VNIC_ID) 4587 mode |= T4_FILTER_VNIC; 4588 4589 if (fconf & F_PORT) 4590 mode |= T4_FILTER_PORT; 4591 4592 if (fconf & F_FCOE) 4593 mode |= T4_FILTER_FCoE; 4594 4595 return (mode); 4596 } 4597 4598 static uint32_t 4599 mode_to_fconf(uint32_t mode) 4600 { 4601 uint32_t fconf = 0; 4602 4603 if (mode & T4_FILTER_IP_FRAGMENT) 4604 fconf |= F_FRAGMENTATION; 4605 4606 if (mode & T4_FILTER_MPS_HIT_TYPE) 4607 fconf |= F_MPSHITTYPE; 4608 4609 if (mode & T4_FILTER_MAC_IDX) 4610 fconf |= F_MACMATCH; 4611 4612 if (mode & T4_FILTER_ETH_TYPE) 4613 fconf |= F_ETHERTYPE; 4614 4615 if (mode & T4_FILTER_IP_PROTO) 4616 fconf |= F_PROTOCOL; 4617 4618 if (mode & T4_FILTER_IP_TOS) 4619 fconf |= F_TOS; 4620 4621 if (mode & T4_FILTER_VLAN) 4622 fconf |= F_VLAN; 4623 4624 if (mode & T4_FILTER_VNIC) 4625 fconf |= F_VNIC_ID; 4626 4627 if (mode & T4_FILTER_PORT) 4628 fconf |= F_PORT; 4629 4630 if (mode & T4_FILTER_FCoE) 4631 fconf |= F_FCOE; 4632 4633 return (fconf); 4634 } 4635 4636 static uint32_t 4637 fspec_to_fconf(struct t4_filter_specification *fs) 4638 { 4639 uint32_t fconf = 0; 4640 4641 if (fs->val.frag || fs->mask.frag) 4642 fconf |= F_FRAGMENTATION; 4643 4644 if (fs->val.matchtype || fs->mask.matchtype) 4645 fconf |= F_MPSHITTYPE; 4646 4647 if (fs->val.macidx || fs->mask.macidx) 4648 fconf |= F_MACMATCH; 4649 4650 if (fs->val.ethtype || fs->mask.ethtype) 4651 fconf |= F_ETHERTYPE; 4652 4653 if (fs->val.proto || fs->mask.proto) 4654 fconf |= F_PROTOCOL; 4655 4656 if (fs->val.tos || fs->mask.tos) 4657 fconf |= F_TOS; 4658 4659 if (fs->val.vlan_vld || fs->mask.vlan_vld) 4660 fconf |= F_VLAN; 4661 4662 if (fs->val.vnic_vld || fs->mask.vnic_vld) 4663 fconf |= F_VNIC_ID; 4664 4665 if (fs->val.iport || fs->mask.iport) 4666 fconf |= F_PORT; 4667 4668 if (fs->val.fcoe || fs->mask.fcoe) 4669 fconf |= F_FCOE; 4670 4671 return (fconf); 4672 } 4673 4674 static int 4675 get_filter_mode(struct adapter *sc, uint32_t *mode) 4676 { 4677 uint32_t fconf; 4678 4679 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 4680 A_TP_VLAN_PRI_MAP); 4681 4682 if (sc->filter_mode != fconf) { 4683 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 4684 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 4685 sc->filter_mode = fconf; 4686 } 4687 4688 *mode = fconf_to_mode(sc->filter_mode); 4689 4690 return (0); 4691 } 4692 4693 static int 4694 set_filter_mode(struct adapter *sc, uint32_t mode) 4695 { 4696 uint32_t fconf; 4697 int rc; 4698 4699 fconf = mode_to_fconf(mode); 4700 4701 ADAPTER_LOCK(sc); 4702 if (IS_BUSY(sc)) { 4703 rc = EAGAIN; 4704 goto done; 4705 } 4706 4707 if (sc->tids.ftids_in_use > 0) { 4708 rc = EBUSY; 4709 goto done; 4710 } 4711 4712 #ifdef TCP_OFFLOAD 4713 if (sc->offload_map) { 4714 rc = EBUSY; 4715 goto done; 4716 } 4717 #endif 4718 4719 #ifdef notyet 4720 rc = -t4_set_filter_mode(sc, fconf); 4721 if (rc == 0) 4722 sc->filter_mode = fconf; 4723 #else 4724 rc = ENOTSUP; 4725 #endif 4726 4727 done: 4728 ADAPTER_UNLOCK(sc); 4729 return (rc); 4730 } 4731 4732 static inline uint64_t 4733 get_filter_hits(struct adapter *sc, uint32_t fid) 4734 { 4735 uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4736 uint64_t hits; 4737 4738 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0), 4739 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 4740 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0)); 4741 hits = t4_read_reg64(sc, MEMWIN0_BASE + 16); 4742 4743 return (be64toh(hits)); 4744 } 4745 4746 static int 4747 get_filter(struct adapter *sc, struct t4_filter *t) 4748 { 4749 int i, nfilters = sc->tids.nftids; 4750 struct filter_entry *f; 4751 4752 ADAPTER_LOCK_ASSERT_OWNED(sc); 4753 4754 if (IS_BUSY(sc)) 4755 return (EAGAIN); 4756 4757 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 4758 t->idx >= nfilters) { 4759 t->idx = 0xffffffff; 4760 return (0); 4761 } 4762 4763 f = &sc->tids.ftid_tab[t->idx]; 4764 for (i = t->idx; i < nfilters; i++, f++) { 4765 if (f->valid) { 4766 t->idx = i; 4767 t->l2tidx = f->l2t ? f->l2t->idx : 0; 4768 t->smtidx = f->smtidx; 4769 if (f->fs.hitcnts) 4770 t->hits = get_filter_hits(sc, t->idx); 4771 else 4772 t->hits = UINT64_MAX; 4773 t->fs = f->fs; 4774 4775 return (0); 4776 } 4777 } 4778 4779 t->idx = 0xffffffff; 4780 return (0); 4781 } 4782 4783 static int 4784 set_filter(struct adapter *sc, struct t4_filter *t) 4785 { 4786 unsigned int nfilters, nports; 4787 struct filter_entry *f; 4788 int i; 4789 4790 ADAPTER_LOCK_ASSERT_OWNED(sc); 4791 4792 nfilters = sc->tids.nftids; 4793 nports = sc->params.nports; 4794 4795 if (nfilters == 0) 4796 return (ENOTSUP); 4797 4798 if (!(sc->flags & FULL_INIT_DONE)) 4799 return (EAGAIN); 4800 4801 if (t->idx >= nfilters) 4802 return (EINVAL); 4803 4804 /* Validate against the global filter mode */ 4805 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) 4806 return (E2BIG); 4807 4808 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) 4809 return (EINVAL); 4810 4811 if (t->fs.val.iport >= nports) 4812 return (EINVAL); 4813 4814 /* Can't specify an iq if not steering to it */ 4815 if (!t->fs.dirsteer && t->fs.iq) 4816 return (EINVAL); 4817 4818 /* IPv6 filter idx must be 4 aligned */ 4819 if (t->fs.type == 1 && 4820 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) 4821 return (EINVAL); 4822 4823 if (sc->tids.ftid_tab == NULL) { 4824 KASSERT(sc->tids.ftids_in_use == 0, 4825 ("%s: no memory allocated but filters_in_use > 0", 4826 __func__)); 4827 4828 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 4829 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 4830 if (sc->tids.ftid_tab == NULL) 4831 return (ENOMEM); 4832 } 4833 4834 for (i = 0; i < 4; i++) { 4835 f = &sc->tids.ftid_tab[t->idx + i]; 4836 4837 if (f->pending || f->valid) 4838 return (EBUSY); 4839 if (f->locked) 4840 return (EPERM); 4841 4842 if (t->fs.type == 0) 4843 break; 4844 } 4845 4846 f = &sc->tids.ftid_tab[t->idx]; 4847 f->fs = t->fs; 4848 4849 return set_filter_wr(sc, t->idx); 4850 } 4851 4852 static int 4853 del_filter(struct adapter *sc, struct t4_filter *t) 4854 { 4855 unsigned int nfilters; 4856 struct filter_entry *f; 4857 4858 ADAPTER_LOCK_ASSERT_OWNED(sc); 4859 4860 if (IS_BUSY(sc)) 4861 return (EAGAIN); 4862 4863 nfilters = sc->tids.nftids; 4864 4865 if (nfilters == 0) 4866 return (ENOTSUP); 4867 4868 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 4869 t->idx >= nfilters) 4870 return (EINVAL); 4871 4872 if (!(sc->flags & FULL_INIT_DONE)) 4873 return (EAGAIN); 4874 4875 f = &sc->tids.ftid_tab[t->idx]; 4876 4877 if (f->pending) 4878 return (EBUSY); 4879 if (f->locked) 4880 return (EPERM); 4881 4882 if (f->valid) { 4883 t->fs = f->fs; /* extra info for the caller */ 4884 return del_filter_wr(sc, t->idx); 4885 } 4886 4887 return (0); 4888 } 4889 4890 static void 4891 clear_filter(struct filter_entry *f) 4892 { 4893 if (f->l2t) 4894 t4_l2t_release(f->l2t); 4895 4896 bzero(f, sizeof (*f)); 4897 } 4898 4899 static int 4900 set_filter_wr(struct adapter *sc, int fidx) 4901 { 4902 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4903 struct wrqe *wr; 4904 struct fw_filter_wr *fwr; 4905 unsigned int ftid; 4906 4907 ADAPTER_LOCK_ASSERT_OWNED(sc); 4908 4909 if (f->fs.newdmac || f->fs.newvlan) { 4910 /* This filter needs an L2T entry; allocate one. */ 4911 f->l2t = t4_l2t_alloc_switching(sc->l2t); 4912 if (f->l2t == NULL) 4913 return (EAGAIN); 4914 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 4915 f->fs.dmac)) { 4916 t4_l2t_release(f->l2t); 4917 f->l2t = NULL; 4918 return (ENOMEM); 4919 } 4920 } 4921 4922 ftid = sc->tids.ftid_base + fidx; 4923 4924 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 4925 if (wr == NULL) 4926 return (ENOMEM); 4927 4928 fwr = wrtod(wr); 4929 bzero(fwr, sizeof (*fwr)); 4930 4931 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 4932 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 4933 fwr->tid_to_iq = 4934 htobe32(V_FW_FILTER_WR_TID(ftid) | 4935 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 4936 V_FW_FILTER_WR_NOREPLY(0) | 4937 V_FW_FILTER_WR_IQ(f->fs.iq)); 4938 fwr->del_filter_to_l2tix = 4939 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 4940 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 4941 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 4942 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 4943 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 4944 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 4945 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 4946 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 4947 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 4948 f->fs.newvlan == VLAN_REWRITE) | 4949 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 4950 f->fs.newvlan == VLAN_REWRITE) | 4951 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 4952 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 4953 V_FW_FILTER_WR_PRIO(f->fs.prio) | 4954 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 4955 fwr->ethtype = htobe16(f->fs.val.ethtype); 4956 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 4957 fwr->frag_to_ovlan_vldm = 4958 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 4959 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 4960 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 4961 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 4962 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 4963 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 4964 fwr->smac_sel = 0; 4965 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 4966 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 4967 fwr->maci_to_matchtypem = 4968 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 4969 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 4970 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 4971 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 4972 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 4973 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 4974 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 4975 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 4976 fwr->ptcl = f->fs.val.proto; 4977 fwr->ptclm = f->fs.mask.proto; 4978 fwr->ttyp = f->fs.val.tos; 4979 fwr->ttypm = f->fs.mask.tos; 4980 fwr->ivlan = htobe16(f->fs.val.vlan); 4981 fwr->ivlanm = htobe16(f->fs.mask.vlan); 4982 fwr->ovlan = htobe16(f->fs.val.vnic); 4983 fwr->ovlanm = htobe16(f->fs.mask.vnic); 4984 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 4985 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 4986 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 4987 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 4988 fwr->lp = htobe16(f->fs.val.dport); 4989 fwr->lpm = htobe16(f->fs.mask.dport); 4990 fwr->fp = htobe16(f->fs.val.sport); 4991 fwr->fpm = htobe16(f->fs.mask.sport); 4992 if (f->fs.newsmac) 4993 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 4994 4995 f->pending = 1; 4996 sc->tids.ftids_in_use++; 4997 4998 t4_wrq_tx(sc, wr); 4999 return (0); 5000 } 5001 5002 static int 5003 del_filter_wr(struct adapter *sc, int fidx) 5004 { 5005 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 5006 struct wrqe *wr; 5007 struct fw_filter_wr *fwr; 5008 unsigned int ftid; 5009 5010 ADAPTER_LOCK_ASSERT_OWNED(sc); 5011 5012 ftid = sc->tids.ftid_base + fidx; 5013 5014 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 5015 if (wr == NULL) 5016 return (ENOMEM); 5017 fwr = wrtod(wr); 5018 bzero(fwr, sizeof (*fwr)); 5019 5020 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 5021 5022 f->pending = 1; 5023 t4_wrq_tx(sc, wr); 5024 return (0); 5025 } 5026 5027 int 5028 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5029 { 5030 struct adapter *sc = iq->adapter; 5031 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 5032 unsigned int idx = GET_TID(rpl); 5033 5034 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 5035 rss->opcode)); 5036 5037 if (idx >= sc->tids.ftid_base && 5038 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 5039 unsigned int rc = G_COOKIE(rpl->cookie); 5040 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 5041 5042 ADAPTER_LOCK(sc); 5043 if (rc == FW_FILTER_WR_FLT_ADDED) { 5044 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 5045 f->pending = 0; /* asynchronous setup completed */ 5046 f->valid = 1; 5047 } else { 5048 if (rc != FW_FILTER_WR_FLT_DELETED) { 5049 /* Add or delete failed, display an error */ 5050 log(LOG_ERR, 5051 "filter %u setup failed with error %u\n", 5052 idx, rc); 5053 } 5054 5055 clear_filter(f); 5056 sc->tids.ftids_in_use--; 5057 } 5058 ADAPTER_UNLOCK(sc); 5059 } 5060 5061 return (0); 5062 } 5063 5064 static int 5065 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 5066 { 5067 int rc = EINVAL; 5068 5069 if (cntxt->cid > M_CTXTQID) 5070 return (rc); 5071 5072 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 5073 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 5074 return (rc); 5075 5076 if (sc->flags & FW_OK) { 5077 ADAPTER_LOCK(sc); /* Avoid parallel t4_wr_mbox */ 5078 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 5079 &cntxt->data[0]); 5080 ADAPTER_UNLOCK(sc); 5081 } 5082 5083 if (rc != 0) { 5084 /* Read via firmware failed or wasn't even attempted */ 5085 5086 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, 5087 &cntxt->data[0]); 5088 } 5089 5090 return (rc); 5091 } 5092 5093 static int 5094 read_card_mem(struct adapter *sc, struct t4_mem_range *mr) 5095 { 5096 uint32_t base, size, lo, hi, win, off, remaining, i, n; 5097 uint32_t *buf, *b; 5098 int rc; 5099 5100 /* reads are in multiples of 32 bits */ 5101 if (mr->addr & 3 || mr->len & 3 || mr->len == 0) 5102 return (EINVAL); 5103 5104 /* 5105 * We don't want to deal with potential holes so we mandate that the 5106 * requested region must lie entirely within one of the 3 memories. 5107 */ 5108 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5109 if (lo & F_EDRAM0_ENABLE) { 5110 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5111 base = G_EDRAM0_BASE(hi) << 20; 5112 size = G_EDRAM0_SIZE(hi) << 20; 5113 if (size > 0 && 5114 mr->addr >= base && mr->addr < base + size && 5115 mr->addr + mr->len <= base + size) 5116 goto proceed; 5117 } 5118 if (lo & F_EDRAM1_ENABLE) { 5119 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5120 base = G_EDRAM1_BASE(hi) << 20; 5121 size = G_EDRAM1_SIZE(hi) << 20; 5122 if (size > 0 && 5123 mr->addr >= base && mr->addr < base + size && 5124 mr->addr + mr->len <= base + size) 5125 goto proceed; 5126 } 5127 if (lo & F_EXT_MEM_ENABLE) { 5128 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5129 base = G_EXT_MEM_BASE(hi) << 20; 5130 size = G_EXT_MEM_SIZE(hi) << 20; 5131 if (size > 0 && 5132 mr->addr >= base && mr->addr < base + size && 5133 mr->addr + mr->len <= base + size) 5134 goto proceed; 5135 } 5136 return (ENXIO); 5137 5138 proceed: 5139 buf = b = malloc(mr->len, M_CXGBE, M_WAITOK); 5140 5141 /* 5142 * Position the PCIe window (we use memwin2) to the 16B aligned area 5143 * just at/before the requested region. 5144 */ 5145 win = mr->addr & ~0xf; 5146 off = mr->addr - win; /* offset of the requested region in the win */ 5147 remaining = mr->len; 5148 5149 while (remaining) { 5150 t4_write_reg(sc, 5151 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 5152 t4_read_reg(sc, 5153 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 5154 5155 /* number of bytes that we'll copy in the inner loop */ 5156 n = min(remaining, MEMWIN2_APERTURE - off); 5157 5158 for (i = 0; i < n; i += 4, remaining -= 4) 5159 *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i); 5160 5161 win += MEMWIN2_APERTURE; 5162 off = 0; 5163 } 5164 5165 rc = copyout(buf, mr->data, mr->len); 5166 free(buf, M_CXGBE); 5167 5168 return (rc); 5169 } 5170 5171 static int 5172 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 5173 { 5174 int rc; 5175 5176 ADAPTER_LOCK_ASSERT_OWNED(sc); /* for mbox */ 5177 5178 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 5179 return (EINVAL); 5180 5181 if (i2cd->len > 1) { 5182 /* XXX: need fw support for longer reads in one go */ 5183 return (ENOTSUP); 5184 } 5185 5186 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 5187 i2cd->offset, &i2cd->data[0]); 5188 5189 return (rc); 5190 } 5191 5192 int 5193 t4_os_find_pci_capability(struct adapter *sc, int cap) 5194 { 5195 int i; 5196 5197 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 5198 } 5199 5200 int 5201 t4_os_pci_save_state(struct adapter *sc) 5202 { 5203 device_t dev; 5204 struct pci_devinfo *dinfo; 5205 5206 dev = sc->dev; 5207 dinfo = device_get_ivars(dev); 5208 5209 pci_cfg_save(dev, dinfo, 0); 5210 return (0); 5211 } 5212 5213 int 5214 t4_os_pci_restore_state(struct adapter *sc) 5215 { 5216 device_t dev; 5217 struct pci_devinfo *dinfo; 5218 5219 dev = sc->dev; 5220 dinfo = device_get_ivars(dev); 5221 5222 pci_cfg_restore(dev, dinfo); 5223 return (0); 5224 } 5225 5226 void 5227 t4_os_portmod_changed(const struct adapter *sc, int idx) 5228 { 5229 struct port_info *pi = sc->port[idx]; 5230 static const char *mod_str[] = { 5231 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 5232 }; 5233 5234 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 5235 if_printf(pi->ifp, "transceiver unplugged.\n"); 5236 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 5237 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 5238 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 5239 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 5240 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 5241 if_printf(pi->ifp, "%s transceiver inserted.\n", 5242 mod_str[pi->mod_type]); 5243 } else { 5244 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 5245 pi->mod_type); 5246 } 5247 } 5248 5249 void 5250 t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 5251 { 5252 struct port_info *pi = sc->port[idx]; 5253 struct ifnet *ifp = pi->ifp; 5254 5255 if (link_stat) { 5256 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 5257 if_link_state_change(ifp, LINK_STATE_UP); 5258 } else 5259 if_link_state_change(ifp, LINK_STATE_DOWN); 5260 } 5261 5262 void 5263 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 5264 { 5265 struct adapter *sc; 5266 5267 mtx_lock(&t4_list_lock); 5268 SLIST_FOREACH(sc, &t4_list, link) { 5269 /* 5270 * func should not make any assumptions about what state sc is 5271 * in - the only guarantee is that sc->sc_lock is a valid lock. 5272 */ 5273 func(sc, arg); 5274 } 5275 mtx_unlock(&t4_list_lock); 5276 } 5277 5278 static int 5279 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 5280 { 5281 return (0); 5282 } 5283 5284 static int 5285 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 5286 { 5287 return (0); 5288 } 5289 5290 static int 5291 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 5292 struct thread *td) 5293 { 5294 int rc; 5295 struct adapter *sc = dev->si_drv1; 5296 5297 rc = priv_check(td, PRIV_DRIVER); 5298 if (rc != 0) 5299 return (rc); 5300 5301 switch (cmd) { 5302 case CHELSIO_T4_GETREG: { 5303 struct t4_reg *edata = (struct t4_reg *)data; 5304 5305 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5306 return (EFAULT); 5307 5308 if (edata->size == 4) 5309 edata->val = t4_read_reg(sc, edata->addr); 5310 else if (edata->size == 8) 5311 edata->val = t4_read_reg64(sc, edata->addr); 5312 else 5313 return (EINVAL); 5314 5315 break; 5316 } 5317 case CHELSIO_T4_SETREG: { 5318 struct t4_reg *edata = (struct t4_reg *)data; 5319 5320 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5321 return (EFAULT); 5322 5323 if (edata->size == 4) { 5324 if (edata->val & 0xffffffff00000000) 5325 return (EINVAL); 5326 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 5327 } else if (edata->size == 8) 5328 t4_write_reg64(sc, edata->addr, edata->val); 5329 else 5330 return (EINVAL); 5331 break; 5332 } 5333 case CHELSIO_T4_REGDUMP: { 5334 struct t4_regdump *regs = (struct t4_regdump *)data; 5335 int reglen = T4_REGDUMP_SIZE; 5336 uint8_t *buf; 5337 5338 if (regs->len < reglen) { 5339 regs->len = reglen; /* hint to the caller */ 5340 return (ENOBUFS); 5341 } 5342 5343 regs->len = reglen; 5344 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 5345 t4_get_regs(sc, regs, buf); 5346 rc = copyout(buf, regs->data, reglen); 5347 free(buf, M_CXGBE); 5348 break; 5349 } 5350 case CHELSIO_T4_GET_FILTER_MODE: 5351 rc = get_filter_mode(sc, (uint32_t *)data); 5352 break; 5353 case CHELSIO_T4_SET_FILTER_MODE: 5354 rc = set_filter_mode(sc, *(uint32_t *)data); 5355 break; 5356 case CHELSIO_T4_GET_FILTER: 5357 ADAPTER_LOCK(sc); 5358 rc = get_filter(sc, (struct t4_filter *)data); 5359 ADAPTER_UNLOCK(sc); 5360 break; 5361 case CHELSIO_T4_SET_FILTER: 5362 ADAPTER_LOCK(sc); 5363 rc = set_filter(sc, (struct t4_filter *)data); 5364 ADAPTER_UNLOCK(sc); 5365 break; 5366 case CHELSIO_T4_DEL_FILTER: 5367 ADAPTER_LOCK(sc); 5368 rc = del_filter(sc, (struct t4_filter *)data); 5369 ADAPTER_UNLOCK(sc); 5370 break; 5371 case CHELSIO_T4_GET_SGE_CONTEXT: 5372 rc = get_sge_context(sc, (struct t4_sge_context *)data); 5373 break; 5374 case CHELSIO_T4_LOAD_FW: { 5375 struct t4_data *fw = (struct t4_data *)data; 5376 uint8_t *fw_data; 5377 5378 if (sc->flags & FULL_INIT_DONE) 5379 return (EBUSY); 5380 5381 fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT); 5382 if (fw_data == NULL) 5383 return (ENOMEM); 5384 5385 rc = copyin(fw->data, fw_data, fw->len); 5386 if (rc == 0) 5387 rc = -t4_load_fw(sc, fw_data, fw->len); 5388 5389 free(fw_data, M_CXGBE); 5390 break; 5391 } 5392 case CHELSIO_T4_GET_MEM: 5393 rc = read_card_mem(sc, (struct t4_mem_range *)data); 5394 break; 5395 case CHELSIO_T4_GET_I2C: 5396 ADAPTER_LOCK(sc); 5397 rc = read_i2c(sc, (struct t4_i2c_data *)data); 5398 ADAPTER_UNLOCK(sc); 5399 break; 5400 case CHELSIO_T4_CLEAR_STATS: { 5401 u_int port_id = *(uint32_t *)data; 5402 5403 if (port_id >= sc->params.nports) 5404 return (EINVAL); 5405 5406 t4_clr_port_stats(sc, port_id); 5407 break; 5408 } 5409 default: 5410 rc = EINVAL; 5411 } 5412 5413 return (rc); 5414 } 5415 5416 #ifdef TCP_OFFLOAD 5417 static int 5418 toe_capability(struct port_info *pi, int enable) 5419 { 5420 int rc; 5421 struct adapter *sc = pi->adapter; 5422 5423 ADAPTER_LOCK_ASSERT_OWNED(sc); 5424 5425 if (!is_offload(sc)) 5426 return (ENODEV); 5427 5428 if (enable) { 5429 if (!(sc->flags & FULL_INIT_DONE)) { 5430 log(LOG_WARNING, 5431 "You must enable a cxgbe interface first\n"); 5432 return (EAGAIN); 5433 } 5434 5435 if (isset(&sc->offload_map, pi->port_id)) 5436 return (0); 5437 5438 if (!(sc->flags & TOM_INIT_DONE)) { 5439 rc = t4_activate_uld(sc, ULD_TOM); 5440 if (rc == EAGAIN) { 5441 log(LOG_WARNING, 5442 "You must kldload t4_tom.ko before trying " 5443 "to enable TOE on a cxgbe interface.\n"); 5444 } 5445 if (rc != 0) 5446 return (rc); 5447 KASSERT(sc->tom_softc != NULL, 5448 ("%s: TOM activated but softc NULL", __func__)); 5449 KASSERT(sc->flags & TOM_INIT_DONE, 5450 ("%s: TOM activated but flag not set", __func__)); 5451 } 5452 5453 setbit(&sc->offload_map, pi->port_id); 5454 } else { 5455 if (!isset(&sc->offload_map, pi->port_id)) 5456 return (0); 5457 5458 KASSERT(sc->flags & TOM_INIT_DONE, 5459 ("%s: TOM never initialized?", __func__)); 5460 clrbit(&sc->offload_map, pi->port_id); 5461 } 5462 5463 return (0); 5464 } 5465 5466 /* 5467 * Add an upper layer driver to the global list. 5468 */ 5469 int 5470 t4_register_uld(struct uld_info *ui) 5471 { 5472 int rc = 0; 5473 struct uld_info *u; 5474 5475 mtx_lock(&t4_uld_list_lock); 5476 SLIST_FOREACH(u, &t4_uld_list, link) { 5477 if (u->uld_id == ui->uld_id) { 5478 rc = EEXIST; 5479 goto done; 5480 } 5481 } 5482 5483 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 5484 ui->refcount = 0; 5485 done: 5486 mtx_unlock(&t4_uld_list_lock); 5487 return (rc); 5488 } 5489 5490 int 5491 t4_unregister_uld(struct uld_info *ui) 5492 { 5493 int rc = EINVAL; 5494 struct uld_info *u; 5495 5496 mtx_lock(&t4_uld_list_lock); 5497 5498 SLIST_FOREACH(u, &t4_uld_list, link) { 5499 if (u == ui) { 5500 if (ui->refcount > 0) { 5501 rc = EBUSY; 5502 goto done; 5503 } 5504 5505 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 5506 rc = 0; 5507 goto done; 5508 } 5509 } 5510 done: 5511 mtx_unlock(&t4_uld_list_lock); 5512 return (rc); 5513 } 5514 5515 int 5516 t4_activate_uld(struct adapter *sc, int id) 5517 { 5518 int rc = EAGAIN; 5519 struct uld_info *ui; 5520 5521 mtx_lock(&t4_uld_list_lock); 5522 5523 SLIST_FOREACH(ui, &t4_uld_list, link) { 5524 if (ui->uld_id == id) { 5525 rc = ui->activate(sc); 5526 if (rc == 0) 5527 ui->refcount++; 5528 goto done; 5529 } 5530 } 5531 done: 5532 mtx_unlock(&t4_uld_list_lock); 5533 5534 return (rc); 5535 } 5536 5537 int 5538 t4_deactivate_uld(struct adapter *sc, int id) 5539 { 5540 int rc = EINVAL; 5541 struct uld_info *ui; 5542 5543 mtx_lock(&t4_uld_list_lock); 5544 5545 SLIST_FOREACH(ui, &t4_uld_list, link) { 5546 if (ui->uld_id == id) { 5547 rc = ui->deactivate(sc); 5548 if (rc == 0) 5549 ui->refcount--; 5550 goto done; 5551 } 5552 } 5553 done: 5554 mtx_unlock(&t4_uld_list_lock); 5555 5556 return (rc); 5557 } 5558 #endif 5559 5560 /* 5561 * Come up with reasonable defaults for some of the tunables, provided they're 5562 * not set by the user (in which case we'll use the values as is). 5563 */ 5564 static void 5565 tweak_tunables(void) 5566 { 5567 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 5568 5569 if (t4_ntxq10g < 1) 5570 t4_ntxq10g = min(nc, NTXQ_10G); 5571 5572 if (t4_ntxq1g < 1) 5573 t4_ntxq1g = min(nc, NTXQ_1G); 5574 5575 if (t4_nrxq10g < 1) 5576 t4_nrxq10g = min(nc, NRXQ_10G); 5577 5578 if (t4_nrxq1g < 1) 5579 t4_nrxq1g = min(nc, NRXQ_1G); 5580 5581 #ifdef TCP_OFFLOAD 5582 if (t4_nofldtxq10g < 1) 5583 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 5584 5585 if (t4_nofldtxq1g < 1) 5586 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 5587 5588 if (t4_nofldrxq10g < 1) 5589 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 5590 5591 if (t4_nofldrxq1g < 1) 5592 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 5593 5594 if (t4_toecaps_allowed == -1) 5595 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 5596 #else 5597 if (t4_toecaps_allowed == -1) 5598 t4_toecaps_allowed = 0; 5599 #endif 5600 5601 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 5602 t4_tmr_idx_10g = TMR_IDX_10G; 5603 5604 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 5605 t4_pktc_idx_10g = PKTC_IDX_10G; 5606 5607 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 5608 t4_tmr_idx_1g = TMR_IDX_1G; 5609 5610 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 5611 t4_pktc_idx_1g = PKTC_IDX_1G; 5612 5613 if (t4_qsize_txq < 128) 5614 t4_qsize_txq = 128; 5615 5616 if (t4_qsize_rxq < 128) 5617 t4_qsize_rxq = 128; 5618 while (t4_qsize_rxq & 7) 5619 t4_qsize_rxq++; 5620 5621 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 5622 } 5623 5624 static int 5625 t4_mod_event(module_t mod, int cmd, void *arg) 5626 { 5627 int rc = 0; 5628 5629 switch (cmd) { 5630 case MOD_LOAD: 5631 t4_sge_modload(); 5632 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 5633 SLIST_INIT(&t4_list); 5634 #ifdef TCP_OFFLOAD 5635 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 5636 SLIST_INIT(&t4_uld_list); 5637 #endif 5638 tweak_tunables(); 5639 break; 5640 5641 case MOD_UNLOAD: 5642 #ifdef TCP_OFFLOAD 5643 mtx_lock(&t4_uld_list_lock); 5644 if (!SLIST_EMPTY(&t4_uld_list)) { 5645 rc = EBUSY; 5646 mtx_unlock(&t4_uld_list_lock); 5647 break; 5648 } 5649 mtx_unlock(&t4_uld_list_lock); 5650 mtx_destroy(&t4_uld_list_lock); 5651 #endif 5652 mtx_lock(&t4_list_lock); 5653 if (!SLIST_EMPTY(&t4_list)) { 5654 rc = EBUSY; 5655 mtx_unlock(&t4_list_lock); 5656 break; 5657 } 5658 mtx_unlock(&t4_list_lock); 5659 mtx_destroy(&t4_list_lock); 5660 break; 5661 } 5662 5663 return (rc); 5664 } 5665 5666 static devclass_t t4_devclass; 5667 static devclass_t cxgbe_devclass; 5668 5669 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0); 5670 MODULE_VERSION(t4nex, 1); 5671 5672 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 5673 MODULE_VERSION(cxgbe, 1); 5674