Lines Matching +full:temperature +full:- +full:lookup +full:- +full:table
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
308 * Each tunable is set to a default value here if it's known at compile-time.
309 * Otherwise it is set to -n as an indication to tweak_tunables() that it should
326 int t4_ntxq = -NTXQ;
332 int t4_nrxq = -NRXQ;
338 static int t4_ntxq_vi = -NTXQ_VI;
343 static int t4_nrxq_vi = -NRXQ_VI;
349 0, "Reserve TX queue 0 of each VI for non-flowid packets");
353 static int t4_nofldtxq = -NOFLDTXQ;
358 static int t4_nofldtxq_vi = -NOFLDTXQ_VI;
365 static int t4_nofldrxq = -NOFLDRXQ;
370 static int t4_nofldrxq_vi = -NOFLDRXQ_VI;
379 #define PKTC_IDX_OFLD (-1)
384 /* 0 means chip/fw default, non-zero number is value in microseconds */
389 /* 0 means chip/fw default, non-zero number is value in microseconds */
394 /* 0 means chip/fw default, non-zero number is # of keepalives before abort */
399 /* 0 means chip/fw default, non-zero number is value in microseconds */
404 /* 0 means chip/fw default, non-zero number is value in microseconds */
409 /* 0 means chip/fw default, non-zero number is # of rexmt before abort */
414 /* -1 means chip/fw default, other values are raw backoff values to use */
416 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
472 static int t4_nnmtxq = -NNMTXQ;
477 static int t4_nnmrxq = -NNMRXQ;
482 static int t4_nnmtxq_vi = -NNMTXQ_VI;
487 static int t4_nnmrxq_vi = -NNMRXQ_VI;
501 #define PKTC_IDX (-1)
519 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
523 0, "Interrupt types allowed (bit 0 = INTx, 1 = MSI, 2 = MSI-X)");
529 #define BUILTIN_CF "built-in"
553 * -1 to run with the firmware default. Same as FEC_AUTO (bit 5)
556 static int t4_fec = -1;
561 t4_fec_bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD1\5RSVD2\6auto\7module";
567 * -1 to set FORCE_FEC iff requested_fec != AUTO. Multiple FEC bits are okay.
574 static int t4_force_fec = -1;
580 * -1 to run with the firmware default.
584 static int t4_autoneg = -1;
589 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
590 * encouraged respectively). '-n' is the same as 'n' except the firmware
595 "Firmware auto-install (0 = prohibited, 1 = allowed, 2 = encouraged)");
628 static int t4_toecaps_allowed = -1;
632 static int t4_rdmacaps_allowed = -1;
636 static int t4_cryptocaps_allowed = -1;
640 static int t4_iscsicaps_allowed = -1;
663 * -1: driver should figure out a good value.
668 static int pcie_relaxed_ordering = -1;
694 * Set to non-zero to enable the attack filter. A packet that matches any of
767 &t4_tls_short_records, 0, "Use cipher-only mode for short records.");
771 &t4_tls_partial_ghash, 0, "Use partial GHASH for AES-GCM records.");
786 uint16_t intr_type; /* INTx, MSI, or MSI-X */
957 {0x4400, "Chelsio T440-dbg"},
958 {0x4401, "Chelsio T420-CR"},
959 {0x4402, "Chelsio T422-CR"},
960 {0x4403, "Chelsio T440-CR"},
961 {0x4404, "Chelsio T420-BCH"},
962 {0x4405, "Chelsio T440-BCH"},
963 {0x4406, "Chelsio T440-CH"},
964 {0x4407, "Chelsio T420-SO"},
965 {0x4408, "Chelsio T420-CX"},
966 {0x4409, "Chelsio T420-BT"},
967 {0x440a, "Chelsio T404-BT"},
968 {0x440e, "Chelsio T440-LP-CR"},
971 {0x5400, "Chelsio T580-dbg"},
972 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */
973 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */
974 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */
975 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */
976 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */
977 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */
978 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */
979 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */
980 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */
981 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */
982 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */
983 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */
984 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */
985 {0x5418, "Chelsio T540-BT"}, /* 4 x 10GBaseT */
986 {0x5419, "Chelsio T540-LP-BT"}, /* 4 x 10GBaseT */
987 {0x541a, "Chelsio T540-SO-BT"}, /* 4 x 10GBaseT, nomem */
988 {0x541b, "Chelsio T540-SO-CR"}, /* 4 x 10G, nomem */
991 {0x5483, "Custom T540-CR"},
992 {0x5484, "Custom T540-BT"},
995 {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */
996 {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */
997 {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */
998 {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */
999 {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */
1000 {0x6405, "Chelsio T6225-SO-OCP3"}, /* 2 x 10/25G, nomem */
1001 {0x6406, "Chelsio T6225-OCP3"}, /* 2 x 10/25G */
1002 {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */
1003 {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */
1004 {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */
1005 {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */
1006 {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */
1007 {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */
1008 {0x6414, "Chelsio T62100-SO-OCP3"}, /* 2 x 40/50/100G, nomem */
1009 {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */
1012 {0x6480, "Custom T6225-CR"},
1013 {0x6481, "Custom T62100-CR"},
1014 {0x6482, "Custom T6225-CR"},
1015 {0x6483, "Custom T62100-CR"},
1016 {0x6484, "Custom T64100-CR"},
1017 {0x6485, "Custom T6240-SO"},
1018 {0x6486, "Custom T6225-SO-CR"},
1019 {0x6487, "Custom T6225-CR"},
1022 {0x7400, "Chelsio T72200-DBG"}, /* 2 x 200G, debug */
1029 {0x7407, "Chelsio T72200-FH"}, /* 2 x 40/100/200G, 2 mem */
1031 {0x7409, "Chelsio S7210-BT"}, /* 2 x 10GBASE-T, nomem */
1032 {0x740a, "Chelsio T7450-RC"}, /* 4 x 10/25/50G, 1 mem, RC */
1033 {0x740b, "Chelsio T72200-RC"}, /* 2 x 40/100/200G, 1 mem, RC */
1034 {0x740c, "Chelsio T72200-FH-RC"}, /* 2 x 40/100/200G, 2 mem, RC */
1035 {0x740d, "Chelsio S72200-OCP3"}, /* 2 x 40/100/200G OCP3 */
1036 {0x740e, "Chelsio S7450-OCP3"}, /* 4 x 1/20/25/50G OCP3 */
1037 {0x740f, "Chelsio S7410-BT-OCP3"}, /* 4 x 10GBASE-T OCP3 */
1038 {0x7410, "Chelsio S7210-BT-A"}, /* 2 x 10GBASE-T */
1216 device_printf(sc->dev, "chip id %d is not supported.\n", id);
1217 sc->names = NULL;
1218 } else if (id - CHELSIO_T4 < nitems(devnames))
1219 sc->names = &devnames[id - CHELSIO_T4];
1221 sc->names = &devnames[nitems(devnames) - 1];
1232 parent = device_get_nameunit(sc->dev);
1233 name = sc->names->ifnet_name;
1236 value == pi->port_id)
1239 return (-1);
1257 cur = &sc->cal_info[sc->cal_current];
1258 next_up = (sc->cal_current + 1) % CNT_CAL_INFO;
1259 nex = &sc->cal_info[next_up];
1260 if (__predict_false(sc->cal_count == 0)) {
1262 cur->hw_cur = hw;
1263 cur->sbt_cur = sbt;
1264 sc->cal_count++;
1268 if (cur->hw_cur == hw) {
1270 sc->cal_count = 0;
1271 atomic_store_rel_int(&cur->gen, 0);
1275 seqc_write_begin(&nex->gen);
1276 nex->hw_prev = cur->hw_cur;
1277 nex->sbt_prev = cur->sbt_cur;
1278 nex->hw_cur = hw;
1279 nex->sbt_cur = sbt;
1280 seqc_write_end(&nex->gen);
1281 sc->cal_current = next_up;
1283 callout_reset_sbt_curcpu(&sc->cal_callout, SBT_1S, 0, t4_calibration,
1298 sc->cal_info[i].gen = 0;
1300 sc->cal_current = 0;
1301 sc->cal_count = 0;
1302 sc->cal_gen = 0;
1327 sc->dev = dev;
1328 sysctl_ctx_init(&sc->ctx);
1329 TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
1339 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
1351 sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
1352 sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
1353 sc->traceq = -1;
1354 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
1355 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
1358 snprintf(sc->lockname, sizeof(sc->lockname), "%s",
1360 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
1363 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
1364 TAILQ_INIT(&sc->sfl);
1365 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
1367 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);
1369 sc->policy = NULL;
1370 rw_init(&sc->policy_lock, "connection offload policy");
1372 callout_init(&sc->ktls_tick, 1);
1374 callout_init(&sc->cal_callout, 1);
1376 refcount_init(&sc->vxlan_refcount, 0);
1378 TASK_INIT(&sc->reset_task, 0, reset_adapter_task, sc);
1379 TASK_INIT(&sc->fatal_error_task, 0, fatal_error_task, sc);
1381 sc->ctrlq_oid = SYSCTL_ADD_NODE(&sc->ctx,
1382 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "ctrlq",
1384 sc->fwq_oid = SYSCTL_ADD_NODE(&sc->ctx,
1385 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "fwq",
1392 memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
1393 memset(sc->port_map, 0xff, sizeof(sc->port_map));
1397 rc = -t4_prep_adapter(sc, buf);
1411 sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j);
1412 sc->mbox = sc->pf;
1415 if (sc->names == NULL) {
1434 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
1457 MPASS(sc->flags & FW_OK);
1463 if (sc->flags & MASTER_PF) {
1490 * First pass over all the ports - allocate VIs and initialize some
1497 sc->port[i] = pi;
1500 pi->adapter = sc;
1501 pi->port_id = i;
1504 * pi->nvi's final value is known.
1506 pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE,
1513 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
1517 free(pi->vi, M_CXGBE);
1519 sc->port[i] = NULL;
1523 if (is_bt(pi->port_type))
1524 setbit(&sc->bt_map, pi->hw_port);
1526 MPASS(!isset(&sc->bt_map, pi->hw_port));
1528 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
1530 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
1531 for (j = 0; j < sc->params.tp.lb_nchan; j++)
1532 sc->chan_map[pi->tx_chan + j] = i;
1533 sc->port_map[pi->hw_port] = i;
1542 pi->fcs_reg = -1;
1544 pi->fcs_reg = A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L;
1545 pi->fcs_base = 0;
1548 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1556 pi->flags |= FIXED_IFMEDIA;
1559 pi->dev = device_add_child(dev, sc->names->ifnet_name,
1561 if (pi->dev == NULL) {
1567 pi->vi[0].dev = pi->dev;
1568 device_set_softc(pi->dev, pi);
1574 nports = sc->params.nports;
1580 sc->intr_type = iaq.intr_type;
1581 sc->intr_count = iaq.nirq;
1583 s = &sc->sge;
1584 s->nctrlq = max(sc->params.nports, sc->params.ncores);
1585 s->nrxq = nports * iaq.nrxq;
1586 s->ntxq = nports * iaq.ntxq;
1588 s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi;
1589 s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi;
1591 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */
1592 s->neq += nports; /* ctrl queues: 1 per port */
1593 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
1596 s->nofldtxq = nports * iaq.nofldtxq;
1598 s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi;
1599 s->neq += s->nofldtxq;
1601 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_ofld_txq),
1607 s->nofldrxq = nports * iaq.nofldrxq;
1609 s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi;
1610 s->neq += s->nofldrxq; /* free list */
1611 s->niq += s->nofldrxq;
1613 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
1618 s->nnmrxq = 0;
1619 s->nnmtxq = 0;
1621 s->nnmrxq += nports * iaq.nnmrxq;
1622 s->nnmtxq += nports * iaq.nnmtxq;
1625 s->nnmrxq += nports * (num_vis - 1) * iaq.nnmrxq_vi;
1626 s->nnmtxq += nports * (num_vis - 1) * iaq.nnmtxq_vi;
1628 s->neq += s->nnmtxq + s->nnmrxq;
1629 s->niq += s->nnmrxq;
1631 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
1633 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
1636 MPASS(s->niq <= s->iqmap_sz);
1637 MPASS(s->neq <= s->eqmap_sz);
1639 s->ctrlq = malloc(s->nctrlq * sizeof(struct sge_wrq), M_CXGBE,
1641 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
1643 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
1645 s->iqmap = malloc(s->iqmap_sz * sizeof(struct sge_iq *), M_CXGBE,
1647 s->eqmap = malloc(s->eqmap_sz * sizeof(struct sge_eq *), M_CXGBE,
1650 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
1663 if (sc->vres.key.size != 0)
1664 sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start,
1665 sc->vres.key.size, 32, 0, M_FIRSTFIT | M_WAITOK);
1683 struct port_info *pi = sc->port[i];
1689 pi->nvi = num_vis;
1691 vi->pi = pi;
1692 vi->adapter = sc;
1693 vi->first_intr = -1;
1694 vi->qsize_rxq = t4_qsize_rxq;
1695 vi->qsize_txq = t4_qsize_txq;
1697 vi->first_rxq = rqidx;
1698 vi->first_txq = tqidx;
1699 vi->tmr_idx = t4_tmr_idx;
1700 vi->pktc_idx = t4_pktc_idx;
1701 vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi;
1702 vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi;
1704 rqidx += vi->nrxq;
1705 tqidx += vi->ntxq;
1707 if (j == 0 && vi->ntxq > 1)
1708 vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0;
1710 vi->rsrv_noflowq = 0;
1713 vi->first_ofld_txq = ofld_tqidx;
1714 vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi;
1715 ofld_tqidx += vi->nofldtxq;
1718 vi->ofld_tmr_idx = t4_tmr_idx_ofld;
1719 vi->ofld_pktc_idx = t4_pktc_idx_ofld;
1720 vi->first_ofld_rxq = ofld_rqidx;
1721 vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi;
1723 ofld_rqidx += vi->nofldrxq;
1726 vi->first_nm_rxq = nm_rqidx;
1727 vi->first_nm_txq = nm_tqidx;
1729 vi->nnmrxq = iaq.nnmrxq;
1730 vi->nnmtxq = iaq.nnmtxq;
1732 vi->nnmrxq = iaq.nnmrxq_vi;
1733 vi->nnmtxq = iaq.nnmtxq_vi;
1735 nm_rqidx += vi->nnmrxq;
1736 nm_tqidx += vi->nnmtxq;
1751 * Ensure thread-safe mailbox access (in debug builds).
1757 sc->flags |= CHK_MBOX_ACCESS;
1764 sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1765 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1766 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1767 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1774 if (rc != 0 && sc->cdev) {
1798 pi = sc->port[i];
1799 if (pi != NULL && pi->dev == dev) {
1800 sbuf_printf(sb, "port=%d", pi->port_id);
1813 if (sc->flags & FW_OK)
1827 pi = sc->port[port];
1828 if (pi == NULL || pi->dev == NULL)
1830 *child = pi->dev;
1894 if (sc->cdev) {
1895 destroy_dev(sc->cdev);
1896 sc->cdev = NULL;
1903 sc->flags &= ~CHK_MBOX_ACCESS;
1904 if (sc->flags & FULL_INIT_DONE) {
1905 if (!(sc->flags & IS_VF))
1918 for (i = 0; i < sc->intr_count; i++)
1919 t4_free_irq(sc, &sc->irq[i]);
1921 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1925 pi = sc->port[i];
1927 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1929 mtx_destroy(&pi->pi_lock);
1930 free(pi->vi, M_CXGBE);
1934 callout_stop(&sc->cal_callout);
1935 callout_drain(&sc->cal_callout);
1937 sysctl_ctx_free(&sc->ctx);
1940 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1941 t4_fw_bye(sc, sc->mbox);
1943 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1946 if (sc->regs_res)
1947 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1948 sc->regs_res);
1950 if (sc->udbs_res)
1951 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1952 sc->udbs_res);
1954 if (sc->msix_res)
1955 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1956 sc->msix_res);
1958 if (sc->l2t)
1960 if (sc->smt)
1961 t4_free_smt(sc->smt);
1966 if (sc->key_map)
1967 vmem_destroy(sc->key_map);
1974 free(sc->sge.ofld_txq, M_CXGBE);
1977 free(sc->sge.ofld_rxq, M_CXGBE);
1980 free(sc->sge.nm_rxq, M_CXGBE);
1981 free(sc->sge.nm_txq, M_CXGBE);
1983 free(sc->irq, M_CXGBE);
1984 free(sc->sge.rxq, M_CXGBE);
1985 free(sc->sge.txq, M_CXGBE);
1986 free(sc->sge.ctrlq, M_CXGBE);
1987 free(sc->sge.iqmap, M_CXGBE);
1988 free(sc->sge.eqmap, M_CXGBE);
1989 free(sc->tids.ftid_tab, M_CXGBE);
1990 free(sc->tids.hpftid_tab, M_CXGBE);
1991 free_hftid_hash(&sc->tids);
1992 free(sc->tids.tid_tab, M_CXGBE);
1995 callout_drain(&sc->ktls_tick);
1996 callout_drain(&sc->sfl_callout);
1997 if (mtx_initialized(&sc->tids.ftid_lock)) {
1998 mtx_destroy(&sc->tids.ftid_lock);
1999 cv_destroy(&sc->tids.ftid_cv);
2001 if (mtx_initialized(&sc->tids.atid_lock))
2002 mtx_destroy(&sc->tids.atid_lock);
2003 if (mtx_initialized(&sc->ifp_lock))
2004 mtx_destroy(&sc->ifp_lock);
2006 if (rw_initialized(&sc->policy_lock)) {
2007 rw_destroy(&sc->policy_lock);
2009 if (sc->policy != NULL)
2010 free_offload_policy(sc->policy);
2015 struct memwin *mw = &sc->memwin[i];
2017 if (rw_initialized(&mw->mw_lock))
2018 rw_destroy(&mw->mw_lock);
2021 mtx_destroy(&sc->sfl_lock);
2022 mtx_destroy(&sc->reg_lock);
2023 mtx_destroy(&sc->sc_lock);
2036 if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_STOPPED))) {
2038 __func__, curthread, sc->flags, sc->error_flags);
2042 sc->flags, sc->error_flags);
2045 pi = sc->port[i];
2049 if (pi->up_vis > 0 && pi->link_cfg.link_ok) {
2056 pi->link_cfg.link_ok = false;
2070 if (!atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_STOPPED))) {
2072 __func__, curthread, sc->flags, sc->error_flags);
2076 sc->flags, sc->error_flags);
2079 MPASS((sc->flags & FW_OK) == 0);
2080 MPASS((sc->flags & MASTER_PF) == 0);
2081 MPASS(sc->reset_thread == NULL);
2088 sc->reset_thread = curthread;
2092 sc->reset_thread = NULL;
2093 atomic_set_int(&sc->error_flags, ADAP_STOPPED);
2096 atomic_clear_int(&sc->error_flags, ADAP_FATAL_ERR);
2097 atomic_add_int(&sc->incarnation, 1);
2098 atomic_add_int(&sc->num_resets, 1);
2109 MPASS(sc->reset_thread == curthread);
2110 mtx_lock(&sc->reg_lock);
2111 atomic_clear_int(&sc->error_flags, HW_OFF_LIMITS);
2112 mtx_unlock(&sc->reg_lock);
2116 mtx_lock(&sc->reg_lock);
2117 atomic_set_int(&sc->error_flags, HW_OFF_LIMITS);
2118 mtx_unlock(&sc->reg_lock);
2119 sc->flags &= ~(FW_OK | MASTER_PF);
2120 sc->reset_thread = NULL;
2158 pi = sc->port[i];
2161 pi->vxlan_tcam_entry = false;
2163 vi->xact_addr_filt = -1;
2164 mtx_lock(&vi->tick_mtx);
2165 vi->flags |= VI_SKIP_STATS;
2166 mtx_unlock(&vi->tick_mtx);
2167 if (!(vi->flags & VI_INIT_DONE))
2170 ifp = vi->ifp;
2172 mtx_lock(&vi->tick_mtx);
2173 callout_stop(&vi->tick);
2174 mtx_unlock(&vi->tick_mtx);
2175 callout_drain(&vi->tick);
2183 txq->eq.flags &= ~(EQ_ENABLED | EQ_HW_ALLOCATED);
2188 TXQ_LOCK(&ofld_txq->wrq);
2189 ofld_txq->wrq.eq.flags &= ~EQ_HW_ALLOCATED;
2190 TXQ_UNLOCK(&ofld_txq->wrq);
2194 rxq->iq.flags &= ~IQ_HW_ALLOCATED;
2198 ofld_rxq->iq.flags &= ~IQ_HW_ALLOCATED;
2205 if (sc->flags & FULL_INIT_DONE) {
2207 wrq = &sc->sge.ctrlq[i];
2209 wrq->eq.flags &= ~EQ_HW_ALLOCATED;
2214 if (pi->flags & HAS_TRACEQ) {
2215 pi->flags &= ~HAS_TRACEQ;
2216 sc->traceq = -1;
2217 sc->tracer_valid = 0;
2218 sc->tracer_enabled = 0;
2221 if (sc->flags & FULL_INIT_DONE) {
2223 sc->sge.fwq.flags &= ~IQ_HW_ALLOCATED;
2224 quiesce_iq_fl(sc, &sc->sge.fwq, NULL);
2228 callout_stop(&sc->cal_callout);
2229 callout_drain(&sc->cal_callout);
2303 o->flags = sc->flags;
2305 o->nbmcaps = sc->nbmcaps;
2306 o->linkcaps = sc->linkcaps;
2307 o->switchcaps = sc->switchcaps;
2308 o->nvmecaps = sc->nvmecaps;
2309 o->niccaps = sc->niccaps;
2310 o->toecaps = sc->toecaps;
2311 o->rdmacaps = sc->rdmacaps;
2312 o->cryptocaps = sc->cryptocaps;
2313 o->iscsicaps = sc->iscsicaps;
2314 o->fcoecaps = sc->fcoecaps;
2316 o->cfcsum = sc->cfcsum;
2317 MPASS(sizeof(o->cfg_file) == sizeof(sc->cfg_file));
2318 memcpy(o->cfg_file, sc->cfg_file, sizeof(o->cfg_file));
2320 o->params = sc->params;
2321 o->vres = sc->vres;
2322 o->tids = sc->tids;
2323 o->sge = sc->sge;
2325 o->rawf_base = sc->rawf_base;
2326 o->nrawf = sc->nrawf;
2338 if (o->c##caps != sc->c##caps) { \
2339 CH_ERR(sc, "%scaps 0x%04x -> 0x%04x.\n", #c, o->c##caps, \
2340 sc->c##caps); \
2357 if (o->cfcsum != sc->cfcsum) {
2358 CH_ERR(sc, "config file %s (0x%x) -> %s (0x%x)\n", o->cfg_file,
2359 o->cfcsum, sc->cfg_file, sc->cfcsum);
2364 if (o->p != sc->p) { \
2365 CH_ERR(sc, #name " %d -> %d\n", o->p, sc->p); \
2457 MPASS(sc->flags & FW_OK);
2459 if (sc->flags & MASTER_PF) {
2478 pi = sc->port[i];
2480 MPASS(pi->vi != NULL);
2481 MPASS(pi->vi[0].dev == pi->dev);
2483 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
2486 "failed to re-initialize port %d: %d\n", i, rc);
2489 MPASS(sc->chan_map[pi->tx_chan] == i);
2501 "failed to re-allocate extra VI: %d\n", rc);
2514 if (sc->flags & FULL_INIT_DONE) {
2517 CH_ERR(sc, "failed to re-initialize adapter: %d\n", rc);
2521 if (sc->vxlan_refcount > 0)
2525 pi = sc->port[i];
2527 mtx_lock(&vi->tick_mtx);
2528 vi->flags &= ~VI_SKIP_STATS;
2529 mtx_unlock(&vi->tick_mtx);
2530 if (!(vi->flags & VI_INIT_DONE))
2534 CH_ERR(vi, "failed to re-initialize "
2538 if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
2539 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
2540 t4_set_trace_rss_control(sc, pi->tx_chan, sc->traceq);
2541 pi->flags |= HAS_TRACEQ;
2544 ifp = vi->ifp;
2556 CH_ERR(vi, "failed to re-configure MAC: %d\n", rc);
2559 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true,
2562 CH_ERR(vi, "failed to re-enable VI: %d\n", rc);
2567 txq->eq.flags |= EQ_ENABLED;
2570 mtx_lock(&vi->tick_mtx);
2571 callout_schedule(&vi->tick, hz);
2572 mtx_unlock(&vi->tick_mtx);
2575 if (pi->up_vis > 0) {
2580 if (pi->link_cfg.link_ok)
2588 pi = sc->port[i];
2590 if (!(vi->flags & VI_INIT_DONE))
2592 ifp = vi->ifp;
2597 CH_ERR(vi, "failed to re-configure MCAST MACs: %d\n", rc);
2662 MPASS(sc->error_flags & HW_OFF_LIMITS);
2663 bus_space_write_4(sc->bt, sc->bh, A_PL_RST,
2672 device_t pdev = device_get_parent(sc->dev);
2718 device_t pdev = device_get_parent(sc->dev);
2793 const int flags = sc->flags;
2794 const int eflags = sc->error_flags;
2802 "flags 0x%08x -> 0x%08x, err_flags 0x%08x -> 0x%08x.\n",
2803 rc, flags, sc->flags, eflags, sc->error_flags);
2812 device_set_descf(dev, "port %d", pi->port_id);
2828 struct sysctl_ctx_list *ctx = &vi->ctx;
2831 struct adapter *sc = vi->adapter;
2834 children = SYSCTL_CHILDREN(device_get_sysctl_tree(vi->dev));
2835 vi->rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rxq",
2837 vi->txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "txq",
2840 vi->nm_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_rxq",
2842 vi->nm_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_txq",
2846 vi->ofld_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_rxq",
2850 vi->ofld_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_txq",
2854 vi->xact_addr_filt = -1;
2855 mtx_init(&vi->tick_mtx, "vi tick", NULL, MTX_DEF);
2856 callout_init_mtx(&vi->tick, &vi->tick_mtx, 0);
2857 if (sc->flags & IS_VF || t4_tx_vm_wr != 0)
2858 vi->flags |= TX_USES_VM_WR;
2862 vi->ifp = ifp;
2872 if (vi->pi->nvi > 1 || sc->flags & IS_VF)
2896 if (vi->nofldrxq != 0)
2900 if (is_ethoffload(sc) && vi->nofldtxq != 0) {
2907 if (vi->flags & TX_USES_VM_WR)
2912 if (is_ethoffload(sc) && vi->nofldtxq != 0)
2919 if (sc->flags & KERN_TLS_ON || !is_t6(sc))
2924 ether_ifattach(ifp, vi->hw_addr);
2926 if (vi->nnmrxq != 0)
2930 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
2934 sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq);
2937 sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq);
2940 sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq);
2946 sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq);
2951 vi->nnmtxq, vi->nnmrxq);
2963 vi->pfil = pfil_head_register(&pa);
2970 struct adapter *sc = pi->adapter;
2974 sysctl_ctx_init(&pi->ctx);
2976 cxgbe_vi_attach(dev, &pi->vi[0]);
2981 vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, DEVICE_UNIT_ANY);
2982 if (vi->dev == NULL) {
2986 device_set_softc(vi->dev, vi);
2999 if_t ifp = vi->ifp;
3001 if (vi->pfil != NULL) {
3002 pfil_head_unregister(vi->pfil);
3003 vi->pfil = NULL;
3014 callout_drain(&vi->tick);
3015 mtx_destroy(&vi->tick_mtx);
3016 sysctl_ctx_free(&vi->ctx);
3019 if_free(vi->ifp);
3020 vi->ifp = NULL;
3027 struct adapter *sc = pi->adapter;
3035 sysctl_ctx_free(&pi->ctx);
3036 begin_vi_detach(sc, &pi->vi[0]);
3037 if (pi->flags & HAS_TRACEQ) {
3038 sc->traceq = -1; /* cloner should not create ifnet */
3041 cxgbe_vi_detach(&pi->vi[0]);
3042 ifmedia_removeall(&pi->media);
3043 end_vi_detach(sc, &pi->vi[0]);
3052 struct adapter *sc = vi->adapter;
3065 struct port_info *pi = vi->pi;
3066 struct adapter *sc = pi->adapter;
3072 mtu = ifr->ifr_mtu;
3080 if (vi->flags & VI_INIT_DONE) {
3101 flags = vi->if_flags;
3110 vi->if_flags = if_getflags(ifp);
3132 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
3142 "tso4 disabled due to -txcsum.\n");
3154 "tso6 disabled due to -txcsum6.\n");
3194 rxq->iq.flags |= IQ_LRO_ENABLED;
3196 rxq->iq.flags &= ~IQ_LRO_ENABLED;
3219 /* Need to find out how to disable auto-mtu-inflation */
3236 rxq->iq.flags |= IQ_RX_TIMESTAMP;
3238 rxq->iq.flags &= ~IQ_RX_TIMESTAMP;
3277 rc = ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
3300 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
3319 struct port_info *pi = vi->pi;
3326 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
3328 if (m->m_pkthdr.csum_flags & CSUM_SND_TAG)
3329 MPASS(m->m_pkthdr.snd_tag->ifp == ifp);
3332 if (__predict_false(pi->link_cfg.link_ok == false)) {
3337 rc = parse_pkt(&m, vi->flags & TX_USES_VM_WR);
3346 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
3351 sc = vi->adapter;
3352 txq = &sc->sge.txq[vi->first_txq];
3354 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
3355 vi->rsrv_noflowq);
3358 rc = mp_ring_enqueue(txq->r, items, 1, 256);
3373 if (vi->flags & VI_INIT_DONE) {
3376 txq->eq.flags |= EQ_QFLUSH;
3378 while (!mp_ring_is_idle(txq->r)) {
3379 mp_ring_check_drainage(txq->r, 4096);
3383 txq->eq.flags &= ~EQ_QFLUSH;
3394 struct fw_vi_stats_vf *s = &vi->stats;
3396 mtx_lock(&vi->tick_mtx);
3398 mtx_unlock(&vi->tick_mtx);
3402 return (s->rx_bcast_frames + s->rx_mcast_frames +
3403 s->rx_ucast_frames);
3405 return (s->rx_err_frames);
3407 return (s->tx_bcast_frames + s->tx_mcast_frames +
3408 s->tx_ucast_frames + s->tx_offload_frames);
3410 return (s->tx_drop_frames);
3412 return (s->rx_bcast_bytes + s->rx_mcast_bytes +
3413 s->rx_ucast_bytes);
3415 return (s->tx_bcast_bytes + s->tx_mcast_bytes +
3416 s->tx_ucast_bytes + s->tx_offload_bytes);
3418 return (s->rx_mcast_frames);
3420 return (s->tx_mcast_frames);
3425 if (vi->flags & VI_INIT_DONE) {
3430 drops += counter_u64_fetch(txq->r->dropped);
3446 struct port_info *pi = vi->pi;
3447 struct port_stats *s = &pi->stats;
3449 mtx_lock(&vi->tick_mtx);
3451 mtx_unlock(&vi->tick_mtx);
3455 return (s->rx_frames);
3458 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
3459 s->rx_fcs_err + s->rx_len_err);
3462 return (s->tx_frames);
3465 return (s->tx_error_frames);
3468 return (s->rx_octets);
3471 return (s->tx_octets);
3474 return (s->rx_mcast_frames);
3477 return (s->tx_mcast_frames);
3480 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
3481 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
3482 s->rx_trunc3 + pi->tnl_cong_drops);
3487 drops = s->tx_drop;
3488 if (vi->flags & VI_INIT_DONE) {
3493 drops += counter_u64_fetch(txq->r->dropped);
3512 switch (params->hdr.type) {
3523 if (is_t6(vi->pi->adapter))
3545 struct port_info *pi = vi->pi;
3546 struct ifmedia *ifm = &pi->media;
3547 struct link_config *lc = &pi->link_cfg;
3548 struct adapter *sc = pi->adapter;
3555 if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) {
3557 if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
3561 lc->requested_aneg = AUTONEG_ENABLE;
3562 lc->requested_speed = 0;
3563 lc->requested_fc |= PAUSE_AUTONEG;
3565 lc->requested_aneg = AUTONEG_DISABLE;
3566 lc->requested_speed =
3567 ifmedia_baudrate(ifm->ifm_media) / 1000000;
3568 lc->requested_fc = 0;
3569 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE)
3570 lc->requested_fc |= PAUSE_RX;
3571 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE)
3572 lc->requested_fc |= PAUSE_TX;
3574 if (pi->up_vis > 0 && hw_all_ok(sc)) {
3595 switch(pi->port_type) {
3656 switch (pi->mod_type) {
3749 struct port_info *pi = vi->pi;
3750 struct adapter *sc = pi->adapter;
3751 struct link_config *lc = &pi->link_cfg;
3757 if (pi->up_vis == 0 && hw_all_ok(sc)) {
3770 ifmr->ifm_status = IFM_AVALID;
3771 if (lc->link_ok == false)
3773 ifmr->ifm_status |= IFM_ACTIVE;
3776 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
3777 ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE);
3778 if (lc->fc & PAUSE_RX)
3779 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
3780 if (lc->fc & PAUSE_TX)
3781 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
3782 ifmr->ifm_active |= port_mword(pi, speed_to_fwcap(lc->speed));
3793 device_set_descf(dev, "port %d vi %td", vi->pi->port_id,
3794 vi - vi->pi->vi);
3807 index = vi - pi->vi;
3811 device_get_nameunit(vi->dev)));
3813 rc = t4_alloc_vi_func(sc, sc->mbox, pi->hw_port, sc->pf, 0, 1,
3814 vi->hw_addr, &vi->rss_size, &vi->vfvld, &vi->vin, func, 0);
3817 "for port %d: %d\n", index, pi->port_id, -rc);
3818 return (-rc);
3820 vi->viid = rc;
3822 if (vi->rss_size == 1) {
3824 * This VI didn't get a slice of the RSS table. Reduce the
3829 device_printf(vi->dev, "RSS table not available.\n");
3830 vi->rss_base = 0xffff;
3837 V_FW_PARAMS_PARAM_YZ(vi->viid);
3838 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
3840 vi->rss_base = 0xffff;
3842 MPASS((val >> 16) == vi->rss_size);
3843 vi->rss_base = val & 0xffff;
3858 pi = vi->pi;
3859 sc = pi->adapter;
3881 sc = vi->adapter;
3885 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
3899 panic("%s: panic on fatal error", device_get_nameunit(sc->dev));
3908 if (atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_CIM_ERR))) {
3933 const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0;
3936 if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_FATAL_ERR)))
3948 atomic_set_int(&sc->error_flags, ADAP_CIM_ERR);
3952 device_get_nameunit(sc->dev), fw_error);
3953 taskqueue_enqueue(reset_tq, &sc->fatal_error_task);
3967 sc->regs_rid = PCIR_BAR(0);
3968 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3969 &sc->regs_rid, RF_ACTIVE);
3970 if (sc->regs_res == NULL) {
3971 device_printf(sc->dev, "cannot map registers.\n");
3974 sc->bt = rman_get_bustag(sc->regs_res);
3975 sc->bh = rman_get_bushandle(sc->regs_res);
3976 sc->mmio_len = rman_get_size(sc->regs_res);
3977 setbit(&sc->doorbells, DOORBELL_KDB);
3979 sc->msix_rid = PCIR_BAR(4);
3980 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3981 &sc->msix_rid, RF_ACTIVE);
3982 if (sc->msix_res == NULL) {
3983 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
3998 if (is_t4(sc) && sc->rdmacaps == 0)
4001 sc->udbs_rid = PCIR_BAR(2);
4002 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
4003 &sc->udbs_rid, RF_ACTIVE);
4004 if (sc->udbs_res == NULL) {
4005 device_printf(sc->dev, "cannot map doorbell BAR.\n");
4008 sc->udbs_base = rman_get_virtual(sc->udbs_res);
4011 setbit(&sc->doorbells, DOORBELL_UDB);
4025 rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
4026 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
4028 clrbit(&sc->doorbells, DOORBELL_UDB);
4029 setbit(&sc->doorbells, DOORBELL_WCWR);
4030 setbit(&sc->doorbells, DOORBELL_UDBWC);
4032 device_printf(sc->dev,
4043 sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0;
4051 if ((sc->doorbells & t4_doorbells_allowed) != 0) {
4052 sc->doorbells &= t4_doorbells_allowed;
4056 sc->doorbells, t4_doorbells_allowed);
4104 for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) {
4105 if (!rw_initialized(&mw->mw_lock)) {
4106 rw_init(&mw->mw_lock, "memory window access");
4107 mw->mw_base = mw_init->base;
4108 mw->mw_aperture = mw_init->aperture;
4109 mw->mw_curpos = 0;
4114 t4_write_reg(sc, reg, (mw->mw_base + bar0) | V_BIR(0) |
4115 V_WINDOW(ilog2(mw->mw_aperture) - 10));
4116 rw_wlock(&mw->mw_lock);
4117 position_memwin(sc, i, mw->mw_curpos);
4118 rw_wunlock(&mw->mw_lock);
4128 * address prior to the requested address. mw->mw_curpos always has the actual
4138 mw = &sc->memwin[idx];
4139 rw_assert(&mw->mw_lock, RA_WLOCKED);
4143 mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */
4145 pf = V_PFNUM(sc->pf);
4146 mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */
4150 val = (mw->mw_curpos >> X_T7_MEMOFST_SHIFT) | pf;
4153 val = mw->mw_curpos | pf;
4172 mw = &sc->memwin[idx];
4174 rw_rlock(&mw->mw_lock);
4175 mw_end = mw->mw_curpos + mw->mw_aperture;
4176 if (addr >= mw_end || addr < mw->mw_curpos) {
4178 if (!rw_try_upgrade(&mw->mw_lock)) {
4179 rw_runlock(&mw->mw_lock);
4180 rw_wlock(&mw->mw_lock);
4182 rw_assert(&mw->mw_lock, RA_WLOCKED);
4184 rw_downgrade(&mw->mw_lock);
4185 mw_end = mw->mw_curpos + mw->mw_aperture;
4187 rw_assert(&mw->mw_lock, RA_RLOCKED);
4190 v = t4_read_reg(sc, mw->mw_base + addr -
4191 mw->mw_curpos);
4195 t4_write_reg(sc, mw->mw_base + addr -
4196 mw->mw_curpos, htole32(v));
4199 len -= 4;
4201 rw_runlock(&mw->mw_lock);
4216 t = &sc->tids;
4217 if (t->natids == 0)
4220 MPASS(t->atid_tab == NULL);
4222 t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE,
4224 mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF);
4225 t->afree = t->atid_tab;
4226 t->atids_in_use = 0;
4227 t->atid_alloc_stopped = false;
4228 for (i = 1; i < t->natids; i++)
4229 t->atid_tab[i - 1].next = &t->atid_tab[i];
4230 t->atid_tab[t->natids - 1].next = NULL;
4238 t = &sc->tids;
4240 KASSERT(t->atids_in_use == 0,
4241 ("%s: %d atids still in use.", __func__, t->atids_in_use));
4243 if (mtx_initialized(&t->atid_lock))
4244 mtx_destroy(&t->atid_lock);
4245 free(t->atid_tab, M_CXGBE);
4246 t->atid_tab = NULL;
4252 struct tid_info *t = &sc->tids;
4254 if (t->natids == 0)
4256 mtx_lock(&t->atid_lock);
4257 t->atid_alloc_stopped = true;
4258 mtx_unlock(&t->atid_lock);
4264 struct tid_info *t = &sc->tids;
4266 if (t->natids == 0)
4268 mtx_lock(&t->atid_lock);
4269 KASSERT(t->atids_in_use == 0,
4270 ("%s: %d atids still in use.", __func__, t->atids_in_use));
4271 t->atid_alloc_stopped = false;
4272 mtx_unlock(&t->atid_lock);
4278 struct tid_info *t = &sc->tids;
4279 int atid = -1;
4281 mtx_lock(&t->atid_lock);
4282 if (t->afree && !t->atid_alloc_stopped) {
4283 union aopen_entry *p = t->afree;
4285 atid = p - t->atid_tab;
4287 t->afree = p->next;
4288 p->data = ctx;
4289 t->atids_in_use++;
4291 mtx_unlock(&t->atid_lock);
4298 struct tid_info *t = &sc->tids;
4300 return (t->atid_tab[atid].data);
4306 struct tid_info *t = &sc->tids;
4307 union aopen_entry *p = &t->atid_tab[atid];
4309 mtx_lock(&t->atid_lock);
4310 p->next = t->afree;
4311 t->afree = p;
4312 t->atids_in_use--;
4313 mtx_unlock(&t->atid_lock);
4344 return ((const struct t4_range *)a)->start -
4345 ((const struct t4_range *)b)->start;
4371 r->size = G_EDRAM0_SIZE(addr_len) << 20;
4372 if (r->size > 0) {
4373 r->start = G_EDRAM0_BASE(addr_len) << 20;
4374 if (addr >= r->start &&
4375 addr + len <= r->start + r->size)
4383 r->size = G_EDRAM1_SIZE(addr_len) << 20;
4384 if (r->size > 0) {
4385 r->start = G_EDRAM1_BASE(addr_len) << 20;
4386 if (addr >= r->start &&
4387 addr + len <= r->start + r->size)
4395 r->size = G_EXT_MEM_SIZE(addr_len) << 20;
4396 if (r->size > 0) {
4397 r->start = G_EXT_MEM_BASE(addr_len) << 20;
4398 if (addr >= r->start &&
4399 addr + len <= r->start + r->size)
4407 r->size = G_EXT_MEM1_SIZE(addr_len) << 20;
4408 if (r->size > 0) {
4409 r->start = G_EXT_MEM1_BASE(addr_len) << 20;
4410 if (addr >= r->start &&
4411 addr + len <= r->start + r->size)
4423 /* Start from index 0 and examine the next n - 1 entries. */
4425 for (remaining = n - 1; remaining > 0; remaining--, r++) {
4427 MPASS(r->size > 0); /* r is a valid entry. */
4429 MPASS(next->size > 0); /* and so is the next one. */
4431 while (r->start + r->size >= next->start) {
4433 r->size = max(r->start + r->size,
4434 next->start + next->size) - r->start;
4435 n--; /* One fewer entry in total. */
4436 if (--remaining == 0)
4446 MPASS(next->size > 0); /* must be valid */
4450 * This so that the foo->size assertion in the
4456 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) *
4466 if (addr >= r->start &&
4467 addr + len <= r->start + r->size)
4545 struct devlog_params *dparams = &sc->params.devlog;
4548 rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
4549 dparams->size, &dparams->addr);
4558 iaq->nirq = T4_EXTRA_INTR;
4559 iaq->nirq += nports * max(iaq->nrxq, iaq->nnmrxq);
4560 iaq->nirq += nports * iaq->nofldrxq;
4561 iaq->nirq += nports * (iaq->num_vis - 1) *
4562 max(iaq->nrxq_vi, iaq->nnmrxq_vi);
4563 iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi;
4574 const int nports = sc->params.nports;
4580 iaq->intr_type = itype;
4581 iaq->num_vis = t4_num_vis;
4582 iaq->ntxq = t4_ntxq;
4583 iaq->ntxq_vi = t4_ntxq_vi;
4584 iaq->nrxq = t4_nrxq;
4585 iaq->nrxq_vi = t4_nrxq_vi;
4588 if (sc->params.tid_qid_sel_mask == 0) {
4589 iaq->nofldtxq = t4_nofldtxq;
4590 iaq->nofldtxq_vi = t4_nofldtxq_vi;
4592 iaq->nofldtxq = roundup(t4_nofldtxq, sc->params.ncores);
4593 iaq->nofldtxq_vi = roundup(t4_nofldtxq_vi,
4594 sc->params.ncores);
4595 if (iaq->nofldtxq != t4_nofldtxq)
4596 device_printf(sc->dev,
4597 "nofldtxq updated (%d -> %d) for correct"
4599 t4_nofldtxq, iaq->nofldtxq,
4600 sc->params.ncores);
4601 if (iaq->num_vis > 1 &&
4602 iaq->nofldtxq_vi != t4_nofldtxq_vi)
4603 device_printf(sc->dev,
4604 "nofldtxq_vi updated (%d -> %d) for correct"
4606 t4_nofldtxq_vi, iaq->nofldtxq_vi,
4607 sc->params.ncores);
4613 iaq->nofldrxq = t4_nofldrxq;
4614 iaq->nofldrxq_vi = t4_nofldrxq_vi;
4619 iaq->nnmtxq = t4_nnmtxq;
4620 iaq->nnmrxq = t4_nnmrxq;
4623 iaq->nnmtxq_vi = t4_nnmtxq_vi;
4624 iaq->nnmrxq_vi = t4_nnmrxq_vi;
4629 if (iaq->nirq <= navail &&
4630 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4632 * This is the normal case -- there are enough interrupts for
4642 while (iaq->num_vis > 1) {
4643 iaq->num_vis--;
4645 if (iaq->nirq <= navail &&
4646 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4647 device_printf(sc->dev, "virtual interfaces per port "
4651 iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq,
4652 iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi,
4653 itype, navail, iaq->nirq);
4661 MPASS(iaq->num_vis == 1);
4662 iaq->ntxq_vi = iaq->nrxq_vi = 0;
4663 iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
4664 iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
4665 if (iaq->num_vis != t4_num_vis) {
4666 device_printf(sc->dev, "extra virtual interfaces disabled. "
4669 iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi,
4670 iaq->nnmrxq_vi, itype, navail, iaq->nirq);
4679 if (iaq->nrxq > 1) {
4680 iaq->nrxq = rounddown_pow_of_two(iaq->nrxq - 1);
4681 if (iaq->nnmrxq > iaq->nrxq)
4682 iaq->nnmrxq = iaq->nrxq;
4684 if (iaq->nofldrxq > 1)
4685 iaq->nofldrxq >>= 1;
4687 old_nirq = iaq->nirq;
4689 if (iaq->nirq <= navail &&
4690 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4691 device_printf(sc->dev, "running with reduced number of "
4694 "itype %d, navail %u, nirq %d.\n", iaq->nrxq,
4695 iaq->nofldrxq, itype, navail, iaq->nirq);
4698 } while (old_nirq != iaq->nirq);
4701 device_printf(sc->dev, "running with minimal number of queues. "
4703 iaq->nirq = 1;
4704 iaq->nrxq = 1;
4705 iaq->ntxq = 1;
4706 if (iaq->nofldrxq > 0) {
4707 iaq->nofldrxq = 1;
4708 iaq->nofldtxq = 1;
4709 if (sc->params.tid_qid_sel_mask == 0)
4710 iaq->nofldtxq = 1;
4712 iaq->nofldtxq = sc->params.ncores;
4714 iaq->nnmtxq = 0;
4715 iaq->nnmrxq = 0;
4717 MPASS(iaq->num_vis > 0);
4718 if (iaq->num_vis > 1) {
4719 MPASS(iaq->nrxq_vi > 0);
4720 MPASS(iaq->ntxq_vi > 0);
4722 MPASS(iaq->nirq > 0);
4723 MPASS(iaq->nrxq > 0);
4724 MPASS(iaq->ntxq > 0);
4726 MPASS(powerof2(iaq->nirq));
4727 if (sc->params.tid_qid_sel_mask != 0)
4728 MPASS(iaq->nofldtxq % sc->params.ncores == 0);
4742 navail = pci_msix_count(sc->dev);
4744 navail = pci_msi_count(sc->dev);
4752 nalloc = iaq->nirq;
4755 rc = pci_alloc_msix(sc->dev, &nalloc);
4757 rc = pci_alloc_msi(sc->dev, &nalloc);
4760 if (nalloc == iaq->nirq)
4767 device_printf(sc->dev, "fewer vectors than requested, "
4769 itype, iaq->nirq, nalloc);
4770 pci_release_msi(sc->dev);
4775 device_printf(sc->dev,
4777 itype, rc, iaq->nirq, nalloc);
4780 device_printf(sc->dev,
4782 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
4783 pci_msix_count(sc->dev), pci_msi_count(sc->dev));
4910 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
4917 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
4918 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
4939 device_printf(sc->dev,
4945 *dcfg = firmware_get(fw_info->kld_name);
4948 *fw = firmware_get(fw_info->fw_mod_name);
4977 const uint32_t c = be32toh(card_fw->fw_ver);
4985 fw_install = t4_fw_install < 0 ? -t4_fw_install : t4_fw_install;
4991 device_printf(sc->dev,
4993 " will use compiled-in firmware version for"
4997 memcpy(&bundled_fw, fw->data, sizeof(bundled_fw));
5006 if ((sc->flags & FW_OK) == 0) {
5042 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
5058 device_printf(sc->dev,
5065 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
5070 rc = sc->flags & FW_OK ? 0 : ENOENT;
5073 k = be32toh(((const struct fw_hdr *)fw->data)->fw_ver);
5076 device_printf(sc->dev,
5083 rc = sc->flags & FW_OK ? 0 : EINVAL;
5087 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
5094 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
5096 device_printf(sc->dev, "failed to install firmware: %d\n", rc);
5100 memcpy(card_fw, fw->data, sizeof(*card_fw));
5125 device_printf(sc->dev,
5130 drv_fw = &fw_info->fw_h;
5135 rc = -t4_get_fw_hdr(sc, card_fw);
5137 device_printf(sc->dev,
5150 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
5152 rc = -rc;
5153 device_printf(sc->dev,
5163 MPASS(be32toh(card_fw->flags) & FW_HDR_FLAGS_RESET_HALT);
5164 sc->flags |= FW_OK; /* The firmware responded to the FW_HELLO. */
5166 if (rc == sc->pf) {
5167 sc->flags |= MASTER_PF;
5180 device_printf(sc->dev, "couldn't be master(%d), "
5190 device_printf(sc->dev, "PF%d is master, device state %d. "
5192 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", rc);
5193 sc->cfcsum = 0;
5197 if (rc != 0 && sc->flags & FW_OK) {
5198 t4_fw_bye(sc, sc->mbox);
5199 sc->flags &= ~FW_OK;
5219 if (pci_get_device(sc->dev) == 0x440a)
5227 device_printf(sc->dev,
5232 cfdata = dcfg->data;
5233 cflen = dcfg->datasize & ~3;
5239 device_printf(sc->dev,
5245 snprintf(s, sizeof(s), "%s_%s", fw_info->kld_name, cfg_file);
5249 device_printf(sc->dev,
5255 cfdata = rcfg->data;
5256 cflen = rcfg->datasize & ~3;
5260 device_printf(sc->dev,
5269 device_printf(sc->dev,
5318 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
5320 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
5343 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5346 device_printf(sc->dev,
5359 device_printf(sc->dev,
5364 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
5366 device_printf(sc->dev, "failed to pre-process config file: %d "
5374 device_printf(sc->dev,
5378 sc->cfcsum = cfcsum;
5379 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", cfg_file);
5386 caps.x##caps &= htobe16(caps_allowed->x##caps); \
5403 * to cope with the situation in non-debug builds by disabling
5416 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
5418 device_printf(sc->dev,
5427 rc = -t4_fw_initialize(sc, sc->mbox);
5429 device_printf(sc->dev, "fw_initialize failed: %d.\n", rc);
5448 MPASS(sc->flags & MASTER_PF);
5464 fallback = sc->debug_flags & DF_DISABLE_CFG_RETRY ? false : true;
5470 device_printf(sc->dev,
5496 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
5497 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
5498 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
5499 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
5500 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
5502 snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u",
5503 G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers),
5504 G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers),
5505 G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers),
5506 G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers));
5508 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
5509 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
5510 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
5511 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
5512 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
5514 snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u",
5515 G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers),
5516 G_FW_HDR_FW_VER_MINOR(sc->params.er_vers),
5517 G_FW_HDR_FW_VER_MICRO(sc->params.er_vers),
5518 G_FW_HDR_FW_VER_BUILD(sc->params.er_vers));
5522 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5524 device_printf(sc->dev,
5529 sc->params.portvec = val[0];
5530 sc->params.nports = bitcount32(val[0]);
5531 sc->params.vpd.cclk = val[1];
5534 rc = -t4_init_devlog_ncores_params(sc, 1);
5538 device_printf(sc->dev,
5558 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5561 sc->params.fw_vers < FW_VERSION32(1, 20, 1, 0)) {
5565 device_printf(sc->dev,
5571 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5573 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m,
5576 device_printf(sc->dev,
5585 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5587 sc->params.viid_smt_extn_support = true;
5589 sc->params.viid_smt_extn_support = false;
5614 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val);
5616 device_printf(sc->dev,
5621 sc->sge.iq_start = val[0];
5622 sc->sge.eq_start = val[1];
5624 sc->tids.ftid_base = val[2];
5625 sc->tids.ftid_end = val[3];
5626 sc->tids.nftids = val[3] - val[2] + 1;
5628 sc->vres.l2t.start = val[4];
5629 sc->vres.l2t.size = val[5] - val[4] + 1;
5631 if (sc->vres.l2t.size > 0)
5633 sc->params.core_vdd = val[6];
5637 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5639 device_printf(sc->dev,
5643 MPASS((int)val[0] >= sc->sge.iq_start);
5644 sc->sge.iqmap_sz = val[0] - sc->sge.iq_start + 1;
5645 MPASS((int)val[1] >= sc->sge.eq_start);
5646 sc->sge.eqmap_sz = val[1] - sc->sge.eq_start + 1;
5650 sc->tids.tid_base = t4_read_reg(sc,
5655 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5657 device_printf(sc->dev,
5662 sc->tids.hpftid_base = val[0];
5663 sc->tids.hpftid_end = val[1];
5664 sc->tids.nhpftids = val[1] - val[0] + 1;
5670 MPASS(sc->tids.hpftid_base == 0);
5671 MPASS(sc->tids.tid_base == sc->tids.nhpftids);
5676 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5678 device_printf(sc->dev,
5683 sc->rawf_base = val[0];
5684 sc->nrawf = val[1] - val[0] + 1;
5688 if (sc->params.ncores > 1) {
5692 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5693 sc->params.tid_qid_sel_mask = rc == 0 ? val[0] : 0;
5707 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5709 sc->params.mps_bg_map = val[0];
5711 sc->params.mps_bg_map = UINT32_MAX; /* Not a legal value. */
5715 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5717 sc->params.tp_ch_map = val[0];
5719 sc->params.tp_ch_map = UINT32_MAX; /* Not a legal value. */
5723 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5725 sc->params.tx_tp_ch_map = val[0];
5727 sc->params.tx_tp_ch_map = UINT32_MAX; /* Not a legal value. */
5734 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5736 sc->params.filter2_wr_support = val[0] != 0;
5738 sc->params.filter2_wr_support = 0;
5745 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5747 sc->params.ulptx_memwrite_dsgl = val[0] != 0;
5749 sc->params.ulptx_memwrite_dsgl = false;
5753 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5755 sc->params.fr_nsmr_tpte_wr_support = val[0] != 0;
5757 sc->params.fr_nsmr_tpte_wr_support = false;
5761 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5763 sc->params.dev_512sgl_mr = val[0] != 0;
5765 sc->params.dev_512sgl_mr = false;
5768 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5770 sc->params.max_pkts_per_eth_tx_pkts_wr = val[0];
5772 sc->params.max_pkts_per_eth_tx_pkts_wr = 15;
5775 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5778 sc->params.nsched_cls = val[0];
5780 sc->params.nsched_cls = sc->chip_params->nsched_cls;
5787 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
5789 device_printf(sc->dev,
5795 sc->x = htobe16(caps.x); \
5808 if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) {
5810 MPASS(sc->toecaps == 0);
5811 sc->toecaps = 0;
5814 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5816 device_printf(sc->dev,
5820 sc->tids.ntids = val[0];
5821 if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
5822 MPASS(sc->tids.ntids >= sc->tids.nhpftids);
5823 sc->tids.ntids -= sc->tids.nhpftids;
5825 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
5826 sc->params.hash_filter = 1;
5828 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
5832 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
5834 device_printf(sc->dev,
5839 sc->tids.etid_base = val[0];
5840 sc->tids.etid_end = val[1];
5841 sc->tids.netids = val[1] - val[0] + 1;
5842 sc->params.eo_wr_cred = val[2];
5843 sc->params.ethoffload = 1;
5846 if (sc->toecaps) {
5847 /* query offload-related parameters */
5854 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5856 device_printf(sc->dev,
5860 sc->tids.ntids = val[0];
5861 if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
5862 MPASS(sc->tids.ntids >= sc->tids.nhpftids);
5863 sc->tids.ntids -= sc->tids.nhpftids;
5865 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
5867 sc->tids.stid_base = val[1];
5868 sc->tids.nstids = val[2] - val[1] + 1;
5870 sc->vres.ddp.start = val[3];
5871 sc->vres.ddp.size = val[4] - val[3] + 1;
5872 sc->params.ofldq_wr_cred = val[5];
5873 sc->params.offload = 1;
5876 * The firmware attempts memfree TOE configuration for -SO cards
5883 sc->iscsicaps = 0;
5884 sc->rdmacaps = 0;
5886 if (sc->rdmacaps) {
5893 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5895 device_printf(sc->dev,
5899 sc->vres.stag.start = val[0];
5900 sc->vres.stag.size = val[1] - val[0] + 1;
5901 sc->vres.rq.start = val[2];
5902 sc->vres.rq.size = val[3] - val[2] + 1;
5903 sc->vres.pbl.start = val[4];
5904 sc->vres.pbl.size = val[5] - val[4] + 1;
5912 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5914 device_printf(sc->dev,
5918 sc->vres.qp.start = val[0];
5919 sc->vres.qp.size = val[1] - val[0] + 1;
5920 sc->vres.cq.start = val[2];
5921 sc->vres.cq.size = val[3] - val[2] + 1;
5922 sc->vres.ocq.start = val[4];
5923 sc->vres.ocq.size = val[5] - val[4] + 1;
5929 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
5931 device_printf(sc->dev,
5935 sc->vres.srq.start = val[0];
5936 sc->vres.srq.size = val[1] - val[0] + 1;
5937 sc->params.max_ordird_qp = val[2];
5938 sc->params.max_ird_adapter = val[3];
5940 if (sc->iscsicaps) {
5943 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5945 device_printf(sc->dev,
5949 sc->vres.iscsi.start = val[0];
5950 sc->vres.iscsi.size = val[1] - val[0] + 1;
5952 if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) {
5955 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5957 device_printf(sc->dev,
5961 sc->vres.key.start = val[0];
5962 sc->vres.key.size = val[1] - val[0] + 1;
5971 t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
5972 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd);
5993 callout_schedule_sbt(&sc->ktls_tick, SBT_1MS, 0, C_HARDCLOCK);
6005 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, ¶m);
6013 sc->flags |= KERN_TLS_ON;
6014 callout_reset_sbt(&sc->ktls_tick, SBT_1MS, 0, ktls_tick, sc,
6017 sc->flags &= ~KERN_TLS_ON;
6018 callout_stop(&sc->ktls_tick);
6036 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
6041 if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val) == 0)
6042 sc->params.port_caps32 = 1;
6045 val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1);
6047 V_MASKFILTER(val - 1));
6123 if (t4_toe_rexmt_backoff[i] != -1) {
6142 sc->tlst.inline_keys = t4_tls_inline_keys;
6144 sc->tlst.combo_wrs = t4_tls_combo_wrs;
6147 sc->tlst.short_records = t4_tls_short_records;
6148 sc->tlst.partial_ghash = t4_tls_partial_ghash;
6161 struct adapter_params *p = &sc->params;
6163 device_set_descf(sc->dev, "Chelsio %s", p->vpd.id);
6192 ifm = &pi->media;
6193 if (ifm->ifm_cur != NULL &&
6194 IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE)
6197 lc = &pi->link_cfg;
6198 if (lc->requested_aneg != AUTONEG_DISABLE &&
6199 lc->pcaps & FW_PORT_CAP32_ANEG) {
6204 if (lc->requested_fc & PAUSE_TX)
6206 if (lc->requested_fc & PAUSE_RX)
6208 if (lc->requested_speed == 0)
6209 speed = port_top_speed(pi) * 1000; /* Gbps -> Mbps */
6211 speed = lc->requested_speed;
6223 return (pi->port_type == FW_PORT_TYPE_BT_SGMII ||
6224 pi->port_type == FW_PORT_TYPE_BT_XFI ||
6225 pi->port_type == FW_PORT_TYPE_BT_XAUI ||
6226 pi->port_type == FW_PORT_TYPE_KX4 ||
6227 pi->port_type == FW_PORT_TYPE_KX ||
6228 pi->port_type == FW_PORT_TYPE_KR ||
6229 pi->port_type == FW_PORT_TYPE_BP_AP ||
6230 pi->port_type == FW_PORT_TYPE_BP4_AP ||
6231 pi->port_type == FW_PORT_TYPE_BP40_BA ||
6232 pi->port_type == FW_PORT_TYPE_KR4_100G ||
6233 pi->port_type == FW_PORT_TYPE_KR_SFP28 ||
6234 pi->port_type == FW_PORT_TYPE_KR_XLAUI);
6247 if (pi->flags & FIXED_IFMEDIA)
6253 ifm = &pi->media;
6255 lc = &pi->link_cfg;
6256 ss = G_FW_PORT_CAP32_SPEED(lc->pcaps); /* Supported Speeds */
6260 MPASS(LIST_EMPTY(&ifm->ifm_list));
6282 if (lc->pcaps & FW_PORT_CAP32_ANEG)
6294 struct link_config *lc = &pi->link_cfg;
6298 lc->requested_caps = 0;
6299 lc->requested_speed = 0;
6302 lc->requested_aneg = AUTONEG_DISABLE;
6304 lc->requested_aneg = AUTONEG_ENABLE;
6306 lc->requested_aneg = AUTONEG_AUTO;
6308 lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX |
6312 lc->requested_fec = FEC_AUTO;
6314 lc->requested_fec = FEC_NONE;
6316 /* -1 is handled by the FEC_AUTO block above and not here. */
6317 lc->requested_fec = t4_fec &
6319 if (lc->requested_fec == 0)
6320 lc->requested_fec = FEC_AUTO;
6323 lc->force_fec = -1;
6325 lc->force_fec = 1;
6327 lc->force_fec = 0;
6338 struct link_config *lc = &pi->link_cfg;
6344 if (lc->requested_speed != 0) {
6345 fwspeed = speed_to_fwcap(lc->requested_speed);
6346 if ((fwspeed & lc->pcaps) == 0) {
6348 lc->requested_speed = 0;
6353 MPASS(lc->requested_aneg == AUTONEG_ENABLE ||
6354 lc->requested_aneg == AUTONEG_DISABLE ||
6355 lc->requested_aneg == AUTONEG_AUTO);
6356 if (lc->requested_aneg == AUTONEG_ENABLE &&
6357 !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
6359 lc->requested_aneg = AUTONEG_AUTO;
6363 MPASS((lc->requested_fc & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) == 0);
6364 if (lc->requested_fc & PAUSE_TX &&
6365 !(lc->pcaps & FW_PORT_CAP32_FC_TX)) {
6367 lc->requested_fc &= ~PAUSE_TX;
6369 if (lc->requested_fc & PAUSE_RX &&
6370 !(lc->pcaps & FW_PORT_CAP32_FC_RX)) {
6372 lc->requested_fc &= ~PAUSE_RX;
6374 if (!(lc->requested_fc & PAUSE_AUTONEG) &&
6375 !(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE)) {
6377 lc->requested_fc |= PAUSE_AUTONEG;
6381 if ((lc->requested_fec & FEC_RS &&
6382 !(lc->pcaps & FW_PORT_CAP32_FEC_RS)) ||
6383 (lc->requested_fec & FEC_BASER_RS &&
6384 !(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS))) {
6386 lc->requested_fec = FEC_AUTO;
6399 struct adapter *sc = pi->adapter;
6400 struct link_config *lc = &pi->link_cfg;
6407 if (lc->requested_aneg == AUTONEG_ENABLE)
6408 MPASS(lc->pcaps & FW_PORT_CAP32_ANEG);
6409 if (!(lc->requested_fc & PAUSE_AUTONEG))
6410 MPASS(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE);
6411 if (lc->requested_fc & PAUSE_TX)
6412 MPASS(lc->pcaps & FW_PORT_CAP32_FC_TX);
6413 if (lc->requested_fc & PAUSE_RX)
6414 MPASS(lc->pcaps & FW_PORT_CAP32_FC_RX);
6415 if (lc->requested_fec & FEC_RS)
6416 MPASS(lc->pcaps & FW_PORT_CAP32_FEC_RS);
6417 if (lc->requested_fec & FEC_BASER_RS)
6418 MPASS(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS);
6420 if (!(sc->flags & IS_VF)) {
6421 rc = -t4_link_l1cfg(sc, sc->mbox, pi->hw_port, lc);
6423 device_printf(pi->dev, "l1cfg failed: %d\n", rc);
6429 * An L1_CFG will almost always result in a link-change event if the
6437 if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG))
6438 lc->fc = lc->requested_fc & (PAUSE_TX | PAUSE_RX);
6457 struct vi_info *vi = if_getsoftc(ctx->ifp);
6458 struct port_info *pi = vi->pi;
6459 struct adapter *sc = pi->adapter;
6461 if (ctx->rc < 0)
6464 ctx->mcaddr[ctx->i] = LLADDR(sdl);
6465 MPASS(ETHER_IS_MULTICAST(ctx->mcaddr[ctx->i]));
6466 ctx->i++;
6468 if (ctx->i == FW_MAC_EXACT_CHUNK) {
6469 ctx->rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, ctx->del,
6470 ctx->i, ctx->mcaddr, NULL, &ctx->hash, 0);
6471 if (ctx->rc < 0) {
6474 for (j = 0; j < ctx->i; j++) {
6475 if_printf(ctx->ifp,
6479 ctx->mcaddr[j][0], ctx->mcaddr[j][1],
6480 ctx->mcaddr[j][2], ctx->mcaddr[j][3],
6481 ctx->mcaddr[j][4], ctx->mcaddr[j][5],
6482 -ctx->rc);
6486 ctx->del = 0;
6487 ctx->i = 0;
6502 struct port_info *pi = vi->pi;
6503 struct adapter *sc = pi->adapter;
6504 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
6523 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
6536 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
6537 ucaddr, true, &vi->smt_idx);
6539 rc = -rc;
6543 vi->xact_addr_filt = rc;
6568 rc = -ctx.rc;
6572 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
6576 rc = -rc;
6593 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, ctx.hash, 0);
6599 pi->vxlan_tcam_entry = false;
6603 if (IS_MAIN_VI(vi) && sc->vxlan_refcount > 0 &&
6604 pi->vxlan_tcam_entry == false) {
6605 rc = t4_alloc_raw_mac_filt(sc, vi->viid, match_all_mac,
6606 match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id,
6609 rc = -rc;
6613 MPASS(rc == sc->rawf_base + pi->port_id);
6615 pi->vxlan_tcam_entry = true;
6654 if (mtx_sleep(&sc->flags, &sc->sc_lock,
6664 sc->last_op = wmesg;
6665 sc->last_op_thr = curthread;
6666 sc->last_op_flags = flags;
6686 wakeup(&sc->flags);
6688 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
6691 sc->last_op = "t4detach";
6692 sc->last_op_thr = curthread;
6693 sc->last_op_flags = 0;
6705 wakeup(&sc->flags);
6723 wakeup(&sc->flags);
6730 struct port_info *pi = vi->pi;
6731 struct adapter *sc = pi->adapter;
6732 if_t ifp = vi->ifp;
6741 if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_init(sc)) != 0))
6744 if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0))
6752 if (pi->up_vis == 0) {
6759 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
6773 txq->eq.flags |= EQ_ENABLED;
6780 if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
6781 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
6782 t4_set_trace_rss_control(sc, pi->tx_chan, sc->traceq);
6783 pi->flags |= HAS_TRACEQ;
6787 pi->up_vis++;
6789 if (pi->link_cfg.link_ok)
6793 mtx_lock(&vi->tick_mtx);
6794 if (vi->pi->nvi > 1 || sc->flags & IS_VF)
6795 callout_reset(&vi->tick, hz, vi_tick, vi);
6797 callout_reset(&vi->tick, hz, cxgbe_tick, vi);
6798 mtx_unlock(&vi->tick_mtx);
6812 struct port_info *pi = vi->pi;
6813 struct adapter *sc = pi->adapter;
6814 if_t ifp = vi->ifp;
6820 if (!(vi->flags & VI_INIT_DONE)) {
6824 "vi->flags 0x%016lx, if_flags 0x%08x, "
6825 "if_drv_flags 0x%08x\n", vi->flags, if_getflags(ifp),
6838 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
6846 txq->eq.flags &= ~EQ_ENABLED;
6850 mtx_lock(&vi->tick_mtx);
6851 callout_stop(&vi->tick);
6852 mtx_unlock(&vi->tick_mtx);
6860 pi->up_vis--;
6861 if (pi->up_vis > 0) {
6866 pi->link_cfg.link_ok = false;
6867 pi->link_cfg.speed = 0;
6868 pi->link_cfg.link_down_rc = 255;
6877 * will walk the entire sc->irq list and clean up whatever is valid.
6887 struct sge *sge = &sc->sge;
6902 irq = &sc->irq[0];
6903 rid = sc->intr_type == INTR_INTX ? 0 : 1;
6908 if (sc->flags & IS_VF)
6909 KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports,
6912 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
6916 if (!(sc->flags & IS_VF)) {
6925 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
6932 pi = sc->port[p];
6934 vi->first_intr = rid - 1;
6936 if (vi->nnmrxq > 0) {
6937 int n = max(vi->nrxq, vi->nnmrxq);
6939 rxq = &sge->rxq[vi->first_rxq];
6941 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
6946 if (q < vi->nrxq)
6947 irq->rxq = rxq++;
6949 if (q < vi->nnmrxq)
6950 irq->nm_rxq = nm_rxq++;
6952 if (irq->nm_rxq != NULL &&
6953 irq->rxq == NULL) {
6956 t4_nm_intr, irq->nm_rxq, s);
6958 if (irq->nm_rxq != NULL &&
6959 irq->rxq != NULL) {
6965 if (irq->rxq != NULL &&
6966 irq->nm_rxq == NULL) {
6969 t4_intr, irq->rxq, s);
6974 if (q < vi->nrxq) {
6975 bus_bind_intr(sc->dev, irq->res,
6981 vi->nintr++;
6992 bus_bind_intr(sc->dev, irq->res,
6997 vi->nintr++;
7009 vi->nintr++;
7014 MPASS(irq == &sc->irq[sc->intr_count]);
7031 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
7033 t4_write_rss_key(sc, &rss_key[0], -1, 1);
7054 MPASS(sc->params.nports <= nitems(sc->tq));
7055 for (i = 0; i < sc->params.nports; i++) {
7056 if (sc->tq[i] != NULL)
7058 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
7059 taskqueue_thread_enqueue, &sc->tq[i]);
7060 if (sc->tq[i] == NULL) {
7064 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
7065 device_get_nameunit(sc->dev), i);
7068 if (!(sc->flags & IS_VF)) {
7082 KASSERT((sc->flags & FULL_INIT_DONE) == 0,
7089 sc->flags |= FULL_INIT_DONE;
7104 for (i = 0; i < nitems(sc->tq); i++) {
7105 if (sc->tq[i] == NULL)
7107 taskqueue_free(sc->tq[i]);
7108 sc->tq[i] = NULL;
7111 sc->flags &= ~FULL_INIT_DONE;
7156 * enabling any 4-tuple hash is nonsense configuration.
7185 struct adapter *sc = vi->adapter;
7204 * Setup RSS for this VI. Save a copy of the RSS table for later use.
7206 if (vi->nrxq > vi->rss_size) {
7207 CH_ALERT(vi, "nrxq (%d) > hw RSS table size (%d); "
7208 "some queues will never receive traffic.\n", vi->nrxq,
7209 vi->rss_size);
7210 } else if (vi->rss_size % vi->nrxq) {
7211 CH_ALERT(vi, "nrxq (%d), hw RSS table size (%d); "
7212 "expect uneven traffic distribution.\n", vi->nrxq,
7213 vi->rss_size);
7216 if (vi->nrxq != nbuckets) {
7218 "performance will be impacted.\n", vi->nrxq, nbuckets);
7221 if (vi->rss == NULL)
7222 vi->rss = malloc(vi->rss_size * sizeof (*vi->rss), M_CXGBE,
7224 for (i = 0; i < vi->rss_size;) {
7227 j %= vi->nrxq;
7228 rxq = &sc->sge.rxq[vi->first_rxq + j];
7229 vi->rss[i++] = rxq->iq.abs_id;
7232 vi->rss[i++] = rxq->iq.abs_id;
7233 if (i == vi->rss_size)
7239 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size,
7240 vi->rss, vi->rss_size);
7247 vi->hashen = hashconfig_to_hashen(hashconfig);
7254 extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig;
7271 CH_ALERT(vi, "IPv4 2-tuple hashing forced on.\n");
7273 CH_ALERT(vi, "TCP/IPv4 4-tuple hashing forced on.\n");
7275 CH_ALERT(vi, "IPv6 2-tuple hashing forced on.\n");
7277 CH_ALERT(vi, "TCP/IPv6 4-tuple hashing forced on.\n");
7279 CH_ALERT(vi, "UDP/IPv4 4-tuple hashing forced on.\n");
7281 CH_ALERT(vi, "UDP/IPv6 4-tuple hashing forced on.\n");
7283 vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
7288 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, vi->rss[0],
7303 ASSERT_SYNCHRONIZED_OP(vi->adapter);
7304 KASSERT((vi->flags & VI_INIT_DONE) == 0,
7311 vi->flags |= VI_INIT_DONE;
7323 if (vi->flags & VI_INIT_DONE) {
7325 free(vi->rss, M_CXGBE);
7326 free(vi->nm_rss, M_CXGBE);
7330 vi->flags &= ~VI_INIT_DONE;
7336 struct sge_eq *eq = &txq->eq;
7337 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
7339 MPASS(eq->flags & EQ_SW_ALLOCATED);
7340 MPASS(!(eq->flags & EQ_ENABLED));
7343 while (!mp_ring_is_idle(txq->r)) {
7344 mp_ring_check_drainage(txq->r, 4096);
7347 MPASS(txq->txp.npkt == 0);
7349 if (eq->flags & EQ_HW_ALLOCATED) {
7355 while (spg->cidx != htobe16(eq->pidx))
7357 while (eq->cidx != eq->pidx)
7365 while (eq->cidx != eq->pidx) {
7369 txsd = &txq->sdesc[eq->cidx];
7370 for (m = txsd->m; m != NULL; m = nextpkt) {
7371 nextpkt = m->m_nextpkt;
7372 m->m_nextpkt = NULL;
7375 IDXINCR(eq->cidx, txsd->desc_used, eq->sidx);
7377 spg->pidx = spg->cidx = htobe16(eq->cidx);
7388 while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) {
7389 STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
7391 wrq->nwr_pending--;
7392 wrq->ndesc_needed -= howmany(wr->wr_len, EQ_ESIZE);
7396 MPASS(wrq->nwr_pending == 0);
7397 MPASS(wrq->ndesc_needed == 0);
7398 wrq->nwr_pending = 0;
7399 wrq->ndesc_needed = 0;
7407 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
7411 MPASS(iq->flags & IQ_HAS_FL);
7413 mtx_lock(&sc->sfl_lock);
7415 fl->flags |= FL_DOOMED;
7417 callout_stop(&sc->sfl_callout);
7418 mtx_unlock(&sc->sfl_lock);
7420 KASSERT((fl->flags & FL_STARVING) == 0,
7424 if (!(iq->flags & IQ_HW_ALLOCATED))
7438 struct adapter *sc = vi->adapter;
7448 if (!(vi->flags & VI_INIT_DONE))
7457 quiesce_wrq(&ofld_txq->wrq);
7462 quiesce_iq_fl(sc, &rxq->iq, &rxq->fl);
7467 quiesce_iq_fl(sc, &ofld_rxq->iq, &ofld_rxq->fl);
7478 irq->rid = rid;
7479 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
7481 if (irq->res == NULL) {
7482 device_printf(sc->dev,
7487 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
7488 NULL, handler, arg, &irq->tag);
7490 device_printf(sc->dev,
7494 bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name);
7502 if (irq->tag)
7503 bus_teardown_intr(sc->dev, irq->res, irq->tag);
7504 if (irq->res)
7505 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
7516 regs->version = chip_id(sc) | chip_rev(sc) << 10;
7517 t4_get_regs(sc, buf, regs->len);
7544 if (sc->flags & IS_VF) {
7548 mtx_assert(&sc->reg_lock, MA_OWNED);
7564 if (!(sc->flags & IS_VF))
7565 mtx_lock(&sc->reg_lock);
7566 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES);
7567 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES);
7568 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES);
7569 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES);
7570 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES);
7571 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES);
7572 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES);
7573 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES);
7574 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
7575 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES);
7576 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES);
7577 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES);
7578 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES);
7579 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES);
7580 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES);
7581 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES);
7582 if (!(sc->flags & IS_VF))
7583 mtx_unlock(&sc->reg_lock);
7606 mtx_assert(&vi->tick_mtx, MA_OWNED);
7608 if (vi->flags & VI_SKIP_STATS)
7613 if (timevalcmp(&tv, &vi->last_refreshed, <))
7616 t4_get_vi_stats(vi->adapter, vi->vin, &vi->stats);
7617 getmicrotime(&vi->last_refreshed);
7629 mtx_assert(&vi->tick_mtx, MA_OWNED);
7631 if (vi->flags & VI_SKIP_STATS)
7636 if (timevalcmp(&tv, &vi->last_refreshed, <))
7639 pi = vi->pi;
7640 sc = vi->adapter;
7642 t4_get_port_stats(sc, pi->hw_port, &pi->stats);
7643 chan_map = pi->rx_e_chan_map;
7645 i = ffs(chan_map) - 1;
7646 mtx_lock(&sc->reg_lock);
7649 mtx_unlock(&sc->reg_lock);
7653 pi->tnl_cong_drops = tnl_cong_drops;
7654 getmicrotime(&vi->last_refreshed);
7663 mtx_assert(&vi->tick_mtx, MA_OWNED);
7666 callout_schedule(&vi->tick, hz);
7674 mtx_assert(&vi->tick_mtx, MA_OWNED);
7677 callout_schedule(&vi->tick, hz);
7724 sc->params.ncores, "# of active CIM cores");
7726 for (i = 0; i < sc->params.ncores; i++) {
7782 MPASS(qcount <= sc->chip_params->cim_num_ibq);
7813 MPASS(qcount <= sc->chip_params->cim_num_obq);
7850 struct sysctl_ctx_list *ctx = &sc->ctx;
7858 oid = device_get_sysctl_tree(sc->dev);
7861 sc->sc_do_rxcopy = 1;
7863 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
7866 sc->params.nports, "# of ports");
7870 (uintptr_t)&sc->doorbells, sysctl_bitfield_8b, "A",
7874 sc->params.vpd.cclk, "core clock frequency (in KHz)");
7878 sc->params.sge.timer_val, sizeof(sc->params.sge.timer_val),
7883 sc->params.sge.counter_val, sizeof(sc->params.sge.counter_val),
7888 sc->lro_timeout = 100;
7890 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
7893 &sc->debug_flags, 0, "flags to enable runtime debugging");
7896 CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");
7899 CTLFLAG_RD, sc->fw_version, 0, "firmware version");
7901 if (sc->flags & IS_VF)
7908 CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number");
7911 CTLFLAG_RD, sc->params.vpd.pn, 0, "part number");
7914 CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change");
7917 CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version");
7920 CTLFLAG_RD, sc->params.vpd.na, 0, "network address");
7923 sc->er_version, 0, "expansion ROM version");
7926 sc->bs_version, 0, "bootstrap firmware version");
7929 NULL, sc->params.scfg_vers, "serial config version");
7932 NULL, sc->params.vpd_vers, "VPD version");
7935 CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
7938 sc->cfcsum, "config file checksum");
7943 (uintptr_t)&sc->name, sysctl_bitfield_16b, "A", \
7959 NULL, sc->tids.nftids, "number of filters");
7961 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
7963 sysctl_temperature, "I", "chip temperature (in Celsius)");
7966 sysctl_reset_sensor, "I", "reset the chip's temperature sensor.");
7981 &sc->swintr, 0, "software triggered interrupts");
8007 sysctl_ddp_stats, "A", "non-TCP DDP statistics");
8014 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, -1,
8027 sysctl_l2t, "A", "hardware L2 table");
8031 sysctl_smt, "A", "hardware source MAC table");
8036 sysctl_clip, "A", "active CLIP table entries");
8113 CTLFLAG_RW, &sc->tlst.inline_keys, 0, "Always pass TLS "
8119 CTLFLAG_RW, &sc->tlst.combo_wrs, 0, "Attempt to "
8124 CTLFLAG_RW, &sc->tlst.short_records, 0,
8125 "Use cipher-only mode for short records.");
8127 CTLFLAG_RW, &sc->tlst.partial_ghash, 0,
8128 "Use partial GHASH for AES-GCM records.");
8145 sc->tt.cong_algorithm = -1;
8147 CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control "
8148 "(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, "
8151 sc->tt.sndbuf = -1;
8153 &sc->tt.sndbuf, 0, "hardware send buffer");
8155 sc->tt.ddp = 0;
8157 CTLFLAG_RW | CTLFLAG_SKIP, &sc->tt.ddp, 0, "");
8159 &sc->tt.ddp, 0, "Enable zero-copy aio_read(2)");
8161 sc->tt.rx_coalesce = -1;
8163 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
8165 sc->tt.tls = 1;
8170 sc->tt.tx_align = -1;
8172 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
8174 sc->tt.tx_zcopy = 0;
8176 CTLFLAG_RW, &sc->tt.tx_zcopy, 0,
8177 "Enable zero-copy aio_write(2)");
8179 sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading;
8182 &sc->tt.cop_managed_offloading, 0,
8185 sc->tt.autorcvbuf_inc = 16 * 1024;
8187 CTLFLAG_RW, &sc->tt.autorcvbuf_inc, 0,
8190 sc->tt.update_hc_on_pmtu_change = 1;
8193 &sc->tt.update_hc_on_pmtu_change, 0,
8196 sc->tt.iso = 1;
8198 &sc->tt.iso, 0, "Enable iSCSI segmentation offload");
8288 struct sysctl_ctx_list *ctx = &vi->ctx;
8295 oid = device_get_sysctl_tree(vi->dev);
8299 vi->viid, "VI identifer");
8301 &vi->nrxq, 0, "# of rx queues");
8303 &vi->ntxq, 0, "# of tx queues");
8305 &vi->first_rxq, 0, "index of first rx queue");
8307 &vi->first_txq, 0, "index of first tx queue");
8309 vi->rss_base, "start of RSS indirection table");
8311 vi->rss_size, "size of RSS indirection table");
8317 "Reserve queue 0 for non-flowid packets");
8320 if (vi->adapter->flags & IS_VF) {
8321 MPASS(vi->flags & TX_USES_VM_WR);
8331 if (vi->nofldrxq != 0) {
8333 &vi->nofldrxq, 0,
8336 CTLFLAG_RD, &vi->first_ofld_rxq, 0,
8349 if (vi->nofldtxq != 0) {
8351 &vi->nofldtxq, 0,
8354 CTLFLAG_RD, &vi->first_ofld_txq, 0,
8359 if (vi->nnmrxq != 0) {
8361 &vi->nnmrxq, 0, "# of netmap rx queues");
8363 &vi->nnmtxq, 0, "# of netmap tx queues");
8365 CTLFLAG_RD, &vi->first_nm_rxq, 0,
8368 CTLFLAG_RD, &vi->first_nm_txq, 0,
8391 struct sysctl_ctx_list *ctx = &pi->ctx;
8394 struct adapter *sc = pi->adapter;
8402 oid = device_get_sysctl_tree(pi->dev);
8408 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
8409 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
8411 sysctl_btphy, "I", "PHY temperature (in Celsius)");
8434 "autonegotiation (-1 = not supported)");
8440 &pi->link_cfg.requested_caps, 0, "L1 config requested by driver");
8442 &pi->link_cfg.pcaps, 0, "port capabilities");
8444 &pi->link_cfg.acaps, 0, "advertised capabilities");
8446 &pi->link_cfg.lpacaps, 0, "link partner advertised capabilities");
8451 pi->mps_bg_map, "MPS buffer group map");
8453 NULL, pi->rx_e_chan_map, "TP rx e-channel map");
8455 pi->tx_chan, "TP tx c-channel");
8457 pi->rx_chan, "TP rx c-channel");
8459 if (sc->flags & IS_VF)
8470 CTLFLAG_RW, &pi->sched_params->pktsize, 0,
8471 "pktsize for per-flow cl-rl (0 means up to the driver )");
8473 CTLFLAG_RW, &pi->sched_params->burstsize, 0,
8474 "burstsize for per-flow cl-rl (0 means up to the driver)");
8475 for (i = 0; i < sc->params.nsched_cls; i++) {
8476 struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i];
8483 CTLFLAG_RD, &tc->state, 0, "current state");
8486 (uintptr_t)&tc->flags, sysctl_bitfield_8b, "A", "flags");
8488 CTLFLAG_RD, &tc->refcount, 0, "references to this class");
8491 (pi->port_id << 16) | i, sysctl_tc_params, "A",
8502 &pi->tx_parse_error, 0,
8506 if (sc->params.tp.lb_mode) { \
8514 t4_port_reg(sc, pi->tx_chan, A_MPS_PORT_STAT_##stat##_L), \
8553 CTLFLAG_RD, &pi->stats.rx_fcs_err,
8586 if (pi->mps_bg_map & 1) {
8588 "# drops due to buffer-group 0 overflows");
8590 "# of buffer-group 0 truncated packets");
8592 if (pi->mps_bg_map & 2) {
8594 "# drops due to buffer-group 1 overflows");
8596 "# of buffer-group 1 truncated packets");
8598 if (pi->mps_bg_map & 4) {
8600 "# drops due to buffer-group 2 overflows");
8602 "# of buffer-group 2 truncated packets");
8604 if (pi->mps_bg_map & 8) {
8606 "# drops due to buffer-group 3 overflows");
8608 "# of buffer-group 3 truncated packets");
8620 for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
8670 struct adapter *sc = pi->adapter;
8674 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
8681 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e,
8700 val = vi->rsrv_noflowq;
8702 if (rc != 0 || req->newptr == NULL)
8705 if ((val >= 1) && (vi->ntxq > 1))
8706 vi->rsrv_noflowq = 1;
8708 vi->rsrv_noflowq = 0;
8717 struct adapter *sc = vi->adapter;
8720 MPASS(!(sc->flags & IS_VF));
8722 val = vi->flags & TX_USES_VM_WR ? 1 : 0;
8724 if (rc != 0 || req->newptr == NULL)
8736 else if (if_getdrvflags(vi->ifp) & IFF_DRV_RUNNING) {
8744 struct port_info *pi = vi->pi;
8747 uint8_t npkt = sc->params.max_pkts_per_eth_tx_pkts_wr;
8750 vi->flags |= TX_USES_VM_WR;
8751 if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_VM_TSO);
8753 V_TXPKT_INTF(pi->hw_port));
8754 if (!(sc->flags & IS_VF))
8755 npkt--;
8757 vi->flags &= ~TX_USES_VM_WR;
8758 if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_TSO);
8760 V_TXPKT_INTF(pi->hw_port) | V_TXPKT_PF(sc->pf) |
8761 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld));
8764 txq->cpl_ctrl0 = ctrl0;
8765 txq->txp.max_npkt = npkt;
8776 struct adapter *sc = vi->adapter;
8781 idx = vi->tmr_idx;
8784 if (rc != 0 || req->newptr == NULL)
8795 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
8798 atomic_store_rel_8(&rxq->iq.intr_params, v);
8800 rxq->iq.intr_params = v;
8803 vi->tmr_idx = idx;
8813 struct adapter *sc = vi->adapter;
8816 idx = vi->pktc_idx;
8819 if (rc != 0 || req->newptr == NULL)
8822 if (idx < -1 || idx >= SGE_NCOUNTERS)
8830 if (vi->flags & VI_INIT_DONE)
8833 vi->pktc_idx = idx;
8843 struct adapter *sc = vi->adapter;
8846 qsize = vi->qsize_rxq;
8849 if (rc != 0 || req->newptr == NULL)
8860 if (vi->flags & VI_INIT_DONE)
8863 vi->qsize_rxq = qsize;
8873 struct adapter *sc = vi->adapter;
8876 qsize = vi->qsize_txq;
8879 if (rc != 0 || req->newptr == NULL)
8890 if (vi->flags & VI_INIT_DONE)
8893 vi->qsize_txq = qsize;
8903 struct adapter *sc = pi->adapter;
8904 struct link_config *lc = &pi->link_cfg;
8907 if (req->newptr == NULL) {
8915 if (lc->link_ok) {
8916 sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX | PAUSE_RX)) |
8917 (lc->requested_fc & PAUSE_AUTONEG), bits);
8919 sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX |
8928 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX |
8940 n = s[0] - '0';
8944 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
8950 lc->requested_fc = n;
8952 if (pi->up_vis > 0)
8967 struct link_config *lc = &pi->link_cfg;
8974 if (lc->link_ok)
8975 sbuf_printf(sb, "%b", lc->fec, t4_fec_bits);
8988 struct adapter *sc = pi->adapter;
8989 struct link_config *lc = &pi->link_cfg;
8993 if (req->newptr == NULL) {
9000 sbuf_printf(sb, "%b", lc->requested_fec, t4_fec_bits);
9008 lc->requested_fec == FEC_AUTO ? -1 :
9009 lc->requested_fec & (M_FW_PORT_CAP32_FEC | FEC_MODULE));
9021 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
9026 old = lc->requested_fec;
9028 lc->requested_fec = FEC_AUTO;
9030 lc->requested_fec = FEC_NONE;
9032 if ((lc->pcaps |
9034 lc->pcaps) {
9038 lc->requested_fec = n & (M_FW_PORT_CAP32_FEC |
9043 if (pi->up_vis > 0) {
9046 lc->requested_fec = old;
9064 struct adapter *sc = pi->adapter;
9065 struct link_config *lc = &pi->link_cfg;
9083 if (pi->up_vis == 0) {
9093 fec = lc->fec_hint;
9094 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE ||
9095 !fec_supported(lc->pcaps)) {
9116 struct adapter *sc = pi->adapter;
9117 struct link_config *lc = &pi->link_cfg;
9120 if (lc->pcaps & FW_PORT_CAP32_ANEG)
9121 val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1;
9123 val = -1;
9125 if (rc != 0 || req->newptr == NULL)
9134 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
9139 if (val == AUTONEG_ENABLE && !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
9143 lc->requested_aneg = val;
9146 if (pi->up_vis > 0)
9160 struct adapter *sc = pi->adapter;
9161 struct link_config *lc = &pi->link_cfg;
9164 val = lc->force_fec;
9165 MPASS(val >= -1 && val <= 1);
9167 if (rc != 0 || req->newptr == NULL)
9169 if (!(lc->pcaps & FW_PORT_CAP32_FORCE_FEC))
9171 if (val < -1 || val > 1)
9174 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4ff");
9178 lc->force_fec = val;
9181 if (pi->up_vis > 0)
9196 mtx_lock(&sc->reg_lock);
9203 mtx_unlock(&sc->reg_lock);
9213 struct adapter *sc = pi->adapter;
9217 mtx_lock(&sc->reg_lock);
9222 for (i = 0; i < sc->params.tp.lb_nchan; i++) {
9224 t4_port_reg(sc, pi->tx_chan + i, reg));
9228 mtx_unlock(&sc->reg_lock);
9250 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
9256 /* unknown is returned as 0 but we display -1 in that case */
9257 t = val == 0 ? -1 : val;
9270 if (sc->params.core_vdd == 0) {
9281 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1,
9287 sc->params.core_vdd = val;
9290 return (sysctl_handle_int(oidp, &sc->params.core_vdd, 0, req));
9300 v = sc->sensor_resets;
9302 if (rc != 0 || req->newptr == NULL || v <= 0)
9305 if (sc->params.fw_vers < FW_VERSION32(1, 24, 7, 0) ||
9319 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
9323 sc->sensor_resets++;
9336 KASSERT(coreid < sc->params.ncores,
9348 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
9388 mtx_lock(&sc->reg_lock);
9393 mtx_unlock(&sc->reg_lock);
9404 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
9424 KASSERT(qid >= 0 && qid < sc->chip_params->cim_num_ibq,
9426 KASSERT(coreid >= 0 && coreid < sc->params.ncores,
9431 mtx_lock(&sc->reg_lock);
9433 rc = -ENXIO;
9436 mtx_unlock(&sc->reg_lock);
9438 rc = -rc;
9469 KASSERT(qid >= 0 && qid < sc->chip_params->cim_num_obq,
9471 KASSERT(coreid >= 0 && coreid < sc->params.ncores,
9476 mtx_lock(&sc->reg_lock);
9478 rc = -ENXIO;
9481 mtx_unlock(&sc->reg_lock);
9483 rc = -rc;
9516 for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) {
9546 for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) {
9576 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
9581 mtx_lock(&sc->reg_lock);
9585 rc = -t4_cim_read_core(sc, 1, coreid, A_UP_UP_DBG_LA_CFG, 1,
9588 rc = -t4_cim_read_la_core(sc, coreid, buf, NULL);
9590 mtx_unlock(&sc->reg_lock);
9624 device_get_nameunit(sc->dev),
9631 device_get_nameunit(sc->dev),
9647 device_get_nameunit(sc->dev));
9655 device_get_nameunit(sc->dev), sbuf_data(&sb));
9664 atomic_set_int(&sc->error_flags, ADAP_CIM_ERR);
9684 mtx_lock(&sc->reg_lock);
9689 mtx_unlock(&sc->reg_lock);
9732 mtx_lock(&sc->reg_lock);
9737 mtx_unlock(&sc->reg_lock);
9775 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */
9776 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
9777 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */
9782 cim_num_obq = sc->chip_params->cim_num_obq;
9792 mtx_lock(&sc->reg_lock);
9796 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
9798 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq,
9804 mtx_unlock(&sc->reg_lock);
9823 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
9846 "TP0", "TP1", "TP2", "TP3", "ULP", "SGE0", "SGE1", "NC-SI",
9850 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", "SGE0-RX",
9859 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "RSVD", "SGE0-RX",
9866 mtx_lock(&sc->reg_lock);
9870 rc = -t4_cim_read_core(sc, 1, coreid,
9875 rc = -t4_cim_read_core(sc, 1, coreid,
9883 rc = -t4_cim_read_core(sc, 1, coreid, addr, 1,
9891 mtx_unlock(&sc->reg_lock);
9918 coreid == 0 ? qname_obq_t7[i - CIM_NUM_IBQ_T7] :
9919 qname_obq_sec_t7[i - CIM_NUM_IBQ_T7],
9943 mtx_lock(&sc->reg_lock);
9948 mtx_unlock(&sc->reg_lock);
9952 if (sc->chip_params->nchan > 2) {
9986 mtx_lock(&sc->reg_lock);
9991 mtx_unlock(&sc->reg_lock);
10016 mtx_lock(&sc->reg_lock);
10021 mtx_unlock(&sc->reg_lock);
10075 struct devlog_params *dparams = &sc->params.devlog;
10080 KASSERT(coreid >= 0 && coreid < sc->params.ncores,
10083 if (dparams->addr == 0)
10086 size = dparams->size / sc->params.ncores;
10087 addr = dparams->addr + coreid * size;
10094 mtx_lock(&sc->reg_lock);
10099 mtx_unlock(&sc->reg_lock);
10107 if (e->timestamp == 0)
10110 e->timestamp = be64toh(e->timestamp);
10111 e->seqno = be32toh(e->seqno);
10113 e->params[j] = be32toh(e->params[j]);
10115 if (e->timestamp < ftstamp) {
10116 ftstamp = e->timestamp;
10130 if (e->timestamp == 0)
10134 e->seqno, e->timestamp,
10135 (e->level < nitems(devlog_level_strings) ?
10136 devlog_level_strings[e->level] : "UNKNOWN"),
10137 (e->facility < nitems(devlog_facility_strings) ?
10138 devlog_facility_strings[e->facility] : "UNKNOWN"));
10139 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
10140 e->params[2], e->params[3], e->params[4],
10141 e->params[5], e->params[6], e->params[7]);
10161 if (coreid == -1) {
10162 /* -1 means all cores */
10163 for (i = rc = 0; i < sc->params.ncores && rc == 0; i++) {
10164 if (sc->params.ncores > 0)
10169 KASSERT(coreid >= 0 && coreid < sc->params.ncores,
10187 device_get_nameunit(sc->dev));
10190 for (i = rc = 0; i < sc->params.ncores && rc == 0; i++) {
10191 if (sc->params.ncores > 0)
10198 device_get_nameunit(sc->dev), sbuf_data(&sb));
10210 int i, nchan = sc->chip_params->nchan;
10213 mtx_lock(&sc->reg_lock);
10220 mtx_unlock(&sc->reg_lock);
10269 mtx_lock(&sc->reg_lock);
10271 mtx_unlock(&sc->reg_lock);
10279 mtx_unlock(&sc->reg_lock);
10286 sbuf_printf(sb, "\n %u %-5s %u ", i,
10334 for (i = 0; i < sc->chip_params->nchan; i += 2) {
10335 mtx_lock(&sc->reg_lock);
10342 mtx_unlock(&sc->reg_lock);
10352 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
10368 struct link_config *lc = &pi->link_cfg;
10375 if (lc->link_ok || lc->link_down_rc == 255)
10378 sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc));
10395 const uint64_t v1 = ((const struct mem_desc *)a)->base;
10396 const uint64_t v2 = ((const struct mem_desc *)b)->base;
10399 return (-1);
10414 size = to - from + 1;
10419 sbuf_printf(sb, "%-18s 0x%012jx-0x%012jx [%ju]\n", name,
10422 sbuf_printf(sb, "%-18s 0x%08jx-0x%08jx [%ju]\n", name,
10445 "ULPTX state:", "RoCE RRQ region:", "On-chip queues:",
10464 mtx_lock(&sc->reg_lock);
10569 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
10570 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
10571 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
10572 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
10573 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
10574 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
10575 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
10576 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
10577 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
10580 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
10581 md->limit = md->base - 1 +
10586 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
10587 md->limit = md->base - 1 +
10594 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
10596 md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR);
10597 md->limit = 0;
10599 md->base = 0;
10600 md->idx = nitems(region); /* hide it */
10606 md->base = (uint64_t)t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT) << shift; \
10607 md->limit = (uint64_t)t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) << shift; \
10608 md->limit += (1 << shift) - 1; \
10613 md->base = 0; \
10614 md->idx = nitems(region); \
10651 md->base = 0;
10653 md->idx = nitems(region);
10666 md->base = t4_read_reg(sc, A_SGE_DBVFIFO_BADDR);
10667 md->limit = md->base + size - 1;
10669 md->idx = nitems(region);
10673 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
10674 md->limit = 0;
10676 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
10677 md->limit = 0;
10682 md->base = lo;
10684 md->base = 0;
10685 md->idx = nitems(region);
10689 md->base = sc->vres.ocq.start;
10690 if (sc->vres.ocq.size)
10691 md->limit = md->base + sc->vres.ocq.size - 1;
10693 md->idx = nitems(region); /* hide it */
10696 /* add any address-space holes, there can be up to 3 */
10697 for (n = 0; n < i - 1; n++)
10699 (md++)->base = avail[n].limit;
10701 (md++)->base = avail[n].limit;
10703 n = md - mem;
10709 avail[lo].limit - 1);
10716 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
10722 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
10723 if (hi != lo - 1) {
10729 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
10730 if (hi != lo - 1)
10756 sbuf_printf(sb, "%u p-structs (%u free)\n",
10776 for (i = 0; i < sc->chip_params->nchan; i++) {
10794 mtx_unlock(&sc->reg_lock);
10826 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
10831 mtx_lock(&sc->reg_lock);
10838 mtx_unlock(&sc->reg_lock);
10844 mtx_lock(&sc->reg_lock);
10851 mtx_unlock(&sc->reg_lock);
10859 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
10881 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
10924 " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF"
10929 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
10940 ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1);
10941 mtx_lock(&sc->reg_lock);
10951 mtx_unlock(&sc->reg_lock);
10971 mtx_lock(&sc->reg_lock);
10981 mtx_unlock(&sc->reg_lock);
10996 mtx_lock(&sc->reg_lock);
11003 mtx_unlock(&sc->reg_lock);
11009 "%012jx %06x %06x - - %3c"
11015 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
11018 "%012jx - - ", i, addr[0], addr[1],
11025 sbuf_printf(sb, " - N ");
11027 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d",
11031 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
11055 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
11104 " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF"
11109 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
11122 ctl |= V_CTLTCAMINDEX(i - 256) | V_T7_CTLTCAMSEL(1);
11130 ctl |= V_CTLTCAMINDEX(i - 512) | V_T7_CTLTCAMSEL(1);
11132 ctl |= V_CTLTCAMINDEX(i - 1024) | V_T7_CTLTCAMSEL(2);
11135 mtx_lock(&sc->reg_lock);
11145 mtx_unlock(&sc->reg_lock);
11165 mtx_lock(&sc->reg_lock);
11175 mtx_unlock(&sc->reg_lock);
11190 mtx_lock(&sc->reg_lock);
11204 mtx_unlock(&sc->reg_lock);
11210 "%012jx %06x %06x - - %3c"
11216 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
11219 "%012jx - - ", i, addr[0], addr[1],
11226 sbuf_printf(sb, " - N ");
11228 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d",
11232 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
11255 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
11299 mtx_lock(&sc->reg_lock);
11304 mtx_unlock(&sc->reg_lock);
11342 mtx_lock(&sc->reg_lock);
11351 mtx_unlock(&sc->reg_lock);
11361 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
11367 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
11374 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
11376 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
11384 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
11386 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
11393 sbuf_printf(sb, "%-40s %u\n", "ReqWrite", stats[i++]);
11394 sbuf_printf(sb, "%-40s %u\n", "ReqReadInv", stats[i++]);
11395 sbuf_printf(sb, "%-40s %u\n", "ReqReadNoInv", stats[i++]);
11396 sbuf_printf(sb, "%-40s %u\n", "Write Split Request",
11398 sbuf_printf(sb, "%-40s %u\n",
11400 sbuf_printf(sb, "%-40s %u\n",
11403 sbuf_printf(sb, "%-40s %u\n", "Write Hit", stats[i++]);
11404 sbuf_printf(sb, "%-40s %u\n", "Normal Read Hit",
11406 sbuf_printf(sb, "%-40s %u\n", "Feedback Read Hit",
11408 sbuf_printf(sb, "%-40s %u\n", "Normal Read Hit Full Avail",
11410 sbuf_printf(sb, "%-40s %u\n", "Normal Read Hit Full UnAvail",
11412 sbuf_printf(sb, "%-40s %u\n",
11415 sbuf_printf(sb, "%-40s %u\n", "FB Read Hit Full Avail",
11417 sbuf_printf(sb, "%-40s %u\n", "FB Read Hit Full UnAvail",
11419 sbuf_printf(sb, "%-40s %u\n", "FB Read Hit Partial Avail",
11421 sbuf_printf(sb, "%-40s %u\n", "Normal Read Full Free",
11423 sbuf_printf(sb, "%-40s %u\n",
11424 "Normal Read Part-avail Mul-Regions",
11426 sbuf_printf(sb, "%-40s %u\n",
11427 "FB Read Part-avail Mul-Regions",
11429 sbuf_printf(sb, "%-40s %u\n", "Write Miss FL Used",
11431 sbuf_printf(sb, "%-40s %u\n", "Write Miss LRU Used",
11433 sbuf_printf(sb, "%-40s %u\n",
11434 "Write Miss LRU-Multiple Evict", stats[i++]);
11435 sbuf_printf(sb, "%-40s %u\n",
11437 sbuf_printf(sb, "%-40s %u\n",
11439 sbuf_printf(sb, "%-40s %u\n", "Write Overflow Eviction",
11441 sbuf_printf(sb, "%-40s %u", "Read Overflow Eviction",
11460 mtx_lock(&sc->reg_lock);
11465 mtx_unlock(&sc->reg_lock);
11491 mtx_lock(&sc->reg_lock);
11496 mtx_unlock(&sc->reg_lock);
11528 struct tid_info *t = &sc->tids;
11535 if (t->natids) {
11536 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
11537 t->atids_in_use);
11540 if (t->nhpftids) {
11541 sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n",
11542 t->hpftid_base, t->hpftid_end, t->hpftids_in_use);
11545 if (t->ntids) {
11548 mtx_lock(&sc->reg_lock);
11561 mtx_unlock(&sc->reg_lock);
11568 sbuf_printf(sb, "%u-%u, ", t->tid_base, x - 1);
11569 sbuf_printf(sb, "%u-%u", y, t->ntids - 1);
11571 sbuf_printf(sb, "%u-%u", t->tid_base, t->tid_base +
11572 t->ntids - 1);
11575 atomic_load_acq_int(&t->tids_in_use));
11578 if (t->nstids) {
11579 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
11580 t->stid_base + t->nstids - 1, t->stids_in_use);
11583 if (t->nftids) {
11584 sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base,
11585 t->ftid_end, t->ftids_in_use);
11588 if (t->netids) {
11589 sbuf_printf(sb, "ETID range: %u-%u, in use: %u\n", t->etid_base,
11590 t->etid_base + t->netids - 1, t->etids_in_use);
11593 mtx_lock(&sc->reg_lock);
11600 mtx_unlock(&sc->reg_lock);
11623 mtx_lock(&sc->reg_lock);
11628 mtx_unlock(&sc->reg_lock);
11636 if (sc->chip_params->nchan > 2) {
11701 mtx_lock(&sc->reg_lock);
11706 mtx_unlock(&sc->reg_lock);
11714 if (sc->chip_params->nchan > 2) {
11741 struct tp_params *tpp = &sc->params.tp;
11745 mask = tpp->la_mask >> 16;
11747 if (rc != 0 || req->newptr == NULL)
11751 mtx_lock(&sc->reg_lock);
11755 tpp->la_mask = mask << 16;
11757 tpp->la_mask);
11759 mtx_unlock(&sc->reg_lock);
11776 while (f->name) {
11777 uint64_t mask = (1ULL << f->width) - 1;
11778 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
11779 ((uintmax_t)v >> f->start) & mask);
11939 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
11950 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
11971 mtx_lock(&sc->reg_lock);
11990 mtx_unlock(&sc->reg_lock);
12013 mtx_lock(&sc->reg_lock);
12018 mtx_unlock(&sc->reg_lock);
12026 if (sc->chip_params->nchan > 2) {
12063 mtx_lock(&sc->reg_lock);
12068 mtx_unlock(&sc->reg_lock);
12097 mtx_lock(&sc->reg_lock);
12105 mtx_unlock(&sc->reg_lock);
12142 rc = bus_get_cpus(sc->dev, op, sizeof(cpuset), &cpuset);
12165 val = atomic_load_int(&sc->num_resets);
12167 if (rc != 0 || req->newptr == NULL)
12172 atomic_store_int(&sc->num_resets, 0);
12182 taskqueue_enqueue(reset_tq, &sc->reset_task);
12194 v = sc->tt.tls;
12196 if (rc != 0 || req->newptr == NULL)
12199 if (v != 0 && !(sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS))
12208 sc->tt.tls = !!v;
12210 for_each_vi(sc->port[i], j, vi) {
12211 if (vi->flags & VI_INIT_DONE)
12212 t4_update_fl_bufsize(vi->ifp);
12242 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
12244 mtx_lock(&sc->reg_lock);
12246 res = (u_int)-1;
12249 mtx_unlock(&sc->reg_lock);
12250 if (res == (u_int)-1)
12281 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
12283 mtx_lock(&sc->reg_lock);
12292 mtx_unlock(&sc->reg_lock);
12308 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
12315 mtx_lock(&sc->reg_lock);
12327 mtx_unlock(&sc->reg_lock);
12347 mtx_lock(&sc->reg_lock);
12354 mtx_unlock(&sc->reg_lock);
12372 mtx_lock(&sc->reg_lock);
12379 mtx_unlock(&sc->reg_lock);
12390 struct adapter *sc = vi->adapter;
12395 idx = vi->ofld_tmr_idx;
12398 if (rc != 0 || req->newptr == NULL)
12409 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->ofld_pktc_idx != -1);
12412 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
12414 ofld_rxq->iq.intr_params = v;
12417 vi->ofld_tmr_idx = idx;
12427 struct adapter *sc = vi->adapter;
12430 idx = vi->ofld_pktc_idx;
12433 if (rc != 0 || req->newptr == NULL)
12436 if (idx < -1 || idx >= SGE_NCOUNTERS)
12444 if (vi->flags & VI_INIT_DONE)
12447 vi->ofld_pktc_idx = idx;
12460 if (len < sc->chip_params->sge_ctxt_size)
12477 if (sc->flags & FW_OK) {
12478 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cid, mem_id, data);
12487 rc = -t4_sge_ctxt_rd_bd(sc, cid, mem_id, data);
12514 if (sc->flags & FULL_INIT_DONE &&
12515 (sc->debug_flags & DF_LOAD_FW_ANYTIME) == 0) {
12520 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
12522 rc = copyin(fw->data, fw_data, fw->len);
12524 rc = -t4_load_fw(sc, fw_data, fw->len);
12547 if (cfg->len == 0) {
12549 rc = -t4_load_cfg(sc, NULL, 0);
12553 cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK);
12555 rc = copyin(cfg->data, cfg_data, cfg->len);
12557 rc = -t4_load_cfg(sc, cfg_data, cfg->len);
12572 if (br->len > 1024 * 1024)
12575 if (br->pf_offset == 0) {
12577 if (br->pfidx_addr > 7)
12579 offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr,
12581 } else if (br->pf_offset == 1) {
12583 offset = G_OFFSET(br->pfidx_addr);
12597 if (br->len == 0) {
12599 rc = -t4_load_boot(sc, NULL, offset, 0);
12603 br_data = malloc(br->len, M_CXGBE, M_WAITOK);
12605 rc = copyin(br->data, br_data, br->len);
12607 rc = -t4_load_boot(sc, br_data, offset, br->len);
12630 if (bc->len == 0) {
12632 rc = -t4_load_bootcfg(sc, NULL, 0);
12636 bc_data = malloc(bc->len, M_CXGBE, M_WAITOK);
12638 rc = copyin(bc->data, bc_data, bc->len);
12640 rc = -t4_load_bootcfg(sc, bc_data, bc->len);
12656 buf = malloc(dump->len, M_CXGBE, M_NOWAIT | M_ZERO);
12667 cudbg->adap = sc;
12668 cudbg->print = (cudbg_print_cb)printf;
12671 device_printf(sc->dev, "%s: wr_flash %u, len %u, data %p.\n",
12672 __func__, dump->wr_flash, dump->len, dump->data);
12675 if (dump->wr_flash)
12676 cudbg->use_flash = 1;
12677 MPASS(sizeof(cudbg->dbg_bitmap) == sizeof(dump->bitmap));
12678 memcpy(cudbg->dbg_bitmap, dump->bitmap, sizeof(cudbg->dbg_bitmap));
12680 rc = cudbg_collect(handle, buf, &dump->len);
12684 rc = copyout(buf, dump->data, dump->len);
12700 r = &op->rule[0];
12701 for (i = 0; i < op->nrules; i++, r++) {
12702 free(r->bpf_prog.bf_insns, M_CXGBE);
12704 free(op->rule, M_CXGBE);
12721 if (uop->nrules == 0) {
12725 } else if (uop->nrules > 256) { /* arbitrary */
12731 op->nrules = uop->nrules;
12732 len = op->nrules * sizeof(struct offload_rule);
12733 op->rule = malloc(len, M_CXGBE, M_ZERO | M_WAITOK);
12734 rc = copyin(uop->rule, op->rule, len);
12736 free(op->rule, M_CXGBE);
12741 r = &op->rule[0];
12742 for (i = 0; i < op->nrules; i++, r++) {
12745 if (r->open_type != OPEN_TYPE_LISTEN &&
12746 r->open_type != OPEN_TYPE_ACTIVE &&
12747 r->open_type != OPEN_TYPE_PASSIVE &&
12748 r->open_type != OPEN_TYPE_DONTCARE) {
12755 op->nrules = i;
12761 s = &r->settings;
12762 if ((s->offload != 0 && s->offload != 1) ||
12763 s->cong_algo < -1 || s->cong_algo > CONG_ALG_HIGHSPEED ||
12764 s->sched_class < -1 ||
12765 s->sched_class >= sc->params.nsched_cls) {
12770 bf = &r->bpf_prog;
12771 u = bf->bf_insns; /* userspace ptr */
12772 bf->bf_insns = NULL;
12773 if (bf->bf_len == 0) {
12777 len = bf->bf_len * sizeof(*bf->bf_insns);
12778 bf->bf_insns = malloc(len, M_CXGBE, M_ZERO | M_WAITOK);
12779 rc = copyin(u, bf->bf_insns, len);
12783 if (!bpf_validate(bf->bf_insns, bf->bf_len)) {
12789 rw_wlock(&sc->policy_lock);
12790 old = sc->policy;
12791 sc->policy = op;
12792 rw_wunlock(&sc->policy_lock);
12807 mtx_lock(&sc->reg_lock);
12811 rc = validate_mem_range(sc, mr->addr, mr->len);
12812 mtx_unlock(&sc->reg_lock);
12816 buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK);
12817 addr = mr->addr;
12818 remaining = mr->len;
12819 dst = (void *)mr->data;
12823 mtx_lock(&sc->reg_lock);
12828 mtx_unlock(&sc->reg_lock);
12837 remaining -= n;
12851 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
12854 if (i2cd->len > sizeof(i2cd->data))
12863 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
12864 i2cd->offset, i2cd->len, &i2cd->data[0]);
12886 if (port_id >= sc->params.nports)
12888 pi = sc->port[port_id];
12892 mtx_lock(&sc->reg_lock);
12895 t4_clr_port_stats(sc, pi->hw_port);
12897 if (pi->fcs_reg != -1)
12898 pi->fcs_base = t4_read_reg64(sc,
12899 t4_port_reg(sc, pi->tx_chan, pi->fcs_reg));
12901 pi->stats.rx_fcs_err = 0;
12904 if (vi->flags & VI_INIT_DONE)
12905 t4_clr_vi_stats(sc, vi->vin);
12907 chan_map = pi->rx_e_chan_map;
12910 i = ffs(chan_map) - 1;
12916 mtx_unlock(&sc->reg_lock);
12917 pi->tx_parse_error = 0;
12918 pi->tnl_cong_drops = 0;
12925 if (vi->flags & VI_INIT_DONE) {
12929 rxq->lro.lro_queued = 0;
12930 rxq->lro.lro_flushed = 0;
12932 rxq->rxcsum = 0;
12933 rxq->vlan_extraction = 0;
12934 rxq->vxlan_rxcsum = 0;
12936 rxq->fl.cl_allocated = 0;
12937 rxq->fl.cl_recycled = 0;
12938 rxq->fl.cl_fast_recycled = 0;
12942 txq->txcsum = 0;
12943 txq->tso_wrs = 0;
12944 txq->vlan_insertion = 0;
12945 txq->imm_wrs = 0;
12946 txq->sgl_wrs = 0;
12947 txq->txpkt_wrs = 0;
12948 txq->txpkts0_wrs = 0;
12949 txq->txpkts1_wrs = 0;
12950 txq->txpkts0_pkts = 0;
12951 txq->txpkts1_pkts = 0;
12952 txq->txpkts_flush = 0;
12953 txq->raw_wrs = 0;
12954 txq->vxlan_tso_wrs = 0;
12955 txq->vxlan_txcsum = 0;
12956 txq->kern_tls_records = 0;
12957 txq->kern_tls_short = 0;
12958 txq->kern_tls_partial = 0;
12959 txq->kern_tls_full = 0;
12960 txq->kern_tls_octets = 0;
12961 txq->kern_tls_waste = 0;
12962 txq->kern_tls_header = 0;
12963 txq->kern_tls_fin_short = 0;
12964 txq->kern_tls_cbc = 0;
12965 txq->kern_tls_gcm = 0;
12967 txq->kern_tls_options = 0;
12968 txq->kern_tls_fin = 0;
12970 txq->kern_tls_ghash_received = 0;
12971 txq->kern_tls_ghash_requested = 0;
12972 txq->kern_tls_lso = 0;
12973 txq->kern_tls_partial_ghash = 0;
12974 txq->kern_tls_splitmode = 0;
12975 txq->kern_tls_trailer = 0;
12977 mp_ring_reset_stats(txq->r);
12982 ofld_txq->wrq.tx_wrs_direct = 0;
12983 ofld_txq->wrq.tx_wrs_copied = 0;
12984 counter_u64_zero(ofld_txq->tx_iscsi_pdus);
12985 counter_u64_zero(ofld_txq->tx_iscsi_octets);
12986 counter_u64_zero(ofld_txq->tx_iscsi_iso_wrs);
12987 counter_u64_zero(ofld_txq->tx_aio_jobs);
12988 counter_u64_zero(ofld_txq->tx_aio_octets);
12989 counter_u64_zero(ofld_txq->tx_toe_tls_records);
12990 counter_u64_zero(ofld_txq->tx_toe_tls_octets);
12995 ofld_rxq->fl.cl_allocated = 0;
12996 ofld_rxq->fl.cl_recycled = 0;
12997 ofld_rxq->fl.cl_fast_recycled = 0;
12999 ofld_rxq->rx_iscsi_ddp_setup_ok);
13001 ofld_rxq->rx_iscsi_ddp_setup_error);
13002 ofld_rxq->rx_iscsi_ddp_pdus = 0;
13003 ofld_rxq->rx_iscsi_ddp_octets = 0;
13004 ofld_rxq->rx_iscsi_fl_pdus = 0;
13005 ofld_rxq->rx_iscsi_fl_octets = 0;
13006 ofld_rxq->rx_aio_ddp_jobs = 0;
13007 ofld_rxq->rx_aio_ddp_octets = 0;
13008 ofld_rxq->rx_toe_tls_records = 0;
13009 ofld_rxq->rx_toe_tls_octets = 0;
13010 ofld_rxq->rx_toe_ddp_octets = 0;
13011 counter_u64_zero(ofld_rxq->ddp_buffer_alloc);
13012 counter_u64_zero(ofld_rxq->ddp_buffer_reuse);
13013 counter_u64_zero(ofld_rxq->ddp_buffer_free);
13018 wrq = &sc->sge.ctrlq[pi->port_id];
13019 wrq->tx_wrs_direct = 0;
13020 wrq->tx_wrs_copied = 0;
13034 bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr));
13050 bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr));
13062 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
13068 struct adapter *sc = pi->adapter;
13076 KASSERT((pi->flags & FIXED_IFMEDIA) == 0,
13077 ("%s: port_type %u", __func__, pi->port_type));
13079 vi = &pi->vi[0];
13083 if (pi->mod_type != FW_PORT_MOD_TYPE_NONE) {
13091 ifp = vi->ifp;
13092 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
13094 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
13096 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
13098 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
13100 port_top_speed(pi), mod_str[pi->mod_type]);
13103 pi->mod_type);
13112 struct link_config *lc = &pi->link_cfg;
13113 struct adapter *sc = pi->adapter;
13119 if (lc->link_ok) {
13120 if (lc->speed > 25000 ||
13121 (lc->speed == 25000 && lc->fec == FEC_RS))
13122 pi->fcs_reg = A_MAC_PORT_AFRAMECHECKSEQUENCEERRORS;
13124 pi->fcs_reg = A_MAC_PORT_MTIP_1G10G_RX_CRCERRORS;
13125 pi->fcs_base = t4_read_reg64(sc,
13126 t4_port_reg(sc, pi->tx_chan, pi->fcs_reg));
13127 pi->stats.rx_fcs_err = 0;
13129 pi->fcs_reg = -1;
13132 MPASS(pi->fcs_reg != -1);
13133 MPASS(pi->fcs_base == 0);
13137 ifp = vi->ifp;
13141 if (lc->link_ok) {
13142 if_setbaudrate(ifp, IF_Mbps(lc->speed));
13159 * in - the only guarantee is that sc->sc_lock is a valid lock.
13171 struct adapter *sc = dev->si_drv1;
13181 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
13184 mtx_lock(&sc->reg_lock);
13187 else if (edata->size == 4)
13188 edata->val = t4_read_reg(sc, edata->addr);
13189 else if (edata->size == 8)
13190 edata->val = t4_read_reg64(sc, edata->addr);
13193 mtx_unlock(&sc->reg_lock);
13200 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
13203 mtx_lock(&sc->reg_lock);
13206 else if (edata->size == 4) {
13207 if (edata->val & 0xffffffff00000000)
13209 t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
13210 } else if (edata->size == 8)
13211 t4_write_reg64(sc, edata->addr, edata->val);
13214 mtx_unlock(&sc->reg_lock);
13223 if (regs->len < reglen) {
13224 regs->len = reglen; /* hint to the caller */
13228 regs->len = reglen;
13230 mtx_lock(&sc->reg_lock);
13235 mtx_unlock(&sc->reg_lock);
13237 rc = copyout(buf, regs->data, reglen);
13262 rc = get_sge_context(sc, ctxt->mem_id, ctxt->cid,
13263 sizeof(ctxt->data), &ctxt->data[0]);
13314 rc = get_sge_context(sc, ctxt->mem_id, ctxt->cid,
13315 sizeof(ctxt->data), &ctxt->data[0]);
13330 struct port_info *pi = vi->pi;
13331 struct adapter *sc = pi->adapter;
13342 if (sc->flags & KERN_TLS_ON && is_t6(sc)) {
13353 p = sc->port[i];
13355 if (if_getcapenable(v->ifp) & IFCAP_TXTLS) {
13358 device_get_nameunit(v->dev));
13374 if ((if_getcapenable(vi->ifp) & IFCAP_TOE) != 0) {
13384 if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0))
13386 if (!(pi->vi[0].flags & VI_INIT_DONE) &&
13387 ((rc = vi_init(&pi->vi[0])) != 0))
13390 if (isset(&sc->offload_map, pi->port_id)) {
13392 MPASS(pi->uld_vis > 0);
13393 pi->uld_vis++;
13406 KASSERT(sc->tom_softc != NULL,
13418 if (pi->uld_vis++ == 0)
13419 setbit(&sc->offload_map, pi->port_id);
13421 if ((if_getcapenable(vi->ifp) & IFCAP_TOE) == 0) {
13425 MPASS(isset(&sc->offload_map, pi->port_id));
13426 MPASS(pi->uld_vis > 0);
13427 if (--pi->uld_vis == 0)
13428 clrbit(&sc->offload_map, pi->port_id);
13479 if (!(sc->flags & FULL_INIT_DONE)) {
13489 rc = t4_uld_list[id]->uld_activate(sc);
13491 setbit(&sc->active_ulds, id);
13512 rc = t4_uld_list[id]->uld_deactivate(sc);
13514 clrbit(&sc->active_ulds, id);
13533 rc = t4_uld_list[i]->uld_deactivate(sc);
13536 clrbit(&sc->active_ulds, i);
13554 t4_uld_list[i]->uld_stop == NULL)
13556 (void) t4_uld_list[i]->uld_stop(sc);
13572 t4_uld_list[i]->uld_restart == NULL)
13574 (void) t4_uld_list[i]->uld_restart(sc);
13586 return (isset(&sc->active_ulds, id));
13604 if (sc->flags & KERN_TLS_ON)
13606 if (sc->offload_map != 0) {
13635 nq = *t < 0 ? -*t : c;
13678 if (t4_toecaps_allowed == -1)
13681 if (t4_toecaps_allowed == -1)
13686 if (t4_rdmacaps_allowed == -1) {
13691 if (t4_iscsicaps_allowed == -1) {
13700 if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS)
13703 if (t4_rdmacaps_allowed == -1)
13706 if (t4_iscsicaps_allowed == -1)
13720 if (t4_pktc_idx < -1 || t4_pktc_idx >= SGE_NCOUNTERS)
13734 * Number of VIs to create per-port. The first VI is the "main" regular
13768 base = sc->memwin[2].mw_base;
13774 pf = V_PFNUM(sc->pf);
13777 off = addr - win_pos;
13794 len -= sizeof(buf);
13815 struct devlog_params *dparams = &sc->params.devlog;
13820 if (dparams->start == 0) {
13825 nentries = dparams->size / sizeof(struct fw_devlog_e);
13826 m = fwmtype_to_hwmtype(dparams->memtype);
13829 first = -1;
13831 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
13846 if (first == -1)
13851 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
14007 t4_write_reg(sc, A_MPS_RX_VXLAN_TYPE, V_VXLAN(sc->vxlan_port) |
14010 pi = sc->port[i];
14011 if (pi->vxlan_tcam_entry == true)
14013 rc = t4_alloc_raw_mac_filt(sc, pi->vi[0].viid, match_all_mac,
14014 match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id,
14017 rc = -rc;
14018 CH_ERR(&pi->vi[0],
14021 MPASS(rc == sc->rawf_base + pi->port_id);
14022 pi->vxlan_tcam_entry = true;
14032 if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5)
14037 if (sc->vxlan_refcount == 0) {
14038 sc->vxlan_port = v->port;
14039 sc->vxlan_refcount = 1;
14042 } else if (sc->vxlan_port == v->port) {
14043 sc->vxlan_refcount++;
14047 sc->vxlan_port, v->port);
14057 if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5)
14067 if (sc->vxlan_port != v->port)
14069 if (sc->vxlan_refcount == 0) {
14071 "ignoring attempt to stop it again.\n", sc->vxlan_port);
14072 } else if (--sc->vxlan_refcount == 0 && !hw_off_limits(sc))
14166 if (--loaded == 0) {