Lines Matching +full:tcam +full:- +full:based
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright © 2021-2023 Dmitry Salychev
35 * high-functioning network interface. The DPNI supports features that are
112 mtx_assert(&(__sc)->lock, MA_NOTOWNED); \
113 mtx_lock(&(__sc)->lock); \
116 mtx_assert(&(__sc)->lock, MA_OWNED); \
117 mtx_unlock(&(__sc)->lock); \
120 mtx_assert(&(__sc)->lock, MA_OWNED); \
124 (&(sc)->channels[(chan)]->txc_queue.tx_rings[(tc)])
126 MALLOC_DEFINE(M_DPAA2_TXB, "dpaa2_txb", "DPAA2 DMA-mapped buffer (Tx)");
142 #define DPAA2_ETH_MFL (ETHER_MAX_LEN_JUMBO + ETHER_VLAN_ENCAP_LEN - \
179 #define DPAA2_NI_BUF_ADDR_MASK (0x1FFFFFFFFFFFFul) /* 49-bit addresses max. */
198 /* Enables TCAM for Flow Steering and QoS look-ups. */
213 /* L3-L4 network traffic flow hash options. */
500 sc->dev = dev; in dpaa2_ni_attach()
501 sc->ifp = NULL; in dpaa2_ni_attach()
502 sc->miibus = NULL; in dpaa2_ni_attach()
503 sc->mii = NULL; in dpaa2_ni_attach()
504 sc->media_status = 0; in dpaa2_ni_attach()
505 sc->if_flags = 0; in dpaa2_ni_attach()
506 sc->link_state = LINK_STATE_UNKNOWN; in dpaa2_ni_attach()
507 sc->buf_align = 0; in dpaa2_ni_attach()
510 sc->rx_anomaly_frames = 0; in dpaa2_ni_attach()
511 sc->rx_single_buf_frames = 0; in dpaa2_ni_attach()
512 sc->rx_sg_buf_frames = 0; in dpaa2_ni_attach()
513 sc->rx_enq_rej_frames = 0; in dpaa2_ni_attach()
514 sc->rx_ieoi_err_frames = 0; in dpaa2_ni_attach()
515 sc->tx_single_buf_frames = 0; in dpaa2_ni_attach()
516 sc->tx_sg_frames = 0; in dpaa2_ni_attach()
518 DPAA2_ATOMIC_XCHG(&sc->buf_num, 0); in dpaa2_ni_attach()
519 DPAA2_ATOMIC_XCHG(&sc->buf_free, 0); in dpaa2_ni_attach()
521 sc->rxd_dmat = NULL; in dpaa2_ni_attach()
522 sc->qos_dmat = NULL; in dpaa2_ni_attach()
524 sc->qos_kcfg.dmap = NULL; in dpaa2_ni_attach()
525 sc->qos_kcfg.paddr = 0; in dpaa2_ni_attach()
526 sc->qos_kcfg.vaddr = NULL; in dpaa2_ni_attach()
528 sc->rxd_kcfg.dmap = NULL; in dpaa2_ni_attach()
529 sc->rxd_kcfg.paddr = 0; in dpaa2_ni_attach()
530 sc->rxd_kcfg.vaddr = NULL; in dpaa2_ni_attach()
532 sc->mac.dpmac_id = 0; in dpaa2_ni_attach()
533 sc->mac.phy_dev = NULL; in dpaa2_ni_attach()
534 memset(sc->mac.addr, 0, ETHER_ADDR_LEN); in dpaa2_ni_attach()
536 error = bus_alloc_resources(sc->dev, dpaa2_ni_spec, sc->res); in dpaa2_ni_attach()
544 mcp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_MCP_RID(0)]); in dpaa2_ni_attach()
546 dinfo->portal = mcp_dinfo->portal; in dpaa2_ni_attach()
548 mtx_init(&sc->lock, device_get_nameunit(dev), "dpaa2_ni", MTX_DEF); in dpaa2_ni_attach()
552 sc->ifp = ifp; in dpaa2_ni_attach()
553 if_initname(ifp, DPAA2_NI_IFNAME, device_get_unit(sc->dev)); in dpaa2_ni_attach()
568 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_attach()
571 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_attach()
574 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_attach()
577 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_attach()
585 * XXX-DSL: Release new buffers on Buffer Pool State Change Notification in dpaa2_ni_attach()
590 sc->bp_taskq = taskqueue_create(tq_name, M_WAITOK, in dpaa2_ni_attach()
591 taskqueue_thread_enqueue, &sc->bp_taskq); in dpaa2_ni_attach()
592 taskqueue_start_threads(&sc->bp_taskq, 1, PI_NET, "%s", tq_name); in dpaa2_ni_attach()
594 /* sc->cleanup_taskq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK, */ in dpaa2_ni_attach()
595 /* taskqueue_thread_enqueue, &sc->cleanup_taskq); */ in dpaa2_ni_attach()
596 /* taskqueue_start_threads(&sc->cleanup_taskq, 1, PI_NET, */ in dpaa2_ni_attach()
631 ether_ifattach(sc->ifp, sc->mac.addr); in dpaa2_ni_attach()
632 callout_init(&sc->mii_callout, 0); in dpaa2_ni_attach()
650 ifmr->ifm_count = 0; in dpaa2_ni_fixed_media_status()
651 ifmr->ifm_mask = 0; in dpaa2_ni_fixed_media_status()
652 ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; in dpaa2_ni_fixed_media_status()
653 ifmr->ifm_current = ifmr->ifm_active = in dpaa2_ni_fixed_media_status()
654 sc->fixed_ifmedia.ifm_cur->ifm_media; in dpaa2_ni_fixed_media_status()
657 * In non-PHY usecases, we need to signal link state up, otherwise in dpaa2_ni_fixed_media_status()
683 sc->fixed_link = true; in dpaa2_ni_setup_fixed_link()
685 ifmedia_init(&sc->fixed_ifmedia, 0, dpaa2_ni_media_change, in dpaa2_ni_setup_fixed_link()
687 ifmedia_add(&sc->fixed_ifmedia, IFM_ETHER | IFM_1000_T, 0, NULL); in dpaa2_ni_setup_fixed_link()
688 ifmedia_set(&sc->fixed_ifmedia, IFM_ETHER | IFM_1000_T); in dpaa2_ni_setup_fixed_link()
720 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_setup()
723 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_setup()
726 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_setup()
729 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_setup()
734 error = DPAA2_CMD_NI_GET_API_VERSION(dev, child, &cmd, &sc->api_major, in dpaa2_ni_setup()
735 &sc->api_minor); in dpaa2_ni_setup()
743 "need >= %u.%u\n", __func__, sc->api_major, sc->api_minor, in dpaa2_ni_setup()
753 __func__, dinfo->id); in dpaa2_ni_setup()
758 error = DPAA2_CMD_NI_GET_ATTRIBUTES(dev, child, &cmd, &sc->attr); in dpaa2_ni_setup()
761 "id=%d\n", __func__, dinfo->id); in dpaa2_ni_setup()
766 "wriop_version=%#x\n", sc->attr.options, sc->attr.num.queues, in dpaa2_ni_setup()
767 sc->attr.num.channels, sc->attr.wriop_ver); in dpaa2_ni_setup()
769 "cgs_groups=%d\n", sc->attr.num.rx_tcs, sc->attr.num.tx_tcs, in dpaa2_ni_setup()
770 sc->attr.num.cgs); in dpaa2_ni_setup()
772 "fs=%d\n", sc->attr.entries.mac, sc->attr.entries.vlan, in dpaa2_ni_setup()
773 sc->attr.entries.qos, sc->attr.entries.fs); in dpaa2_ni_setup()
775 sc->attr.key_size.qos, sc->attr.key_size.fs); in dpaa2_ni_setup()
794 ep1_desc.obj_id = dinfo->id; in dpaa2_ni_setup()
796 ep1_desc.type = dinfo->dtype; in dpaa2_ni_setup()
818 sc->mac.dpmac_id = ep2_desc.obj_id; in dpaa2_ni_setup()
827 error = DPAA2_CMD_MAC_OPEN(sc->dev, child, in dpaa2_ni_setup()
828 DPAA2_CMD_TK(&cmd, rc_token), sc->mac.dpmac_id, in dpaa2_ni_setup()
839 sc->mac.dpmac_id); in dpaa2_ni_setup()
847 "error=%d\n", __func__, dinfo->id, in dpaa2_ni_setup()
863 &sc->mac.phy_dev, sc->mac.dpmac_id); in dpaa2_ni_setup()
866 sc->mac.phy_dev, dev); in dpaa2_ni_setup()
873 sc->mac.phy_dev), error); in dpaa2_ni_setup()
878 sc->mac.phy_dev, &sc->mac.phy_loc); in dpaa2_ni_setup()
887 sc->mac.phy_dev), error); in dpaa2_ni_setup()
891 error = mii_attach(sc->mac.phy_dev, in dpaa2_ni_setup()
892 &sc->miibus, sc->ifp, in dpaa2_ni_setup()
895 BMSR_DEFCAPMASK, sc->mac.phy_loc, in dpaa2_ni_setup()
905 sc->mii = device_get_softc(sc->miibus); in dpaa2_ni_setup()
984 /* Calculate number of the channels based on the allocated resources */ in dpaa2_ni_setup_channels()
986 if (!sc->res[DPAA2_NI_IO_RID(i)]) { in dpaa2_ni_setup_channels()
992 if (!sc->res[DPAA2_NI_CON_RID(i)]) { in dpaa2_ni_setup_channels()
997 sc->chan_n = num_chan > DPAA2_MAX_CHANNELS in dpaa2_ni_setup_channels()
999 sc->chan_n = sc->chan_n > sc->attr.num.queues in dpaa2_ni_setup_channels()
1000 ? sc->attr.num.queues : sc->chan_n; in dpaa2_ni_setup_channels()
1002 KASSERT(sc->chan_n > 0u, ("%s: positive number of channels expected: " in dpaa2_ni_setup_channels()
1003 "chan_n=%d", __func__, sc->chan_n)); in dpaa2_ni_setup_channels()
1005 device_printf(dev, "channels=%d\n", sc->chan_n); in dpaa2_ni_setup_channels()
1007 for (i = 0; i < sc->chan_n; i++) { in dpaa2_ni_setup_channels()
1008 iodev = (device_t)rman_get_start(sc->res[DPAA2_NI_IO_RID(i)]); in dpaa2_ni_setup_channels()
1009 condev = (device_t)rman_get_start(sc->res[DPAA2_NI_CON_RID(i)]); in dpaa2_ni_setup_channels()
1011 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); in dpaa2_ni_setup_channels()
1014 &sc->channels[i], i, dpaa2_ni_cleanup_task); in dpaa2_ni_setup_channels()
1023 error = dpaa2_chan_setup_fq(dev, sc->channels[0], DPAA2_NI_QUEUE_RX_ERR); in dpaa2_ni_setup_channels()
1055 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_bind()
1058 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_bind()
1061 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_bind()
1064 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_bind()
1069 bp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); in dpaa2_ni_bind()
1074 pools_cfg.pools[0].bp_obj_id = bp_info->id; in dpaa2_ni_bind()
1076 pools_cfg.pools[0].buf_sz = sc->buf_sz; in dpaa2_ni_bind()
1107 for (uint32_t i = 0; i < sc->chan_n; i++) { in dpaa2_ni_bind()
1108 chan = sc->channels[i]; in dpaa2_ni_bind()
1111 for (uint32_t j = 0; j < chan->rxq_n; j++) { in dpaa2_ni_bind()
1112 error = dpaa2_ni_setup_rx_flow(dev, &chan->rx_queues[j]); in dpaa2_ni_bind()
1121 error = dpaa2_ni_setup_tx_flow(dev, &chan->txc_queue); in dpaa2_ni_bind()
1130 error = dpaa2_ni_setup_rx_err_flow(dev, &sc->rxe_queue); in dpaa2_ni_bind()
1142 &sc->tx_qdid); in dpaa2_ni_bind()
1171 * Have the interface implicitly distribute traffic based on the default in dpaa2_ni_setup_rx_dist()
1192 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_setup_rx_flow()
1195 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_setup_rx_flow()
1198 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_setup_rx_flow()
1201 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_setup_rx_flow()
1206 con_info = device_get_ivars(fq->chan->con_dev); in dpaa2_ni_setup_rx_flow()
1209 queue_cfg.tc = fq->tc; in dpaa2_ni_setup_rx_flow()
1210 queue_cfg.idx = fq->flowid; in dpaa2_ni_setup_rx_flow()
1219 fq->fqid = queue_cfg.fqid; in dpaa2_ni_setup_rx_flow()
1221 queue_cfg.dest_id = con_info->id; in dpaa2_ni_setup_rx_flow()
1238 "user_ctx=%#jx\n", fq->flowid, fq->tc, fq->chan->id, in dpaa2_ni_setup_rx_flow()
1239 fq->fqid, (uint64_t) fq); in dpaa2_ni_setup_rx_flow()
1260 struct dpaa2_channel *ch = fq->chan; in dpaa2_ni_setup_tx_flow()
1274 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_setup_tx_flow()
1277 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_setup_tx_flow()
1280 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_setup_tx_flow()
1283 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_setup_tx_flow()
1288 con_info = device_get_ivars(fq->chan->con_dev); in dpaa2_ni_setup_tx_flow()
1290 KASSERT(sc->attr.num.tx_tcs <= DPAA2_MAX_TCS, in dpaa2_ni_setup_tx_flow()
1292 sc->attr.num.tx_tcs)); in dpaa2_ni_setup_tx_flow()
1298 for (int i = 0; i < sc->attr.num.tx_tcs; i++) { in dpaa2_ni_setup_tx_flow()
1301 queue_cfg.idx = fq->flowid; in dpaa2_ni_setup_tx_flow()
1302 queue_cfg.chan_id = fq->chan->id; in dpaa2_ni_setup_tx_flow()
1312 tx = &fq->tx_rings[i]; in dpaa2_ni_setup_tx_flow()
1313 tx->fq = fq; in dpaa2_ni_setup_tx_flow()
1314 tx->fqid = queue_cfg.fqid; in dpaa2_ni_setup_tx_flow()
1315 tx->txid = tx_rings_n; in dpaa2_ni_setup_tx_flow()
1319 "fqid=%d\n", fq->flowid, i, fq->chan->id, in dpaa2_ni_setup_tx_flow()
1323 mtx_init(&tx->lock, "dpaa2_tx_ring", NULL, MTX_DEF); in dpaa2_ni_setup_tx_flow()
1326 tx->br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT, in dpaa2_ni_setup_tx_flow()
1327 &tx->lock); in dpaa2_ni_setup_tx_flow()
1328 if (tx->br == NULL) { in dpaa2_ni_setup_tx_flow()
1330 " (2) fqid=%d\n", __func__, tx->fqid); in dpaa2_ni_setup_tx_flow()
1339 DPAA2_BUF_INIT_TAGOPT(buf, ch->tx_dmat, tx); in dpaa2_ni_setup_tx_flow()
1341 buf->sgt = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB, in dpaa2_ni_setup_tx_flow()
1344 DPAA2_BUF_INIT_TAGOPT(buf->sgt, ch->sgt_dmat, buf); in dpaa2_ni_setup_tx_flow()
1349 buf_ring_enqueue(tx->br, buf); in dpaa2_ni_setup_tx_flow()
1356 fq->tx_qdbin = queue_cfg.qdbin; in dpaa2_ni_setup_tx_flow()
1360 queue_cfg.idx = fq->flowid; in dpaa2_ni_setup_tx_flow()
1369 fq->fqid = queue_cfg.fqid; in dpaa2_ni_setup_tx_flow()
1371 queue_cfg.dest_id = con_info->id; in dpaa2_ni_setup_tx_flow()
1413 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_setup_rx_err_flow()
1416 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_setup_rx_err_flow()
1419 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_setup_rx_err_flow()
1422 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_setup_rx_err_flow()
1427 con_info = device_get_ivars(fq->chan->con_dev); in dpaa2_ni_setup_rx_err_flow()
1430 queue_cfg.tc = fq->tc; /* ignored */ in dpaa2_ni_setup_rx_err_flow()
1431 queue_cfg.idx = fq->flowid; /* ignored */ in dpaa2_ni_setup_rx_err_flow()
1439 fq->fqid = queue_cfg.fqid; in dpaa2_ni_setup_rx_err_flow()
1441 queue_cfg.dest_id = con_info->id; in dpaa2_ni_setup_rx_err_flow()
1484 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_setup_irqs()
1487 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_setup_irqs()
1490 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_setup_irqs()
1493 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_setup_irqs()
1503 if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, in dpaa2_ni_setup_irqs()
1504 &sc->irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) { in dpaa2_ni_setup_irqs()
1509 if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE, in dpaa2_ni_setup_irqs()
1510 NULL, dpaa2_ni_intr, sc, &sc->intr)) { in dpaa2_ni_setup_irqs()
1551 val = pci_msi_count(sc->dev); in dpaa2_ni_setup_msi()
1553 device_printf(sc->dev, "MSI: actual=%d, expected=%d\n", val, in dpaa2_ni_setup_msi()
1557 if (pci_alloc_msi(sc->dev, &val) != 0) in dpaa2_ni_setup_msi()
1561 sc->irq_rid[i] = i + 1; in dpaa2_ni_setup_msi()
1572 const bool en_rxcsum = if_getcapenable(sc->ifp) & IFCAP_RXCSUM; in dpaa2_ni_setup_if_caps()
1573 const bool en_txcsum = if_getcapenable(sc->ifp) & IFCAP_TXCSUM; in dpaa2_ni_setup_if_caps()
1574 device_t pdev = device_get_parent(sc->dev); in dpaa2_ni_setup_if_caps()
1575 device_t dev = sc->dev; in dpaa2_ni_setup_if_caps()
1585 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_setup_if_caps()
1588 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_setup_if_caps()
1591 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_setup_if_caps()
1594 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_setup_if_caps()
1648 const bool en_promisc = if_getflags(sc->ifp) & IFF_PROMISC; in dpaa2_ni_setup_if_flags()
1649 const bool en_allmulti = if_getflags(sc->ifp) & IFF_ALLMULTI; in dpaa2_ni_setup_if_flags()
1650 device_t pdev = device_get_parent(sc->dev); in dpaa2_ni_setup_if_flags()
1651 device_t dev = sc->dev; in dpaa2_ni_setup_if_flags()
1661 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_setup_if_flags()
1664 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_setup_if_flags()
1667 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_setup_if_flags()
1670 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_setup_if_flags()
1710 ctx = device_get_sysctl_ctx(sc->dev); in dpaa2_ni_setup_sysctls()
1711 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); in dpaa2_ni_setup_sysctls()
1723 CTLFLAG_RD, &sc->rx_anomaly_frames, in dpaa2_ni_setup_sysctls()
1726 CTLFLAG_RD, &sc->rx_single_buf_frames, in dpaa2_ni_setup_sysctls()
1729 CTLFLAG_RD, &sc->rx_sg_buf_frames, in dpaa2_ni_setup_sysctls()
1732 CTLFLAG_RD, &sc->rx_enq_rej_frames, in dpaa2_ni_setup_sysctls()
1735 CTLFLAG_RD, &sc->rx_ieoi_err_frames, in dpaa2_ni_setup_sysctls()
1738 CTLFLAG_RD, &sc->tx_single_buf_frames, in dpaa2_ni_setup_sysctls()
1741 CTLFLAG_RD, &sc->tx_sg_frames, in dpaa2_ni_setup_sysctls()
1751 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); in dpaa2_ni_setup_sysctls()
1757 for (int i = 0; i < sc->chan_n; i++) { in dpaa2_ni_setup_sysctls()
1765 CTLFLAG_RD, &sc->channels[i]->tx_frames, in dpaa2_ni_setup_sysctls()
1768 CTLFLAG_RD, &sc->channels[i]->tx_dropped, in dpaa2_ni_setup_sysctls()
1778 device_t dev = sc->dev; in dpaa2_ni_setup_dma()
1781 KASSERT((sc->buf_align == BUF_ALIGN) || (sc->buf_align == BUF_ALIGN_V1), in dpaa2_ni_setup_dma()
1782 ("unexpected buffer alignment: %d\n", sc->buf_align)); in dpaa2_ni_setup_dma()
1794 &sc->rxd_dmat); in dpaa2_ni_setup_dma()
1810 &sc->qos_dmat); in dpaa2_ni_setup_dma()
1838 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_set_buf_layout()
1841 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_set_buf_layout()
1844 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_set_buf_layout()
1846 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " in dpaa2_ni_set_buf_layout()
1847 "error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_set_buf_layout()
1856 sc->buf_align = (sc->attr.wriop_ver == WRIOP_VERSION(0, 0, 0) || in dpaa2_ni_set_buf_layout()
1857 sc->attr.wriop_ver == WRIOP_VERSION(1, 0, 0)) in dpaa2_ni_set_buf_layout()
1864 sc->buf_sz = ALIGN_DOWN(DPAA2_RX_BUF_SIZE, sc->buf_align); in dpaa2_ni_set_buf_layout()
1868 sc->buf_sz, sc->buf_align); in dpaa2_ni_set_buf_layout()
1874 * ADDR -> |---------------------| in dpaa2_ni_set_buf_layout()
1876 * |---------------------| in dpaa2_ni_set_buf_layout()
1878 * |---------------------| in dpaa2_ni_set_buf_layout()
1880 * ADDR + OFFSET -> |---------------------| in dpaa2_ni_set_buf_layout()
1886 * |---------------------| in dpaa2_ni_set_buf_layout()
1888 * |---------------------| in dpaa2_ni_set_buf_layout()
1907 /* Tx-confirmation buffer layout */ in dpaa2_ni_set_buf_layout()
1923 error = DPAA2_CMD_NI_GET_TX_DATA_OFF(dev, child, &cmd, &sc->tx_data_off); in dpaa2_ni_set_buf_layout()
1931 device_printf(dev, "Tx data offset=%d\n", sc->tx_data_off); in dpaa2_ni_set_buf_layout()
1933 if ((sc->tx_data_off % 64) != 0) { in dpaa2_ni_set_buf_layout()
1935 "of 64 bytes\n", sc->tx_data_off); in dpaa2_ni_set_buf_layout()
1941 * ADDR -> |---------------------| in dpaa2_ni_set_buf_layout()
1943 * |---------------------| in dpaa2_ni_set_buf_layout()
1945 * |---------------------| in dpaa2_ni_set_buf_layout()
1946 * | DATA HEADROOM | OFFSET-BUF_RX_HWA_SIZE in dpaa2_ni_set_buf_layout()
1947 * ADDR + OFFSET -> |---------------------| in dpaa2_ni_set_buf_layout()
1953 * |---------------------| in dpaa2_ni_set_buf_layout()
1955 * |---------------------| in dpaa2_ni_set_buf_layout()
1961 buf_layout.fd_align = sc->buf_align; in dpaa2_ni_set_buf_layout()
1962 buf_layout.head_size = sc->tx_data_off - BUF_RX_HWA_SIZE - BUF_SWA_SIZE; in dpaa2_ni_set_buf_layout()
2012 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_set_pause_frame()
2015 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_set_pause_frame()
2018 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_set_pause_frame()
2020 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " in dpaa2_ni_set_pause_frame()
2021 "error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_set_pause_frame()
2043 sc->link_options = link_cfg.options; in dpaa2_ni_set_pause_frame()
2066 struct dpaa2_buf *buf = &sc->qos_kcfg; in dpaa2_ni_set_qos_table()
2071 if (sc->attr.num.rx_tcs == 1 || in dpaa2_ni_set_qos_table()
2072 !(sc->attr.options & DPNI_OPT_HAS_KEY_MASKING)) { in dpaa2_ni_set_qos_table()
2085 if (__predict_true(buf->dmat == NULL)) { in dpaa2_ni_set_qos_table()
2086 buf->dmat = sc->qos_dmat; in dpaa2_ni_set_qos_table()
2089 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, in dpaa2_ni_set_qos_table()
2090 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); in dpaa2_ni_set_qos_table()
2097 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, in dpaa2_ni_set_qos_table()
2098 ETH_QOS_KCFG_BUF_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr, in dpaa2_ni_set_qos_table()
2108 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_set_qos_table()
2111 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_set_qos_table()
2114 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_set_qos_table()
2116 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " in dpaa2_ni_set_qos_table()
2117 "error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_set_qos_table()
2124 tbl.kcfg_busaddr = buf->paddr; in dpaa2_ni_set_qos_table()
2152 if_t ifp = sc->ifp; in dpaa2_ni_set_mac_addr()
2164 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_set_mac_addr()
2167 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_set_mac_addr()
2170 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_set_mac_addr()
2172 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " in dpaa2_ni_set_mac_addr()
2173 "error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_set_mac_addr()
2207 sc->mac.addr[i] = mac_addr[i]; in dpaa2_ni_set_mac_addr()
2224 sc->mac.addr[i] = mac_addr[i]; in dpaa2_ni_set_mac_addr()
2228 sc->mac.addr[i] = dpni_mac_addr[i]; in dpaa2_ni_set_mac_addr()
2253 if (sc->fixed_link || sc->mii == NULL) { in dpaa2_ni_miibus_statchg()
2256 if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) == 0) { in dpaa2_ni_miibus_statchg()
2261 * sc->link_state and not trigger the update once needed, in dpaa2_ni_miibus_statchg()
2269 * cannot rely on ifp->if_linkstate here. in dpaa2_ni_miibus_statchg()
2271 if (sc->mii->mii_media_status & IFM_AVALID) { in dpaa2_ni_miibus_statchg()
2272 if (sc->mii->mii_media_status & IFM_ACTIVE) { in dpaa2_ni_miibus_statchg()
2281 if (link_state != sc->link_state) { in dpaa2_ni_miibus_statchg()
2282 sc->link_state = link_state; in dpaa2_ni_miibus_statchg()
2286 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, in dpaa2_ni_miibus_statchg()
2290 "container: id=%d, error=%d\n", __func__, rcinfo->id, in dpaa2_ni_miibus_statchg()
2294 error = DPAA2_CMD_MAC_OPEN(dev, child, &cmd, sc->mac.dpmac_id, in dpaa2_ni_miibus_statchg()
2297 device_printf(sc->dev, "%s: failed to open DPMAC: " in dpaa2_ni_miibus_statchg()
2298 "id=%d, error=%d\n", __func__, sc->mac.dpmac_id, in dpaa2_ni_miibus_statchg()
2306 mac_link.supported = sc->mii->mii_media.ifm_media; in dpaa2_ni_miibus_statchg()
2307 mac_link.advert = sc->mii->mii_media.ifm_media; in dpaa2_ni_miibus_statchg()
2319 device_printf(sc->dev, "%s: failed to set DPMAC " in dpaa2_ni_miibus_statchg()
2321 sc->mac.dpmac_id, error); in dpaa2_ni_miibus_statchg()
2345 if (sc->mii) { in dpaa2_ni_media_change_locked()
2346 mii_mediachg(sc->mii); in dpaa2_ni_media_change_locked()
2347 sc->media_status = sc->mii->mii_media.ifm_media; in dpaa2_ni_media_change_locked()
2348 } else if (sc->fixed_link) { in dpaa2_ni_media_change_locked()
2349 if_printf(sc->ifp, "%s: can't change media in fixed mode\n", in dpaa2_ni_media_change_locked()
2377 if (sc->mii) { in dpaa2_ni_media_status()
2378 mii_pollstat(sc->mii); in dpaa2_ni_media_status()
2379 ifmr->ifm_active = sc->mii->mii_media_active; in dpaa2_ni_media_status()
2380 ifmr->ifm_status = sc->mii->mii_media_status; in dpaa2_ni_media_status()
2394 if (sc->mii) { in dpaa2_ni_media_tick()
2395 mii_tick(sc->mii); in dpaa2_ni_media_tick()
2396 if (sc->media_status != sc->mii->mii_media.ifm_media) { in dpaa2_ni_media_tick()
2398 __func__, sc->mii->mii_media.ifm_media); in dpaa2_ni_media_tick()
2399 dpaa2_ni_media_change(sc->ifp); in dpaa2_ni_media_tick()
2404 callout_reset(&sc->mii_callout, hz, dpaa2_ni_media_tick, sc); in dpaa2_ni_media_tick()
2411 if_t ifp = sc->ifp; in dpaa2_ni_init()
2412 device_t pdev = device_get_parent(sc->dev); in dpaa2_ni_init()
2413 device_t dev = sc->dev; in dpaa2_ni_init()
2430 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_init()
2433 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_init()
2436 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_init()
2439 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_init()
2453 if (sc->mii) { in dpaa2_ni_init()
2460 callout_reset(&sc->mii_callout, hz, dpaa2_ni_media_tick, sc); in dpaa2_ni_init()
2488 fqid = m->m_pkthdr.flowid; in dpaa2_ni_transmit()
2489 for (int i = 0; i < sc->chan_n; i++) { in dpaa2_ni_transmit()
2490 ch = sc->channels[i]; in dpaa2_ni_transmit()
2491 for (int j = 0; j < ch->rxq_n; j++) { in dpaa2_ni_transmit()
2492 if (fqid == ch->rx_queues[j].fqid) { in dpaa2_ni_transmit()
2493 chidx = ch->flowid; in dpaa2_ni_transmit()
2504 ch = sc->channels[chidx]; in dpaa2_ni_transmit()
2505 error = buf_ring_enqueue(ch->xmit_br, m); in dpaa2_ni_transmit()
2509 taskqueue_enqueue(ch->cleanup_tq, &ch->cleanup_task); in dpaa2_ni_transmit()
2527 device_t pdev = device_get_parent(sc->dev); in dpaa2_ni_ioctl()
2528 device_t dev = sc->dev; in dpaa2_ni_ioctl()
2539 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_ioctl()
2542 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_ioctl()
2545 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_ioctl()
2548 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_ioctl()
2555 mtu = ifr->ifr_mtu; in dpaa2_ni_ioctl()
2576 changed = if_getcapenable(ifp) ^ ifr->ifr_reqcap; in dpaa2_ni_ioctl()
2578 if ((ifr->ifr_reqcap & changed) & IFCAP_HWCSUM) { in dpaa2_ni_ioctl()
2595 changed = if_getflags(ifp) ^ sc->if_flags; in dpaa2_ni_ioctl()
2609 sc->if_flags = if_getflags(ifp); in dpaa2_ni_ioctl()
2628 if (sc->mii) in dpaa2_ni_ioctl()
2629 rc = ifmedia_ioctl(ifp, ifr, &sc->mii->mii_media, c); in dpaa2_ni_ioctl()
2630 else if(sc->fixed_link) { in dpaa2_ni_ioctl()
2631 rc = ifmedia_ioctl(ifp, ifr, &sc->fixed_ifmedia, c); in dpaa2_ni_ioctl()
2656 device_t pdev = device_get_parent(sc->dev); in dpaa2_ni_update_mac_filters()
2657 device_t dev = sc->dev; in dpaa2_ni_update_mac_filters()
2667 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_update_mac_filters()
2670 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_update_mac_filters()
2673 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_update_mac_filters()
2676 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_update_mac_filters()
2707 struct dpaa2_ni_softc *sc = if_getsoftc(ctx->ifp); in dpaa2_ni_add_maddr()
2708 device_t pdev = device_get_parent(sc->dev); in dpaa2_ni_add_maddr()
2709 device_t dev = sc->dev; in dpaa2_ni_add_maddr()
2717 if (ctx->error != 0) { in dpaa2_ni_add_maddr()
2724 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, in dpaa2_ni_add_maddr()
2728 "container: id=%d, error=%d\n", __func__, rcinfo->id, in dpaa2_ni_add_maddr()
2732 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, in dpaa2_ni_add_maddr()
2736 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_add_maddr()
2742 ctx->error = DPAA2_CMD_NI_ADD_MAC_ADDR(dev, child, &cmd, in dpaa2_ni_add_maddr()
2750 if (ctx->error != 0) { in dpaa2_ni_add_maddr()
2753 "mode\n", __func__, ctx->nent); in dpaa2_ni_add_maddr()
2757 if_setflagbits(ctx->ifp, IFF_ALLMULTI, 0); in dpaa2_ni_add_maddr()
2758 sc->if_flags |= IFF_ALLMULTI; in dpaa2_ni_add_maddr()
2759 ctx->error = dpaa2_ni_setup_if_flags(sc); in dpaa2_ni_add_maddr()
2764 ctx->nent++; in dpaa2_ni_add_maddr()
2774 device_t pdev = device_get_parent(sc->dev); in dpaa2_ni_intr()
2775 device_t dev = sc->dev; in dpaa2_ni_intr()
2786 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_intr()
2789 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_intr()
2792 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_intr()
2795 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_intr()
2802 device_printf(sc->dev, "%s: failed to obtain IRQ status: " in dpaa2_ni_intr()
2816 * NOTE: Should not be re-entrant for the same channel. It is achieved by
2817 * enqueuing the cleanup routine on a single-threaded taskqueue.
2823 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); in dpaa2_ni_cleanup_task()
2830 if (__predict_false((if_getdrvflags(sc->ifp) & in dpaa2_ni_cleanup_task()
2840 /* Re-arm channel to generate CDAN */ in dpaa2_ni_cleanup_task()
2841 error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, &ch->ctx); in dpaa2_ni_cleanup_task()
2844 __func__, ch->id, error); in dpaa2_ni_cleanup_task()
2854 struct dpaa2_io_softc *iosc = device_get_softc(ch->io_dev); in dpaa2_ni_rx_cleanup()
2855 struct dpaa2_swp *swp = iosc->swp; in dpaa2_ni_rx_cleanup()
2857 struct dpaa2_buf *buf = &ch->store; in dpaa2_ni_rx_cleanup()
2862 error = dpaa2_swp_pull(swp, ch->id, buf, DPAA2_ETH_STORE_FRAMES); in dpaa2_ni_rx_cleanup()
2864 device_printf(ch->ni_dev, "%s: failed to pull frames: " in dpaa2_ni_rx_cleanup()
2865 "chan_id=%d, error=%d\n", __func__, ch->id, error); in dpaa2_ni_rx_cleanup()
2873 device_printf(ch->ni_dev, "%s: timeout to consume " in dpaa2_ni_rx_cleanup()
2874 "frames: chan_id=%d\n", __func__, ch->id); in dpaa2_ni_rx_cleanup()
2876 } while (--budget); in dpaa2_ni_rx_cleanup()
2878 return (DPAA2_RX_BUDGET - budget); in dpaa2_ni_rx_cleanup()
2884 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); in dpaa2_ni_tx_cleanup()
2885 struct dpaa2_ni_tx_ring *tx = &ch->txc_queue.tx_rings[0]; in dpaa2_ni_tx_cleanup()
2890 mtx_assert(&ch->xmit_mtx, MA_NOTOWNED); in dpaa2_ni_tx_cleanup()
2891 mtx_lock(&ch->xmit_mtx); in dpaa2_ni_tx_cleanup()
2892 m = buf_ring_dequeue_sc(ch->xmit_br); in dpaa2_ni_tx_cleanup()
2893 mtx_unlock(&ch->xmit_mtx); in dpaa2_ni_tx_cleanup()
2901 } while (--budget); in dpaa2_ni_tx_cleanup()
2903 return (DPAA2_TX_BUDGET - budget); in dpaa2_ni_tx_cleanup()
2910 device_t dev = sc->dev; in dpaa2_ni_tx()
2911 struct dpaa2_ni_fq *fq = tx->fq; in dpaa2_ni_tx()
2919 mtx_assert(&tx->lock, MA_NOTOWNED); in dpaa2_ni_tx()
2920 mtx_lock(&tx->lock); in dpaa2_ni_tx()
2921 buf = buf_ring_dequeue_sc(tx->br); in dpaa2_ni_tx()
2922 mtx_unlock(&tx->lock); in dpaa2_ni_tx()
2929 buf->m = m; in dpaa2_ni_tx()
2930 sgt = buf->sgt; in dpaa2_ni_tx()
2934 struct dpaa2_ni_tx_ring *btx = (struct dpaa2_ni_tx_ring *)buf->opt; in dpaa2_ni_tx()
2935 KASSERT(buf->opt == tx, ("%s: unexpected Tx ring", __func__)); in dpaa2_ni_tx()
2936 KASSERT(btx->fq->chan == ch, ("%s: unexpected channel", __func__)); in dpaa2_ni_tx()
2939 BPF_MTAP(sc->ifp, m); in dpaa2_ni_tx()
2941 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, &nsegs, in dpaa2_ni_tx()
2948 fq->chan->tx_dropped++; in dpaa2_ni_tx()
2952 buf->m = m = md; in dpaa2_ni_tx()
2953 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, in dpaa2_ni_tx()
2958 fq->chan->tx_dropped++; in dpaa2_ni_tx()
2967 fq->chan->tx_dropped++; in dpaa2_ni_tx()
2974 rc = DPAA2_SWP_ENQ_MULTIPLE_FQ(fq->chan->io_dev, tx->fqid, &fd, 1); in dpaa2_ni_tx()
2980 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREWRITE); in dpaa2_ni_tx()
2981 bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_PREWRITE); in dpaa2_ni_tx()
2984 fq->chan->tx_dropped++; in dpaa2_ni_tx()
2987 fq->chan->tx_frames++; in dpaa2_ni_tx()
2992 bus_dmamap_unload(buf->dmat, buf->dmap); in dpaa2_ni_tx()
2993 if (sgt->paddr != 0) { in dpaa2_ni_tx()
2994 bus_dmamap_unload(sgt->dmat, sgt->dmap); in dpaa2_ni_tx()
2997 m_freem(buf->m); in dpaa2_ni_tx()
2998 buf_ring_enqueue(tx->br, buf); in dpaa2_ni_tx()
3019 if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) { in dpaa2_ni_consume_frames()
3020 fd = &dq->fdr.fd; in dpaa2_ni_consume_frames()
3021 fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx; in dpaa2_ni_consume_frames()
3023 switch (fq->type) { in dpaa2_ni_consume_frames()
3040 if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) { in dpaa2_ni_consume_frames()
3041 fd = &dq->fdr.fd; in dpaa2_ni_consume_frames()
3042 fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx; in dpaa2_ni_consume_frames()
3044 switch (fq->type) { in dpaa2_ni_consume_frames()
3071 KASSERT(chan->store_idx < chan->store_sz, ("%s: store_idx(%d) >= " in dpaa2_ni_consume_frames()
3072 "store_sz(%d)", __func__, chan->store_idx, chan->store_sz)); in dpaa2_ni_consume_frames()
3095 bus_addr_t paddr = (bus_addr_t)fd->addr; in dpaa2_ni_rx()
3097 struct dpaa2_buf *buf = fa->buf; in dpaa2_ni_rx()
3098 struct dpaa2_channel *bch = (struct dpaa2_channel *)buf->opt; in dpaa2_ni_rx()
3099 struct dpaa2_ni_softc *sc = device_get_softc(bch->ni_dev); in dpaa2_ni_rx()
3107 KASSERT(fa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); in dpaa2_ni_rx()
3114 if (__predict_false(paddr != buf->paddr)) { in dpaa2_ni_rx()
3116 __func__, paddr, buf->paddr); in dpaa2_ni_rx()
3121 sc->rx_enq_rej_frames++; in dpaa2_ni_rx()
3124 sc->rx_ieoi_err_frames++; in dpaa2_ni_rx()
3131 sc->rx_single_buf_frames++; in dpaa2_ni_rx()
3134 sc->rx_sg_buf_frames++; in dpaa2_ni_rx()
3140 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); in dpaa2_ni_rx()
3141 mtx_lock(&bch->dma_mtx); in dpaa2_ni_rx()
3143 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD); in dpaa2_ni_rx()
3144 bus_dmamap_unload(buf->dmat, buf->dmap); in dpaa2_ni_rx()
3145 m = buf->m; in dpaa2_ni_rx()
3147 buf_data = (uint8_t *)buf->vaddr + dpaa2_ni_fd_offset(fd); in dpaa2_ni_rx()
3148 /* Prepare buffer to be re-cycled */ in dpaa2_ni_rx()
3149 buf->m = NULL; in dpaa2_ni_rx()
3150 buf->paddr = 0; in dpaa2_ni_rx()
3151 buf->vaddr = NULL; in dpaa2_ni_rx()
3152 buf->seg.ds_addr = 0; in dpaa2_ni_rx()
3153 buf->seg.ds_len = 0; in dpaa2_ni_rx()
3154 buf->nseg = 0; in dpaa2_ni_rx()
3156 mtx_unlock(&bch->dma_mtx); in dpaa2_ni_rx()
3158 m->m_flags |= M_PKTHDR; in dpaa2_ni_rx()
3159 m->m_data = buf_data; in dpaa2_ni_rx()
3160 m->m_len = buf_len; in dpaa2_ni_rx()
3161 m->m_pkthdr.len = buf_len; in dpaa2_ni_rx()
3162 m->m_pkthdr.rcvif = sc->ifp; in dpaa2_ni_rx()
3163 m->m_pkthdr.flowid = fq->fqid; in dpaa2_ni_rx()
3166 if (ctx->head == NULL) { in dpaa2_ni_rx()
3167 KASSERT(ctx->tail == NULL, ("%s: tail already given?", __func__)); in dpaa2_ni_rx()
3168 ctx->head = m; in dpaa2_ni_rx()
3169 ctx->tail = m; in dpaa2_ni_rx()
3171 KASSERT(ctx->head != NULL, ("%s: head is NULL", __func__)); in dpaa2_ni_rx()
3172 ctx->tail->m_nextpkt = m; in dpaa2_ni_rx()
3173 ctx->tail = m; in dpaa2_ni_rx()
3175 ctx->cnt++; in dpaa2_ni_rx()
3177 if (ctx->last) { in dpaa2_ni_rx()
3178 ctx->tail->m_nextpkt = NULL; in dpaa2_ni_rx()
3179 if_input(sc->ifp, ctx->head); in dpaa2_ni_rx()
3183 ch->recycled[ch->recycled_n++] = buf; in dpaa2_ni_rx()
3185 /* Re-seed and release recycled buffers back to the pool */ in dpaa2_ni_rx()
3186 if (ch->recycled_n == DPAA2_SWP_BUFS_PER_CMD) { in dpaa2_ni_rx()
3188 taskqueue_enqueue(sc->bp_taskq, &ch->bp_task); in dpaa2_ni_rx()
3190 for (int i = 0; i < ch->recycled_n; i++) { in dpaa2_ni_rx()
3191 buf = ch->recycled[i]; in dpaa2_ni_rx()
3192 bch = (struct dpaa2_channel *)buf->opt; in dpaa2_ni_rx()
3194 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); in dpaa2_ni_rx()
3195 mtx_lock(&bch->dma_mtx); in dpaa2_ni_rx()
3196 error = dpaa2_buf_seed_rxb(sc->dev, buf, in dpaa2_ni_rx()
3197 DPAA2_RX_BUF_SIZE, &bch->dma_mtx); in dpaa2_ni_rx()
3198 mtx_unlock(&bch->dma_mtx); in dpaa2_ni_rx()
3207 released[released_n++] = buf->paddr; in dpaa2_ni_rx()
3211 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); in dpaa2_ni_rx()
3214 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, in dpaa2_ni_rx()
3217 device_printf(sc->dev, "%s: failed to release buffers " in dpaa2_ni_rx()
3221 ch->recycled_n = 0; in dpaa2_ni_rx()
3234 bus_addr_t paddr = (bus_addr_t)fd->addr; in dpaa2_ni_rx_err()
3236 struct dpaa2_buf *buf = fa->buf; in dpaa2_ni_rx_err()
3237 struct dpaa2_channel *bch = (struct dpaa2_channel *)buf->opt; in dpaa2_ni_rx_err()
3238 struct dpaa2_ni_softc *sc = device_get_softc(bch->ni_dev); in dpaa2_ni_rx_err()
3243 KASSERT(fa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); in dpaa2_ni_rx_err()
3250 if (__predict_false(paddr != buf->paddr)) { in dpaa2_ni_rx_err()
3252 __func__, paddr, buf->paddr); in dpaa2_ni_rx_err()
3256 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); in dpaa2_ni_rx_err()
3260 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, &paddr, 1); in dpaa2_ni_rx_err()
3262 device_printf(sc->dev, "%s: failed to release frame buffer to " in dpaa2_ni_rx_err()
3277 bus_addr_t paddr = (bus_addr_t)fd->addr; in dpaa2_ni_tx_conf()
3279 struct dpaa2_buf *buf = fa->buf; in dpaa2_ni_tx_conf()
3280 struct dpaa2_buf *sgt = buf->sgt; in dpaa2_ni_tx_conf()
3281 struct dpaa2_ni_tx_ring *tx = (struct dpaa2_ni_tx_ring *)buf->opt; in dpaa2_ni_tx_conf()
3282 struct dpaa2_channel *bch = tx->fq->chan; in dpaa2_ni_tx_conf()
3284 KASSERT(fa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); in dpaa2_ni_tx_conf()
3293 if (paddr != buf->paddr) { in dpaa2_ni_tx_conf()
3295 __func__, paddr, buf->paddr); in dpaa2_ni_tx_conf()
3298 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); in dpaa2_ni_tx_conf()
3299 mtx_lock(&bch->dma_mtx); in dpaa2_ni_tx_conf()
3301 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTWRITE); in dpaa2_ni_tx_conf()
3302 bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_POSTWRITE); in dpaa2_ni_tx_conf()
3303 bus_dmamap_unload(buf->dmat, buf->dmap); in dpaa2_ni_tx_conf()
3304 bus_dmamap_unload(sgt->dmat, sgt->dmap); in dpaa2_ni_tx_conf()
3305 m_freem(buf->m); in dpaa2_ni_tx_conf()
3306 buf->m = NULL; in dpaa2_ni_tx_conf()
3307 buf->paddr = 0; in dpaa2_ni_tx_conf()
3308 buf->vaddr = NULL; in dpaa2_ni_tx_conf()
3309 sgt->paddr = 0; in dpaa2_ni_tx_conf()
3311 mtx_unlock(&bch->dma_mtx); in dpaa2_ni_tx_conf()
3314 buf_ring_enqueue(tx->br, buf); in dpaa2_ni_tx_conf()
3326 if (sc->api_major == major) { in dpaa2_ni_cmp_api_version()
3327 return sc->api_minor - minor; in dpaa2_ni_cmp_api_version()
3329 return sc->api_major - major; in dpaa2_ni_cmp_api_version()
3339 struct dpaa2_buf *sgt = buf->sgt; in dpaa2_ni_build_fd()
3345 KASSERT(buf->opt != NULL, ("%s: no Tx ring?", __func__)); in dpaa2_ni_build_fd()
3347 KASSERT(sgt->vaddr != NULL, ("%s: no S/G vaddr?", __func__)); in dpaa2_ni_build_fd()
3353 sge = (struct dpaa2_sg_entry *)sgt->vaddr + sc->tx_data_off; in dpaa2_ni_build_fd()
3359 sge[i-1].offset_fmt |= 0x8000u; /* set final entry flag */ in dpaa2_ni_build_fd()
3361 KASSERT(sgt->paddr == 0, ("%s: paddr(%#jx) != 0", __func__, in dpaa2_ni_build_fd()
3362 sgt->paddr)); in dpaa2_ni_build_fd()
3364 error = bus_dmamap_load(sgt->dmat, sgt->dmap, sgt->vaddr, in dpaa2_ni_build_fd()
3365 DPAA2_TX_SGT_SZ, dpaa2_dmamap_oneseg_cb, &sgt->paddr, in dpaa2_ni_build_fd()
3368 device_printf(sc->dev, "%s: bus_dmamap_load() failed: " in dpaa2_ni_build_fd()
3373 buf->paddr = sgt->paddr; in dpaa2_ni_build_fd()
3374 buf->vaddr = sgt->vaddr; in dpaa2_ni_build_fd()
3375 sc->tx_sg_frames++; /* for sysctl(9) */ in dpaa2_ni_build_fd()
3380 fa = (struct dpaa2_fa *)sgt->vaddr; in dpaa2_ni_build_fd()
3381 fa->magic = DPAA2_MAGIC; in dpaa2_ni_build_fd()
3382 fa->buf = buf; in dpaa2_ni_build_fd()
3384 fd->addr = buf->paddr; in dpaa2_ni_build_fd()
3385 fd->data_length = (uint32_t)buf->m->m_pkthdr.len; in dpaa2_ni_build_fd()
3386 fd->bpid_ivp_bmt = 0; in dpaa2_ni_build_fd()
3387 fd->offset_fmt_sl = 0x2000u | sc->tx_data_off; in dpaa2_ni_build_fd()
3388 fd->ctrl = 0x00800000u; in dpaa2_ni_build_fd()
3396 return ((fd->ctrl >> DPAA2_NI_FD_ERR_SHIFT) & DPAA2_NI_FD_ERR_MASK); in dpaa2_ni_fd_err()
3403 return (fd->data_length & DPAA2_NI_FD_LEN_MASK); in dpaa2_ni_fd_data_len()
3405 return (fd->data_length); in dpaa2_ni_fd_data_len()
3411 return ((enum dpaa2_fd_format)((fd->offset_fmt_sl >> in dpaa2_ni_fd_format()
3418 return (((fd->offset_fmt_sl >> DPAA2_NI_FD_SL_SHIFT) in dpaa2_ni_fd_short_len()
3425 return (fd->offset_fmt_sl & DPAA2_NI_FD_OFFSET_MASK); in dpaa2_ni_fd_offset()
3435 struct dpni_stat *stat = &dpni_stat_sysctls[oidp->oid_number]; in dpaa2_ni_collect_stats()
3436 device_t pdev = device_get_parent(sc->dev); in dpaa2_ni_collect_stats()
3437 device_t dev = sc->dev; in dpaa2_ni_collect_stats()
3449 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); in dpaa2_ni_collect_stats()
3452 "id=%d, error=%d\n", __func__, rcinfo->id, error); in dpaa2_ni_collect_stats()
3455 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); in dpaa2_ni_collect_stats()
3458 "id=%d, error=%d\n", __func__, dinfo->id, error); in dpaa2_ni_collect_stats()
3462 error = DPAA2_CMD_NI_GET_STATISTICS(dev, child, &cmd, stat->page, 0, cnt); in dpaa2_ni_collect_stats()
3464 result = cnt[stat->cnt]; in dpaa2_ni_collect_stats()
3478 uint32_t buf_num = DPAA2_ATOMIC_READ(&sc->buf_num); in dpaa2_ni_collect_buf_num()
3487 uint32_t buf_free = DPAA2_ATOMIC_READ(&sc->buf_free); in dpaa2_ni_collect_buf_free()
3499 if (!(sc->attr.num.queues > 1)) { in dpaa2_ni_set_hash()
3526 struct dpaa2_buf *buf = &sc->rxd_kcfg; in dpaa2_ni_set_dist_key()
3531 if (__predict_true(buf->dmat == NULL)) { in dpaa2_ni_set_dist_key()
3532 buf->dmat = sc->rxd_dmat; in dpaa2_ni_set_dist_key()
3551 key->type = DPKG_EXTRACT_FROM_HDR; in dpaa2_ni_set_dist_key()
3552 key->extract.from_hdr.prot = dist_fields[i].cls_prot; in dpaa2_ni_set_dist_key()
3553 key->extract.from_hdr.type = DPKG_FULL_FIELD; in dpaa2_ni_set_dist_key()
3554 key->extract.from_hdr.field = dist_fields[i].cls_field; in dpaa2_ni_set_dist_key()
3558 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, in dpaa2_ni_set_dist_key()
3559 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); in dpaa2_ni_set_dist_key()
3566 error = dpaa2_ni_prepare_key_cfg(&cls_cfg, (uint8_t *)buf->vaddr); in dpaa2_ni_set_dist_key()
3574 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, in dpaa2_ni_set_dist_key()
3575 DPAA2_CLASSIFIER_DMA_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr, in dpaa2_ni_set_dist_key()
3578 device_printf(sc->dev, "%s: failed to map a buffer for Rx " in dpaa2_ni_set_dist_key()
3586 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, in dpaa2_ni_set_dist_key()
3590 "container: id=%d, error=%d\n", __func__, rcinfo->id, in dpaa2_ni_set_dist_key()
3594 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, in dpaa2_ni_set_dist_key()
3598 "interface: id=%d, error=%d\n", __func__, dinfo->id, in dpaa2_ni_set_dist_key()
3604 sc->attr.num.queues, 0, DPAA2_NI_DIST_MODE_HASH, buf->paddr); in dpaa2_ni_set_dist_key()
3634 if (cfg->num_extracts > DPKG_MAX_NUM_OF_EXTRACTS) in dpaa2_ni_prepare_key_cfg()
3638 dpni_ext->num_extracts = cfg->num_extracts; in dpaa2_ni_prepare_key_cfg()
3640 for (i = 0; i < cfg->num_extracts; i++) { in dpaa2_ni_prepare_key_cfg()
3641 extr = &dpni_ext->extracts[i]; in dpaa2_ni_prepare_key_cfg()
3643 switch (cfg->extracts[i].type) { in dpaa2_ni_prepare_key_cfg()
3645 extr->prot = cfg->extracts[i].extract.from_hdr.prot; in dpaa2_ni_prepare_key_cfg()
3646 extr->efh_type = in dpaa2_ni_prepare_key_cfg()
3647 cfg->extracts[i].extract.from_hdr.type & 0x0Fu; in dpaa2_ni_prepare_key_cfg()
3648 extr->size = cfg->extracts[i].extract.from_hdr.size; in dpaa2_ni_prepare_key_cfg()
3649 extr->offset = cfg->extracts[i].extract.from_hdr.offset; in dpaa2_ni_prepare_key_cfg()
3650 extr->field = cfg->extracts[i].extract.from_hdr.field; in dpaa2_ni_prepare_key_cfg()
3651 extr->hdr_index = in dpaa2_ni_prepare_key_cfg()
3652 cfg->extracts[i].extract.from_hdr.hdr_index; in dpaa2_ni_prepare_key_cfg()
3655 extr->size = cfg->extracts[i].extract.from_data.size; in dpaa2_ni_prepare_key_cfg()
3656 extr->offset = in dpaa2_ni_prepare_key_cfg()
3657 cfg->extracts[i].extract.from_data.offset; in dpaa2_ni_prepare_key_cfg()
3660 extr->size = cfg->extracts[i].extract.from_parse.size; in dpaa2_ni_prepare_key_cfg()
3661 extr->offset = in dpaa2_ni_prepare_key_cfg()
3662 cfg->extracts[i].extract.from_parse.offset; in dpaa2_ni_prepare_key_cfg()
3668 extr->num_of_byte_masks = cfg->extracts[i].num_of_byte_masks; in dpaa2_ni_prepare_key_cfg()
3669 extr->extract_type = cfg->extracts[i].type & 0x0Fu; in dpaa2_ni_prepare_key_cfg()
3672 extr->masks[j].mask = cfg->extracts[i].masks[j].mask; in dpaa2_ni_prepare_key_cfg()
3673 extr->masks[j].offset = in dpaa2_ni_prepare_key_cfg()
3674 cfg->extracts[i].masks[j].offset; in dpaa2_ni_prepare_key_cfg()