Lines Matching +full:led +full:- +full:7 +full:seg
1 /*-
2 * Copyright (c) 2014-2018, Matthew Macy <mmacy@mattmacy.io>
82 #include <dev/led/led.h>
107 * - Prefetching in tx cleaning should perhaps be a tunable. The distance ahead
110 * - NFLX's m_free path is dominated by vm-based M_EXT manipulation which
112 * - small packet forwarding which is just returning a single mbuf to
119 * - private structures
120 * - iflib private utility functions
121 * - ifnet functions
122 * - vlan registry and other exported functions
123 * - iflib public core functions
224 return (ctx->ifc_softc); in iflib_get_softc()
231 return (ctx->ifc_dev); in iflib_get_dev()
238 return (ctx->ifc_ifp); in iflib_get_ifp()
245 return (ctx->ifc_mediap); in iflib_get_media()
252 bcopy(mac, ctx->ifc_mac.octet, ETHER_ADDR_LEN); in iflib_set_mac()
259 return (&ctx->ifc_softc_ctx); in iflib_get_softc_ctx()
266 return (ctx->ifc_sctx); in iflib_get_sctx()
273 return (ctx->ifc_sysctl_extra_msix_vectors); in iflib_get_extra_msix_vectors_sysctl()
276 #define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2)
280 #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
281 #define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF)
309 /* this should really scale with ring size - this is a fairly arbitrary value */
426 used = pidx - cidx; in get_inuse()
428 used = size - cidx + pidx; in get_inuse()
439 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq-…
442 ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
506 pi_pad->pkt_val[0] = 0; pi_pad->pkt_val[1] = 0; pi_pad->pkt_val[2] = 0; in pkt_info_zero()
507 pi_pad->pkt_val[3] = 0; pi_pad->pkt_val[4] = 0; pi_pad->pkt_val[5] = 0; in pkt_info_zero()
509 pi_pad->pkt_val[6] = 0; pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0; in pkt_info_zero()
510 pi_pad->pkt_val[9] = 0; pi_pad->pkt_val[10] = 0; in pkt_info_zero()
522 ri_pad->rxd_val[i] = 0; in rxd_info_zero()
523 ri_pad->rxd_val[i + 1] = 0; in rxd_info_zero()
524 ri_pad->rxd_val[i + 2] = 0; in rxd_info_zero()
525 ri_pad->rxd_val[i + 3] = 0; in rxd_info_zero()
528 ri_pad->rxd_val[RXD_INFO_SIZE - 1] = 0; in rxd_info_zero()
536 #define IF_BAD_DMA ((bus_addr_t)-1)
538 #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))
540 #define CTX_LOCK_INIT(_sc) sx_init(&(_sc)->ifc_ctx_sx, "iflib ctx lock")
541 #define CTX_LOCK(ctx) sx_xlock(&(ctx)->ifc_ctx_sx)
542 #define CTX_UNLOCK(ctx) sx_xunlock(&(ctx)->ifc_ctx_sx)
543 #define CTX_LOCK_DESTROY(ctx) sx_destroy(&(ctx)->ifc_ctx_sx)
545 #define STATE_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_state_mtx, _name, "iflib state lock", MTX_…
546 #define STATE_LOCK(ctx) mtx_lock(&(ctx)->ifc_state_mtx)
547 #define STATE_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_state_mtx)
548 #define STATE_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_state_mtx)
550 #define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx)
551 #define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)
553 /* Our boot-time initialization hook */
745 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_num_rx_descs()
746 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_num_rx_descs()
747 uint16_t first_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; in iflib_num_rx_descs()
749 return (scctx->isc_nrxd[first_rxq]); in iflib_num_rx_descs()
755 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_num_tx_descs()
756 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_num_tx_descs()
757 uint16_t first_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; in iflib_num_tx_descs()
759 return (scctx->isc_ntxd[first_txq]); in iflib_num_tx_descs()
773 * device-specific sysctl variables:
805 if_t ifp = na->ifp; in iflib_netmap_register()
817 * ifp->if_transmit. This is done once the device has been stopped in iflib_netmap_register()
841 if_t ifp = na->ifp; in iflib_netmap_config()
843 iflib_rxq_t rxq = &ctx->ifc_rxqs[0]; in iflib_netmap_config()
844 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_config()
846 info->num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets; in iflib_netmap_config()
847 info->num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets; in iflib_netmap_config()
848 info->num_tx_descs = iflib_num_tx_descs(ctx); in iflib_netmap_config()
849 info->num_rx_descs = iflib_num_rx_descs(ctx); in iflib_netmap_config()
850 info->rx_buf_maxsize = fl->ifl_buf_size; in iflib_netmap_config()
852 info->num_tx_rings, info->num_rx_rings, info->num_tx_descs, in iflib_netmap_config()
853 info->num_rx_descs, info->rx_buf_maxsize); in iflib_netmap_config()
861 struct netmap_adapter *na = kring->na; in netmap_fl_refill()
862 u_int const lim = kring->nkr_num_slots - 1; in netmap_fl_refill()
863 struct netmap_ring *ring = kring->ring; in netmap_fl_refill()
866 if_ctx_t ctx = rxq->ifr_ctx; in netmap_fl_refill()
867 iflib_fl_t fl = &rxq->ifr_fl[0]; in netmap_fl_refill()
879 * such a way to keep fl->ifl_pidx and kring->nr_hwcur in sync in netmap_fl_refill()
880 * (except for kring->nkr_hwofs). These may be less than in netmap_fl_refill()
881 * kring->nkr_num_slots if netmap_reset() was called while in netmap_fl_refill()
887 * (fl->ifl_pidx - 1) % N (included), to avoid the NIC tail/prod in netmap_fl_refill()
892 n = kring->nkr_num_slots - nm_kr_rxspace(kring); in netmap_fl_refill()
894 n = kring->rhead - kring->nr_hwcur; in netmap_fl_refill()
898 n += kring->nkr_num_slots; in netmap_fl_refill()
902 map = fl->ifl_sds.ifsd_map; in netmap_fl_refill()
903 nic_i = fl->ifl_pidx; in netmap_fl_refill()
911 MPASS(nm_i == kring->nr_hwtail); in netmap_fl_refill()
913 MPASS(nm_i == kring->nr_hwcur); in netmap_fl_refill()
921 for (i = 0; n > 0 && i < IFLIB_MAX_RX_REFRESH; n--, i++) { in netmap_fl_refill()
922 struct netmap_slot *slot = &ring->slot[nm_i]; in netmap_fl_refill()
931 fl->ifl_bus_addrs[i] = paddr + in netmap_fl_refill()
933 fl->ifl_rxd_idxs[i] = nic_i; in netmap_fl_refill()
936 netmap_load_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
938 } else if (slot->flags & NS_BUF_CHANGED) { in netmap_fl_refill()
940 netmap_reload_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
943 bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i], in netmap_fl_refill()
945 slot->flags &= ~NS_BUF_CHANGED; in netmap_fl_refill()
953 ctx->isc_rxd_refill(ctx->ifc_softc, &iru); in netmap_fl_refill()
955 fl->ifl_pidx = nic_i; in netmap_fl_refill()
960 MPASS(nm_i == kring->rhead); in netmap_fl_refill()
961 kring->nr_hwcur = nm_i; in netmap_fl_refill()
963 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in netmap_fl_refill()
965 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, in netmap_fl_refill()
978 * Userspace wants to send packets up to the one before kring->rhead,
979 * kernel knows kring->nr_hwcur is the first unsent packet.
991 struct netmap_adapter *na = kring->na; in iflib_netmap_txsync()
992 if_t ifp = na->ifp; in iflib_netmap_txsync()
993 struct netmap_ring *ring = kring->ring; in iflib_netmap_txsync()
996 u_int const lim = kring->nkr_num_slots - 1; in iflib_netmap_txsync()
997 u_int const head = kring->rhead; in iflib_netmap_txsync()
1005 u_int report_frequency = kring->nkr_num_slots >> 1; in iflib_netmap_txsync()
1006 /* device-specific */ in iflib_netmap_txsync()
1008 iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id]; in iflib_netmap_txsync()
1010 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_netmap_txsync()
1035 nm_i = kring->nr_hwcur; in iflib_netmap_txsync()
1038 int nic_i_start = -1, flags = 0; in iflib_netmap_txsync()
1040 pi.ipi_segs = txq->ift_segs; in iflib_netmap_txsync()
1041 pi.ipi_qsidx = kring->ring_id; in iflib_netmap_txsync()
1044 __builtin_prefetch(&ring->slot[nm_i]); in iflib_netmap_txsync()
1045 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]); in iflib_netmap_txsync()
1046 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]); in iflib_netmap_txsync()
1049 struct netmap_slot *slot = &ring->slot[nm_i]; in iflib_netmap_txsync()
1051 u_int len = slot->len; in iflib_netmap_txsync()
1055 flags |= (slot->flags & NS_REPORT || in iflib_netmap_txsync()
1073 if (!(slot->flags & NS_MOREFRAG)) { in iflib_netmap_txsync()
1081 ctx->isc_txd_encap(ctx->ifc_softc, &pi); in iflib_netmap_txsync()
1088 /* Reinit per-packet info for the next one. */ in iflib_netmap_txsync()
1090 nic_i_start = -1; in iflib_netmap_txsync()
1094 __builtin_prefetch(&ring->slot[nm_i + 1]); in iflib_netmap_txsync()
1095 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]); in iflib_netmap_txsync()
1096 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]); in iflib_netmap_txsync()
1100 if (slot->flags & NS_BUF_CHANGED) { in iflib_netmap_txsync()
1102 netmap_reload_map(na, txq->ift_buf_tag, in iflib_netmap_txsync()
1103 txq->ift_sds.ifsd_map[nic_i], addr); in iflib_netmap_txsync()
1106 bus_dmamap_sync(txq->ift_buf_tag, in iflib_netmap_txsync()
1107 txq->ift_sds.ifsd_map[nic_i], in iflib_netmap_txsync()
1110 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED | NS_MOREFRAG); in iflib_netmap_txsync()
1114 kring->nr_hwcur = nm_i; in iflib_netmap_txsync()
1117 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_netmap_txsync()
1121 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i); in iflib_netmap_txsync()
1129 * trigger a per-tx-queue timer to try again later. in iflib_netmap_txsync()
1131 if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { in iflib_netmap_txsync()
1134 nic_i = txq->ift_cidx_processed; in iflib_netmap_txsync()
1135 kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); in iflib_netmap_txsync()
1139 if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) in iflib_netmap_txsync()
1140 if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { in iflib_netmap_txsync()
1141 callout_reset_sbt_on(&txq->ift_netmap_timer, in iflib_netmap_txsync()
1144 txq->ift_netmap_timer.c_cpu, 0); in iflib_netmap_txsync()
1159 * On call, kring->rhead is the first packet that userspace wants
1160 * to keep, and kring->rcur is the wakeup point.
1161 * The kernel has previously reported packets up to kring->rtail.
1169 struct netmap_adapter *na = kring->na; in iflib_netmap_rxsync()
1170 struct netmap_ring *ring = kring->ring; in iflib_netmap_rxsync()
1171 if_t ifp = na->ifp; in iflib_netmap_rxsync()
1175 u_int const lim = kring->nkr_num_slots - 1; in iflib_netmap_rxsync()
1176 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; in iflib_netmap_rxsync()
1180 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_netmap_rxsync()
1181 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_netmap_rxsync()
1182 iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id]; in iflib_netmap_rxsync()
1183 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_rxsync()
1192 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_netmap_rxsync()
1204 * nic_i = fl->ifl_cidx; in iflib_netmap_rxsync()
1205 * nm_i = kring->nr_hwtail (previous) in iflib_netmap_rxsync()
1207 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size in iflib_netmap_rxsync()
1209 * fl->ifl_cidx is set to 0 on a ring reinit in iflib_netmap_rxsync()
1212 uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim); in iflib_netmap_rxsync()
1213 bool have_rxcq = sctx->isc_flags & IFLIB_HAS_RXCQ; in iflib_netmap_rxsync()
1222 cidxp = &rxq->ifr_cq_cidx; in iflib_netmap_rxsync()
1224 cidxp = &fl->ifl_cidx; in iflib_netmap_rxsync()
1225 avail = ctx->isc_rxd_available(ctx->ifc_softc, in iflib_netmap_rxsync()
1226 rxq->ifr_id, *cidxp, USHRT_MAX); in iflib_netmap_rxsync()
1228 nic_i = fl->ifl_cidx; in iflib_netmap_rxsync()
1230 MPASS(nm_i == kring->nr_hwtail); in iflib_netmap_rxsync()
1231 for (n = 0; avail > 0 && nm_i != hwtail_lim; n++, avail--) { in iflib_netmap_rxsync()
1233 ri.iri_frags = rxq->ifr_frags; in iflib_netmap_rxsync()
1234 ri.iri_qsidx = kring->ring_id; in iflib_netmap_rxsync()
1235 ri.iri_ifp = ctx->ifc_ifp; in iflib_netmap_rxsync()
1238 error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); in iflib_netmap_rxsync()
1241 ring->slot[nm_i].len = 0; in iflib_netmap_rxsync()
1242 ring->slot[nm_i].flags = 0; in iflib_netmap_rxsync()
1244 ring->slot[nm_i].len = ri.iri_frags[i].irf_len; in iflib_netmap_rxsync()
1245 if (i == (ri.iri_nfrags - 1)) { in iflib_netmap_rxsync()
1246 ring->slot[nm_i].len -= crclen; in iflib_netmap_rxsync()
1247 ring->slot[nm_i].flags = 0; in iflib_netmap_rxsync()
1253 ring->slot[nm_i].flags = NS_MOREFRAG; in iflib_netmap_rxsync()
1256 bus_dmamap_sync(fl->ifl_buf_tag, in iflib_netmap_rxsync()
1257 fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); in iflib_netmap_rxsync()
1259 fl->ifl_cidx = nic_i = nm_next(nic_i, lim); in iflib_netmap_rxsync()
1264 while (*cidxp >= scctx->isc_nrxd[0]) in iflib_netmap_rxsync()
1265 *cidxp -= scctx->isc_nrxd[0]; in iflib_netmap_rxsync()
1275 kring->nr_hwtail = nm_i; in iflib_netmap_rxsync()
1277 kring->nr_kflags &= ~NKR_PENDINTR; in iflib_netmap_rxsync()
1281 * (kring->nr_hwcur to head excluded), in iflib_netmap_rxsync()
1285 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size in iflib_netmap_rxsync()
1298 if_ctx_t ctx = if_getsoftc(na->ifp); in iflib_netmap_intr()
1316 na.ifp = ctx->ifc_ifp; in iflib_netmap_attach()
1318 MPASS(ctx->ifc_softc_ctx.isc_ntxqsets); in iflib_netmap_attach()
1319 MPASS(ctx->ifc_softc_ctx.isc_nrxqsets); in iflib_netmap_attach()
1328 na.num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets; in iflib_netmap_attach()
1329 na.num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets; in iflib_netmap_attach()
1336 struct netmap_adapter *na = NA(ctx->ifc_ifp); in iflib_netmap_txq_init()
1339 slot = netmap_reset(na, NR_TX, txq->ift_id, 0); in iflib_netmap_txq_init()
1342 for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) { in iflib_netmap_txq_init()
1350 int si = netmap_idx_n2k(na->tx_rings[txq->ift_id], i); in iflib_netmap_txq_init()
1351 netmap_load_map(na, txq->ift_buf_tag, txq->ift_sds.ifsd_map[i], in iflib_netmap_txq_init()
1360 struct netmap_adapter *na = NA(ctx->ifc_ifp); in iflib_netmap_rxq_init()
1364 slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0); in iflib_netmap_rxq_init()
1367 kring = na->rx_rings[rxq->ifr_id]; in iflib_netmap_rxq_init()
1376 if_ctx_t ctx = txq->ift_ctx; in iflib_netmap_timer()
1382 netmap_tx_irq(ctx->ifc_ifp, txq->ift_id); in iflib_netmap_timer()
1430 fl = &rxq->ifr_fl[flid]; in iru_init()
1431 iru->iru_paddrs = fl->ifl_bus_addrs; in iru_init()
1432 iru->iru_idxs = fl->ifl_rxd_idxs; in iru_init()
1433 iru->iru_qsidx = rxq->ifr_id; in iru_init()
1434 iru->iru_buf_size = fl->ifl_buf_size; in iru_init()
1435 iru->iru_flidx = fl->ifl_id; in iru_init()
1448 BUS_SPACE_MAXADDR : (1ULL << (width)) - 1ULL)
1454 device_t dev = ctx->ifc_dev; in iflib_dma_alloc_align()
1457 lowaddr = DMA_WIDTH_TO_BUS_LOWADDR(ctx->ifc_softc_ctx.isc_dma_width); in iflib_dma_alloc_align()
1470 &dma->idi_tag); in iflib_dma_alloc_align()
1478 err = bus_dmamem_alloc(dma->idi_tag, (void **)&dma->idi_vaddr, in iflib_dma_alloc_align()
1479 BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->idi_map); in iflib_dma_alloc_align()
1487 dma->idi_paddr = IF_BAD_DMA; in iflib_dma_alloc_align()
1488 err = bus_dmamap_load(dma->idi_tag, dma->idi_map, dma->idi_vaddr, in iflib_dma_alloc_align()
1489 size, _iflib_dmamap_cb, &dma->idi_paddr, mapflags | BUS_DMA_NOWAIT); in iflib_dma_alloc_align()
1490 if (err || dma->idi_paddr == IF_BAD_DMA) { in iflib_dma_alloc_align()
1497 dma->idi_size = size; in iflib_dma_alloc_align()
1501 bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map); in iflib_dma_alloc_align()
1503 bus_dma_tag_destroy(dma->idi_tag); in iflib_dma_alloc_align()
1505 dma->idi_tag = NULL; in iflib_dma_alloc_align()
1513 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_dma_alloc()
1515 KASSERT(sctx->isc_q_align != 0, ("alignment value not initialized")); in iflib_dma_alloc()
1517 return (iflib_dma_alloc_align(ctx, size, sctx->isc_q_align, dma, mapflags)); in iflib_dma_alloc()
1539 if (dma->idi_tag == NULL) in iflib_dma_free()
1541 if (dma->idi_paddr != IF_BAD_DMA) { in iflib_dma_free()
1542 bus_dmamap_sync(dma->idi_tag, dma->idi_map, in iflib_dma_free()
1544 bus_dmamap_unload(dma->idi_tag, dma->idi_map); in iflib_dma_free()
1545 dma->idi_paddr = IF_BAD_DMA; in iflib_dma_free()
1547 if (dma->idi_vaddr != NULL) { in iflib_dma_free()
1548 bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map); in iflib_dma_free()
1549 dma->idi_vaddr = NULL; in iflib_dma_free()
1551 bus_dma_tag_destroy(dma->idi_tag); in iflib_dma_free()
1552 dma->idi_tag = NULL; in iflib_dma_free()
1569 struct grouptask *gtask = info->ifi_task; in iflib_fast_intr()
1573 if (info->ifi_filter != NULL) { in iflib_fast_intr()
1574 result = info->ifi_filter(info->ifi_filter_arg); in iflib_fast_intr()
1587 struct grouptask *gtask = info->ifi_task; in iflib_fast_intr_rxtx()
1589 iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; in iflib_fast_intr_rxtx()
1597 if (info->ifi_filter != NULL) { in iflib_fast_intr_rxtx()
1598 result = info->ifi_filter(info->ifi_filter_arg); in iflib_fast_intr_rxtx()
1603 ctx = rxq->ifr_ctx; in iflib_fast_intr_rxtx()
1604 sc = ctx->ifc_softc; in iflib_fast_intr_rxtx()
1606 intr_legacy = !!(ctx->ifc_flags & IFC_LEGACY); in iflib_fast_intr_rxtx()
1607 MPASS(rxq->ifr_ntxqirq); in iflib_fast_intr_rxtx()
1608 for (i = 0; i < rxq->ifr_ntxqirq; i++) { in iflib_fast_intr_rxtx()
1609 txqid = rxq->ifr_txqid[i]; in iflib_fast_intr_rxtx()
1610 txq = &ctx->ifc_txqs[txqid]; in iflib_fast_intr_rxtx()
1611 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_fast_intr_rxtx()
1613 if (!ctx->isc_txd_credits_update(sc, txqid, false)) { in iflib_fast_intr_rxtx()
1620 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_fast_intr_rxtx()
1622 if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) in iflib_fast_intr_rxtx()
1623 cidx = rxq->ifr_cq_cidx; in iflib_fast_intr_rxtx()
1625 cidx = rxq->ifr_fl[0].ifl_cidx; in iflib_fast_intr_rxtx()
1632 IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); in iflib_fast_intr_rxtx()
1644 if_ctx_t ctx = info->ifi_ctx; in iflib_fast_intr_ctx()
1648 if (info->ifi_filter != NULL) { in iflib_fast_intr_ctx()
1649 result = info->ifi_filter(info->ifi_filter_arg); in iflib_fast_intr_ctx()
1654 taskqueue_enqueue(ctx->ifc_tq, &ctx->ifc_admin_task); in iflib_fast_intr_ctx()
1665 device_t dev = ctx->ifc_dev; in _iflib_irq_alloc()
1669 if (ctx->ifc_flags & IFC_LEGACY) in _iflib_irq_alloc()
1679 irq->ii_res = res; in _iflib_irq_alloc()
1680 KASSERT(filter == NULL || handler == NULL, ("filter and handler can't both be non-NULL")); in _iflib_irq_alloc()
1691 irq->ii_tag = tag; in _iflib_irq_alloc()
1698 * mbuf map. TX DMA maps (non-TSO/TSO) and TX mbuf map are kept in a
1707 if_ctx_t ctx = txq->ift_ctx; in iflib_txsd_alloc()
1708 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_txsd_alloc()
1709 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_txsd_alloc()
1710 device_t dev = ctx->ifc_dev; in iflib_txsd_alloc()
1716 nsegments = scctx->isc_tx_nsegments; in iflib_txsd_alloc()
1717 ntsosegments = scctx->isc_tx_tso_segments_max; in iflib_txsd_alloc()
1718 tsomaxsize = scctx->isc_tx_tso_size_max; in iflib_txsd_alloc()
1719 if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_VLAN_MTU) in iflib_txsd_alloc()
1721 MPASS(scctx->isc_ntxd[0] > 0); in iflib_txsd_alloc()
1722 MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0); in iflib_txsd_alloc()
1724 if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) { in iflib_txsd_alloc()
1726 MPASS(sctx->isc_tso_maxsize >= tsomaxsize); in iflib_txsd_alloc()
1729 lowaddr = DMA_WIDTH_TO_BUS_LOWADDR(scctx->isc_dma_width); in iflib_txsd_alloc()
1739 sctx->isc_tx_maxsize, /* maxsize */ in iflib_txsd_alloc()
1741 sctx->isc_tx_maxsegsize, /* maxsegsize */ in iflib_txsd_alloc()
1745 &txq->ift_buf_tag))) { in iflib_txsd_alloc()
1748 (uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize); in iflib_txsd_alloc()
1751 tso = (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) != 0; in iflib_txsd_alloc()
1759 sctx->isc_tso_maxsegsize,/* maxsegsize */ in iflib_txsd_alloc()
1763 &txq->ift_tso_buf_tag))) { in iflib_txsd_alloc()
1770 if (!(txq->ift_sds.ifsd_m = in iflib_txsd_alloc()
1772 scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_txsd_alloc()
1781 if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc( in iflib_txsd_alloc()
1782 sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], in iflib_txsd_alloc()
1789 if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc( in iflib_txsd_alloc()
1790 sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], in iflib_txsd_alloc()
1797 for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { in iflib_txsd_alloc()
1798 err = bus_dmamap_create(txq->ift_buf_tag, 0, in iflib_txsd_alloc()
1799 &txq->ift_sds.ifsd_map[i]); in iflib_txsd_alloc()
1806 err = bus_dmamap_create(txq->ift_tso_buf_tag, 0, in iflib_txsd_alloc()
1807 &txq->ift_sds.ifsd_tso_map[i]); in iflib_txsd_alloc()
1825 if (txq->ift_sds.ifsd_map != NULL) { in iflib_txsd_destroy()
1826 map = txq->ift_sds.ifsd_map[i]; in iflib_txsd_destroy()
1827 bus_dmamap_sync(txq->ift_buf_tag, map, BUS_DMASYNC_POSTWRITE); in iflib_txsd_destroy()
1828 bus_dmamap_unload(txq->ift_buf_tag, map); in iflib_txsd_destroy()
1829 bus_dmamap_destroy(txq->ift_buf_tag, map); in iflib_txsd_destroy()
1830 txq->ift_sds.ifsd_map[i] = NULL; in iflib_txsd_destroy()
1833 if (txq->ift_sds.ifsd_tso_map != NULL) { in iflib_txsd_destroy()
1834 map = txq->ift_sds.ifsd_tso_map[i]; in iflib_txsd_destroy()
1835 bus_dmamap_sync(txq->ift_tso_buf_tag, map, in iflib_txsd_destroy()
1837 bus_dmamap_unload(txq->ift_tso_buf_tag, map); in iflib_txsd_destroy()
1838 bus_dmamap_destroy(txq->ift_tso_buf_tag, map); in iflib_txsd_destroy()
1839 txq->ift_sds.ifsd_tso_map[i] = NULL; in iflib_txsd_destroy()
1846 if_ctx_t ctx = txq->ift_ctx; in iflib_txq_destroy()
1848 for (int i = 0; i < txq->ift_size; i++) in iflib_txq_destroy()
1851 if (txq->ift_br != NULL) { in iflib_txq_destroy()
1852 ifmp_ring_free(txq->ift_br); in iflib_txq_destroy()
1853 txq->ift_br = NULL; in iflib_txq_destroy()
1856 mtx_destroy(&txq->ift_mtx); in iflib_txq_destroy()
1858 if (txq->ift_sds.ifsd_map != NULL) { in iflib_txq_destroy()
1859 free(txq->ift_sds.ifsd_map, M_IFLIB); in iflib_txq_destroy()
1860 txq->ift_sds.ifsd_map = NULL; in iflib_txq_destroy()
1862 if (txq->ift_sds.ifsd_tso_map != NULL) { in iflib_txq_destroy()
1863 free(txq->ift_sds.ifsd_tso_map, M_IFLIB); in iflib_txq_destroy()
1864 txq->ift_sds.ifsd_tso_map = NULL; in iflib_txq_destroy()
1866 if (txq->ift_sds.ifsd_m != NULL) { in iflib_txq_destroy()
1867 free(txq->ift_sds.ifsd_m, M_IFLIB); in iflib_txq_destroy()
1868 txq->ift_sds.ifsd_m = NULL; in iflib_txq_destroy()
1870 if (txq->ift_buf_tag != NULL) { in iflib_txq_destroy()
1871 bus_dma_tag_destroy(txq->ift_buf_tag); in iflib_txq_destroy()
1872 txq->ift_buf_tag = NULL; in iflib_txq_destroy()
1874 if (txq->ift_tso_buf_tag != NULL) { in iflib_txq_destroy()
1875 bus_dma_tag_destroy(txq->ift_tso_buf_tag); in iflib_txq_destroy()
1876 txq->ift_tso_buf_tag = NULL; in iflib_txq_destroy()
1878 if (txq->ift_ifdi != NULL) { in iflib_txq_destroy()
1879 free(txq->ift_ifdi, M_IFLIB); in iflib_txq_destroy()
1888 mp = &txq->ift_sds.ifsd_m[i]; in iflib_txsd_free()
1892 if (txq->ift_sds.ifsd_map != NULL) { in iflib_txsd_free()
1893 bus_dmamap_sync(txq->ift_buf_tag, in iflib_txsd_free()
1894 txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE); in iflib_txsd_free()
1895 bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]); in iflib_txsd_free()
1897 if (txq->ift_sds.ifsd_tso_map != NULL) { in iflib_txsd_free()
1898 bus_dmamap_sync(txq->ift_tso_buf_tag, in iflib_txsd_free()
1899 txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE); in iflib_txsd_free()
1900 bus_dmamap_unload(txq->ift_tso_buf_tag, in iflib_txsd_free()
1901 txq->ift_sds.ifsd_tso_map[i]); in iflib_txsd_free()
1911 if_ctx_t ctx = txq->ift_ctx; in iflib_txq_setup()
1912 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_txq_setup()
1913 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_txq_setup()
1918 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_txq_setup()
1920 txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ; in iflib_txq_setup()
1923 txq->ift_cidx_processed = 0; in iflib_txq_setup()
1924 txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0; in iflib_txq_setup()
1925 txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset]; in iflib_txq_setup()
1927 for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) in iflib_txq_setup()
1928 bzero((void *)di->idi_vaddr, di->idi_size); in iflib_txq_setup()
1930 IFDI_TXQ_SETUP(ctx, txq->ift_id); in iflib_txq_setup()
1931 for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) in iflib_txq_setup()
1932 bus_dmamap_sync(di->idi_tag, di->idi_map, in iflib_txq_setup()
1951 if_ctx_t ctx = rxq->ifr_ctx; in iflib_rxsd_alloc()
1952 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_rxsd_alloc()
1953 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_rxsd_alloc()
1954 device_t dev = ctx->ifc_dev; in iflib_rxsd_alloc()
1959 MPASS(scctx->isc_nrxd[0] > 0); in iflib_rxsd_alloc()
1960 MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0); in iflib_rxsd_alloc()
1962 lowaddr = DMA_WIDTH_TO_BUS_LOWADDR(scctx->isc_dma_width); in iflib_rxsd_alloc()
1964 fl = rxq->ifr_fl; in iflib_rxsd_alloc()
1965 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { in iflib_rxsd_alloc()
1966 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ in iflib_rxsd_alloc()
1973 sctx->isc_rx_maxsize, /* maxsize */ in iflib_rxsd_alloc()
1974 sctx->isc_rx_nsegments, /* nsegments */ in iflib_rxsd_alloc()
1975 sctx->isc_rx_maxsegsize, /* maxsegsize */ in iflib_rxsd_alloc()
1979 &fl->ifl_buf_tag); in iflib_rxsd_alloc()
1987 if (!(fl->ifl_sds.ifsd_m = in iflib_rxsd_alloc()
1989 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
1997 if (!(fl->ifl_sds.ifsd_cl = in iflib_rxsd_alloc()
1999 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
2007 if (!(fl->ifl_sds.ifsd_ba = in iflib_rxsd_alloc()
2009 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
2019 if (!(fl->ifl_sds.ifsd_map = in iflib_rxsd_alloc()
2020 …(bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOW… in iflib_rxsd_alloc()
2026 for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { in iflib_rxsd_alloc()
2027 err = bus_dmamap_create(fl->ifl_buf_tag, 0, in iflib_rxsd_alloc()
2028 &fl->ifl_sds.ifsd_map[i]); in iflib_rxsd_alloc()
2048 bus_dma_segment_t seg; member
2057 cb_arg->error = error; in _rxq_refill_cb()
2058 cb_arg->seg = segs[0]; in _rxq_refill_cb()
2059 cb_arg->nseg = nseg; in _rxq_refill_cb()
2063 * iflib_fl_refill - refill an rxq free-buffer list
2068 * (Re)populate an rxq free-buffer list with up to @count new packet buffers.
2085 MPASS(count <= fl->ifl_size - fl->ifl_credits - 1); in iflib_fl_refill()
2087 sd_m = fl->ifl_sds.ifsd_m; in iflib_fl_refill()
2088 sd_map = fl->ifl_sds.ifsd_map; in iflib_fl_refill()
2089 sd_cl = fl->ifl_sds.ifsd_cl; in iflib_fl_refill()
2090 sd_ba = fl->ifl_sds.ifsd_ba; in iflib_fl_refill()
2091 pidx = fl->ifl_pidx; in iflib_fl_refill()
2093 frag_idx = fl->ifl_fragidx; in iflib_fl_refill()
2094 credits = fl->ifl_credits; in iflib_fl_refill()
2099 MPASS(credits + n <= fl->ifl_size); in iflib_fl_refill()
2101 if (pidx < fl->ifl_cidx) in iflib_fl_refill()
2102 MPASS(pidx + n <= fl->ifl_cidx); in iflib_fl_refill()
2103 if (pidx == fl->ifl_cidx && (credits < fl->ifl_size)) in iflib_fl_refill()
2104 MPASS(fl->ifl_gen == 0); in iflib_fl_refill()
2105 if (pidx > fl->ifl_cidx) in iflib_fl_refill()
2106 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx); in iflib_fl_refill()
2111 iru_init(&iru, fl->ifl_rxq, fl->ifl_id); in iflib_fl_refill()
2112 while (n-- > 0) { in iflib_fl_refill()
2120 bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, in iflib_fl_refill()
2123 bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); in iflib_fl_refill()
2126 cl = uma_zalloc(fl->ifl_zone, M_NOWAIT); in iflib_fl_refill()
2132 err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2133 cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, in iflib_fl_refill()
2136 uma_zfree(fl->ifl_zone, cl); in iflib_fl_refill()
2140 sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; in iflib_fl_refill()
2143 fl->ifl_cl_enqueued++; in iflib_fl_refill()
2148 bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2157 bit_set(fl->ifl_rx_bitmap, frag_idx); in iflib_fl_refill()
2159 fl->ifl_m_enqueued++; in iflib_fl_refill()
2163 fl->ifl_rxd_idxs[i] = frag_idx; in iflib_fl_refill()
2164 fl->ifl_bus_addrs[i] = bus_addr; in iflib_fl_refill()
2167 MPASS(credits <= fl->ifl_size); in iflib_fl_refill()
2168 if (++idx == fl->ifl_size) { in iflib_fl_refill()
2170 fl->ifl_gen = 1; in iflib_fl_refill()
2177 ctx->isc_rxd_refill(ctx->ifc_softc, &iru); in iflib_fl_refill()
2178 fl->ifl_pidx = idx; in iflib_fl_refill()
2179 fl->ifl_credits = credits; in iflib_fl_refill()
2185 if (n < count - 1) { in iflib_fl_refill()
2189 ctx->isc_rxd_refill(ctx->ifc_softc, &iru); in iflib_fl_refill()
2190 fl->ifl_pidx = idx; in iflib_fl_refill()
2191 fl->ifl_credits = credits; in iflib_fl_refill()
2194 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_refill()
2196 ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, in iflib_fl_refill()
2197 fl->ifl_id, fl->ifl_pidx); in iflib_fl_refill()
2198 if (__predict_true(bit_test(fl->ifl_rx_bitmap, frag_idx))) { in iflib_fl_refill()
2199 fl->ifl_fragidx = frag_idx + 1; in iflib_fl_refill()
2200 if (fl->ifl_fragidx == fl->ifl_size) in iflib_fl_refill()
2201 fl->ifl_fragidx = 0; in iflib_fl_refill()
2203 fl->ifl_fragidx = frag_idx; in iflib_fl_refill()
2207 return (n == -1 ? 0 : IFLIB_RXEOF_EMPTY); in iflib_fl_refill()
2219 * driver to the NIC (RDT - 1 is thus the last valid one). in iflib_fl_refill_all()
2224 int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; in iflib_fl_refill_all()
2226 …int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - … in iflib_fl_refill_all()
2229 MPASS(fl->ifl_credits <= fl->ifl_size); in iflib_fl_refill_all()
2243 in_detach = !!(ctx->ifc_flags & IFC_IN_DETACH); in iflib_in_detach()
2251 iflib_dma_info_t idi = fl->ifl_ifdi; in iflib_fl_bufs_free()
2255 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2256 struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; in iflib_fl_bufs_free()
2257 caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; in iflib_fl_bufs_free()
2260 sd_map = fl->ifl_sds.ifsd_map[i]; in iflib_fl_bufs_free()
2261 bus_dmamap_sync(fl->ifl_buf_tag, sd_map, in iflib_fl_bufs_free()
2263 bus_dmamap_unload(fl->ifl_buf_tag, sd_map); in iflib_fl_bufs_free()
2264 uma_zfree(fl->ifl_zone, *sd_cl); in iflib_fl_bufs_free()
2275 fl->ifl_m_dequeued++; in iflib_fl_bufs_free()
2276 fl->ifl_cl_dequeued++; in iflib_fl_bufs_free()
2280 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2281 MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); in iflib_fl_bufs_free()
2282 MPASS(fl->ifl_sds.ifsd_m[i] == NULL); in iflib_fl_bufs_free()
2288 fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; in iflib_fl_bufs_free()
2289 bzero(idi->idi_vaddr, idi->idi_size); in iflib_fl_bufs_free()
2300 iflib_rxq_t rxq = fl->ifl_rxq; in iflib_fl_setup()
2301 if_ctx_t ctx = rxq->ifr_ctx; in iflib_fl_setup()
2302 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_fl_setup()
2305 bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1); in iflib_fl_setup()
2311 MPASS(fl->ifl_credits == 0); in iflib_fl_setup()
2312 qidx = rxq->ifr_fl_offset + fl->ifl_id; in iflib_fl_setup()
2313 if (scctx->isc_rxd_buf_size[qidx] != 0) in iflib_fl_setup()
2314 fl->ifl_buf_size = scctx->isc_rxd_buf_size[qidx]; in iflib_fl_setup()
2316 fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz; in iflib_fl_setup()
2318 * ifl_buf_size may be a driver-supplied value, so pull it up in iflib_fl_setup()
2321 fl->ifl_buf_size = iflib_get_mbuf_size_for(fl->ifl_buf_size); in iflib_fl_setup()
2322 if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size) in iflib_fl_setup()
2323 ctx->ifc_max_fl_buf_size = fl->ifl_buf_size; in iflib_fl_setup()
2324 fl->ifl_cltype = m_gettype(fl->ifl_buf_size); in iflib_fl_setup()
2325 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_fl_setup()
2328 * Avoid pre-allocating zillions of clusters to an idle card in iflib_fl_setup()
2333 MPASS(fl->ifl_size > 0); in iflib_fl_setup()
2334 (void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size - 1)); in iflib_fl_setup()
2335 if (min(128, fl->ifl_size - 1) != fl->ifl_credits) in iflib_fl_setup()
2341 MPASS(fl->ifl_ifdi != NULL); in iflib_fl_setup()
2342 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_setup()
2358 if (rxq->ifr_fl != NULL) { in iflib_rx_sds_free()
2359 for (i = 0; i < rxq->ifr_nfl; i++) { in iflib_rx_sds_free()
2360 fl = &rxq->ifr_fl[i]; in iflib_rx_sds_free()
2361 if (fl->ifl_buf_tag != NULL) { in iflib_rx_sds_free()
2362 if (fl->ifl_sds.ifsd_map != NULL) { in iflib_rx_sds_free()
2363 for (j = 0; j < fl->ifl_size; j++) { in iflib_rx_sds_free()
2365 fl->ifl_buf_tag, in iflib_rx_sds_free()
2366 fl->ifl_sds.ifsd_map[j], in iflib_rx_sds_free()
2369 fl->ifl_buf_tag, in iflib_rx_sds_free()
2370 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2372 fl->ifl_buf_tag, in iflib_rx_sds_free()
2373 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2376 bus_dma_tag_destroy(fl->ifl_buf_tag); in iflib_rx_sds_free()
2377 fl->ifl_buf_tag = NULL; in iflib_rx_sds_free()
2379 free(fl->ifl_sds.ifsd_m, M_IFLIB); in iflib_rx_sds_free()
2380 free(fl->ifl_sds.ifsd_cl, M_IFLIB); in iflib_rx_sds_free()
2381 free(fl->ifl_sds.ifsd_ba, M_IFLIB); in iflib_rx_sds_free()
2382 free(fl->ifl_sds.ifsd_map, M_IFLIB); in iflib_rx_sds_free()
2383 free(fl->ifl_rx_bitmap, M_IFLIB); in iflib_rx_sds_free()
2384 fl->ifl_sds.ifsd_m = NULL; in iflib_rx_sds_free()
2385 fl->ifl_sds.ifsd_cl = NULL; in iflib_rx_sds_free()
2386 fl->ifl_sds.ifsd_ba = NULL; in iflib_rx_sds_free()
2387 fl->ifl_sds.ifsd_map = NULL; in iflib_rx_sds_free()
2388 fl->ifl_rx_bitmap = NULL; in iflib_rx_sds_free()
2390 free(rxq->ifr_fl, M_IFLIB); in iflib_rx_sds_free()
2391 rxq->ifr_fl = NULL; in iflib_rx_sds_free()
2392 free(rxq->ifr_ifdi, M_IFLIB); in iflib_rx_sds_free()
2393 rxq->ifr_ifdi = NULL; in iflib_rx_sds_free()
2394 rxq->ifr_cq_cidx = 0; in iflib_rx_sds_free()
2405 if_ctx_t ctx = txq->ift_ctx; in iflib_timer()
2406 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; in iflib_timer()
2409 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) in iflib_timer()
2417 if (this_tick - txq->ift_last_timer_tick >= iflib_timer_default) { in iflib_timer()
2418 txq->ift_last_timer_tick = this_tick; in iflib_timer()
2419 IFDI_TIMER(ctx, txq->ift_id); in iflib_timer()
2420 if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) && in iflib_timer()
2421 ((txq->ift_cleaned_prev == txq->ift_cleaned) || in iflib_timer()
2422 (sctx->isc_pause_frames == 0))) in iflib_timer()
2425 if (txq->ift_qstatus != IFLIB_QUEUE_IDLE && in iflib_timer()
2426 ifmp_ring_is_stalled(txq->ift_br)) { in iflib_timer()
2427 KASSERT(ctx->ifc_link_state == LINK_STATE_UP, in iflib_timer()
2429 txq->ift_qstatus = IFLIB_QUEUE_HUNG; in iflib_timer()
2431 txq->ift_cleaned_prev = txq->ift_cleaned; in iflib_timer()
2434 if (txq->ift_db_pending) in iflib_timer()
2435 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_timer()
2437 sctx->isc_pause_frames = 0; in iflib_timer()
2438 if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) in iflib_timer()
2439 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, in iflib_timer()
2440 txq, txq->ift_timer.c_cpu); in iflib_timer()
2444 device_printf(ctx->ifc_dev, in iflib_timer()
2445 "Watchdog timeout (TX: %d desc avail: %d pidx: %d) -- resetting\n", in iflib_timer()
2446 txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx); in iflib_timer()
2448 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in iflib_timer()
2449 ctx->ifc_flags |= (IFC_DO_WATCHDOG | IFC_DO_RESET); in iflib_timer()
2467 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; in iflib_calc_rx_mbuf_sz()
2473 ctx->ifc_rx_mbuf_sz = in iflib_calc_rx_mbuf_sz()
2474 iflib_get_mbuf_size_for(sctx->isc_max_frame_size); in iflib_calc_rx_mbuf_sz()
2481 return (ctx->ifc_rx_mbuf_sz); in iflib_get_rx_mbuf_sz()
2487 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_init_locked()
2488 if_t ifp = ctx->ifc_ifp; in iflib_init_locked()
2503 tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP); in iflib_init_locked()
2504 tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP); in iflib_init_locked()
2516 for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) { in iflib_init_locked()
2518 callout_stop(&txq->ift_timer); in iflib_init_locked()
2520 callout_stop(&txq->ift_netmap_timer); in iflib_init_locked()
2538 for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_init_locked()
2543 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_init_locked()
2545 device_printf(ctx->ifc_dev, in iflib_init_locked()
2546 "setting up free list %d failed - " in iflib_init_locked()
2553 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); in iflib_init_locked()
2555 txq = ctx->ifc_txqs; in iflib_init_locked()
2556 for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) in iflib_init_locked()
2557 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq, in iflib_init_locked()
2558 txq->ift_timer.c_cpu); in iflib_init_locked()
2560 /* Re-enable txsync/rxsync. */ in iflib_init_locked()
2591 iflib_txq_t txq = ctx->ifc_txqs; in iflib_stop()
2592 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_stop()
2593 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_stop()
2594 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_stop()
2600 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in iflib_stop()
2612 netmap_disable_all_rings(ctx->ifc_ifp); in iflib_stop()
2616 for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) { in iflib_stop()
2620 callout_stop(&txq->ift_timer); in iflib_stop()
2622 callout_stop(&txq->ift_netmap_timer); in iflib_stop()
2629 for (j = 0; j < txq->ift_size; j++) { in iflib_stop()
2632 txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0; in iflib_stop()
2633 txq->ift_in_use = txq->ift_gen = txq->ift_no_desc_avail = 0; in iflib_stop()
2634 if (sctx->isc_flags & IFLIB_PRESERVE_TX_INDICES) in iflib_stop()
2635 txq->ift_cidx = txq->ift_pidx; in iflib_stop()
2637 txq->ift_cidx = txq->ift_pidx = 0; in iflib_stop()
2639 txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0; in iflib_stop()
2640 txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0; in iflib_stop()
2641 txq->ift_pullups = 0; in iflib_stop()
2642 ifmp_ring_reset_stats(txq->ift_br); in iflib_stop()
2643 for (j = 0, di = txq->ift_ifdi; j < sctx->isc_ntxqs; j++, di++) in iflib_stop()
2644 bzero((void *)di->idi_vaddr, di->idi_size); in iflib_stop()
2646 for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_stop()
2647 if (rxq->ifr_task.gt_taskqueue != NULL) in iflib_stop()
2648 gtaskqueue_drain(rxq->ifr_task.gt_taskqueue, in iflib_stop()
2649 &rxq->ifr_task.gt_task); in iflib_stop()
2651 rxq->ifr_cq_cidx = 0; in iflib_stop()
2652 for (j = 0, di = rxq->ifr_ifdi; j < sctx->isc_nrxqs; j++, di++) in iflib_stop()
2653 bzero((void *)di->idi_vaddr, di->idi_size); in iflib_stop()
2655 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_stop()
2667 nrxd = fl->ifl_size; in calc_next_rxd()
2668 size = fl->ifl_rxd_size; in calc_next_rxd()
2669 start = fl->ifl_ifdi->idi_vaddr; in calc_next_rxd()
2683 int nrxd = fl->ifl_size; in prefetch_pkts()
2686 nextptr = (cidx + CACHE_PTR_INCREMENT) & (nrxd - 1); in prefetch_pkts()
2687 prefetch(&fl->ifl_sds.ifsd_m[nextptr]); in prefetch_pkts()
2688 prefetch(&fl->ifl_sds.ifsd_cl[nextptr]); in prefetch_pkts()
2691 prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd - 1)]); in prefetch_pkts()
2692 prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd - 1)]); in prefetch_pkts()
2693 prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd - 1)]); in prefetch_pkts()
2694 prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd - 1)]); in prefetch_pkts()
2695 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd - 1)]); in prefetch_pkts()
2696 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd - 1)]); in prefetch_pkts()
2697 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd - 1)]); in prefetch_pkts()
2698 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd - 1)]); in prefetch_pkts()
2712 flid = irf->irf_flid; in rxd_frag_to_sd()
2713 cidx = irf->irf_idx; in rxd_frag_to_sd()
2714 fl = &rxq->ifr_fl[flid]; in rxd_frag_to_sd()
2715 sd->ifsd_fl = fl; in rxd_frag_to_sd()
2716 sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; in rxd_frag_to_sd()
2717 fl->ifl_credits--; in rxd_frag_to_sd()
2719 fl->ifl_m_dequeued++; in rxd_frag_to_sd()
2721 if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) in rxd_frag_to_sd()
2723 next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size - 1); in rxd_frag_to_sd()
2724 prefetch(&fl->ifl_sds.ifsd_map[next]); in rxd_frag_to_sd()
2725 map = fl->ifl_sds.ifsd_map[cidx]; in rxd_frag_to_sd()
2727 bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); in rxd_frag_to_sd()
2729 if (rxq->pfil != NULL && PFIL_HOOKED_IN(rxq->pfil) && pf_rv != NULL && in rxd_frag_to_sd()
2730 irf->irf_len != 0) { in rxd_frag_to_sd()
2731 payload = *sd->ifsd_cl; in rxd_frag_to_sd()
2732 payload += ri->iri_pad; in rxd_frag_to_sd()
2733 len = ri->iri_len - ri->iri_pad; in rxd_frag_to_sd()
2734 *pf_rv = pfil_mem_in(rxq->pfil, payload, len, ri->iri_ifp, &m); in rxd_frag_to_sd()
2756 m = fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2757 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2763 m = fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2764 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2769 if (unload && irf->irf_len != 0) in rxd_frag_to_sd()
2770 bus_dmamap_unload(fl->ifl_buf_tag, map); in rxd_frag_to_sd()
2771 fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size - 1); in rxd_frag_to_sd()
2772 if (__predict_false(fl->ifl_cidx == 0)) in rxd_frag_to_sd()
2773 fl->ifl_gen = 0; in rxd_frag_to_sd()
2774 bit_clear(fl->ifl_rx_bitmap, cidx); in rxd_frag_to_sd()
2792 m = rxd_frag_to_sd(rxq, &ri->iri_frags[i], !consumed, sd, in assemble_segments()
2795 MPASS(*sd->ifsd_cl != NULL); in assemble_segments()
2798 * Exclude zero-length frags & frags from in assemble_segments()
2801 if (ri->iri_frags[i].irf_len == 0 || consumed || in assemble_segments()
2817 padlen = ri->iri_pad; in assemble_segments()
2820 mt->m_next = m; in assemble_segments()
2825 cl = *sd->ifsd_cl; in assemble_segments()
2826 *sd->ifsd_cl = NULL; in assemble_segments()
2830 m_cljset(m, cl, sd->ifsd_fl->ifl_cltype); in assemble_segments()
2834 m->m_data += padlen; in assemble_segments()
2835 ri->iri_len -= padlen; in assemble_segments()
2836 m->m_len = ri->iri_frags[i].irf_len; in assemble_segments()
2837 } while (++i < ri->iri_nfrags); in assemble_segments()
2853 if (ri->iri_nfrags == 1 && in iflib_rxd_pkt_get()
2854 ri->iri_frags[0].irf_len != 0 && in iflib_rxd_pkt_get()
2855 ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) { in iflib_rxd_pkt_get()
2856 m = rxd_frag_to_sd(rxq, &ri->iri_frags[0], false, &sd, in iflib_rxd_pkt_get()
2863 if (!IP_ALIGNED(m) && ri->iri_pad == 0) in iflib_rxd_pkt_get()
2864 m->m_data += 2; in iflib_rxd_pkt_get()
2866 memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); in iflib_rxd_pkt_get()
2867 m->m_len = ri->iri_frags[0].irf_len; in iflib_rxd_pkt_get()
2868 m->m_data += ri->iri_pad; in iflib_rxd_pkt_get()
2869 ri->iri_len -= ri->iri_pad; in iflib_rxd_pkt_get()
2878 m->m_pkthdr.len = ri->iri_len; in iflib_rxd_pkt_get()
2879 m->m_pkthdr.rcvif = ri->iri_ifp; in iflib_rxd_pkt_get()
2880 m->m_flags |= ri->iri_flags; in iflib_rxd_pkt_get()
2881 m->m_pkthdr.ether_vtag = ri->iri_vtag; in iflib_rxd_pkt_get()
2882 m->m_pkthdr.flowid = ri->iri_flowid; in iflib_rxd_pkt_get()
2884 m->m_pkthdr.numa_domain = if_getnumadomain(ri->iri_ifp); in iflib_rxd_pkt_get()
2886 M_HASHTYPE_SET(m, ri->iri_rsstype); in iflib_rxd_pkt_get()
2887 m->m_pkthdr.csum_flags = ri->iri_csum_flags; in iflib_rxd_pkt_get()
2888 m->m_pkthdr.csum_data = ri->iri_csum_data; in iflib_rxd_pkt_get()
2896 CURVNET_SET(if_getvnet(lc->ifp)); in iflib_get_ip_forwarding()
2917 switch (eh->ether_type) { in iflib_check_lro_possible()
2942 GROUPTASK_ENQUEUE(&rxq->ifr_task); in _task_fn_rx_watchdog()
2949 if_ctx_t ctx = rxq->ifr_ctx; in iflib_rxeof()
2950 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_rxeof()
2951 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_rxeof()
2970 ifp = ctx->ifc_ifp; in iflib_rxeof()
2974 if (sctx->isc_flags & IFLIB_HAS_RXCQ) in iflib_rxeof()
2975 cidxp = &rxq->ifr_cq_cidx; in iflib_rxeof()
2977 cidxp = &rxq->ifr_fl[0].ifl_cidx; in iflib_rxeof()
2979 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2996 ri.iri_qsidx = rxq->ifr_id; in iflib_rxeof()
2999 ri.iri_frags = rxq->ifr_frags; in iflib_rxeof()
3000 err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); in iflib_rxeof()
3006 if (sctx->isc_flags & IFLIB_HAS_RXCQ) { in iflib_rxeof()
3009 /* XXX NB: shurd - check if this is still safe */ in iflib_rxeof()
3010 while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) in iflib_rxeof()
3011 rxq->ifr_cq_cidx -= scctx->isc_nrxd[0]; in iflib_rxeof()
3021 avail--; in iflib_rxeof()
3022 budget_left--; in iflib_rxeof()
3029 /* imm_pkt: -- cxgb */ in iflib_rxeof()
3033 mt->m_nextpkt = m; in iflib_rxeof()
3039 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
3044 iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding); in iflib_rxeof()
3048 mh = mh->m_nextpkt; in iflib_rxeof()
3049 m->m_nextpkt = NULL; in iflib_rxeof()
3064 if ((m->m_pkthdr.csum_flags & (CSUM_L4_CALC | CSUM_L4_VALID)) == in iflib_rxeof()
3066 if (lro_possible && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0) in iflib_rxeof()
3080 mt->m_nextpkt = m; in iflib_rxeof()
3095 tcp_lro_flush_all(&rxq->ifr_lc); in iflib_rxeof()
3102 ctx->ifc_flags |= IFC_DO_RESET; in iflib_rxeof()
3108 #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq) - 1)
3113 qidx_t minthresh = txq->ift_size / 8; in txq_max_db_deferred()
3127 qidx_t minthresh = txq->ift_size / 8; in txq_max_rs_deferred()
3128 if (txq->ift_in_use > 4 * minthresh) in txq_max_rs_deferred()
3130 if (txq->ift_in_use > 2 * minthresh) in txq_max_rs_deferred()
3132 if (txq->ift_in_use > minthresh) in txq_max_rs_deferred()
3137 #define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
3138 #define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)
3146 #define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
3147 #define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
3148 #define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ct…
3149 #define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_…
3152 #define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
3153 #define MAX_TX_DESC(ctx) MAX((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max, \
3154 (ctx)->ifc_softc_ctx.isc_tx_nsegments)
3159 if_ctx_t ctx = txq->ift_ctx; in iflib_txd_db_check()
3162 max = TXQ_MAX_DB_DEFERRED(txq, txq->ift_in_use); in iflib_txd_db_check()
3165 if (ring || (txq->ift_db_pending >= max) || (TXQ_AVAIL(txq) <= MAX_TX_DESC(ctx) + 2)) { in iflib_txd_db_check()
3172 dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx; in iflib_txd_db_check()
3173 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_txd_db_check()
3175 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval); in iflib_txd_db_check()
3180 txq->ift_db_pending = txq->ift_npending = 0; in iflib_txd_db_check()
3191 pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx); in print_pkt()
3193 pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag); in print_pkt()
3195 pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto); in print_pkt()
3199 #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO)
3200 #define IS_TX_OFFLOAD4(pi) ((pi)->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP_TSO))
3201 #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO)
3202 #define IS_TX_OFFLOAD6(pi) ((pi)->ipi_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_TSO))
3220 if (__predict_false(m->m_len < sizeof(*eh))) { in iflib_parse_ether_header()
3226 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { in iflib_parse_ether_header()
3227 pi->ipi_etype = ntohs(eh->evl_proto); in iflib_parse_ether_header()
3228 pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; in iflib_parse_ether_header()
3230 pi->ipi_etype = ntohs(eh->evl_encap_proto); in iflib_parse_ether_header()
3231 pi->ipi_ehdrlen = ETHER_HDR_LEN; in iflib_parse_ether_header()
3263 /* Fills out pi->ipi_etype */ in iflib_parse_header_partial()
3269 switch (pi->ipi_etype) { in iflib_parse_header_partial()
3277 miniplen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip)); in iflib_parse_header_partial()
3278 if (__predict_false(m->m_len < miniplen)) { in iflib_parse_header_partial()
3283 if (m->m_len == pi->ipi_ehdrlen) { in iflib_parse_header_partial()
3284 n = m->m_next; in iflib_parse_header_partial()
3287 if (n->m_len >= sizeof(*ip)) { in iflib_parse_header_partial()
3288 ip = (struct ip *)n->m_data; in iflib_parse_header_partial()
3293 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header_partial()
3299 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header_partial()
3302 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header_partial()
3306 pi->ipi_ip_hlen = ip->ip_hl << 2; in iflib_parse_header_partial()
3307 pi->ipi_ipproto = ip->ip_p; in iflib_parse_header_partial()
3308 pi->ipi_ip_tos = ip->ip_tos; in iflib_parse_header_partial()
3309 pi->ipi_flags |= IPI_TX_IPV4; in iflib_parse_header_partial()
3319 if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) { in iflib_parse_header_partial()
3321 if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL)) in iflib_parse_header_partial()
3324 ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header_partial()
3327 pi->ipi_ip_hlen = sizeof(struct ip6_hdr); in iflib_parse_header_partial()
3328 pi->ipi_ipproto = ip6->ip6_nxt; in iflib_parse_header_partial()
3329 pi->ipi_ip_tos = IPV6_TRAFFIC_CLASS(ip6); in iflib_parse_header_partial()
3330 pi->ipi_flags |= IPI_TX_IPV6; in iflib_parse_header_partial()
3336 pi->ipi_csum_flags &= ~CSUM_OFFLOAD; in iflib_parse_header_partial()
3337 pi->ipi_ip_hlen = 0; in iflib_parse_header_partial()
3349 if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx; in iflib_parse_header()
3354 if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) && in iflib_parse_header()
3365 /* Fills out pi->ipi_etype */ in iflib_parse_header()
3366 err = iflib_parse_ether_header(pi, mp, &txq->ift_pullups); in iflib_parse_header()
3371 switch (pi->ipi_etype) { in iflib_parse_header()
3380 minthlen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip) + sizeof(*th)); in iflib_parse_header()
3381 if (__predict_false(m->m_len < minthlen)) { in iflib_parse_header()
3386 if (m->m_len == pi->ipi_ehdrlen) { in iflib_parse_header()
3387 n = m->m_next; in iflib_parse_header()
3389 if (n->m_len >= sizeof(*ip)) { in iflib_parse_header()
3390 ip = (struct ip *)n->m_data; in iflib_parse_header()
3391 if (n->m_len >= (ip->ip_hl << 2) + sizeof(*th)) in iflib_parse_header()
3392 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); in iflib_parse_header()
3394 txq->ift_pullups++; in iflib_parse_header()
3397 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header()
3400 txq->ift_pullups++; in iflib_parse_header()
3403 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header()
3404 if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th)) in iflib_parse_header()
3405 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); in iflib_parse_header()
3408 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header()
3409 if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th)) in iflib_parse_header()
3410 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); in iflib_parse_header()
3412 pi->ipi_ip_hlen = ip->ip_hl << 2; in iflib_parse_header()
3413 pi->ipi_ipproto = ip->ip_p; in iflib_parse_header()
3414 pi->ipi_ip_tos = ip->ip_tos; in iflib_parse_header()
3415 pi->ipi_flags |= IPI_TX_IPV4; in iflib_parse_header()
3419 if (__predict_true(pi->ipi_ipproto == IPPROTO_TCP)) { in iflib_parse_header()
3421 txq->ift_pullups++; in iflib_parse_header()
3422 if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL)) in iflib_parse_header()
3424 th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen); in iflib_parse_header()
3426 pi->ipi_tcp_hflags = tcp_get_flags(th); in iflib_parse_header()
3427 pi->ipi_tcp_hlen = th->th_off << 2; in iflib_parse_header()
3428 pi->ipi_tcp_seq = th->th_seq; in iflib_parse_header()
3431 if (__predict_false(ip->ip_p != IPPROTO_TCP)) in iflib_parse_header()
3436 pi->ipi_csum_flags |= (CSUM_IP_TCP | CSUM_IP); in iflib_parse_header()
3437 th->th_sum = in_pseudo(ip->ip_src.s_addr, in iflib_parse_header()
3438 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); in iflib_parse_header()
3439 pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; in iflib_parse_header()
3440 if (sctx->isc_flags & IFLIB_TSO_INIT_IP) { in iflib_parse_header()
3441 ip->ip_sum = 0; in iflib_parse_header()
3442 ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz); in iflib_parse_header()
3446 if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP)) in iflib_parse_header()
3447 ip->ip_sum = 0; in iflib_parse_header()
3455 struct ip6_hdr *ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header()
3457 pi->ipi_ip_hlen = sizeof(struct ip6_hdr); in iflib_parse_header()
3459 if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) { in iflib_parse_header()
3460 txq->ift_pullups++; in iflib_parse_header()
3461 if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL)) in iflib_parse_header()
3464 th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen); in iflib_parse_header()
3466 /* XXX-BZ this will go badly in case of ext hdrs. */ in iflib_parse_header()
3467 pi->ipi_ipproto = ip6->ip6_nxt; in iflib_parse_header()
3468 pi->ipi_ip_tos = IPV6_TRAFFIC_CLASS(ip6); in iflib_parse_header()
3469 pi->ipi_flags |= IPI_TX_IPV6; in iflib_parse_header()
3473 if (pi->ipi_ipproto == IPPROTO_TCP) { in iflib_parse_header()
3474 …if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) { in iflib_parse_header()
3475 txq->ift_pullups++; in iflib_parse_header()
3476 …if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcph… in iflib_parse_header()
3479 pi->ipi_tcp_hflags = tcp_get_flags(th); in iflib_parse_header()
3480 pi->ipi_tcp_hlen = th->th_off << 2; in iflib_parse_header()
3481 pi->ipi_tcp_seq = th->th_seq; in iflib_parse_header()
3484 if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP)) in iflib_parse_header()
3489 pi->ipi_csum_flags |= CSUM_IP6_TCP; in iflib_parse_header()
3490 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); in iflib_parse_header()
3491 pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; in iflib_parse_header()
3498 pi->ipi_csum_flags &= ~CSUM_OFFLOAD; in iflib_parse_header()
3499 pi->ipi_ip_hlen = 0; in iflib_parse_header()
3518 ifsd_m = txq->ift_sds.ifsd_m; in iflib_remove_mbuf()
3519 ntxd = txq->ift_size; in iflib_remove_mbuf()
3520 pidx = txq->ift_pidx & (ntxd - 1); in iflib_remove_mbuf()
3521 ifsd_m = txq->ift_sds.ifsd_m; in iflib_remove_mbuf()
3524 bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]); in iflib_remove_mbuf()
3525 if (txq->ift_sds.ifsd_tso_map != NULL) in iflib_remove_mbuf()
3526 bus_dmamap_unload(txq->ift_tso_buf_tag, in iflib_remove_mbuf()
3527 txq->ift_sds.ifsd_tso_map[pidx]); in iflib_remove_mbuf()
3529 txq->ift_dequeued++; in iflib_remove_mbuf()
3541 ntxd = txq->ift_size; in calc_next_txd()
3542 size = txq->ift_txd_size[qid]; in calc_next_txd()
3543 start = txq->ift_ifdi[qid].idi_vaddr; in calc_next_txd()
3581 for (n = min_frame_size - (*m_head)->m_pkthdr.len; in iflib_ether_pad()
3582 n > 0; n -= sizeof(pad)) in iflib_ether_pad()
3612 ctx = txq->ift_ctx; in iflib_encap()
3613 sctx = ctx->ifc_sctx; in iflib_encap()
3614 scctx = &ctx->ifc_softc_ctx; in iflib_encap()
3615 segs = txq->ift_segs; in iflib_encap()
3616 ntxd = txq->ift_size; in iflib_encap()
3623 cidx = txq->ift_cidx; in iflib_encap()
3624 pidx = txq->ift_pidx; in iflib_encap()
3625 if (ctx->ifc_flags & IFC_PREFETCH) { in iflib_encap()
3626 next = (cidx + CACHE_PTR_INCREMENT) & (ntxd - 1); in iflib_encap()
3627 if (!(ctx->ifc_flags & IFLIB_HAS_TXCQ)) { in iflib_encap()
3633 prefetch(&txq->ift_sds.ifsd_m[next]); in iflib_encap()
3634 prefetch(&txq->ift_sds.ifsd_map[next]); in iflib_encap()
3635 next = (cidx + CACHE_LINE_SIZE) & (ntxd - 1); in iflib_encap()
3637 map = txq->ift_sds.ifsd_map[pidx]; in iflib_encap()
3638 ifsd_m = txq->ift_sds.ifsd_m; in iflib_encap()
3640 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { in iflib_encap()
3641 buf_tag = txq->ift_tso_buf_tag; in iflib_encap()
3642 max_segs = scctx->isc_tx_tso_segments_max; in iflib_encap()
3643 map = txq->ift_sds.ifsd_tso_map[pidx]; in iflib_encap()
3647 buf_tag = txq->ift_buf_tag; in iflib_encap()
3648 max_segs = scctx->isc_tx_nsegments; in iflib_encap()
3649 map = txq->ift_sds.ifsd_map[pidx]; in iflib_encap()
3651 if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) && in iflib_encap()
3652 __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) { in iflib_encap()
3653 err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size); in iflib_encap()
3662 pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG | M_BCAST | M_MCAST)); in iflib_encap()
3664 pi.ipi_qsidx = txq->ift_id; in iflib_encap()
3665 pi.ipi_len = m_head->m_pkthdr.len; in iflib_encap()
3666 pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags; in iflib_encap()
3667 pi.ipi_vtag = M_HAS_VLANTAG(m_head) ? m_head->m_pkthdr.ether_vtag : 0; in iflib_encap()
3693 txq->ift_mbuf_defrag++; in iflib_encap()
3708 txq->ift_no_tx_dma_setup++; in iflib_encap()
3711 txq->ift_no_tx_dma_setup++; in iflib_encap()
3717 txq->ift_map_failed++; in iflib_encap()
3725 * descriptors - this does not hold true on all drivers, e.g. in iflib_encap()
3729 txq->ift_no_desc_avail++; in iflib_encap()
3733 if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0) in iflib_encap()
3734 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_encap()
3743 txq->ift_rs_pending += nsegs + 1; in iflib_encap()
3744 if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) || in iflib_encap()
3745 iflib_no_tx_batch || (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx) + 2) { in iflib_encap()
3747 txq->ift_rs_pending = 0; in iflib_encap()
3753 MPASS(pidx >= 0 && pidx < txq->ift_size); in iflib_encap()
3757 if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) { in iflib_encap()
3760 MPASS(pi.ipi_new_pidx < txq->ift_size); in iflib_encap()
3762 ndesc = pi.ipi_new_pidx - pi.ipi_pidx; in iflib_encap()
3764 ndesc += txq->ift_size; in iflib_encap()
3765 txq->ift_gen = 1; in iflib_encap()
3774 txq->ift_in_use += ndesc; in iflib_encap()
3775 txq->ift_db_pending += ndesc; in iflib_encap()
3781 txq->ift_pidx = pi.ipi_new_pidx; in iflib_encap()
3782 txq->ift_npending += pi.ipi_ndescs; in iflib_encap()
3786 txq->ift_txd_encap_efbig++; in iflib_encap()
3795 * err can't possibly be non-zero here, so we don't neet to test it in iflib_encap()
3801 txq->ift_mbuf_defrag_failed++; in iflib_encap()
3802 txq->ift_map_failed++; in iflib_encap()
3817 cidx = txq->ift_cidx; in iflib_tx_desc_free()
3818 gen = txq->ift_gen; in iflib_tx_desc_free()
3819 qsize = txq->ift_size; in iflib_tx_desc_free()
3820 mask = qsize - 1; in iflib_tx_desc_free()
3821 ifsd_m = txq->ift_sds.ifsd_m; in iflib_tx_desc_free()
3822 do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH); in iflib_tx_desc_free()
3824 while (n-- > 0) { in iflib_tx_desc_free()
3831 if (m->m_pkthdr.csum_flags & CSUM_TSO) { in iflib_tx_desc_free()
3832 bus_dmamap_sync(txq->ift_tso_buf_tag, in iflib_tx_desc_free()
3833 txq->ift_sds.ifsd_tso_map[cidx], in iflib_tx_desc_free()
3835 bus_dmamap_unload(txq->ift_tso_buf_tag, in iflib_tx_desc_free()
3836 txq->ift_sds.ifsd_tso_map[cidx]); in iflib_tx_desc_free()
3838 bus_dmamap_sync(txq->ift_buf_tag, in iflib_tx_desc_free()
3839 txq->ift_sds.ifsd_map[cidx], in iflib_tx_desc_free()
3841 bus_dmamap_unload(txq->ift_buf_tag, in iflib_tx_desc_free()
3842 txq->ift_sds.ifsd_map[cidx]); in iflib_tx_desc_free()
3845 MPASS(m->m_nextpkt == NULL); in iflib_tx_desc_free()
3849 txq->ift_dequeued++; in iflib_tx_desc_free()
3858 txq->ift_cidx = cidx; in iflib_tx_desc_free()
3859 txq->ift_gen = gen; in iflib_tx_desc_free()
3866 if_ctx_t ctx = txq->ift_ctx; in iflib_completed_tx_reclaim()
3869 MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size); in iflib_completed_tx_reclaim()
3872 * Need a rate-limiting check so that this isn't called every time in iflib_completed_tx_reclaim()
3877 if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) { in iflib_completed_tx_reclaim()
3881 txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments, in iflib_completed_tx_reclaim()
3888 txq->ift_cleaned += reclaim; in iflib_completed_tx_reclaim()
3889 txq->ift_in_use -= reclaim; in iflib_completed_tx_reclaim()
3900 size = r->size; in _ring_peek_one()
3901 next = (cidx + CACHE_PTR_INCREMENT) & (size - 1); in _ring_peek_one()
3902 items = __DEVOLATILE(struct mbuf **, &r->items[0]); in _ring_peek_one()
3904 prefetch(items[(cidx + offset) & (size - 1)]); in _ring_peek_one()
3907 prefetch2cachelines(items[(cidx + offset + 1) & (size - 1)]); in _ring_peek_one()
3908 prefetch2cachelines(items[(cidx + offset + 2) & (size - 1)]); in _ring_peek_one()
3909 prefetch2cachelines(items[(cidx + offset + 3) & (size - 1)]); in _ring_peek_one()
3911 return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size - 1)])); in _ring_peek_one()
3918 ifmp_ring_check_drainage(txq->ift_br, budget); in iflib_txq_check_drain()
3924 iflib_txq_t txq = r->cookie; in iflib_txq_can_drain()
3925 if_ctx_t ctx = txq->ift_ctx; in iflib_txq_can_drain()
3929 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_txq_can_drain()
3931 return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, in iflib_txq_can_drain()
3938 iflib_txq_t txq = r->cookie; in iflib_txq_drain()
3939 if_ctx_t ctx = txq->ift_ctx; in iflib_txq_drain()
3940 if_t ifp = ctx->ifc_ifp; in iflib_txq_drain()
3952 rang = iflib_txd_db_check(txq, reclaimed && txq->ift_db_pending); in iflib_txq_drain()
3953 avail = IDXDIFF(pidx, cidx, r->size); in iflib_txq_drain()
3955 if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) { in iflib_txq_drain()
3961 if (__predict_true(r->items[(cidx + i) & (r->size - 1)] != (void *)txq)) in iflib_txq_drain()
3962 m_freem(r->items[(cidx + i) & (r->size - 1)]); in iflib_txq_drain()
3963 r->items[(cidx + i) & (r->size - 1)] = NULL; in iflib_txq_drain()
3968 if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) { in iflib_txq_drain()
3969 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_txq_drain()
3971 callout_stop(&txq->ift_timer); in iflib_txq_drain()
3981 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_txq_drain()
3987 avail, ctx->ifc_flags, TXQ_AVAIL(txq)); in iflib_txq_drain()
3989 do_prefetch = (ctx->ifc_flags & IFC_PREFETCH); in iflib_txq_drain()
3992 int rem = do_prefetch ? count - i : 0; in iflib_txq_drain()
4010 /* no room - bail out */ in iflib_txq_drain()
4014 /* we can't send this packet - skip it */ in iflib_txq_drain()
4020 bytes_sent += m->m_pkthdr.len; in iflib_txq_drain()
4021 mcast_sent += !!(m->m_flags & M_MCAST); in iflib_txq_drain()
4029 /* deliberate use of bitwise or to avoid gratuitous short-circuit */ in iflib_txq_drain()
4056 txq = r->cookie; in iflib_txq_drain_free()
4058 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_txq_drain_free()
4060 callout_stop(&txq->ift_timer); in iflib_txq_drain_free()
4063 avail = IDXDIFF(pidx, cidx, r->size); in iflib_txq_drain_free()
4065 mp = _ring_peek_one(r, cidx, i, avail - i); in iflib_txq_drain_free()
4080 r = txq->ift_br; in iflib_ifmp_purge()
4081 r->drain = iflib_txq_drain_free; in iflib_ifmp_purge()
4082 r->can_drain = iflib_txq_drain_always; in iflib_ifmp_purge()
4084 ifmp_ring_check_drainage(r, r->size); in iflib_ifmp_purge()
4086 r->drain = iflib_txq_drain; in iflib_ifmp_purge()
4087 r->can_drain = iflib_txq_can_drain; in iflib_ifmp_purge()
4094 if_ctx_t ctx = txq->ift_ctx; in _task_fn_tx()
4095 if_t ifp = ctx->ifc_ifp; in _task_fn_tx()
4096 int abdicate = ctx->ifc_sysctl_tx_abdicate; in _task_fn_tx()
4099 txq->ift_cpu_exec_count[curcpu]++; in _task_fn_tx()
4105 netmap_tx_irq(ifp, txq->ift_id)) in _task_fn_tx()
4112 if (txq->ift_db_pending) in _task_fn_tx()
4113 ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); in _task_fn_tx()
4115 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); in _task_fn_tx()
4120 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); in _task_fn_tx()
4124 if (ctx->ifc_flags & IFC_LEGACY) in _task_fn_tx()
4127 IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); in _task_fn_tx()
4134 if_ctx_t ctx = rxq->ifr_ctx; in _task_fn_rx()
4143 rxq->ifr_cpu_exec_count[curcpu]++; in _task_fn_rx()
4146 if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) in _task_fn_rx()
4149 nmirq = netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work); in _task_fn_rx()
4155 budget = ctx->ifc_sysctl_rx_budget; in _task_fn_rx()
4163 if (ctx->ifc_flags & IFC_LEGACY) in _task_fn_rx()
4166 IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); in _task_fn_rx()
4169 if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) in _task_fn_rx()
4173 GROUPTASK_ENQUEUE(&rxq->ifr_task); in _task_fn_rx()
4175 callout_reset_curcpu(&rxq->ifr_watchdog, 1, &_task_fn_rx_watchdog, rxq); in _task_fn_rx()
4182 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; in _task_fn_admin()
4188 running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); in _task_fn_admin()
4189 oactive = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE); in _task_fn_admin()
4190 do_reset = (ctx->ifc_flags & IFC_DO_RESET); in _task_fn_admin()
4191 do_watchdog = (ctx->ifc_flags & IFC_DO_WATCHDOG); in _task_fn_admin()
4192 in_detach = (ctx->ifc_flags & IFC_IN_DETACH); in _task_fn_admin()
4193 ctx->ifc_flags &= ~(IFC_DO_RESET | IFC_DO_WATCHDOG); in _task_fn_admin()
4196 if ((!running && !oactive) && !(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN)) in _task_fn_admin()
4202 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { in _task_fn_admin()
4204 callout_stop(&txq->ift_timer); in _task_fn_admin()
4207 if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_ADMINCQ) in _task_fn_admin()
4210 ctx->ifc_watchdog_events++; in _task_fn_admin()
4214 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { in _task_fn_admin()
4215 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq, in _task_fn_admin()
4216 txq->ift_timer.c_cpu); in _task_fn_admin()
4225 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) in _task_fn_admin()
4234 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) && in _task_fn_iov()
4235 !(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN)) in _task_fn_iov()
4251 ctx = info->iidi_ctx; in iflib_sysctl_int_delay()
4252 info->iidi_req = req; in iflib_sysctl_int_delay()
4253 info->iidi_oidp = oidp; in iflib_sysctl_int_delay()
4297 MPASS(m->m_nextpkt == NULL); in iflib_if_transmit()
4298 /* ALTQ-enabled interfaces always use queue 0. */ in iflib_if_transmit()
4300 /* Use driver-supplied queue selection method if it exists */ in iflib_if_transmit()
4301 if (ctx->isc_txq_select_v2) { in iflib_if_transmit()
4309 ctx->ifc_txqs[0].ift_pullups += early_pullups; in iflib_if_transmit()
4314 qidx = ctx->isc_txq_select_v2(ctx->ifc_softc, m, &pi); in iflib_if_transmit()
4315 ctx->ifc_txqs[qidx].ift_pullups += early_pullups; in iflib_if_transmit()
4318 else if (ctx->isc_txq_select) in iflib_if_transmit()
4319 qidx = ctx->isc_txq_select(ctx->ifc_softc, m); in iflib_if_transmit()
4325 txq = &ctx->ifc_txqs[qidx]; in iflib_if_transmit()
4328 if (txq->ift_closed) { in iflib_if_transmit()
4330 next = m->m_nextpkt; in iflib_if_transmit()
4331 m->m_nextpkt = NULL; in iflib_if_transmit()
4345 next = next->m_nextpkt; in iflib_if_transmit()
4358 next = next->m_nextpkt; in iflib_if_transmit()
4359 mp[i]->m_nextpkt = NULL; in iflib_if_transmit()
4363 abdicate = ctx->ifc_sysctl_tx_abdicate; in iflib_if_transmit()
4365 err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE, abdicate); in iflib_if_transmit()
4368 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_if_transmit()
4371 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_if_transmit()
4374 txq->ift_closed = TRUE; in iflib_if_transmit()
4376 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); in iflib_if_transmit()
4390 * ALTQ-specific code required in iflib. It is assumed that the overhead of
4409 struct ifaltq *ifq = &ifp->if_snd; /* XXX - DRVAPI */ in iflib_altq_if_start()
4427 IFQ_ENQUEUE(&ifp->if_snd, m, err); /* XXX - DRVAPI */ in iflib_altq_if_transmit()
4441 iflib_txq_t txq = ctx->ifc_txqs; in iflib_if_qflush()
4445 ctx->ifc_flags |= IFC_QFLUSH; in iflib_if_qflush()
4448 while (!(ifmp_ring_is_idle(txq->ift_br) || ifmp_ring_is_stalled(txq->ift_br))) in iflib_if_qflush()
4451 ctx->ifc_flags &= ~IFC_QFLUSH; in iflib_if_qflush()
4480 if (ifa->ifa_addr->sa_family == AF_INET) in iflib_if_ioctl()
4484 if (ifa->ifa_addr->sa_family == AF_INET6) in iflib_if_ioctl()
4504 if (ifr->ifr_mtu == if_getmtu(ifp)) { in iflib_if_ioctl()
4512 if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) { in iflib_if_ioctl()
4514 if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size) in iflib_if_ioctl()
4515 ctx->ifc_flags |= IFC_MULTISEG; in iflib_if_ioctl()
4517 ctx->ifc_flags &= ~IFC_MULTISEG; in iflib_if_ioctl()
4519 err = if_setmtu(ifp, ifr->ifr_mtu); in iflib_if_ioctl()
4531 if ((if_getflags(ifp) ^ ctx->ifc_if_flags) & in iflib_if_ioctl()
4542 ctx->ifc_if_flags = if_getflags(ifp); in iflib_if_ioctl()
4562 err = ifmedia_ioctl(ifp, ifr, ctx->ifc_mediap, command); in iflib_if_ioctl()
4590 mask = ifr->ifr_reqcap ^ oldmask; in iflib_if_ioctl()
4591 mask &= ctx->ifc_softc_ctx.isc_capabilities | IFCAP_MEXTPG; in iflib_if_ioctl()
4604 setmask |= ctx->ifc_softc_ctx.isc_capabilities & in iflib_if_ioctl()
4618 ctx->ifc_softc_ctx.isc_capenable ^= setmask; in iflib_if_ioctl()
4679 /* Re-init to load the changes, if required */ in iflib_vlan_register()
4701 /* Re-init to load the changes, if required */ in iflib_vlan_unregister()
4731 if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC) in iflib_device_probe()
4739 if (sctx->isc_parse_devinfo != NULL) in iflib_device_probe()
4740 sctx->isc_parse_devinfo(&pci_device_id, &pci_subvendor_id, &pci_subdevice_id, &pci_rev_id); in iflib_device_probe()
4742 ent = sctx->isc_vendor_info; in iflib_device_probe()
4743 while (ent->pvi_vendor_id != 0) { in iflib_device_probe()
4744 if (pci_vendor_id != ent->pvi_vendor_id) { in iflib_device_probe()
4748 if ((pci_device_id == ent->pvi_device_id) && in iflib_device_probe()
4749 ((pci_subvendor_id == ent->pvi_subvendor_id) || in iflib_device_probe()
4750 (ent->pvi_subvendor_id == 0)) && in iflib_device_probe()
4751 ((pci_subdevice_id == ent->pvi_subdevice_id) || in iflib_device_probe()
4752 (ent->pvi_subdevice_id == 0)) && in iflib_device_probe()
4753 ((pci_rev_id == ent->pvi_rev_id) || in iflib_device_probe()
4754 (ent->pvi_rev_id == 0))) { in iflib_device_probe()
4755 device_set_desc_copy(dev, ent->pvi_name); in iflib_device_probe()
4757 * ever stops re-probing on best match because the sctx in iflib_device_probe()
4783 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_reset_qvalues()
4784 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_reset_qvalues()
4785 device_t dev = ctx->ifc_dev; in iflib_reset_qvalues()
4788 if (ctx->ifc_sysctl_ntxqs != 0) in iflib_reset_qvalues()
4789 scctx->isc_ntxqsets = ctx->ifc_sysctl_ntxqs; in iflib_reset_qvalues()
4790 if (ctx->ifc_sysctl_nrxqs != 0) in iflib_reset_qvalues()
4791 scctx->isc_nrxqsets = ctx->ifc_sysctl_nrxqs; in iflib_reset_qvalues()
4793 for (i = 0; i < sctx->isc_ntxqs; i++) { in iflib_reset_qvalues()
4794 if (ctx->ifc_sysctl_ntxds[i] != 0) in iflib_reset_qvalues()
4795 scctx->isc_ntxd[i] = ctx->ifc_sysctl_ntxds[i]; in iflib_reset_qvalues()
4797 scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i]; in iflib_reset_qvalues()
4800 for (i = 0; i < sctx->isc_nrxqs; i++) { in iflib_reset_qvalues()
4801 if (ctx->ifc_sysctl_nrxds[i] != 0) in iflib_reset_qvalues()
4802 scctx->isc_nrxd[i] = ctx->ifc_sysctl_nrxds[i]; in iflib_reset_qvalues()
4804 scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i]; in iflib_reset_qvalues()
4807 for (i = 0; i < sctx->isc_nrxqs; i++) { in iflib_reset_qvalues()
4808 if (scctx->isc_nrxd[i] < sctx->isc_nrxd_min[i]) { in iflib_reset_qvalues()
4809 device_printf(dev, "nrxd%d: %d less than nrxd_min %d - resetting to min\n", in iflib_reset_qvalues()
4810 i, scctx->isc_nrxd[i], sctx->isc_nrxd_min[i]); in iflib_reset_qvalues()
4811 scctx->isc_nrxd[i] = sctx->isc_nrxd_min[i]; in iflib_reset_qvalues()
4813 if (scctx->isc_nrxd[i] > sctx->isc_nrxd_max[i]) { in iflib_reset_qvalues()
4814 device_printf(dev, "nrxd%d: %d greater than nrxd_max %d - resetting to max\n", in iflib_reset_qvalues()
4815 i, scctx->isc_nrxd[i], sctx->isc_nrxd_max[i]); in iflib_reset_qvalues()
4816 scctx->isc_nrxd[i] = sctx->isc_nrxd_max[i]; in iflib_reset_qvalues()
4818 if (!powerof2(scctx->isc_nrxd[i])) { in iflib_reset_qvalues()
4819 device_printf(dev, "nrxd%d: %d is not a power of 2 - using default value of %d\n", in iflib_reset_qvalues()
4820 i, scctx->isc_nrxd[i], sctx->isc_nrxd_default[i]); in iflib_reset_qvalues()
4821 scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i]; in iflib_reset_qvalues()
4825 for (i = 0; i < sctx->isc_ntxqs; i++) { in iflib_reset_qvalues()
4826 if (scctx->isc_ntxd[i] < sctx->isc_ntxd_min[i]) { in iflib_reset_qvalues()
4827 device_printf(dev, "ntxd%d: %d less than ntxd_min %d - resetting to min\n", in iflib_reset_qvalues()
4828 i, scctx->isc_ntxd[i], sctx->isc_ntxd_min[i]); in iflib_reset_qvalues()
4829 scctx->isc_ntxd[i] = sctx->isc_ntxd_min[i]; in iflib_reset_qvalues()
4831 if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) { in iflib_reset_qvalues()
4832 device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n", in iflib_reset_qvalues()
4833 i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]); in iflib_reset_qvalues()
4834 scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i]; in iflib_reset_qvalues()
4836 if (!powerof2(scctx->isc_ntxd[i])) { in iflib_reset_qvalues()
4837 device_printf(dev, "ntxd%d: %d is not a power of 2 - using default value of %d\n", in iflib_reset_qvalues()
4838 i, scctx->isc_ntxd[i], sctx->isc_ntxd_default[i]); in iflib_reset_qvalues()
4839 scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i]; in iflib_reset_qvalues()
4855 pa.pa_headname = if_name(ctx->ifc_ifp); in iflib_add_pfil()
4858 for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { in iflib_add_pfil()
4859 rxq->pfil = pfil; in iflib_add_pfil()
4870 rxq = ctx->ifc_rxqs; in iflib_rem_pfil()
4871 pfil = rxq->pfil; in iflib_rem_pfil()
4873 rxq->pfil = NULL; in iflib_rem_pfil()
4880 * Advance forward by n members of the cpuset ctx->ifc_cpus starting from
4890 MPASS(CPU_ISSET(cpuid, &ctx->ifc_cpus)); in cpuid_advance()
4893 MPASS(!CPU_EMPTY(&ctx->ifc_cpus)); in cpuid_advance()
4895 first_valid = CPU_FFS(&ctx->ifc_cpus) - 1; in cpuid_advance()
4896 last_valid = CPU_FLS(&ctx->ifc_cpus) - 1; in cpuid_advance()
4897 n = n % CPU_COUNT(&ctx->ifc_cpus); in cpuid_advance()
4903 } while (!CPU_ISSET(cpuid, &ctx->ifc_cpus)); in cpuid_advance()
4904 n--; in cpuid_advance()
4918 if (grp->cg_children == 0) in find_child_with_core()
4919 return (-1); in find_child_with_core()
4921 MPASS(grp->cg_child); in find_child_with_core()
4922 for (i = 0; i < grp->cg_children; i++) { in find_child_with_core()
4923 if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask)) in find_child_with_core()
4927 return (-1); in find_child_with_core()
4932 * Find an L2 neighbor of the given CPU or return -1 if none found. This
4944 return (-1); in find_l2_neighbor()
4950 while ((i = find_child_with_core(cpu, grp)) != -1) { in find_l2_neighbor()
4956 if (grp->cg_child[i].cg_count <= 1) in find_l2_neighbor()
4957 return (-1); in find_l2_neighbor()
4958 grp = &grp->cg_child[i]; in find_l2_neighbor()
4962 if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE) in find_l2_neighbor()
4963 return (-1); in find_l2_neighbor()
4970 if (CPU_ISSET(i, &grp->cg_mask) && i != cpu) in find_l2_neighbor()
4975 return (-1); in find_l2_neighbor()
4983 return (-1); in find_l2_neighbor()
4989 * ---------------------
4996 * ---------- --------- ------ ------------------------------------------------
4997 * - - X RX and TX queues mapped to consecutive physical
5000 * - X X RX and TX queues mapped to consecutive cores
5003 * X - X RX and TX queues mapped to consecutive physical
5009 * - n/a - RX and TX queues mapped to consecutive cores of
5012 * X n/a - RX and TX queues mapped to consecutive cores of
5019 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in get_cpuid_for_queue()
5022 if (ctx->ifc_sysctl_separate_txrx) { in get_cpuid_for_queue()
5032 if (ctx->ifc_sysctl_use_logical_cores && in get_cpuid_for_queue()
5033 ctx->ifc_cpus_are_physical_cores && in get_cpuid_for_queue()
5034 is_tx && qid < scctx->isc_nrxqsets) { in get_cpuid_for_queue()
5040 if (l2_neighbor != -1) { in get_cpuid_for_queue()
5045 * consecutive-after-RX assignment scheme. in get_cpuid_for_queue()
5055 core_index = scctx->isc_nrxqsets + qid; in get_cpuid_for_queue()
5068 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in get_ctx_core_offset()
5072 unsigned int base_cpuid = ctx->ifc_sysctl_core_offset; in get_ctx_core_offset()
5077 first_valid = CPU_FFS(&ctx->ifc_cpus) - 1; in get_ctx_core_offset()
5078 last_valid = CPU_FLS(&ctx->ifc_cpus) - 1; in get_ctx_core_offset()
5082 * Align the user-chosen base CPU ID to the next valid CPU in get_ctx_core_offset()
5087 * zero-based reference frame, and so we shift the given in get_ctx_core_offset()
5095 /* shift from zero-based to first_valid-based */ in get_ctx_core_offset()
5098 base_cpuid = (base_cpuid - first_valid) % in get_ctx_core_offset()
5099 (last_valid - first_valid + 1); in get_ctx_core_offset()
5101 if (!CPU_ISSET(base_cpuid, &ctx->ifc_cpus)) { in get_ctx_core_offset()
5109 while (!CPU_ISSET(base_cpuid, &ctx->ifc_cpus)) in get_ctx_core_offset()
5126 * neighbors of CPUs that RX queues have been mapped to - in this in get_ctx_core_offset()
5133 for (i = 0; i < scctx->isc_ntxqsets; i++) in get_ctx_core_offset()
5136 for (i = 0; i < scctx->isc_nrxqsets; i++) in get_ctx_core_offset()
5139 CPU_AND(&assigned_cpus, &assigned_cpus, &ctx->ifc_cpus); in get_ctx_core_offset()
5144 if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) { in get_ctx_core_offset()
5145 base_cpuid = op->next_cpuid; in get_ctx_core_offset()
5146 op->next_cpuid = cpuid_advance(ctx, op->next_cpuid, in get_ctx_core_offset()
5148 MPASS(op->refcount < UINT_MAX); in get_ctx_core_offset()
5149 op->refcount++; in get_ctx_core_offset()
5158 device_printf(ctx->ifc_dev, in get_ctx_core_offset()
5161 op->next_cpuid = cpuid_advance(ctx, base_cpuid, in get_ctx_core_offset()
5163 op->refcount = 1; in get_ctx_core_offset()
5164 CPU_COPY(&ctx->ifc_cpus, &op->set); in get_ctx_core_offset()
5180 if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) { in unref_ctx_core_offset()
5181 MPASS(op->refcount > 0); in unref_ctx_core_offset()
5182 op->refcount--; in unref_ctx_core_offset()
5183 if (op->refcount == 0) { in unref_ctx_core_offset()
5208 sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK | M_ZERO); in iflib_device_register()
5210 ctx->ifc_flags |= IFC_SC_ALLOCATED; in iflib_device_register()
5213 ctx->ifc_sctx = sctx; in iflib_device_register()
5214 ctx->ifc_dev = dev; in iflib_device_register()
5215 ctx->ifc_softc = sc; in iflib_device_register()
5223 scctx = &ctx->ifc_softc_ctx; in iflib_device_register()
5224 ifp = ctx->ifc_ifp; in iflib_device_register()
5234 ctx->ifc_txrx = *scctx->isc_txrx; in iflib_device_register()
5236 MPASS(scctx->isc_dma_width <= flsll(BUS_SPACE_MAXADDR)); in iflib_device_register()
5238 if (sctx->isc_flags & IFLIB_DRIVER_MEDIA) in iflib_device_register()
5239 ctx->ifc_mediap = scctx->isc_media; in iflib_device_register()
5242 if (scctx->isc_capabilities & IFCAP_TXCSUM) in iflib_device_register()
5243 MPASS(scctx->isc_tx_csum_flags); in iflib_device_register()
5247 scctx->isc_capabilities | IFCAP_HWSTATS | IFCAP_MEXTPG); in iflib_device_register()
5249 scctx->isc_capenable | IFCAP_HWSTATS | IFCAP_MEXTPG); in iflib_device_register()
5251 …if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_n… in iflib_device_register()
5252 scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; in iflib_device_register()
5253 …if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_n… in iflib_device_register()
5254 scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; in iflib_device_register()
5259 /* XXX change for per-queue sizes */ in iflib_device_register()
5263 if (scctx->isc_tx_nsegments > num_txd / MAX_SINGLE_PACKET_FRACTION) in iflib_device_register()
5264 scctx->isc_tx_nsegments = max(1, num_txd / in iflib_device_register()
5266 if (scctx->isc_tx_tso_segments_max > num_txd / in iflib_device_register()
5268 scctx->isc_tx_tso_segments_max = max(1, in iflib_device_register()
5271 /* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */ in iflib_device_register()
5277 if_sethwtsomax(ifp, min(scctx->isc_tx_tso_size_max, in iflib_device_register()
5287 if_sethwtsomaxsegcount(ifp, scctx->isc_tx_tso_segments_max - 3); in iflib_device_register()
5288 if_sethwtsomaxsegsize(ifp, scctx->isc_tx_tso_segsize_max); in iflib_device_register()
5290 if (scctx->isc_rss_table_size == 0) in iflib_device_register()
5291 scctx->isc_rss_table_size = 64; in iflib_device_register()
5292 scctx->isc_rss_table_mask = scctx->isc_rss_table_size - 1; in iflib_device_register()
5296 ctx->ifc_tq = taskqueue_create_fast(namebuf, M_NOWAIT, in iflib_device_register()
5297 taskqueue_thread_enqueue, &ctx->ifc_tq); in iflib_device_register()
5298 if (ctx->ifc_tq == NULL) { in iflib_device_register()
5303 err = taskqueue_start_threads(&ctx->ifc_tq, 1, PI_NET, "%s", namebuf); in iflib_device_register()
5308 taskqueue_free(ctx->ifc_tq); in iflib_device_register()
5312 TASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx); in iflib_device_register()
5315 if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) { in iflib_device_register()
5317 CPU_COPY(&all_cpus, &ctx->ifc_cpus); in iflib_device_register()
5318 ctx->ifc_cpus_are_physical_cores = false; in iflib_device_register()
5320 ctx->ifc_cpus_are_physical_cores = true; in iflib_device_register()
5321 MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0); in iflib_device_register()
5324 * Now set up MSI or MSI-X, should return us the number of supported in iflib_device_register()
5327 if (sctx->isc_flags & IFLIB_SKIP_MSIX) { in iflib_device_register()
5328 msix = scctx->isc_vectors; in iflib_device_register()
5329 } else if (scctx->isc_msix_bar != 0) in iflib_device_register()
5336 scctx->isc_vectors = 1; in iflib_device_register()
5337 scctx->isc_ntxqsets = 1; in iflib_device_register()
5338 scctx->isc_nrxqsets = 1; in iflib_device_register()
5339 scctx->isc_intr = IFLIB_INTR_LEGACY; in iflib_device_register()
5354 ctx->ifc_sysctl_core_offset = get_ctx_core_offset(ctx); in iflib_device_register()
5358 * When using MSI-X, ensure that ifdi_{r,t}x_queue_intr_enable in iflib_device_register()
5362 kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL, in iflib_device_register()
5364 if (kobj_method == &kobj_desc->deflt) { in iflib_device_register()
5366 "MSI-X requires ifdi_rx_queue_intr_enable method"); in iflib_device_register()
5371 kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL, in iflib_device_register()
5373 if (kobj_method == &kobj_desc->deflt) { in iflib_device_register()
5375 "MSI-X requires ifdi_tx_queue_intr_enable method"); in iflib_device_register()
5381 * Assign the MSI-X vectors. in iflib_device_register()
5391 } else if (scctx->isc_intr != IFLIB_INTR_MSIX) { in iflib_device_register()
5393 if (scctx->isc_intr == IFLIB_INTR_MSI) { in iflib_device_register()
5397 if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) { in iflib_device_register()
5403 "Cannot use iflib with only 1 MSI-X interrupt!\n"); in iflib_device_register()
5409 * It prevents a double-locking panic with iflib_media_status when in iflib_device_register()
5413 ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet); in iflib_device_register()
5430 device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err); in iflib_device_register()
5435 DEBUGNET_SET(ctx->ifc_ifp, iflib); in iflib_device_register()
5437 if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); in iflib_device_register()
5440 ctx->ifc_flags |= IFC_INIT_DONE; in iflib_device_register()
5447 ether_ifdetach(ctx->ifc_ifp); in iflib_device_register()
5449 taskqueue_free(ctx->ifc_tq); in iflib_device_register()
5462 device_set_softc(ctx->ifc_dev, NULL); in iflib_device_register()
5463 if (ctx->ifc_flags & IFC_SC_ALLOCATED) in iflib_device_register()
5464 free(ctx->ifc_softc, M_IFLIB); in iflib_device_register()
5475 if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC) in iflib_device_attach()
5486 if_t ifp = ctx->ifc_ifp; in iflib_device_deregister()
5487 device_t dev = ctx->ifc_dev; in iflib_device_deregister()
5496 device_printf(dev, "SR-IOV in use; detach first.\n"); in iflib_device_deregister()
5502 ctx->ifc_flags |= IFC_IN_DETACH; in iflib_device_deregister()
5516 if (ctx->ifc_led_dev != NULL) in iflib_device_deregister()
5517 led_destroy(ctx->ifc_led_dev); in iflib_device_deregister()
5528 taskqueue_free(ctx->ifc_tq); in iflib_device_deregister()
5529 ctx->ifc_tq = NULL; in iflib_device_deregister()
5531 /* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/ in iflib_device_deregister()
5538 device_set_softc(ctx->ifc_dev, NULL); in iflib_device_deregister()
5539 if (ctx->ifc_flags & IFC_SC_ALLOCATED) in iflib_device_deregister()
5540 free(ctx->ifc_softc, M_IFLIB); in iflib_device_deregister()
5556 for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { in iflib_tqg_detach()
5557 callout_drain(&txq->ift_timer); in iflib_tqg_detach()
5559 callout_drain(&txq->ift_netmap_timer); in iflib_tqg_detach()
5561 if (txq->ift_task.gt_uniq != NULL) in iflib_tqg_detach()
5562 taskqgroup_detach(tqg, &txq->ift_task); in iflib_tqg_detach()
5564 for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { in iflib_tqg_detach()
5565 if (rxq->ifr_task.gt_uniq != NULL) in iflib_tqg_detach()
5566 taskqgroup_detach(tqg, &rxq->ifr_task); in iflib_tqg_detach()
5574 if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { in iflib_free_intr_mem()
5575 iflib_irq_free(ctx, &ctx->ifc_legacy_irq); in iflib_free_intr_mem()
5577 if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { in iflib_free_intr_mem()
5578 pci_release_msi(ctx->ifc_dev); in iflib_free_intr_mem()
5580 if (ctx->ifc_msix_mem != NULL) { in iflib_free_intr_mem()
5581 bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, in iflib_free_intr_mem()
5582 rman_get_rid(ctx->ifc_msix_mem), ctx->ifc_msix_mem); in iflib_free_intr_mem()
5583 ctx->ifc_msix_mem = NULL; in iflib_free_intr_mem()
5622 iflib_txq_t txq = ctx->ifc_txqs; in iflib_device_resume()
5677 * - Start a fast taskqueue thread for each core
5678 * - Start a taskqueue for control operations
5718 MPASS(sctx->isc_tx_maxsize); in _iflib_assert()
5719 MPASS(sctx->isc_tx_maxsegsize); in _iflib_assert()
5721 MPASS(sctx->isc_rx_maxsize); in _iflib_assert()
5722 MPASS(sctx->isc_rx_nsegments); in _iflib_assert()
5723 MPASS(sctx->isc_rx_maxsegsize); in _iflib_assert()
5725 MPASS(sctx->isc_nrxqs >= 1 && sctx->isc_nrxqs <= 8); in _iflib_assert()
5726 for (i = 0; i < sctx->isc_nrxqs; i++) { in _iflib_assert()
5727 MPASS(sctx->isc_nrxd_min[i]); in _iflib_assert()
5728 MPASS(powerof2(sctx->isc_nrxd_min[i])); in _iflib_assert()
5729 MPASS(sctx->isc_nrxd_max[i]); in _iflib_assert()
5730 MPASS(powerof2(sctx->isc_nrxd_max[i])); in _iflib_assert()
5731 MPASS(sctx->isc_nrxd_default[i]); in _iflib_assert()
5732 MPASS(powerof2(sctx->isc_nrxd_default[i])); in _iflib_assert()
5735 MPASS(sctx->isc_ntxqs >= 1 && sctx->isc_ntxqs <= 8); in _iflib_assert()
5736 for (i = 0; i < sctx->isc_ntxqs; i++) { in _iflib_assert()
5737 MPASS(sctx->isc_ntxd_min[i]); in _iflib_assert()
5738 MPASS(powerof2(sctx->isc_ntxd_min[i])); in _iflib_assert()
5739 MPASS(sctx->isc_ntxd_max[i]); in _iflib_assert()
5740 MPASS(powerof2(sctx->isc_ntxd_max[i])); in _iflib_assert()
5741 MPASS(sctx->isc_ntxd_default[i]); in _iflib_assert()
5742 MPASS(powerof2(sctx->isc_ntxd_default[i])); in _iflib_assert()
5750 MPASS(scctx->isc_txrx->ift_txd_encap); in _iflib_pre_assert()
5751 MPASS(scctx->isc_txrx->ift_txd_flush); in _iflib_pre_assert()
5752 MPASS(scctx->isc_txrx->ift_txd_credits_update); in _iflib_pre_assert()
5753 MPASS(scctx->isc_txrx->ift_rxd_available); in _iflib_pre_assert()
5754 MPASS(scctx->isc_txrx->ift_rxd_pkt_get); in _iflib_pre_assert()
5755 MPASS(scctx->isc_txrx->ift_rxd_refill); in _iflib_pre_assert()
5756 MPASS(scctx->isc_txrx->ift_rxd_flush); in _iflib_pre_assert()
5762 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_register()
5763 driver_t *driver = sctx->isc_driver; in iflib_register()
5764 device_t dev = ctx->ifc_dev; in iflib_register()
5770 STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev)); in iflib_register()
5771 ifp = ctx->ifc_ifp = if_alloc_dev(IFT_ETHER, dev); in iflib_register()
5793 ctx->ifc_vlan_attach_event = in iflib_register()
5796 ctx->ifc_vlan_detach_event = in iflib_register()
5800 if ((sctx->isc_flags & IFLIB_DRIVER_MEDIA) == 0) { in iflib_register()
5801 ctx->ifc_mediap = &ctx->ifc_media; in iflib_register()
5802 ifmedia_init(ctx->ifc_mediap, IFM_IMASK, in iflib_register()
5812 if (ctx->ifc_vlan_attach_event != NULL) { in iflib_unregister_vlan_handlers()
5813 EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); in iflib_unregister_vlan_handlers()
5814 ctx->ifc_vlan_attach_event = NULL; in iflib_unregister_vlan_handlers()
5816 if (ctx->ifc_vlan_detach_event != NULL) { in iflib_unregister_vlan_handlers()
5817 EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); in iflib_unregister_vlan_handlers()
5818 ctx->ifc_vlan_detach_event = NULL; in iflib_unregister_vlan_handlers()
5826 if_t ifp = ctx->ifc_ifp; in iflib_deregister()
5829 ifmedia_removeall(&ctx->ifc_media); in iflib_deregister()
5842 /* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/ in iflib_deregister()
5849 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_queues_alloc()
5850 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_queues_alloc()
5851 device_t dev = ctx->ifc_dev; in iflib_queues_alloc()
5852 int nrxqsets = scctx->isc_nrxqsets; in iflib_queues_alloc()
5853 int ntxqsets = scctx->isc_ntxqsets; in iflib_queues_alloc()
5859 uint32_t *rxqsizes = scctx->isc_rxqsizes; in iflib_queues_alloc()
5860 uint32_t *txqsizes = scctx->isc_txqsizes; in iflib_queues_alloc()
5861 uint8_t nrxqs = sctx->isc_nrxqs; in iflib_queues_alloc()
5862 uint8_t ntxqs = sctx->isc_ntxqs; in iflib_queues_alloc()
5863 int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1; in iflib_queues_alloc()
5864 int fl_offset = (sctx->isc_flags & IFLIB_HAS_RXCQ ? 1 : 0); in iflib_queues_alloc()
5874 if (!(ctx->ifc_txqs = in iflib_queues_alloc()
5883 if (!(ctx->ifc_rxqs = in iflib_queues_alloc()
5891 txq = ctx->ifc_txqs; in iflib_queues_alloc()
5892 rxq = ctx->ifc_rxqs; in iflib_queues_alloc()
5907 txq->ift_ifdi = ifdip; in iflib_queues_alloc()
5915 txq->ift_txd_size[j] = scctx->isc_txd_size[j]; in iflib_queues_alloc()
5916 bzero((void *)ifdip->idi_vaddr, txqsizes[j]); in iflib_queues_alloc()
5918 txq->ift_ctx = ctx; in iflib_queues_alloc()
5919 txq->ift_id = i; in iflib_queues_alloc()
5920 if (sctx->isc_flags & IFLIB_HAS_TXCQ) { in iflib_queues_alloc()
5921 txq->ift_br_offset = 1; in iflib_queues_alloc()
5923 txq->ift_br_offset = 0; in iflib_queues_alloc()
5933 snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:TX(%d):callout", in iflib_queues_alloc()
5934 device_get_nameunit(dev), txq->ift_id); in iflib_queues_alloc()
5935 mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); in iflib_queues_alloc()
5936 callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); in iflib_queues_alloc()
5937 txq->ift_timer.c_cpu = cpu; in iflib_queues_alloc()
5939 callout_init_mtx(&txq->ift_netmap_timer, &txq->ift_mtx, 0); in iflib_queues_alloc()
5940 txq->ift_netmap_timer.c_cpu = cpu; in iflib_queues_alloc()
5943 err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, in iflib_queues_alloc()
5954 callout_init(&rxq->ifr_watchdog, 1); in iflib_queues_alloc()
5964 rxq->ifr_ifdi = ifdip; in iflib_queues_alloc()
5966 rxq->ifr_ntxqirq = 1; in iflib_queues_alloc()
5967 rxq->ifr_txqid[0] = i; in iflib_queues_alloc()
5975 bzero((void *)ifdip->idi_vaddr, rxqsizes[j]); in iflib_queues_alloc()
5977 rxq->ifr_ctx = ctx; in iflib_queues_alloc()
5978 rxq->ifr_id = i; in iflib_queues_alloc()
5979 rxq->ifr_fl_offset = fl_offset; in iflib_queues_alloc()
5980 rxq->ifr_nfl = nfree_lists; in iflib_queues_alloc()
5987 rxq->ifr_fl = fl; in iflib_queues_alloc()
5991 fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; in iflib_queues_alloc()
5992 fl[j].ifl_rxd_size = scctx->isc_rxd_size[j]; in iflib_queues_alloc()
6002 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_queues_alloc()
6003 fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, in iflib_queues_alloc()
6011 iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi; in iflib_queues_alloc()
6014 vaddrs[i * ntxqs + j] = di->idi_vaddr; in iflib_queues_alloc()
6015 paddrs[i * ntxqs + j] = di->idi_paddr; in iflib_queues_alloc()
6019 device_printf(ctx->ifc_dev, in iflib_queues_alloc()
6033 iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi; in iflib_queues_alloc()
6036 vaddrs[i * nrxqs + j] = di->idi_vaddr; in iflib_queues_alloc()
6037 paddrs[i * nrxqs + j] = di->idi_paddr; in iflib_queues_alloc()
6041 device_printf(ctx->ifc_dev, in iflib_queues_alloc()
6057 if (ctx->ifc_rxqs != NULL) in iflib_queues_alloc()
6058 free(ctx->ifc_rxqs, M_IFLIB); in iflib_queues_alloc()
6059 ctx->ifc_rxqs = NULL; in iflib_queues_alloc()
6060 if (ctx->ifc_txqs != NULL) in iflib_queues_alloc()
6061 free(ctx->ifc_txqs, M_IFLIB); in iflib_queues_alloc()
6062 ctx->ifc_txqs = NULL; in iflib_queues_alloc()
6070 iflib_txq_t txq = ctx->ifc_txqs; in iflib_tx_structures_setup()
6082 iflib_txq_t txq = ctx->ifc_txqs; in iflib_tx_structures_free()
6083 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_tx_structures_free()
6087 for (j = 0; j < sctx->isc_ntxqs; j++) in iflib_tx_structures_free()
6088 iflib_dma_free(&txq->ift_ifdi[j]); in iflib_tx_structures_free()
6091 free(ctx->ifc_txqs, M_IFLIB); in iflib_tx_structures_free()
6092 ctx->ifc_txqs = NULL; in iflib_tx_structures_free()
6103 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_rx_structures_setup()
6109 for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) { in iflib_rx_structures_setup()
6111 err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp, in iflib_rx_structures_setup()
6113 ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset])); in iflib_rx_structures_setup()
6115 device_printf(ctx->ifc_dev, in iflib_rx_structures_setup()
6120 IFDI_RXQ_SETUP(ctx, rxq->ifr_id); in iflib_rx_structures_setup()
6130 rxq = ctx->ifc_rxqs; in iflib_rx_structures_setup()
6132 tcp_lro_free(&rxq->ifr_lc); in iflib_rx_structures_setup()
6146 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_rx_structures_free()
6147 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_rx_structures_free()
6150 for (i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) { in iflib_rx_structures_free()
6151 for (j = 0; j < sctx->isc_nrxqs; j++) in iflib_rx_structures_free()
6152 iflib_dma_free(&rxq->ifr_ifdi[j]); in iflib_rx_structures_free()
6155 tcp_lro_free(&rxq->ifr_lc); in iflib_rx_structures_free()
6158 free(ctx->ifc_rxqs, M_IFLIB); in iflib_rx_structures_free()
6159 ctx->ifc_rxqs = NULL; in iflib_rx_structures_free()
6172 device_printf(ctx->ifc_dev, "iflib_tx_structures_setup failed: %d\n", err); in iflib_qset_structures_setup()
6177 device_printf(ctx->ifc_dev, "iflib_rx_structures_setup failed: %d\n", err); in iflib_qset_structures_setup()
6200 dev = ctx->ifc_dev; in iflib_irq_set_affinity()
6201 base_cpuid = ctx->ifc_sysctl_core_offset; in iflib_irq_set_affinity()
6204 irq ? irq->ii_res : NULL, name); in iflib_irq_set_affinity()
6210 if (cpuid > ctx->ifc_cpuid_highest) in iflib_irq_set_affinity()
6211 ctx->ifc_cpuid_highest = cpuid; in iflib_irq_set_affinity()
6245 dev = ctx->ifc_dev; in iflib_irq_alloc_generic_subctx()
6246 subdev = subctx->ifc_dev; in iflib_irq_alloc_generic_subctx()
6250 q = &subctx->ifc_rxqs[qid]; in iflib_irq_alloc_generic_subctx()
6251 info = &subctx->ifc_rxqs[qid].ifr_filter_info; in iflib_irq_alloc_generic_subctx()
6252 gtask = &subctx->ifc_rxqs[qid].ifr_task; in iflib_irq_alloc_generic_subctx()
6264 info->ifi_filter = filter; in iflib_irq_alloc_generic_subctx()
6265 info->ifi_filter_arg = filter_arg; in iflib_irq_alloc_generic_subctx()
6266 info->ifi_task = gtask; in iflib_irq_alloc_generic_subctx()
6267 info->ifi_ctx = q; in iflib_irq_alloc_generic_subctx()
6279 if (tqrid != -1) { in iflib_irq_alloc_generic_subctx()
6285 taskqgroup_attach(tqg, gtask, q, dev, irq->ii_res, name); in iflib_irq_alloc_generic_subctx()
6305 info = &ctx->ifc_filter_info; in iflib_irq_alloc_generic()
6311 q = &ctx->ifc_txqs[qid]; in iflib_irq_alloc_generic()
6312 info = &ctx->ifc_txqs[qid].ift_filter_info; in iflib_irq_alloc_generic()
6313 gtask = &ctx->ifc_txqs[qid].ift_task; in iflib_irq_alloc_generic()
6318 ctx->ifc_flags |= IFC_NETMAP_TX_IRQ; in iflib_irq_alloc_generic()
6321 q = &ctx->ifc_rxqs[qid]; in iflib_irq_alloc_generic()
6322 info = &ctx->ifc_rxqs[qid].ifr_filter_info; in iflib_irq_alloc_generic()
6323 gtask = &ctx->ifc_rxqs[qid].ifr_task; in iflib_irq_alloc_generic()
6330 q = &ctx->ifc_rxqs[qid]; in iflib_irq_alloc_generic()
6331 info = &ctx->ifc_rxqs[qid].ifr_filter_info; in iflib_irq_alloc_generic()
6332 gtask = &ctx->ifc_rxqs[qid].ifr_task; in iflib_irq_alloc_generic()
6340 tqrid = -1; in iflib_irq_alloc_generic()
6341 info = &ctx->ifc_filter_info; in iflib_irq_alloc_generic()
6346 device_printf(ctx->ifc_dev, "%s: unknown net intr type\n", in iflib_irq_alloc_generic()
6351 info->ifi_filter = filter; in iflib_irq_alloc_generic()
6352 info->ifi_filter_arg = filter_arg; in iflib_irq_alloc_generic()
6353 info->ifi_task = gtask; in iflib_irq_alloc_generic()
6354 info->ifi_ctx = q; in iflib_irq_alloc_generic()
6356 dev = ctx->ifc_dev; in iflib_irq_alloc_generic()
6365 if (tqrid != -1) { in iflib_irq_alloc_generic()
6371 taskqgroup_attach(tqg, gtask, q, dev, irq->ii_res, name); in iflib_irq_alloc_generic()
6390 q = &ctx->ifc_txqs[qid]; in iflib_softirq_alloc_generic()
6391 gtask = &ctx->ifc_txqs[qid].ift_task; in iflib_softirq_alloc_generic()
6397 q = &ctx->ifc_rxqs[qid]; in iflib_softirq_alloc_generic()
6398 gtask = &ctx->ifc_rxqs[qid].ifr_task; in iflib_softirq_alloc_generic()
6404 TASK_INIT(&ctx->ifc_vflr_task, 0, _task_fn_iov, ctx); in iflib_softirq_alloc_generic()
6411 dev = ctx->ifc_dev; in iflib_softirq_alloc_generic()
6412 taskqgroup_attach(tqg, gtask, q, dev, irq ? irq->ii_res : NULL, in iflib_softirq_alloc_generic()
6421 if (irq->ii_tag) in iflib_irq_free()
6422 bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); in iflib_irq_free()
6424 if (irq->ii_res) in iflib_irq_free()
6425 bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, in iflib_irq_free()
6426 rman_get_rid(irq->ii_res), irq->ii_res); in iflib_irq_free()
6432 iflib_txq_t txq = ctx->ifc_txqs; in iflib_legacy_setup()
6433 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_legacy_setup()
6434 if_irq_t irq = &ctx->ifc_legacy_irq; in iflib_legacy_setup()
6442 info = &rxq->ifr_filter_info; in iflib_legacy_setup()
6443 gtask = &rxq->ifr_task; in iflib_legacy_setup()
6445 rx_only = (ctx->ifc_sctx->isc_flags & IFLIB_SINGLE_IRQ_RX_ONLY) != 0; in iflib_legacy_setup()
6447 ctx->ifc_flags |= IFC_LEGACY; in iflib_legacy_setup()
6448 info->ifi_filter = filter; in iflib_legacy_setup()
6449 info->ifi_filter_arg = filter_arg; in iflib_legacy_setup()
6450 info->ifi_task = gtask; in iflib_legacy_setup()
6451 info->ifi_ctx = rxq; in iflib_legacy_setup()
6453 dev = ctx->ifc_dev; in iflib_legacy_setup()
6460 res = irq->ii_res; in iflib_legacy_setup()
6463 GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq); in iflib_legacy_setup()
6464 taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, dev, res, in iflib_legacy_setup()
6473 ctx->ifc_led_dev = led_create(iflib_led_func, ctx, in iflib_led_create()
6474 device_get_nameunit(ctx->ifc_dev)); in iflib_led_create()
6481 GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task); in iflib_tx_intr_deferred()
6488 GROUPTASK_ENQUEUE(&ctx->ifc_rxqs[rxqid].ifr_task); in iflib_rx_intr_deferred()
6495 taskqueue_enqueue(ctx->ifc_tq, &ctx->ifc_admin_task); in iflib_admin_intr_deferred()
6502 taskqueue_enqueue(ctx->ifc_tq, &ctx->ifc_vflr_task); in iflib_iov_intr_deferred()
6522 taskqueue_enqueue(ctx->ifc_tq, config_task); in iflib_config_task_enqueue()
6528 if_t ifp = ctx->ifc_ifp; in iflib_link_state_change()
6529 iflib_txq_t txq = ctx->ifc_txqs; in iflib_link_state_change()
6534 ctx->ifc_flags |= IFC_PREFETCH; in iflib_link_state_change()
6538 if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) { in iflib_link_state_change()
6539 for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++) in iflib_link_state_change()
6540 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_link_state_change()
6542 ctx->ifc_link_state = link_state; in iflib_link_state_change()
6551 int credits_pre = txq->ift_cidx_processed; in iflib_tx_credits_update()
6554 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_tx_credits_update()
6556 if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0) in iflib_tx_credits_update()
6559 txq->ift_processed += credits; in iflib_tx_credits_update()
6560 txq->ift_cidx_processed += credits; in iflib_tx_credits_update()
6562 MPASS(credits_pre + credits == txq->ift_cidx_processed); in iflib_tx_credits_update()
6563 if (txq->ift_cidx_processed >= txq->ift_size) in iflib_tx_credits_update()
6564 txq->ift_cidx_processed -= txq->ift_size; in iflib_tx_credits_update()
6574 for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++) in iflib_rxd_avail()
6575 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_rxd_avail()
6577 return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx, in iflib_rxd_avail()
6586 info->iidi_ctx = ctx; in iflib_add_int_delay_sysctl()
6587 info->iidi_offset = offset; in iflib_add_int_delay_sysctl()
6588 info->iidi_value = value; in iflib_add_int_delay_sysctl()
6589 SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev), in iflib_add_int_delay_sysctl()
6590 SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)), in iflib_add_int_delay_sysctl()
6599 return (&ctx->ifc_ctx_sx); in iflib_ctx_lock_get()
6605 device_t dev = ctx->ifc_dev; in iflib_msix_init()
6606 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_msix_init()
6607 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_msix_init()
6611 iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs; in iflib_msix_init()
6612 iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs; in iflib_msix_init()
6616 imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets)); in iflib_msix_init()
6619 if (scctx->isc_disable_msix) in iflib_msix_init()
6622 /* First try MSI-X */ in iflib_msix_init()
6625 device_printf(dev, "MSI-X not supported or disabled\n"); in iflib_msix_init()
6629 bar = ctx->ifc_softc_ctx.isc_msix_bar; in iflib_msix_init()
6631 * bar == -1 => "trust me I know what I'm doing" in iflib_msix_init()
6635 * allows shoddy garbage to use MSI-X in this framework. in iflib_msix_init()
6637 if (bar != -1) { in iflib_msix_init()
6638 ctx->ifc_msix_mem = bus_alloc_resource_any(dev, in iflib_msix_init()
6640 if (ctx->ifc_msix_mem == NULL) { in iflib_msix_init()
6641 device_printf(dev, "Unable to map MSI-X table\n"); in iflib_msix_init()
6646 admincnt = sctx->isc_admin_intrcnt; in iflib_msix_init()
6649 queuemsgs = min(msgs - admincnt, 1); in iflib_msix_init()
6651 queuemsgs = msgs - admincnt; in iflib_msix_init()
6658 queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues); in iflib_msix_init()
6662 CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt); in iflib_msix_init()
6668 if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queuemsgs - admincnt) in iflib_msix_init()
6673 if (rx_queues > scctx->isc_nrxqsets) in iflib_msix_init()
6674 rx_queues = scctx->isc_nrxqsets; in iflib_msix_init()
6684 if (tx_queues > scctx->isc_ntxqsets) in iflib_msix_init()
6685 tx_queues = scctx->isc_ntxqsets; in iflib_msix_init()
6687 if (ctx->ifc_sysctl_qs_eq_override == 0) { in iflib_msix_init()
6701 "insufficient number of MSI-X vectors " in iflib_msix_init()
6712 "Unable to allocate sufficient MSI-X vectors " in iflib_msix_init()
6715 if (bar != -1) { in iflib_msix_init()
6717 ctx->ifc_msix_mem); in iflib_msix_init()
6718 ctx->ifc_msix_mem = NULL; in iflib_msix_init()
6722 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", in iflib_msix_init()
6724 scctx->isc_vectors = vectors; in iflib_msix_init()
6725 scctx->isc_nrxqsets = rx_queues; in iflib_msix_init()
6726 scctx->isc_ntxqsets = tx_queues; in iflib_msix_init()
6727 scctx->isc_intr = IFLIB_INTR_MSIX; in iflib_msix_init()
6732 "failed to allocate %d MSI-X vectors, err: %d\n", vectors, in iflib_msix_init()
6734 if (bar != -1) { in iflib_msix_init()
6736 ctx->ifc_msix_mem); in iflib_msix_init()
6737 ctx->ifc_msix_mem = NULL; in iflib_msix_init()
6743 scctx->isc_nrxqsets = 1; in iflib_msix_init()
6744 scctx->isc_ntxqsets = 1; in iflib_msix_init()
6745 scctx->isc_vectors = vectors; in iflib_msix_init()
6748 scctx->isc_intr = IFLIB_INTR_MSI; in iflib_msix_init()
6750 scctx->isc_vectors = 1; in iflib_msix_init()
6752 scctx->isc_intr = IFLIB_INTR_LEGACY; in iflib_msix_init()
6764 uint16_t *state = ((uint16_t *)oidp->oid_arg1); in mp_ring_state_handler()
6805 ndesc = ctx->ifc_sysctl_ntxds; in mp_ndesc_handler()
6806 if (ctx->ifc_sctx) in mp_ndesc_handler()
6807 nqs = ctx->ifc_sctx->isc_ntxqs; in mp_ndesc_handler()
6810 ndesc = ctx->ifc_sysctl_nrxds; in mp_ndesc_handler()
6811 if (ctx->ifc_sctx) in mp_ndesc_handler()
6812 nqs = ctx->ifc_sctx->isc_nrxqs; in mp_ndesc_handler()
6830 if (rc || req->newptr == NULL) in mp_ndesc_handler()
6852 ctx->ifc_sysctl_node = node = SYSCTL_ADD_NODE(ctx_list, child, in iflib_add_device_sysctl_pre()
6858 CTLFLAG_RD, ctx->ifc_sctx->isc_driver_version, "driver version"); in iflib_add_device_sysctl_pre()
6861 CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0, in iflib_add_device_sysctl_pre()
6864 CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0, in iflib_add_device_sysctl_pre()
6867 CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0, in iflib_add_device_sysctl_pre()
6870 CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, in iflib_add_device_sysctl_pre()
6871 "disable MSI-X (default 0)"); in iflib_add_device_sysctl_pre()
6873 CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, "set the RX budget"); in iflib_add_device_sysctl_pre()
6875 CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, in iflib_add_device_sysctl_pre()
6877 ctx->ifc_sysctl_core_offset = CORE_OFFSET_UNSPECIFIED; in iflib_add_device_sysctl_pre()
6879 CTLFLAG_RDTUN, &ctx->ifc_sysctl_core_offset, 0, in iflib_add_device_sysctl_pre()
6882 CTLFLAG_RDTUN, &ctx->ifc_sysctl_separate_txrx, 0, in iflib_add_device_sysctl_pre()
6885 CTLFLAG_RDTUN, &ctx->ifc_sysctl_use_logical_cores, 0, in iflib_add_device_sysctl_pre()
6888 CTLFLAG_RDTUN, &ctx->ifc_sysctl_extra_msix_vectors, 0, in iflib_add_device_sysctl_pre()
6889 …"attempt to reserve the given number of extra MSI-X vectors during driver load for the creation of… in iflib_add_device_sysctl_pre()
6891 CTLFLAG_RDTUN, &ctx->ifc_softc_ctx.isc_vectors, 0, in iflib_add_device_sysctl_pre()
6892 "total # of MSI-X vectors allocated by driver"); in iflib_add_device_sysctl_pre()
6894 /* XXX change for per-queue sizes */ in iflib_add_device_sysctl_pre()
6908 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_add_device_sysctl_post()
6909 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_add_device_sysctl_post()
6923 node = ctx->ifc_sysctl_node; in iflib_add_device_sysctl_post()
6926 if (scctx->isc_ntxqsets > 100) in iflib_add_device_sysctl_post()
6928 else if (scctx->isc_ntxqsets > 10) in iflib_add_device_sysctl_post()
6932 for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) { in iflib_add_device_sysctl_post()
6938 CTLFLAG_RD, &txq->ift_task.gt_cpu, 0, in iflib_add_device_sysctl_post()
6942 CTLFLAG_RD, &txq->ift_dequeued, "total mbufs freed"); in iflib_add_device_sysctl_post()
6944 CTLFLAG_RD, &txq->ift_enqueued, "total mbufs enqueued"); in iflib_add_device_sysctl_post()
6947 CTLFLAG_RD, &txq->ift_mbuf_defrag, in iflib_add_device_sysctl_post()
6950 CTLFLAG_RD, &txq->ift_pullups, in iflib_add_device_sysctl_post()
6954 &txq->ift_mbuf_defrag_failed, "# of times m_defrag failed"); in iflib_add_device_sysctl_post()
6956 "no_desc_avail", CTLFLAG_RD, &txq->ift_no_desc_avail, in iflib_add_device_sysctl_post()
6959 "tx_map_failed", CTLFLAG_RD, &txq->ift_map_failed, in iflib_add_device_sysctl_post()
6962 "txd_encap_efbig", CTLFLAG_RD, &txq->ift_txd_encap_efbig, in iflib_add_device_sysctl_post()
6965 "no_tx_dma_setup", CTLFLAG_RD, &txq->ift_no_tx_dma_setup, in iflib_add_device_sysctl_post()
6968 CTLFLAG_RD, &txq->ift_pidx, 1, "Producer Index"); in iflib_add_device_sysctl_post()
6970 CTLFLAG_RD, &txq->ift_cidx, 1, "Consumer Index"); in iflib_add_device_sysctl_post()
6972 "txq_cidx_processed", CTLFLAG_RD, &txq->ift_cidx_processed, in iflib_add_device_sysctl_post()
6975 CTLFLAG_RD, &txq->ift_in_use, 1, "descriptors in use"); in iflib_add_device_sysctl_post()
6977 "txq_processed", CTLFLAG_RD, &txq->ift_processed, in iflib_add_device_sysctl_post()
6980 CTLFLAG_RD, &txq->ift_cleaned, "total cleaned"); in iflib_add_device_sysctl_post()
6983 __DEVOLATILE(uint64_t *, &txq->ift_br->state), 0, in iflib_add_device_sysctl_post()
6986 "r_enqueues", CTLFLAG_RD, &txq->ift_br->enqueues, in iflib_add_device_sysctl_post()
6989 "r_drops", CTLFLAG_RD, &txq->ift_br->drops, in iflib_add_device_sysctl_post()
6992 "r_starts", CTLFLAG_RD, &txq->ift_br->starts, in iflib_add_device_sysctl_post()
6995 "r_stalls", CTLFLAG_RD, &txq->ift_br->stalls, in iflib_add_device_sysctl_post()
6998 "r_restarts", CTLFLAG_RD, &txq->ift_br->restarts, in iflib_add_device_sysctl_post()
7001 "r_abdications", CTLFLAG_RD, &txq->ift_br->abdications, in iflib_add_device_sysctl_post()
7005 if (scctx->isc_nrxqsets > 100) in iflib_add_device_sysctl_post()
7007 else if (scctx->isc_nrxqsets > 10) in iflib_add_device_sysctl_post()
7011 for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_add_device_sysctl_post()
7017 CTLFLAG_RD, &rxq->ifr_task.gt_cpu, 0, in iflib_add_device_sysctl_post()
7019 if (sctx->isc_flags & IFLIB_HAS_RXCQ) { in iflib_add_device_sysctl_post()
7021 "rxq_cq_cidx", CTLFLAG_RD, &rxq->ifr_cq_cidx, 1, in iflib_add_device_sysctl_post()
7025 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_add_device_sysctl_post()
7032 CTLFLAG_RD, &fl->ifl_pidx, 1, "Producer Index"); in iflib_add_device_sysctl_post()
7034 CTLFLAG_RD, &fl->ifl_cidx, 1, "Consumer Index"); in iflib_add_device_sysctl_post()
7036 CTLFLAG_RD, &fl->ifl_credits, 1, in iflib_add_device_sysctl_post()
7039 CTLFLAG_RD, &fl->ifl_buf_size, 1, "buffer size"); in iflib_add_device_sysctl_post()
7042 "fl_m_enqueued", CTLFLAG_RD, &fl->ifl_m_enqueued, in iflib_add_device_sysctl_post()
7045 "fl_m_dequeued", CTLFLAG_RD, &fl->ifl_m_dequeued, in iflib_add_device_sysctl_post()
7048 "fl_cl_enqueued", CTLFLAG_RD, &fl->ifl_cl_enqueued, in iflib_add_device_sysctl_post()
7051 "fl_cl_dequeued", CTLFLAG_RD, &fl->ifl_cl_dequeued, in iflib_add_device_sysctl_post()
7064 ctx->ifc_flags |= IFC_DO_RESET; in iflib_request_reset()
7074 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { in iflib_fixup_rx()
7075 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); in iflib_fixup_rx()
7076 m->m_data += ETHER_HDR_LEN; in iflib_fixup_rx()
7084 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); in iflib_fixup_rx()
7085 m->m_data += ETHER_HDR_LEN; in iflib_fixup_rx()
7086 m->m_len -= ETHER_HDR_LEN; in iflib_fixup_rx()
7087 n->m_len = ETHER_HDR_LEN; in iflib_fixup_rx()
7089 n->m_next = m; in iflib_fixup_rx()
7104 *ncl = ctx->ifc_rxqs[0].ifr_fl->ifl_size; in iflib_debugnet_init()
7105 *clsize = ctx->ifc_rxqs[0].ifr_fl->ifl_buf_size; in iflib_debugnet_init()
7119 scctx = &ctx->ifc_softc_ctx; in iflib_debugnet_event()
7123 for (i = 0; i < scctx->isc_nrxqsets; i++) { in iflib_debugnet_event()
7124 rxq = &ctx->ifc_rxqs[i]; in iflib_debugnet_event()
7125 for (j = 0; j < rxq->ifr_nfl; j++) { in iflib_debugnet_event()
7126 fl = rxq->ifr_fl; in iflib_debugnet_event()
7127 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_debugnet_event()
7149 txq = &ctx->ifc_txqs[0]; in iflib_debugnet_transmit()
7166 scctx = &ctx->ifc_softc_ctx; in iflib_debugnet_poll()
7172 txq = &ctx->ifc_txqs[0]; in iflib_debugnet_poll()
7176 for (i = 0; i < scctx->isc_nrxqsets; i++) in iflib_debugnet_poll()
7177 (void)iflib_rxeof(&ctx->ifc_rxqs[i], 16 /* XXX */); in iflib_debugnet_poll()