Lines Matching +full:dma +full:- +full:info
1 /*-
2 * Copyright (c) 2014-2018, Matthew Macy <mmacy@mattmacy.io>
107 * - Prefetching in tx cleaning should perhaps be a tunable. The distance ahead
110 * - NFLX's m_free path is dominated by vm-based M_EXT manipulation which
112 * - small packet forwarding which is just returning a single mbuf to
119 * - private structures
120 * - iflib private utility functions
121 * - ifnet functions
122 * - vlan registry and other exported functions
123 * - iflib public core functions
223 return (ctx->ifc_softc); in iflib_get_softc()
230 return (ctx->ifc_dev); in iflib_get_dev()
237 return (ctx->ifc_ifp); in iflib_get_ifp()
244 return (ctx->ifc_mediap); in iflib_get_media()
251 bcopy(mac, ctx->ifc_mac.octet, ETHER_ADDR_LEN); in iflib_set_mac()
258 return (&ctx->ifc_softc_ctx); in iflib_get_softc_ctx()
265 return (ctx->ifc_sctx); in iflib_get_sctx()
272 return (ctx->ifc_sysctl_extra_msix_vectors); in iflib_get_extra_msix_vectors_sysctl()
275 #define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2)
277 #define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr) + CACHE_LINE_SIZE - 1) & (CACHE_LINE_SIZE -…
279 #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
280 #define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF)
308 /* this should really scale with ring size - this is a fairly arbitrary value */
425 used = pidx - cidx; in get_inuse()
427 used = size - cidx + pidx; in get_inuse()
438 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq-…
441 ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
505 pi_pad->pkt_val[0] = 0; pi_pad->pkt_val[1] = 0; pi_pad->pkt_val[2] = 0; in pkt_info_zero()
506 pi_pad->pkt_val[3] = 0; pi_pad->pkt_val[4] = 0; pi_pad->pkt_val[5] = 0; in pkt_info_zero()
508 pi_pad->pkt_val[6] = 0; pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0; in pkt_info_zero()
509 pi_pad->pkt_val[9] = 0; pi_pad->pkt_val[10] = 0; in pkt_info_zero()
521 ri_pad->rxd_val[i] = 0; in rxd_info_zero()
522 ri_pad->rxd_val[i + 1] = 0; in rxd_info_zero()
523 ri_pad->rxd_val[i + 2] = 0; in rxd_info_zero()
524 ri_pad->rxd_val[i + 3] = 0; in rxd_info_zero()
527 ri_pad->rxd_val[RXD_INFO_SIZE - 1] = 0; in rxd_info_zero()
535 #define IF_BAD_DMA ((bus_addr_t)-1)
537 #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))
539 #define CTX_LOCK_INIT(_sc) sx_init(&(_sc)->ifc_ctx_sx, "iflib ctx lock")
540 #define CTX_LOCK(ctx) sx_xlock(&(ctx)->ifc_ctx_sx)
541 #define CTX_UNLOCK(ctx) sx_xunlock(&(ctx)->ifc_ctx_sx)
542 #define CTX_LOCK_DESTROY(ctx) sx_destroy(&(ctx)->ifc_ctx_sx)
544 #define STATE_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_state_mtx, _name, "iflib state lock", MTX_…
545 #define STATE_LOCK(ctx) mtx_lock(&(ctx)->ifc_state_mtx)
546 #define STATE_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_state_mtx)
547 #define STATE_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_state_mtx)
549 #define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx)
550 #define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)
552 /* Our boot-time initialization hook */
744 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_num_rx_descs()
745 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_num_rx_descs()
746 uint16_t first_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; in iflib_num_rx_descs()
748 return (scctx->isc_nrxd[first_rxq]); in iflib_num_rx_descs()
754 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_num_tx_descs()
755 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_num_tx_descs()
756 uint16_t first_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; in iflib_num_tx_descs()
758 return (scctx->isc_ntxd[first_txq]); in iflib_num_tx_descs()
772 * device-specific sysctl variables:
804 if_t ifp = na->ifp; in iflib_netmap_register()
816 * ifp->if_transmit. This is done once the device has been stopped in iflib_netmap_register()
838 iflib_netmap_config(struct netmap_adapter *na, struct nm_config_info *info) in iflib_netmap_config() argument
840 if_t ifp = na->ifp; in iflib_netmap_config()
842 iflib_rxq_t rxq = &ctx->ifc_rxqs[0]; in iflib_netmap_config()
843 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_config()
845 info->num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets; in iflib_netmap_config()
846 info->num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets; in iflib_netmap_config()
847 info->num_tx_descs = iflib_num_tx_descs(ctx); in iflib_netmap_config()
848 info->num_rx_descs = iflib_num_rx_descs(ctx); in iflib_netmap_config()
849 info->rx_buf_maxsize = fl->ifl_buf_size; in iflib_netmap_config()
851 info->num_tx_rings, info->num_rx_rings, info->num_tx_descs, in iflib_netmap_config()
852 info->num_rx_descs, info->rx_buf_maxsize); in iflib_netmap_config()
860 struct netmap_adapter *na = kring->na; in netmap_fl_refill()
861 u_int const lim = kring->nkr_num_slots - 1; in netmap_fl_refill()
862 struct netmap_ring *ring = kring->ring; in netmap_fl_refill()
865 if_ctx_t ctx = rxq->ifr_ctx; in netmap_fl_refill()
866 iflib_fl_t fl = &rxq->ifr_fl[0]; in netmap_fl_refill()
878 * such a way to keep fl->ifl_pidx and kring->nr_hwcur in sync in netmap_fl_refill()
879 * (except for kring->nkr_hwofs). These may be less than in netmap_fl_refill()
880 * kring->nkr_num_slots if netmap_reset() was called while in netmap_fl_refill()
886 * (fl->ifl_pidx - 1) % N (included), to avoid the NIC tail/prod in netmap_fl_refill()
891 n = kring->nkr_num_slots - nm_kr_rxspace(kring); in netmap_fl_refill()
893 n = kring->rhead - kring->nr_hwcur; in netmap_fl_refill()
897 n += kring->nkr_num_slots; in netmap_fl_refill()
901 map = fl->ifl_sds.ifsd_map; in netmap_fl_refill()
902 nic_i = fl->ifl_pidx; in netmap_fl_refill()
910 MPASS(nm_i == kring->nr_hwtail); in netmap_fl_refill()
912 MPASS(nm_i == kring->nr_hwcur); in netmap_fl_refill()
920 for (i = 0; n > 0 && i < IFLIB_MAX_RX_REFRESH; n--, i++) { in netmap_fl_refill()
921 struct netmap_slot *slot = &ring->slot[nm_i]; in netmap_fl_refill()
930 fl->ifl_bus_addrs[i] = paddr + in netmap_fl_refill()
932 fl->ifl_rxd_idxs[i] = nic_i; in netmap_fl_refill()
935 netmap_load_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
937 } else if (slot->flags & NS_BUF_CHANGED) { in netmap_fl_refill()
939 netmap_reload_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
942 bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i], in netmap_fl_refill()
944 slot->flags &= ~NS_BUF_CHANGED; in netmap_fl_refill()
952 ctx->isc_rxd_refill(ctx->ifc_softc, &iru); in netmap_fl_refill()
954 fl->ifl_pidx = nic_i; in netmap_fl_refill()
959 MPASS(nm_i == kring->rhead); in netmap_fl_refill()
960 kring->nr_hwcur = nm_i; in netmap_fl_refill()
962 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in netmap_fl_refill()
964 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, in netmap_fl_refill()
977 * Userspace wants to send packets up to the one before kring->rhead,
978 * kernel knows kring->nr_hwcur is the first unsent packet.
990 struct netmap_adapter *na = kring->na; in iflib_netmap_txsync()
991 if_t ifp = na->ifp; in iflib_netmap_txsync()
992 struct netmap_ring *ring = kring->ring; in iflib_netmap_txsync()
995 u_int const lim = kring->nkr_num_slots - 1; in iflib_netmap_txsync()
996 u_int const head = kring->rhead; in iflib_netmap_txsync()
1004 u_int report_frequency = kring->nkr_num_slots >> 1; in iflib_netmap_txsync()
1005 /* device-specific */ in iflib_netmap_txsync()
1007 iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id]; in iflib_netmap_txsync()
1009 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_netmap_txsync()
1034 nm_i = kring->nr_hwcur; in iflib_netmap_txsync()
1037 int nic_i_start = -1, flags = 0; in iflib_netmap_txsync()
1039 pi.ipi_segs = txq->ift_segs; in iflib_netmap_txsync()
1040 pi.ipi_qsidx = kring->ring_id; in iflib_netmap_txsync()
1043 __builtin_prefetch(&ring->slot[nm_i]); in iflib_netmap_txsync()
1044 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]); in iflib_netmap_txsync()
1045 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]); in iflib_netmap_txsync()
1048 struct netmap_slot *slot = &ring->slot[nm_i]; in iflib_netmap_txsync()
1050 u_int len = slot->len; in iflib_netmap_txsync()
1054 flags |= (slot->flags & NS_REPORT || in iflib_netmap_txsync()
1072 if (!(slot->flags & NS_MOREFRAG)) { in iflib_netmap_txsync()
1080 ctx->isc_txd_encap(ctx->ifc_softc, &pi); in iflib_netmap_txsync()
1087 /* Reinit per-packet info for the next one. */ in iflib_netmap_txsync()
1089 nic_i_start = -1; in iflib_netmap_txsync()
1093 __builtin_prefetch(&ring->slot[nm_i + 1]); in iflib_netmap_txsync()
1094 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]); in iflib_netmap_txsync()
1095 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]); in iflib_netmap_txsync()
1099 if (slot->flags & NS_BUF_CHANGED) { in iflib_netmap_txsync()
1101 netmap_reload_map(na, txq->ift_buf_tag, in iflib_netmap_txsync()
1102 txq->ift_sds.ifsd_map[nic_i], addr); in iflib_netmap_txsync()
1105 bus_dmamap_sync(txq->ift_buf_tag, in iflib_netmap_txsync()
1106 txq->ift_sds.ifsd_map[nic_i], in iflib_netmap_txsync()
1109 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED | NS_MOREFRAG); in iflib_netmap_txsync()
1113 kring->nr_hwcur = nm_i; in iflib_netmap_txsync()
1116 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_netmap_txsync()
1120 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i); in iflib_netmap_txsync()
1128 * trigger a per-tx-queue timer to try again later. in iflib_netmap_txsync()
1130 if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { in iflib_netmap_txsync()
1133 nic_i = txq->ift_cidx_processed; in iflib_netmap_txsync()
1134 kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); in iflib_netmap_txsync()
1138 if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) in iflib_netmap_txsync()
1139 if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { in iflib_netmap_txsync()
1140 callout_reset_sbt_on(&txq->ift_netmap_timer, in iflib_netmap_txsync()
1143 txq->ift_netmap_timer.c_cpu, 0); in iflib_netmap_txsync()
1158 * On call, kring->rhead is the first packet that userspace wants
1159 * to keep, and kring->rcur is the wakeup point.
1160 * The kernel has previously reported packets up to kring->rtail.
1168 struct netmap_adapter *na = kring->na; in iflib_netmap_rxsync()
1169 struct netmap_ring *ring = kring->ring; in iflib_netmap_rxsync()
1170 if_t ifp = na->ifp; in iflib_netmap_rxsync()
1174 u_int const lim = kring->nkr_num_slots - 1; in iflib_netmap_rxsync()
1175 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; in iflib_netmap_rxsync()
1179 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_netmap_rxsync()
1180 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_netmap_rxsync()
1181 iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id]; in iflib_netmap_rxsync()
1182 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_rxsync()
1191 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_netmap_rxsync()
1203 * nic_i = fl->ifl_cidx; in iflib_netmap_rxsync()
1204 * nm_i = kring->nr_hwtail (previous) in iflib_netmap_rxsync()
1206 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size in iflib_netmap_rxsync()
1208 * fl->ifl_cidx is set to 0 on a ring reinit in iflib_netmap_rxsync()
1211 uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim); in iflib_netmap_rxsync()
1212 bool have_rxcq = sctx->isc_flags & IFLIB_HAS_RXCQ; in iflib_netmap_rxsync()
1221 cidxp = &rxq->ifr_cq_cidx; in iflib_netmap_rxsync()
1223 cidxp = &fl->ifl_cidx; in iflib_netmap_rxsync()
1224 avail = ctx->isc_rxd_available(ctx->ifc_softc, in iflib_netmap_rxsync()
1225 rxq->ifr_id, *cidxp, USHRT_MAX); in iflib_netmap_rxsync()
1227 nic_i = fl->ifl_cidx; in iflib_netmap_rxsync()
1229 MPASS(nm_i == kring->nr_hwtail); in iflib_netmap_rxsync()
1230 for (n = 0; avail > 0 && nm_i != hwtail_lim; n++, avail--) { in iflib_netmap_rxsync()
1232 ri.iri_frags = rxq->ifr_frags; in iflib_netmap_rxsync()
1233 ri.iri_qsidx = kring->ring_id; in iflib_netmap_rxsync()
1234 ri.iri_ifp = ctx->ifc_ifp; in iflib_netmap_rxsync()
1237 error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); in iflib_netmap_rxsync()
1240 ring->slot[nm_i].len = 0; in iflib_netmap_rxsync()
1241 ring->slot[nm_i].flags = 0; in iflib_netmap_rxsync()
1243 ring->slot[nm_i].len = ri.iri_frags[i].irf_len; in iflib_netmap_rxsync()
1244 if (i == (ri.iri_nfrags - 1)) { in iflib_netmap_rxsync()
1245 ring->slot[nm_i].len -= crclen; in iflib_netmap_rxsync()
1246 ring->slot[nm_i].flags = 0; in iflib_netmap_rxsync()
1252 ring->slot[nm_i].flags = NS_MOREFRAG; in iflib_netmap_rxsync()
1255 bus_dmamap_sync(fl->ifl_buf_tag, in iflib_netmap_rxsync()
1256 fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); in iflib_netmap_rxsync()
1258 fl->ifl_cidx = nic_i = nm_next(nic_i, lim); in iflib_netmap_rxsync()
1263 while (*cidxp >= scctx->isc_nrxd[0]) in iflib_netmap_rxsync()
1264 *cidxp -= scctx->isc_nrxd[0]; in iflib_netmap_rxsync()
1274 kring->nr_hwtail = nm_i; in iflib_netmap_rxsync()
1276 kring->nr_kflags &= ~NKR_PENDINTR; in iflib_netmap_rxsync()
1280 * (kring->nr_hwcur to head excluded), in iflib_netmap_rxsync()
1284 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size in iflib_netmap_rxsync()
1297 if_ctx_t ctx = if_getsoftc(na->ifp); in iflib_netmap_intr()
1315 na.ifp = ctx->ifc_ifp; in iflib_netmap_attach()
1317 MPASS(ctx->ifc_softc_ctx.isc_ntxqsets); in iflib_netmap_attach()
1318 MPASS(ctx->ifc_softc_ctx.isc_nrxqsets); in iflib_netmap_attach()
1327 na.num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets; in iflib_netmap_attach()
1328 na.num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets; in iflib_netmap_attach()
1335 struct netmap_adapter *na = NA(ctx->ifc_ifp); in iflib_netmap_txq_init()
1338 slot = netmap_reset(na, NR_TX, txq->ift_id, 0); in iflib_netmap_txq_init()
1341 for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) { in iflib_netmap_txq_init()
1349 int si = netmap_idx_n2k(na->tx_rings[txq->ift_id], i); in iflib_netmap_txq_init()
1350 netmap_load_map(na, txq->ift_buf_tag, txq->ift_sds.ifsd_map[i], in iflib_netmap_txq_init()
1359 struct netmap_adapter *na = NA(ctx->ifc_ifp); in iflib_netmap_rxq_init()
1363 slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0); in iflib_netmap_rxq_init()
1366 kring = na->rx_rings[rxq->ifr_id]; in iflib_netmap_rxq_init()
1375 if_ctx_t ctx = txq->ift_ctx; in iflib_netmap_timer()
1381 netmap_tx_irq(ctx->ifc_ifp, txq->ift_id); in iflib_netmap_timer()
1429 fl = &rxq->ifr_fl[flid]; in iru_init()
1430 iru->iru_paddrs = fl->ifl_bus_addrs; in iru_init()
1431 iru->iru_idxs = fl->ifl_rxd_idxs; in iru_init()
1432 iru->iru_qsidx = rxq->ifr_id; in iru_init()
1433 iru->iru_buf_size = fl->ifl_buf_size; in iru_init()
1434 iru->iru_flidx = fl->ifl_id; in iru_init()
1447 BUS_SPACE_MAXADDR : (1ULL << (width)) - 1ULL)
1450 iflib_dma_alloc_align(if_ctx_t ctx, int size, int align, iflib_dma_info_t dma, int mapflags) in iflib_dma_alloc_align() argument
1453 device_t dev = ctx->ifc_dev; in iflib_dma_alloc_align()
1456 lowaddr = DMA_WIDTH_TO_BUS_LOWADDR(ctx->ifc_softc_ctx.isc_dma_width); in iflib_dma_alloc_align()
1469 &dma->idi_tag); in iflib_dma_alloc_align()
1477 err = bus_dmamem_alloc(dma->idi_tag, (void **)&dma->idi_vaddr, in iflib_dma_alloc_align()
1478 BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->idi_map); in iflib_dma_alloc_align()
1486 dma->idi_paddr = IF_BAD_DMA; in iflib_dma_alloc_align()
1487 err = bus_dmamap_load(dma->idi_tag, dma->idi_map, dma->idi_vaddr, in iflib_dma_alloc_align()
1488 size, _iflib_dmamap_cb, &dma->idi_paddr, mapflags | BUS_DMA_NOWAIT); in iflib_dma_alloc_align()
1489 if (err || dma->idi_paddr == IF_BAD_DMA) { in iflib_dma_alloc_align()
1496 dma->idi_size = size; in iflib_dma_alloc_align()
1500 bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map); in iflib_dma_alloc_align()
1502 bus_dma_tag_destroy(dma->idi_tag); in iflib_dma_alloc_align()
1504 dma->idi_tag = NULL; in iflib_dma_alloc_align()
1510 iflib_dma_alloc(if_ctx_t ctx, int size, iflib_dma_info_t dma, int mapflags) in iflib_dma_alloc() argument
1512 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_dma_alloc()
1514 KASSERT(sctx->isc_q_align != 0, ("alignment value not initialized")); in iflib_dma_alloc()
1516 return (iflib_dma_alloc_align(ctx, size, sctx->isc_q_align, dma, mapflags)); in iflib_dma_alloc()
1536 iflib_dma_free(iflib_dma_info_t dma) in iflib_dma_free() argument
1538 if (dma->idi_tag == NULL) in iflib_dma_free()
1540 if (dma->idi_paddr != IF_BAD_DMA) { in iflib_dma_free()
1541 bus_dmamap_sync(dma->idi_tag, dma->idi_map, in iflib_dma_free()
1543 bus_dmamap_unload(dma->idi_tag, dma->idi_map); in iflib_dma_free()
1544 dma->idi_paddr = IF_BAD_DMA; in iflib_dma_free()
1546 if (dma->idi_vaddr != NULL) { in iflib_dma_free()
1547 bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map); in iflib_dma_free()
1548 dma->idi_vaddr = NULL; in iflib_dma_free()
1550 bus_dma_tag_destroy(dma->idi_tag); in iflib_dma_free()
1551 dma->idi_tag = NULL; in iflib_dma_free()
1567 iflib_filter_info_t info = arg; in iflib_fast_intr() local
1568 struct grouptask *gtask = info->ifi_task; in iflib_fast_intr()
1572 if (info->ifi_filter != NULL) { in iflib_fast_intr()
1573 result = info->ifi_filter(info->ifi_filter_arg); in iflib_fast_intr()
1585 iflib_filter_info_t info = arg; in iflib_fast_intr_rxtx() local
1586 struct grouptask *gtask = info->ifi_task; in iflib_fast_intr_rxtx()
1588 iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; in iflib_fast_intr_rxtx()
1596 if (info->ifi_filter != NULL) { in iflib_fast_intr_rxtx()
1597 result = info->ifi_filter(info->ifi_filter_arg); in iflib_fast_intr_rxtx()
1602 ctx = rxq->ifr_ctx; in iflib_fast_intr_rxtx()
1603 sc = ctx->ifc_softc; in iflib_fast_intr_rxtx()
1605 intr_legacy = !!(ctx->ifc_flags & IFC_LEGACY); in iflib_fast_intr_rxtx()
1606 MPASS(rxq->ifr_ntxqirq); in iflib_fast_intr_rxtx()
1607 for (i = 0; i < rxq->ifr_ntxqirq; i++) { in iflib_fast_intr_rxtx()
1608 txqid = rxq->ifr_txqid[i]; in iflib_fast_intr_rxtx()
1609 txq = &ctx->ifc_txqs[txqid]; in iflib_fast_intr_rxtx()
1610 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_fast_intr_rxtx()
1612 if (!ctx->isc_txd_credits_update(sc, txqid, false)) { in iflib_fast_intr_rxtx()
1619 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_fast_intr_rxtx()
1621 if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) in iflib_fast_intr_rxtx()
1622 cidx = rxq->ifr_cq_cidx; in iflib_fast_intr_rxtx()
1624 cidx = rxq->ifr_fl[0].ifl_cidx; in iflib_fast_intr_rxtx()
1631 IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); in iflib_fast_intr_rxtx()
1642 iflib_filter_info_t info = arg; in iflib_fast_intr_ctx() local
1643 struct grouptask *gtask = info->ifi_task; in iflib_fast_intr_ctx()
1647 if (info->ifi_filter != NULL) { in iflib_fast_intr_ctx()
1648 result = info->ifi_filter(info->ifi_filter_arg); in iflib_fast_intr_ctx()
1653 if (gtask->gt_taskqueue != NULL) in iflib_fast_intr_ctx()
1665 device_t dev = ctx->ifc_dev; in _iflib_irq_alloc()
1669 if (ctx->ifc_flags & IFC_LEGACY) in _iflib_irq_alloc()
1679 irq->ii_res = res; in _iflib_irq_alloc()
1680 KASSERT(filter == NULL || handler == NULL, ("filter and handler can't both be non-NULL")); in _iflib_irq_alloc()
1691 irq->ii_tag = tag; in _iflib_irq_alloc()
1697 * Allocate DMA resources for TX buffers as well as memory for the TX
1698 * mbuf map. TX DMA maps (non-TSO/TSO) and TX mbuf map are kept in a
1707 if_ctx_t ctx = txq->ift_ctx; in iflib_txsd_alloc()
1708 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_txsd_alloc()
1709 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_txsd_alloc()
1710 device_t dev = ctx->ifc_dev; in iflib_txsd_alloc()
1716 nsegments = scctx->isc_tx_nsegments; in iflib_txsd_alloc()
1717 ntsosegments = scctx->isc_tx_tso_segments_max; in iflib_txsd_alloc()
1718 tsomaxsize = scctx->isc_tx_tso_size_max; in iflib_txsd_alloc()
1719 if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_VLAN_MTU) in iflib_txsd_alloc()
1721 MPASS(scctx->isc_ntxd[0] > 0); in iflib_txsd_alloc()
1722 MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0); in iflib_txsd_alloc()
1724 if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) { in iflib_txsd_alloc()
1726 MPASS(sctx->isc_tso_maxsize >= tsomaxsize); in iflib_txsd_alloc()
1729 lowaddr = DMA_WIDTH_TO_BUS_LOWADDR(scctx->isc_dma_width); in iflib_txsd_alloc()
1732 * Set up DMA tags for TX buffers. in iflib_txsd_alloc()
1739 sctx->isc_tx_maxsize, /* maxsize */ in iflib_txsd_alloc()
1741 sctx->isc_tx_maxsegsize, /* maxsegsize */ in iflib_txsd_alloc()
1745 &txq->ift_buf_tag))) { in iflib_txsd_alloc()
1746 device_printf(dev, "Unable to allocate TX DMA tag: %d\n", err); in iflib_txsd_alloc()
1748 (uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize); in iflib_txsd_alloc()
1751 tso = (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) != 0; in iflib_txsd_alloc()
1759 sctx->isc_tso_maxsegsize,/* maxsegsize */ in iflib_txsd_alloc()
1763 &txq->ift_tso_buf_tag))) { in iflib_txsd_alloc()
1764 device_printf(dev, "Unable to allocate TSO TX DMA tag: %d\n", in iflib_txsd_alloc()
1770 if (!(txq->ift_sds.ifsd_m = in iflib_txsd_alloc()
1772 scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_txsd_alloc()
1779 * Create the DMA maps for TX buffers. in iflib_txsd_alloc()
1781 if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc( in iflib_txsd_alloc()
1782 sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], in iflib_txsd_alloc()
1785 "Unable to allocate TX buffer DMA map memory\n"); in iflib_txsd_alloc()
1789 if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc( in iflib_txsd_alloc()
1790 sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], in iflib_txsd_alloc()
1797 for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { in iflib_txsd_alloc()
1798 err = bus_dmamap_create(txq->ift_buf_tag, 0, in iflib_txsd_alloc()
1799 &txq->ift_sds.ifsd_map[i]); in iflib_txsd_alloc()
1801 device_printf(dev, "Unable to create TX DMA map\n"); in iflib_txsd_alloc()
1806 err = bus_dmamap_create(txq->ift_tso_buf_tag, 0, in iflib_txsd_alloc()
1807 &txq->ift_sds.ifsd_tso_map[i]); in iflib_txsd_alloc()
1809 device_printf(dev, "Unable to create TSO TX DMA map\n"); in iflib_txsd_alloc()
1825 if (txq->ift_sds.ifsd_map != NULL) { in iflib_txsd_destroy()
1826 map = txq->ift_sds.ifsd_map[i]; in iflib_txsd_destroy()
1827 bus_dmamap_sync(txq->ift_buf_tag, map, BUS_DMASYNC_POSTWRITE); in iflib_txsd_destroy()
1828 bus_dmamap_unload(txq->ift_buf_tag, map); in iflib_txsd_destroy()
1829 bus_dmamap_destroy(txq->ift_buf_tag, map); in iflib_txsd_destroy()
1830 txq->ift_sds.ifsd_map[i] = NULL; in iflib_txsd_destroy()
1833 if (txq->ift_sds.ifsd_tso_map != NULL) { in iflib_txsd_destroy()
1834 map = txq->ift_sds.ifsd_tso_map[i]; in iflib_txsd_destroy()
1835 bus_dmamap_sync(txq->ift_tso_buf_tag, map, in iflib_txsd_destroy()
1837 bus_dmamap_unload(txq->ift_tso_buf_tag, map); in iflib_txsd_destroy()
1838 bus_dmamap_destroy(txq->ift_tso_buf_tag, map); in iflib_txsd_destroy()
1839 txq->ift_sds.ifsd_tso_map[i] = NULL; in iflib_txsd_destroy()
1846 if_ctx_t ctx = txq->ift_ctx; in iflib_txq_destroy()
1848 for (int i = 0; i < txq->ift_size; i++) in iflib_txq_destroy()
1851 if (txq->ift_br != NULL) { in iflib_txq_destroy()
1852 ifmp_ring_free(txq->ift_br); in iflib_txq_destroy()
1853 txq->ift_br = NULL; in iflib_txq_destroy()
1856 mtx_destroy(&txq->ift_mtx); in iflib_txq_destroy()
1858 if (txq->ift_sds.ifsd_map != NULL) { in iflib_txq_destroy()
1859 free(txq->ift_sds.ifsd_map, M_IFLIB); in iflib_txq_destroy()
1860 txq->ift_sds.ifsd_map = NULL; in iflib_txq_destroy()
1862 if (txq->ift_sds.ifsd_tso_map != NULL) { in iflib_txq_destroy()
1863 free(txq->ift_sds.ifsd_tso_map, M_IFLIB); in iflib_txq_destroy()
1864 txq->ift_sds.ifsd_tso_map = NULL; in iflib_txq_destroy()
1866 if (txq->ift_sds.ifsd_m != NULL) { in iflib_txq_destroy()
1867 free(txq->ift_sds.ifsd_m, M_IFLIB); in iflib_txq_destroy()
1868 txq->ift_sds.ifsd_m = NULL; in iflib_txq_destroy()
1870 if (txq->ift_buf_tag != NULL) { in iflib_txq_destroy()
1871 bus_dma_tag_destroy(txq->ift_buf_tag); in iflib_txq_destroy()
1872 txq->ift_buf_tag = NULL; in iflib_txq_destroy()
1874 if (txq->ift_tso_buf_tag != NULL) { in iflib_txq_destroy()
1875 bus_dma_tag_destroy(txq->ift_tso_buf_tag); in iflib_txq_destroy()
1876 txq->ift_tso_buf_tag = NULL; in iflib_txq_destroy()
1878 if (txq->ift_ifdi != NULL) { in iflib_txq_destroy()
1879 free(txq->ift_ifdi, M_IFLIB); in iflib_txq_destroy()
1888 mp = &txq->ift_sds.ifsd_m[i]; in iflib_txsd_free()
1892 if (txq->ift_sds.ifsd_map != NULL) { in iflib_txsd_free()
1893 bus_dmamap_sync(txq->ift_buf_tag, in iflib_txsd_free()
1894 txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE); in iflib_txsd_free()
1895 bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]); in iflib_txsd_free()
1897 if (txq->ift_sds.ifsd_tso_map != NULL) { in iflib_txsd_free()
1898 bus_dmamap_sync(txq->ift_tso_buf_tag, in iflib_txsd_free()
1899 txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE); in iflib_txsd_free()
1900 bus_dmamap_unload(txq->ift_tso_buf_tag, in iflib_txsd_free()
1901 txq->ift_sds.ifsd_tso_map[i]); in iflib_txsd_free()
1911 if_ctx_t ctx = txq->ift_ctx; in iflib_txq_setup()
1912 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_txq_setup()
1913 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_txq_setup()
1918 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_txq_setup()
1920 txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ; in iflib_txq_setup()
1923 txq->ift_cidx_processed = 0; in iflib_txq_setup()
1924 txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0; in iflib_txq_setup()
1925 txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset]; in iflib_txq_setup()
1927 for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) in iflib_txq_setup()
1928 bzero((void *)di->idi_vaddr, di->idi_size); in iflib_txq_setup()
1930 IFDI_TXQ_SETUP(ctx, txq->ift_id); in iflib_txq_setup()
1931 for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) in iflib_txq_setup()
1932 bus_dmamap_sync(di->idi_tag, di->idi_map, in iflib_txq_setup()
1939 * Allocate DMA resources for RX buffers as well as memory for the RX
1941 * map. RX DMA map, RX mbuf map, direct RX cluster pointer map and
1951 if_ctx_t ctx = rxq->ifr_ctx; in iflib_rxsd_alloc()
1952 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_rxsd_alloc()
1953 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_rxsd_alloc()
1954 device_t dev = ctx->ifc_dev; in iflib_rxsd_alloc()
1959 MPASS(scctx->isc_nrxd[0] > 0); in iflib_rxsd_alloc()
1960 MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0); in iflib_rxsd_alloc()
1962 lowaddr = DMA_WIDTH_TO_BUS_LOWADDR(scctx->isc_dma_width); in iflib_rxsd_alloc()
1964 fl = rxq->ifr_fl; in iflib_rxsd_alloc()
1965 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { in iflib_rxsd_alloc()
1966 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ in iflib_rxsd_alloc()
1967 /* Set up DMA tag for RX buffers. */ in iflib_rxsd_alloc()
1973 sctx->isc_rx_maxsize, /* maxsize */ in iflib_rxsd_alloc()
1974 sctx->isc_rx_nsegments, /* nsegments */ in iflib_rxsd_alloc()
1975 sctx->isc_rx_maxsegsize, /* maxsegsize */ in iflib_rxsd_alloc()
1979 &fl->ifl_buf_tag); in iflib_rxsd_alloc()
1982 "Unable to allocate RX DMA tag: %d\n", err); in iflib_rxsd_alloc()
1987 if (!(fl->ifl_sds.ifsd_m = in iflib_rxsd_alloc()
1989 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
1997 if (!(fl->ifl_sds.ifsd_cl = in iflib_rxsd_alloc()
1999 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
2007 if (!(fl->ifl_sds.ifsd_ba = in iflib_rxsd_alloc()
2009 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
2017 * Create the DMA maps for RX buffers. in iflib_rxsd_alloc()
2019 if (!(fl->ifl_sds.ifsd_map = in iflib_rxsd_alloc()
2020 …(bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOW… in iflib_rxsd_alloc()
2022 "Unable to allocate RX buffer DMA map memory\n"); in iflib_rxsd_alloc()
2026 for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { in iflib_rxsd_alloc()
2027 err = bus_dmamap_create(fl->ifl_buf_tag, 0, in iflib_rxsd_alloc()
2028 &fl->ifl_sds.ifsd_map[i]); in iflib_rxsd_alloc()
2030 device_printf(dev, "Unable to create RX buffer DMA map\n"); in iflib_rxsd_alloc()
2057 cb_arg->error = error; in _rxq_refill_cb()
2058 cb_arg->seg = segs[0]; in _rxq_refill_cb()
2059 cb_arg->nseg = nseg; in _rxq_refill_cb()
2063 * iflib_fl_refill - refill an rxq free-buffer list
2068 * (Re)populate an rxq free-buffer list with up to @count new packet buffers.
2085 MPASS(count <= fl->ifl_size - fl->ifl_credits - 1); in iflib_fl_refill()
2087 sd_m = fl->ifl_sds.ifsd_m; in iflib_fl_refill()
2088 sd_map = fl->ifl_sds.ifsd_map; in iflib_fl_refill()
2089 sd_cl = fl->ifl_sds.ifsd_cl; in iflib_fl_refill()
2090 sd_ba = fl->ifl_sds.ifsd_ba; in iflib_fl_refill()
2091 pidx = fl->ifl_pidx; in iflib_fl_refill()
2093 frag_idx = fl->ifl_fragidx; in iflib_fl_refill()
2094 credits = fl->ifl_credits; in iflib_fl_refill()
2099 MPASS(credits + n <= fl->ifl_size); in iflib_fl_refill()
2101 if (pidx < fl->ifl_cidx) in iflib_fl_refill()
2102 MPASS(pidx + n <= fl->ifl_cidx); in iflib_fl_refill()
2103 if (pidx == fl->ifl_cidx && (credits < fl->ifl_size)) in iflib_fl_refill()
2104 MPASS(fl->ifl_gen == 0); in iflib_fl_refill()
2105 if (pidx > fl->ifl_cidx) in iflib_fl_refill()
2106 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx); in iflib_fl_refill()
2111 iru_init(&iru, fl->ifl_rxq, fl->ifl_id); in iflib_fl_refill()
2112 while (n-- > 0) { in iflib_fl_refill()
2120 bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, in iflib_fl_refill()
2123 bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); in iflib_fl_refill()
2126 cl = uma_zalloc(fl->ifl_zone, M_NOWAIT); in iflib_fl_refill()
2132 err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2133 cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, in iflib_fl_refill()
2136 uma_zfree(fl->ifl_zone, cl); in iflib_fl_refill()
2143 fl->ifl_cl_enqueued++; in iflib_fl_refill()
2148 bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2157 bit_set(fl->ifl_rx_bitmap, frag_idx); in iflib_fl_refill()
2159 fl->ifl_m_enqueued++; in iflib_fl_refill()
2163 fl->ifl_rxd_idxs[i] = frag_idx; in iflib_fl_refill()
2164 fl->ifl_bus_addrs[i] = bus_addr; in iflib_fl_refill()
2167 MPASS(credits <= fl->ifl_size); in iflib_fl_refill()
2168 if (++idx == fl->ifl_size) { in iflib_fl_refill()
2170 fl->ifl_gen = 1; in iflib_fl_refill()
2177 ctx->isc_rxd_refill(ctx->ifc_softc, &iru); in iflib_fl_refill()
2178 fl->ifl_pidx = idx; in iflib_fl_refill()
2179 fl->ifl_credits = credits; in iflib_fl_refill()
2185 if (n < count - 1) { in iflib_fl_refill()
2189 ctx->isc_rxd_refill(ctx->ifc_softc, &iru); in iflib_fl_refill()
2190 fl->ifl_pidx = idx; in iflib_fl_refill()
2191 fl->ifl_credits = credits; in iflib_fl_refill()
2194 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_refill()
2196 ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, in iflib_fl_refill()
2197 fl->ifl_id, fl->ifl_pidx); in iflib_fl_refill()
2198 if (__predict_true(bit_test(fl->ifl_rx_bitmap, frag_idx))) { in iflib_fl_refill()
2199 fl->ifl_fragidx = frag_idx + 1; in iflib_fl_refill()
2200 if (fl->ifl_fragidx == fl->ifl_size) in iflib_fl_refill()
2201 fl->ifl_fragidx = 0; in iflib_fl_refill()
2203 fl->ifl_fragidx = frag_idx; in iflib_fl_refill()
2207 return (n == -1 ? 0 : IFLIB_RXEOF_EMPTY); in iflib_fl_refill()
2219 * driver to the NIC (RDT - 1 is thus the last valid one). in iflib_fl_refill_all()
2224 int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; in iflib_fl_refill_all()
2226 …int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - … in iflib_fl_refill_all()
2229 MPASS(fl->ifl_credits <= fl->ifl_size); in iflib_fl_refill_all()
2243 in_detach = !!(ctx->ifc_flags & IFC_IN_DETACH); in iflib_in_detach()
2251 iflib_dma_info_t idi = fl->ifl_ifdi; in iflib_fl_bufs_free()
2255 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2256 struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; in iflib_fl_bufs_free()
2257 caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; in iflib_fl_bufs_free()
2260 sd_map = fl->ifl_sds.ifsd_map[i]; in iflib_fl_bufs_free()
2261 bus_dmamap_sync(fl->ifl_buf_tag, sd_map, in iflib_fl_bufs_free()
2263 bus_dmamap_unload(fl->ifl_buf_tag, sd_map); in iflib_fl_bufs_free()
2264 uma_zfree(fl->ifl_zone, *sd_cl); in iflib_fl_bufs_free()
2275 fl->ifl_m_dequeued++; in iflib_fl_bufs_free()
2276 fl->ifl_cl_dequeued++; in iflib_fl_bufs_free()
2280 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2281 MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); in iflib_fl_bufs_free()
2282 MPASS(fl->ifl_sds.ifsd_m[i] == NULL); in iflib_fl_bufs_free()
2288 fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; in iflib_fl_bufs_free()
2289 bzero(idi->idi_vaddr, idi->idi_size); in iflib_fl_bufs_free()
2300 iflib_rxq_t rxq = fl->ifl_rxq; in iflib_fl_setup()
2301 if_ctx_t ctx = rxq->ifr_ctx; in iflib_fl_setup()
2302 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_fl_setup()
2305 bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1); in iflib_fl_setup()
2311 MPASS(fl->ifl_credits == 0); in iflib_fl_setup()
2312 qidx = rxq->ifr_fl_offset + fl->ifl_id; in iflib_fl_setup()
2313 if (scctx->isc_rxd_buf_size[qidx] != 0) in iflib_fl_setup()
2314 fl->ifl_buf_size = scctx->isc_rxd_buf_size[qidx]; in iflib_fl_setup()
2316 fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz; in iflib_fl_setup()
2318 * ifl_buf_size may be a driver-supplied value, so pull it up in iflib_fl_setup()
2321 fl->ifl_buf_size = iflib_get_mbuf_size_for(fl->ifl_buf_size); in iflib_fl_setup()
2322 if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size) in iflib_fl_setup()
2323 ctx->ifc_max_fl_buf_size = fl->ifl_buf_size; in iflib_fl_setup()
2324 fl->ifl_cltype = m_gettype(fl->ifl_buf_size); in iflib_fl_setup()
2325 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_fl_setup()
2328 * Avoid pre-allocating zillions of clusters to an idle card in iflib_fl_setup()
2333 MPASS(fl->ifl_size > 0); in iflib_fl_setup()
2334 (void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size - 1)); in iflib_fl_setup()
2335 if (min(128, fl->ifl_size - 1) != fl->ifl_credits) in iflib_fl_setup()
2341 MPASS(fl->ifl_ifdi != NULL); in iflib_fl_setup()
2342 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_setup()
2358 if (rxq->ifr_fl != NULL) { in iflib_rx_sds_free()
2359 for (i = 0; i < rxq->ifr_nfl; i++) { in iflib_rx_sds_free()
2360 fl = &rxq->ifr_fl[i]; in iflib_rx_sds_free()
2361 if (fl->ifl_buf_tag != NULL) { in iflib_rx_sds_free()
2362 if (fl->ifl_sds.ifsd_map != NULL) { in iflib_rx_sds_free()
2363 for (j = 0; j < fl->ifl_size; j++) { in iflib_rx_sds_free()
2365 fl->ifl_buf_tag, in iflib_rx_sds_free()
2366 fl->ifl_sds.ifsd_map[j], in iflib_rx_sds_free()
2369 fl->ifl_buf_tag, in iflib_rx_sds_free()
2370 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2372 fl->ifl_buf_tag, in iflib_rx_sds_free()
2373 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2376 bus_dma_tag_destroy(fl->ifl_buf_tag); in iflib_rx_sds_free()
2377 fl->ifl_buf_tag = NULL; in iflib_rx_sds_free()
2379 free(fl->ifl_sds.ifsd_m, M_IFLIB); in iflib_rx_sds_free()
2380 free(fl->ifl_sds.ifsd_cl, M_IFLIB); in iflib_rx_sds_free()
2381 free(fl->ifl_sds.ifsd_ba, M_IFLIB); in iflib_rx_sds_free()
2382 free(fl->ifl_sds.ifsd_map, M_IFLIB); in iflib_rx_sds_free()
2383 free(fl->ifl_rx_bitmap, M_IFLIB); in iflib_rx_sds_free()
2384 fl->ifl_sds.ifsd_m = NULL; in iflib_rx_sds_free()
2385 fl->ifl_sds.ifsd_cl = NULL; in iflib_rx_sds_free()
2386 fl->ifl_sds.ifsd_ba = NULL; in iflib_rx_sds_free()
2387 fl->ifl_sds.ifsd_map = NULL; in iflib_rx_sds_free()
2388 fl->ifl_rx_bitmap = NULL; in iflib_rx_sds_free()
2390 free(rxq->ifr_fl, M_IFLIB); in iflib_rx_sds_free()
2391 rxq->ifr_fl = NULL; in iflib_rx_sds_free()
2392 free(rxq->ifr_ifdi, M_IFLIB); in iflib_rx_sds_free()
2393 rxq->ifr_ifdi = NULL; in iflib_rx_sds_free()
2394 rxq->ifr_cq_cidx = 0; in iflib_rx_sds_free()
2405 if_ctx_t ctx = txq->ift_ctx; in iflib_timer()
2406 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; in iflib_timer()
2409 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) in iflib_timer()
2417 if (this_tick - txq->ift_last_timer_tick >= iflib_timer_default) { in iflib_timer()
2418 txq->ift_last_timer_tick = this_tick; in iflib_timer()
2419 IFDI_TIMER(ctx, txq->ift_id); in iflib_timer()
2420 if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) && in iflib_timer()
2421 ((txq->ift_cleaned_prev == txq->ift_cleaned) || in iflib_timer()
2422 (sctx->isc_pause_frames == 0))) in iflib_timer()
2425 if (txq->ift_qstatus != IFLIB_QUEUE_IDLE && in iflib_timer()
2426 ifmp_ring_is_stalled(txq->ift_br)) { in iflib_timer()
2427 KASSERT(ctx->ifc_link_state == LINK_STATE_UP, in iflib_timer()
2429 txq->ift_qstatus = IFLIB_QUEUE_HUNG; in iflib_timer()
2431 txq->ift_cleaned_prev = txq->ift_cleaned; in iflib_timer()
2434 if (txq->ift_db_pending) in iflib_timer()
2435 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_timer()
2437 sctx->isc_pause_frames = 0; in iflib_timer()
2438 if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) in iflib_timer()
2439 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, in iflib_timer()
2440 txq, txq->ift_timer.c_cpu); in iflib_timer()
2444 device_printf(ctx->ifc_dev, in iflib_timer()
2445 "Watchdog timeout (TX: %d desc avail: %d pidx: %d) -- resetting\n", in iflib_timer()
2446 txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx); in iflib_timer()
2448 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in iflib_timer()
2449 ctx->ifc_flags |= (IFC_DO_WATCHDOG | IFC_DO_RESET); in iflib_timer()
2467 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; in iflib_calc_rx_mbuf_sz()
2473 ctx->ifc_rx_mbuf_sz = in iflib_calc_rx_mbuf_sz()
2474 iflib_get_mbuf_size_for(sctx->isc_max_frame_size); in iflib_calc_rx_mbuf_sz()
2481 return (ctx->ifc_rx_mbuf_sz); in iflib_get_rx_mbuf_sz()
2487 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_init_locked()
2488 if_t ifp = ctx->ifc_ifp; in iflib_init_locked()
2503 tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP); in iflib_init_locked()
2504 tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP); in iflib_init_locked()
2516 for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) { in iflib_init_locked()
2518 callout_stop(&txq->ift_timer); in iflib_init_locked()
2520 callout_stop(&txq->ift_netmap_timer); in iflib_init_locked()
2538 for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_init_locked()
2543 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_init_locked()
2545 device_printf(ctx->ifc_dev, in iflib_init_locked()
2546 "setting up free list %d failed - " in iflib_init_locked()
2553 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); in iflib_init_locked()
2555 txq = ctx->ifc_txqs; in iflib_init_locked()
2556 for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) in iflib_init_locked()
2557 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq, in iflib_init_locked()
2558 txq->ift_timer.c_cpu); in iflib_init_locked()
2560 /* Re-enable txsync/rxsync. */ in iflib_init_locked()
2591 iflib_txq_t txq = ctx->ifc_txqs; in iflib_stop()
2592 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_stop()
2593 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_stop()
2594 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_stop()
2600 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in iflib_stop()
2612 netmap_disable_all_rings(ctx->ifc_ifp); in iflib_stop()
2616 for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) { in iflib_stop()
2620 callout_stop(&txq->ift_timer); in iflib_stop()
2622 callout_stop(&txq->ift_netmap_timer); in iflib_stop()
2629 for (j = 0; j < txq->ift_size; j++) { in iflib_stop()
2632 txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0; in iflib_stop()
2633 txq->ift_in_use = txq->ift_gen = txq->ift_no_desc_avail = 0; in iflib_stop()
2634 if (sctx->isc_flags & IFLIB_PRESERVE_TX_INDICES) in iflib_stop()
2635 txq->ift_cidx = txq->ift_pidx; in iflib_stop()
2637 txq->ift_cidx = txq->ift_pidx = 0; in iflib_stop()
2639 txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0; in iflib_stop()
2640 txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0; in iflib_stop()
2641 txq->ift_pullups = 0; in iflib_stop()
2642 ifmp_ring_reset_stats(txq->ift_br); in iflib_stop()
2643 for (j = 0, di = txq->ift_ifdi; j < sctx->isc_ntxqs; j++, di++) in iflib_stop()
2644 bzero((void *)di->idi_vaddr, di->idi_size); in iflib_stop()
2646 for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_stop()
2647 if (rxq->ifr_task.gt_taskqueue != NULL) in iflib_stop()
2648 gtaskqueue_drain(rxq->ifr_task.gt_taskqueue, in iflib_stop()
2649 &rxq->ifr_task.gt_task); in iflib_stop()
2651 rxq->ifr_cq_cidx = 0; in iflib_stop()
2652 for (j = 0, di = rxq->ifr_ifdi; j < sctx->isc_nrxqs; j++, di++) in iflib_stop()
2653 bzero((void *)di->idi_vaddr, di->idi_size); in iflib_stop()
2655 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_stop()
2667 nrxd = fl->ifl_size; in calc_next_rxd()
2668 size = fl->ifl_rxd_size; in calc_next_rxd()
2669 start = fl->ifl_ifdi->idi_vaddr; in calc_next_rxd()
2683 int nrxd = fl->ifl_size; in prefetch_pkts()
2686 nextptr = (cidx + CACHE_PTR_INCREMENT) & (nrxd - 1); in prefetch_pkts()
2687 prefetch(&fl->ifl_sds.ifsd_m[nextptr]); in prefetch_pkts()
2688 prefetch(&fl->ifl_sds.ifsd_cl[nextptr]); in prefetch_pkts()
2691 prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd - 1)]); in prefetch_pkts()
2692 prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd - 1)]); in prefetch_pkts()
2693 prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd - 1)]); in prefetch_pkts()
2694 prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd - 1)]); in prefetch_pkts()
2695 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd - 1)]); in prefetch_pkts()
2696 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd - 1)]); in prefetch_pkts()
2697 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd - 1)]); in prefetch_pkts()
2698 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd - 1)]); in prefetch_pkts()
2712 flid = irf->irf_flid; in rxd_frag_to_sd()
2713 cidx = irf->irf_idx; in rxd_frag_to_sd()
2714 fl = &rxq->ifr_fl[flid]; in rxd_frag_to_sd()
2715 sd->ifsd_fl = fl; in rxd_frag_to_sd()
2716 sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; in rxd_frag_to_sd()
2717 fl->ifl_credits--; in rxd_frag_to_sd()
2719 fl->ifl_m_dequeued++; in rxd_frag_to_sd()
2721 if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) in rxd_frag_to_sd()
2723 next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size - 1); in rxd_frag_to_sd()
2724 prefetch(&fl->ifl_sds.ifsd_map[next]); in rxd_frag_to_sd()
2725 map = fl->ifl_sds.ifsd_map[cidx]; in rxd_frag_to_sd()
2727 bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); in rxd_frag_to_sd()
2729 if (rxq->pfil != NULL && PFIL_HOOKED_IN(rxq->pfil) && pf_rv != NULL && in rxd_frag_to_sd()
2730 irf->irf_len != 0) { in rxd_frag_to_sd()
2731 payload = *sd->ifsd_cl; in rxd_frag_to_sd()
2732 payload += ri->iri_pad; in rxd_frag_to_sd()
2733 len = ri->iri_len - ri->iri_pad; in rxd_frag_to_sd()
2734 *pf_rv = pfil_mem_in(rxq->pfil, payload, len, ri->iri_ifp, &m); in rxd_frag_to_sd()
2756 m = fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2757 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2763 m = fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2764 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2769 if (unload && irf->irf_len != 0) in rxd_frag_to_sd()
2770 bus_dmamap_unload(fl->ifl_buf_tag, map); in rxd_frag_to_sd()
2771 fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size - 1); in rxd_frag_to_sd()
2772 if (__predict_false(fl->ifl_cidx == 0)) in rxd_frag_to_sd()
2773 fl->ifl_gen = 0; in rxd_frag_to_sd()
2774 bit_clear(fl->ifl_rx_bitmap, cidx); in rxd_frag_to_sd()
2792 m = rxd_frag_to_sd(rxq, &ri->iri_frags[i], !consumed, sd, in assemble_segments()
2795 MPASS(*sd->ifsd_cl != NULL); in assemble_segments()
2798 * Exclude zero-length frags & frags from in assemble_segments()
2801 if (ri->iri_frags[i].irf_len == 0 || consumed || in assemble_segments()
2817 padlen = ri->iri_pad; in assemble_segments()
2820 mt->m_next = m; in assemble_segments()
2825 cl = *sd->ifsd_cl; in assemble_segments()
2826 *sd->ifsd_cl = NULL; in assemble_segments()
2830 m_cljset(m, cl, sd->ifsd_fl->ifl_cltype); in assemble_segments()
2834 m->m_data += padlen; in assemble_segments()
2835 ri->iri_len -= padlen; in assemble_segments()
2836 m->m_len = ri->iri_frags[i].irf_len; in assemble_segments()
2837 } while (++i < ri->iri_nfrags); in assemble_segments()
2853 if (ri->iri_nfrags == 1 && in iflib_rxd_pkt_get()
2854 ri->iri_frags[0].irf_len != 0 && in iflib_rxd_pkt_get()
2855 ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) { in iflib_rxd_pkt_get()
2856 m = rxd_frag_to_sd(rxq, &ri->iri_frags[0], false, &sd, in iflib_rxd_pkt_get()
2863 if (!IP_ALIGNED(m) && ri->iri_pad == 0) in iflib_rxd_pkt_get()
2864 m->m_data += 2; in iflib_rxd_pkt_get()
2866 memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); in iflib_rxd_pkt_get()
2867 m->m_len = ri->iri_frags[0].irf_len; in iflib_rxd_pkt_get()
2868 m->m_data += ri->iri_pad; in iflib_rxd_pkt_get()
2869 ri->iri_len -= ri->iri_pad; in iflib_rxd_pkt_get()
2878 m->m_pkthdr.len = ri->iri_len; in iflib_rxd_pkt_get()
2879 m->m_pkthdr.rcvif = ri->iri_ifp; in iflib_rxd_pkt_get()
2880 m->m_flags |= ri->iri_flags; in iflib_rxd_pkt_get()
2881 m->m_pkthdr.ether_vtag = ri->iri_vtag; in iflib_rxd_pkt_get()
2882 m->m_pkthdr.flowid = ri->iri_flowid; in iflib_rxd_pkt_get()
2883 M_HASHTYPE_SET(m, ri->iri_rsstype); in iflib_rxd_pkt_get()
2884 m->m_pkthdr.csum_flags = ri->iri_csum_flags; in iflib_rxd_pkt_get()
2885 m->m_pkthdr.csum_data = ri->iri_csum_data; in iflib_rxd_pkt_get()
2893 CURVNET_SET(if_getvnet(lc->ifp)); in iflib_get_ip_forwarding()
2914 switch (eh->ether_type) { in iflib_check_lro_possible()
2939 GROUPTASK_ENQUEUE(&rxq->ifr_task); in _task_fn_rx_watchdog()
2946 if_ctx_t ctx = rxq->ifr_ctx; in iflib_rxeof()
2947 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_rxeof()
2948 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_rxeof()
2967 ifp = ctx->ifc_ifp; in iflib_rxeof()
2971 if (sctx->isc_flags & IFLIB_HAS_RXCQ) in iflib_rxeof()
2972 cidxp = &rxq->ifr_cq_cidx; in iflib_rxeof()
2974 cidxp = &rxq->ifr_fl[0].ifl_cidx; in iflib_rxeof()
2976 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2993 ri.iri_qsidx = rxq->ifr_id; in iflib_rxeof()
2996 ri.iri_frags = rxq->ifr_frags; in iflib_rxeof()
2997 err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); in iflib_rxeof()
3003 if (sctx->isc_flags & IFLIB_HAS_RXCQ) { in iflib_rxeof()
3006 /* XXX NB: shurd - check if this is still safe */ in iflib_rxeof()
3007 while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) in iflib_rxeof()
3008 rxq->ifr_cq_cidx -= scctx->isc_nrxd[0]; in iflib_rxeof()
3018 avail--; in iflib_rxeof()
3019 budget_left--; in iflib_rxeof()
3026 /* imm_pkt: -- cxgb */ in iflib_rxeof()
3030 mt->m_nextpkt = m; in iflib_rxeof()
3036 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
3041 iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding); in iflib_rxeof()
3045 mh = mh->m_nextpkt; in iflib_rxeof()
3046 m->m_nextpkt = NULL; in iflib_rxeof()
3061 if ((m->m_pkthdr.csum_flags & (CSUM_L4_CALC | CSUM_L4_VALID)) == in iflib_rxeof()
3063 if (lro_possible && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0) in iflib_rxeof()
3077 mt->m_nextpkt = m; in iflib_rxeof()
3092 tcp_lro_flush_all(&rxq->ifr_lc); in iflib_rxeof()
3099 ctx->ifc_flags |= IFC_DO_RESET; in iflib_rxeof()
3105 #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq) - 1)
3110 qidx_t minthresh = txq->ift_size / 8; in txq_max_db_deferred()
3124 qidx_t minthresh = txq->ift_size / 8; in txq_max_rs_deferred()
3125 if (txq->ift_in_use > 4 * minthresh) in txq_max_rs_deferred()
3127 if (txq->ift_in_use > 2 * minthresh) in txq_max_rs_deferred()
3129 if (txq->ift_in_use > minthresh) in txq_max_rs_deferred()
3134 #define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
3135 #define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)
3143 #define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
3144 #define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
3145 #define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ct…
3146 #define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_…
3149 #define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
3150 #define MAX_TX_DESC(ctx) MAX((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max, \
3151 (ctx)->ifc_softc_ctx.isc_tx_nsegments)
3156 if_ctx_t ctx = txq->ift_ctx; in iflib_txd_db_check()
3159 max = TXQ_MAX_DB_DEFERRED(txq, txq->ift_in_use); in iflib_txd_db_check()
3162 if (ring || (txq->ift_db_pending >= max) || (TXQ_AVAIL(txq) <= MAX_TX_DESC(ctx) + 2)) { in iflib_txd_db_check()
3169 dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx; in iflib_txd_db_check()
3170 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_txd_db_check()
3172 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval); in iflib_txd_db_check()
3177 txq->ift_db_pending = txq->ift_npending = 0; in iflib_txd_db_check()
3188 pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx); in print_pkt()
3190 pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag); in print_pkt()
3192 pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto); in print_pkt()
3196 #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO)
3197 #define IS_TX_OFFLOAD4(pi) ((pi)->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP_TSO))
3198 #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO)
3199 #define IS_TX_OFFLOAD6(pi) ((pi)->ipi_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_TSO))
3217 if (__predict_false(m->m_len < sizeof(*eh))) { in iflib_parse_ether_header()
3223 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { in iflib_parse_ether_header()
3224 pi->ipi_etype = ntohs(eh->evl_proto); in iflib_parse_ether_header()
3225 pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; in iflib_parse_ether_header()
3227 pi->ipi_etype = ntohs(eh->evl_encap_proto); in iflib_parse_ether_header()
3228 pi->ipi_ehdrlen = ETHER_HDR_LEN; in iflib_parse_ether_header()
3260 /* Fills out pi->ipi_etype */ in iflib_parse_header_partial()
3266 switch (pi->ipi_etype) { in iflib_parse_header_partial()
3274 miniplen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip)); in iflib_parse_header_partial()
3275 if (__predict_false(m->m_len < miniplen)) { in iflib_parse_header_partial()
3280 if (m->m_len == pi->ipi_ehdrlen) { in iflib_parse_header_partial()
3281 n = m->m_next; in iflib_parse_header_partial()
3284 if (n->m_len >= sizeof(*ip)) { in iflib_parse_header_partial()
3285 ip = (struct ip *)n->m_data; in iflib_parse_header_partial()
3290 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header_partial()
3296 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header_partial()
3299 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header_partial()
3303 pi->ipi_ip_hlen = ip->ip_hl << 2; in iflib_parse_header_partial()
3304 pi->ipi_ipproto = ip->ip_p; in iflib_parse_header_partial()
3305 pi->ipi_ip_tos = ip->ip_tos; in iflib_parse_header_partial()
3306 pi->ipi_flags |= IPI_TX_IPV4; in iflib_parse_header_partial()
3316 if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) { in iflib_parse_header_partial()
3318 if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL)) in iflib_parse_header_partial()
3321 ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header_partial()
3324 pi->ipi_ip_hlen = sizeof(struct ip6_hdr); in iflib_parse_header_partial()
3325 pi->ipi_ipproto = ip6->ip6_nxt; in iflib_parse_header_partial()
3326 pi->ipi_ip_tos = IPV6_TRAFFIC_CLASS(ip6); in iflib_parse_header_partial()
3327 pi->ipi_flags |= IPI_TX_IPV6; in iflib_parse_header_partial()
3333 pi->ipi_csum_flags &= ~CSUM_OFFLOAD; in iflib_parse_header_partial()
3334 pi->ipi_ip_hlen = 0; in iflib_parse_header_partial()
3346 if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx; in iflib_parse_header()
3351 if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) && in iflib_parse_header()
3362 /* Fills out pi->ipi_etype */ in iflib_parse_header()
3363 err = iflib_parse_ether_header(pi, mp, &txq->ift_pullups); in iflib_parse_header()
3368 switch (pi->ipi_etype) { in iflib_parse_header()
3377 minthlen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip) + sizeof(*th)); in iflib_parse_header()
3378 if (__predict_false(m->m_len < minthlen)) { in iflib_parse_header()
3383 if (m->m_len == pi->ipi_ehdrlen) { in iflib_parse_header()
3384 n = m->m_next; in iflib_parse_header()
3386 if (n->m_len >= sizeof(*ip)) { in iflib_parse_header()
3387 ip = (struct ip *)n->m_data; in iflib_parse_header()
3388 if (n->m_len >= (ip->ip_hl << 2) + sizeof(*th)) in iflib_parse_header()
3389 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); in iflib_parse_header()
3391 txq->ift_pullups++; in iflib_parse_header()
3394 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header()
3397 txq->ift_pullups++; in iflib_parse_header()
3400 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header()
3401 if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th)) in iflib_parse_header()
3402 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); in iflib_parse_header()
3405 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header()
3406 if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th)) in iflib_parse_header()
3407 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); in iflib_parse_header()
3409 pi->ipi_ip_hlen = ip->ip_hl << 2; in iflib_parse_header()
3410 pi->ipi_ipproto = ip->ip_p; in iflib_parse_header()
3411 pi->ipi_ip_tos = ip->ip_tos; in iflib_parse_header()
3412 pi->ipi_flags |= IPI_TX_IPV4; in iflib_parse_header()
3416 if (__predict_true(pi->ipi_ipproto == IPPROTO_TCP)) { in iflib_parse_header()
3418 txq->ift_pullups++; in iflib_parse_header()
3419 if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL)) in iflib_parse_header()
3421 th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen); in iflib_parse_header()
3423 pi->ipi_tcp_hflags = tcp_get_flags(th); in iflib_parse_header()
3424 pi->ipi_tcp_hlen = th->th_off << 2; in iflib_parse_header()
3425 pi->ipi_tcp_seq = th->th_seq; in iflib_parse_header()
3428 if (__predict_false(ip->ip_p != IPPROTO_TCP)) in iflib_parse_header()
3433 pi->ipi_csum_flags |= (CSUM_IP_TCP | CSUM_IP); in iflib_parse_header()
3434 th->th_sum = in_pseudo(ip->ip_src.s_addr, in iflib_parse_header()
3435 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); in iflib_parse_header()
3436 pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; in iflib_parse_header()
3437 if (sctx->isc_flags & IFLIB_TSO_INIT_IP) { in iflib_parse_header()
3438 ip->ip_sum = 0; in iflib_parse_header()
3439 ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz); in iflib_parse_header()
3443 if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP)) in iflib_parse_header()
3444 ip->ip_sum = 0; in iflib_parse_header()
3452 struct ip6_hdr *ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen); in iflib_parse_header()
3454 pi->ipi_ip_hlen = sizeof(struct ip6_hdr); in iflib_parse_header()
3456 if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) { in iflib_parse_header()
3457 txq->ift_pullups++; in iflib_parse_header()
3458 if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL)) in iflib_parse_header()
3461 th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen); in iflib_parse_header()
3463 /* XXX-BZ this will go badly in case of ext hdrs. */ in iflib_parse_header()
3464 pi->ipi_ipproto = ip6->ip6_nxt; in iflib_parse_header()
3465 pi->ipi_ip_tos = IPV6_TRAFFIC_CLASS(ip6); in iflib_parse_header()
3466 pi->ipi_flags |= IPI_TX_IPV6; in iflib_parse_header()
3470 if (pi->ipi_ipproto == IPPROTO_TCP) { in iflib_parse_header()
3471 …if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) { in iflib_parse_header()
3472 txq->ift_pullups++; in iflib_parse_header()
3473 …if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcph… in iflib_parse_header()
3476 pi->ipi_tcp_hflags = tcp_get_flags(th); in iflib_parse_header()
3477 pi->ipi_tcp_hlen = th->th_off << 2; in iflib_parse_header()
3478 pi->ipi_tcp_seq = th->th_seq; in iflib_parse_header()
3481 if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP)) in iflib_parse_header()
3486 pi->ipi_csum_flags |= CSUM_IP6_TCP; in iflib_parse_header()
3487 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); in iflib_parse_header()
3488 pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; in iflib_parse_header()
3495 pi->ipi_csum_flags &= ~CSUM_OFFLOAD; in iflib_parse_header()
3496 pi->ipi_ip_hlen = 0; in iflib_parse_header()
3515 ifsd_m = txq->ift_sds.ifsd_m; in iflib_remove_mbuf()
3516 ntxd = txq->ift_size; in iflib_remove_mbuf()
3517 pidx = txq->ift_pidx & (ntxd - 1); in iflib_remove_mbuf()
3518 ifsd_m = txq->ift_sds.ifsd_m; in iflib_remove_mbuf()
3521 bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]); in iflib_remove_mbuf()
3522 if (txq->ift_sds.ifsd_tso_map != NULL) in iflib_remove_mbuf()
3523 bus_dmamap_unload(txq->ift_tso_buf_tag, in iflib_remove_mbuf()
3524 txq->ift_sds.ifsd_tso_map[pidx]); in iflib_remove_mbuf()
3526 txq->ift_dequeued++; in iflib_remove_mbuf()
3538 ntxd = txq->ift_size; in calc_next_txd()
3539 size = txq->ift_txd_size[qid]; in calc_next_txd()
3540 start = txq->ift_ifdi[qid].idi_vaddr; in calc_next_txd()
3578 for (n = min_frame_size - (*m_head)->m_pkthdr.len; in iflib_ether_pad()
3579 n > 0; n -= sizeof(pad)) in iflib_ether_pad()
3609 ctx = txq->ift_ctx; in iflib_encap()
3610 sctx = ctx->ifc_sctx; in iflib_encap()
3611 scctx = &ctx->ifc_softc_ctx; in iflib_encap()
3612 segs = txq->ift_segs; in iflib_encap()
3613 ntxd = txq->ift_size; in iflib_encap()
3620 cidx = txq->ift_cidx; in iflib_encap()
3621 pidx = txq->ift_pidx; in iflib_encap()
3622 if (ctx->ifc_flags & IFC_PREFETCH) { in iflib_encap()
3623 next = (cidx + CACHE_PTR_INCREMENT) & (ntxd - 1); in iflib_encap()
3624 if (!(ctx->ifc_flags & IFLIB_HAS_TXCQ)) { in iflib_encap()
3630 prefetch(&txq->ift_sds.ifsd_m[next]); in iflib_encap()
3631 prefetch(&txq->ift_sds.ifsd_map[next]); in iflib_encap()
3632 next = (cidx + CACHE_LINE_SIZE) & (ntxd - 1); in iflib_encap()
3634 map = txq->ift_sds.ifsd_map[pidx]; in iflib_encap()
3635 ifsd_m = txq->ift_sds.ifsd_m; in iflib_encap()
3637 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { in iflib_encap()
3638 buf_tag = txq->ift_tso_buf_tag; in iflib_encap()
3639 max_segs = scctx->isc_tx_tso_segments_max; in iflib_encap()
3640 map = txq->ift_sds.ifsd_tso_map[pidx]; in iflib_encap()
3644 buf_tag = txq->ift_buf_tag; in iflib_encap()
3645 max_segs = scctx->isc_tx_nsegments; in iflib_encap()
3646 map = txq->ift_sds.ifsd_map[pidx]; in iflib_encap()
3648 if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) && in iflib_encap()
3649 __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) { in iflib_encap()
3650 err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size); in iflib_encap()
3659 pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG | M_BCAST | M_MCAST)); in iflib_encap()
3661 pi.ipi_qsidx = txq->ift_id; in iflib_encap()
3662 pi.ipi_len = m_head->m_pkthdr.len; in iflib_encap()
3663 pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags; in iflib_encap()
3664 pi.ipi_vtag = M_HAS_VLANTAG(m_head) ? m_head->m_pkthdr.ether_vtag : 0; in iflib_encap()
3690 txq->ift_mbuf_defrag++; in iflib_encap()
3705 txq->ift_no_tx_dma_setup++; in iflib_encap()
3708 txq->ift_no_tx_dma_setup++; in iflib_encap()
3714 txq->ift_map_failed++; in iflib_encap()
3722 * descriptors - this does not hold true on all drivers, e.g. in iflib_encap()
3726 txq->ift_no_desc_avail++; in iflib_encap()
3730 if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0) in iflib_encap()
3731 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_encap()
3740 txq->ift_rs_pending += nsegs + 1; in iflib_encap()
3741 if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) || in iflib_encap()
3742 iflib_no_tx_batch || (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx) + 2) { in iflib_encap()
3744 txq->ift_rs_pending = 0; in iflib_encap()
3750 MPASS(pidx >= 0 && pidx < txq->ift_size); in iflib_encap()
3754 if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) { in iflib_encap()
3757 MPASS(pi.ipi_new_pidx < txq->ift_size); in iflib_encap()
3759 ndesc = pi.ipi_new_pidx - pi.ipi_pidx; in iflib_encap()
3761 ndesc += txq->ift_size; in iflib_encap()
3762 txq->ift_gen = 1; in iflib_encap()
3771 txq->ift_in_use += ndesc; in iflib_encap()
3772 txq->ift_db_pending += ndesc; in iflib_encap()
3778 txq->ift_pidx = pi.ipi_new_pidx; in iflib_encap()
3779 txq->ift_npending += pi.ipi_ndescs; in iflib_encap()
3783 txq->ift_txd_encap_efbig++; in iflib_encap()
3792 * err can't possibly be non-zero here, so we don't neet to test it in iflib_encap()
3798 txq->ift_mbuf_defrag_failed++; in iflib_encap()
3799 txq->ift_map_failed++; in iflib_encap()
3814 cidx = txq->ift_cidx; in iflib_tx_desc_free()
3815 gen = txq->ift_gen; in iflib_tx_desc_free()
3816 qsize = txq->ift_size; in iflib_tx_desc_free()
3817 mask = qsize - 1; in iflib_tx_desc_free()
3818 ifsd_m = txq->ift_sds.ifsd_m; in iflib_tx_desc_free()
3819 do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH); in iflib_tx_desc_free()
3821 while (n-- > 0) { in iflib_tx_desc_free()
3828 if (m->m_pkthdr.csum_flags & CSUM_TSO) { in iflib_tx_desc_free()
3829 bus_dmamap_sync(txq->ift_tso_buf_tag, in iflib_tx_desc_free()
3830 txq->ift_sds.ifsd_tso_map[cidx], in iflib_tx_desc_free()
3832 bus_dmamap_unload(txq->ift_tso_buf_tag, in iflib_tx_desc_free()
3833 txq->ift_sds.ifsd_tso_map[cidx]); in iflib_tx_desc_free()
3835 bus_dmamap_sync(txq->ift_buf_tag, in iflib_tx_desc_free()
3836 txq->ift_sds.ifsd_map[cidx], in iflib_tx_desc_free()
3838 bus_dmamap_unload(txq->ift_buf_tag, in iflib_tx_desc_free()
3839 txq->ift_sds.ifsd_map[cidx]); in iflib_tx_desc_free()
3842 MPASS(m->m_nextpkt == NULL); in iflib_tx_desc_free()
3846 txq->ift_dequeued++; in iflib_tx_desc_free()
3855 txq->ift_cidx = cidx; in iflib_tx_desc_free()
3856 txq->ift_gen = gen; in iflib_tx_desc_free()
3863 if_ctx_t ctx = txq->ift_ctx; in iflib_completed_tx_reclaim()
3866 MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size); in iflib_completed_tx_reclaim()
3869 * Need a rate-limiting check so that this isn't called every time in iflib_completed_tx_reclaim()
3874 if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) { in iflib_completed_tx_reclaim()
3878 txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments, in iflib_completed_tx_reclaim()
3885 txq->ift_cleaned += reclaim; in iflib_completed_tx_reclaim()
3886 txq->ift_in_use -= reclaim; in iflib_completed_tx_reclaim()
3897 size = r->size; in _ring_peek_one()
3898 next = (cidx + CACHE_PTR_INCREMENT) & (size - 1); in _ring_peek_one()
3899 items = __DEVOLATILE(struct mbuf **, &r->items[0]); in _ring_peek_one()
3901 prefetch(items[(cidx + offset) & (size - 1)]); in _ring_peek_one()
3904 prefetch2cachelines(items[(cidx + offset + 1) & (size - 1)]); in _ring_peek_one()
3905 prefetch2cachelines(items[(cidx + offset + 2) & (size - 1)]); in _ring_peek_one()
3906 prefetch2cachelines(items[(cidx + offset + 3) & (size - 1)]); in _ring_peek_one()
3908 return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size - 1)])); in _ring_peek_one()
3915 ifmp_ring_check_drainage(txq->ift_br, budget); in iflib_txq_check_drain()
3921 iflib_txq_t txq = r->cookie; in iflib_txq_can_drain()
3922 if_ctx_t ctx = txq->ift_ctx; in iflib_txq_can_drain()
3926 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_txq_can_drain()
3928 return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, in iflib_txq_can_drain()
3935 iflib_txq_t txq = r->cookie; in iflib_txq_drain()
3936 if_ctx_t ctx = txq->ift_ctx; in iflib_txq_drain()
3937 if_t ifp = ctx->ifc_ifp; in iflib_txq_drain()
3949 rang = iflib_txd_db_check(txq, reclaimed && txq->ift_db_pending); in iflib_txq_drain()
3950 avail = IDXDIFF(pidx, cidx, r->size); in iflib_txq_drain()
3952 if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) { in iflib_txq_drain()
3958 if (__predict_true(r->items[(cidx + i) & (r->size - 1)] != (void *)txq)) in iflib_txq_drain()
3959 m_freem(r->items[(cidx + i) & (r->size - 1)]); in iflib_txq_drain()
3960 r->items[(cidx + i) & (r->size - 1)] = NULL; in iflib_txq_drain()
3965 if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) { in iflib_txq_drain()
3966 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_txq_drain()
3968 callout_stop(&txq->ift_timer); in iflib_txq_drain()
3978 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_txq_drain()
3984 avail, ctx->ifc_flags, TXQ_AVAIL(txq)); in iflib_txq_drain()
3986 do_prefetch = (ctx->ifc_flags & IFC_PREFETCH); in iflib_txq_drain()
3989 int rem = do_prefetch ? count - i : 0; in iflib_txq_drain()
4007 /* no room - bail out */ in iflib_txq_drain()
4011 /* we can't send this packet - skip it */ in iflib_txq_drain()
4017 bytes_sent += m->m_pkthdr.len; in iflib_txq_drain()
4018 mcast_sent += !!(m->m_flags & M_MCAST); in iflib_txq_drain()
4026 /* deliberate use of bitwise or to avoid gratuitous short-circuit */ in iflib_txq_drain()
4053 txq = r->cookie; in iflib_txq_drain_free()
4055 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_txq_drain_free()
4057 callout_stop(&txq->ift_timer); in iflib_txq_drain_free()
4060 avail = IDXDIFF(pidx, cidx, r->size); in iflib_txq_drain_free()
4062 mp = _ring_peek_one(r, cidx, i, avail - i); in iflib_txq_drain_free()
4077 r = txq->ift_br; in iflib_ifmp_purge()
4078 r->drain = iflib_txq_drain_free; in iflib_ifmp_purge()
4079 r->can_drain = iflib_txq_drain_always; in iflib_ifmp_purge()
4081 ifmp_ring_check_drainage(r, r->size); in iflib_ifmp_purge()
4083 r->drain = iflib_txq_drain; in iflib_ifmp_purge()
4084 r->can_drain = iflib_txq_can_drain; in iflib_ifmp_purge()
4091 if_ctx_t ctx = txq->ift_ctx; in _task_fn_tx()
4092 if_t ifp = ctx->ifc_ifp; in _task_fn_tx()
4093 int abdicate = ctx->ifc_sysctl_tx_abdicate; in _task_fn_tx()
4096 txq->ift_cpu_exec_count[curcpu]++; in _task_fn_tx()
4102 netmap_tx_irq(ifp, txq->ift_id)) in _task_fn_tx()
4109 if (txq->ift_db_pending) in _task_fn_tx()
4110 ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); in _task_fn_tx()
4112 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); in _task_fn_tx()
4117 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); in _task_fn_tx()
4121 if (ctx->ifc_flags & IFC_LEGACY) in _task_fn_tx()
4124 IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); in _task_fn_tx()
4131 if_ctx_t ctx = rxq->ifr_ctx; in _task_fn_rx()
4140 rxq->ifr_cpu_exec_count[curcpu]++; in _task_fn_rx()
4143 if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) in _task_fn_rx()
4146 nmirq = netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work); in _task_fn_rx()
4152 budget = ctx->ifc_sysctl_rx_budget; in _task_fn_rx()
4160 if (ctx->ifc_flags & IFC_LEGACY) in _task_fn_rx()
4163 IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); in _task_fn_rx()
4166 if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) in _task_fn_rx()
4170 GROUPTASK_ENQUEUE(&rxq->ifr_task); in _task_fn_rx()
4172 callout_reset_curcpu(&rxq->ifr_watchdog, 1, &_task_fn_rx_watchdog, rxq); in _task_fn_rx()
4179 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; in _task_fn_admin()
4185 running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); in _task_fn_admin()
4186 oactive = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE); in _task_fn_admin()
4187 do_reset = (ctx->ifc_flags & IFC_DO_RESET); in _task_fn_admin()
4188 do_watchdog = (ctx->ifc_flags & IFC_DO_WATCHDOG); in _task_fn_admin()
4189 in_detach = (ctx->ifc_flags & IFC_IN_DETACH); in _task_fn_admin()
4190 ctx->ifc_flags &= ~(IFC_DO_RESET | IFC_DO_WATCHDOG); in _task_fn_admin()
4193 if ((!running && !oactive) && !(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN)) in _task_fn_admin()
4199 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { in _task_fn_admin()
4201 callout_stop(&txq->ift_timer); in _task_fn_admin()
4204 if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_ADMINCQ) in _task_fn_admin()
4207 ctx->ifc_watchdog_events++; in _task_fn_admin()
4211 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { in _task_fn_admin()
4212 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq, in _task_fn_admin()
4213 txq->ift_timer.c_cpu); in _task_fn_admin()
4222 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) in _task_fn_admin()
4231 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) && in _task_fn_iov()
4232 !(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN)) in _task_fn_iov()
4244 if_int_delay_info_t info; in iflib_sysctl_int_delay() local
4247 info = (if_int_delay_info_t)arg1; in iflib_sysctl_int_delay()
4248 ctx = info->iidi_ctx; in iflib_sysctl_int_delay()
4249 info->iidi_req = req; in iflib_sysctl_int_delay()
4250 info->iidi_oidp = oidp; in iflib_sysctl_int_delay()
4252 err = IFDI_SYSCTL_INT_DELAY(ctx, info); in iflib_sysctl_int_delay()
4294 MPASS(m->m_nextpkt == NULL); in iflib_if_transmit()
4295 /* ALTQ-enabled interfaces always use queue 0. */ in iflib_if_transmit()
4297 /* Use driver-supplied queue selection method if it exists */ in iflib_if_transmit()
4298 if (ctx->isc_txq_select_v2) { in iflib_if_transmit()
4306 ctx->ifc_txqs[0].ift_pullups += early_pullups; in iflib_if_transmit()
4311 qidx = ctx->isc_txq_select_v2(ctx->ifc_softc, m, &pi); in iflib_if_transmit()
4312 ctx->ifc_txqs[qidx].ift_pullups += early_pullups; in iflib_if_transmit()
4315 else if (ctx->isc_txq_select) in iflib_if_transmit()
4316 qidx = ctx->isc_txq_select(ctx->ifc_softc, m); in iflib_if_transmit()
4322 txq = &ctx->ifc_txqs[qidx]; in iflib_if_transmit()
4325 if (txq->ift_closed) { in iflib_if_transmit()
4327 next = m->m_nextpkt; in iflib_if_transmit()
4328 m->m_nextpkt = NULL; in iflib_if_transmit()
4342 next = next->m_nextpkt; in iflib_if_transmit()
4355 next = next->m_nextpkt; in iflib_if_transmit()
4356 mp[i]->m_nextpkt = NULL; in iflib_if_transmit()
4360 abdicate = ctx->ifc_sysctl_tx_abdicate; in iflib_if_transmit()
4362 err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE, abdicate); in iflib_if_transmit()
4365 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_if_transmit()
4368 GROUPTASK_ENQUEUE(&txq->ift_task); in iflib_if_transmit()
4371 txq->ift_closed = TRUE; in iflib_if_transmit()
4373 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); in iflib_if_transmit()
4387 * ALTQ-specific code required in iflib. It is assumed that the overhead of
4406 struct ifaltq *ifq = &ifp->if_snd; /* XXX - DRVAPI */ in iflib_altq_if_start()
4424 IFQ_ENQUEUE(&ifp->if_snd, m, err); /* XXX - DRVAPI */ in iflib_altq_if_transmit()
4438 iflib_txq_t txq = ctx->ifc_txqs; in iflib_if_qflush()
4442 ctx->ifc_flags |= IFC_QFLUSH; in iflib_if_qflush()
4445 while (!(ifmp_ring_is_idle(txq->ift_br) || ifmp_ring_is_stalled(txq->ift_br))) in iflib_if_qflush()
4448 ctx->ifc_flags &= ~IFC_QFLUSH; in iflib_if_qflush()
4477 if (ifa->ifa_addr->sa_family == AF_INET) in iflib_if_ioctl()
4481 if (ifa->ifa_addr->sa_family == AF_INET6) in iflib_if_ioctl()
4501 if (ifr->ifr_mtu == if_getmtu(ifp)) { in iflib_if_ioctl()
4509 if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) { in iflib_if_ioctl()
4511 if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size) in iflib_if_ioctl()
4512 ctx->ifc_flags |= IFC_MULTISEG; in iflib_if_ioctl()
4514 ctx->ifc_flags &= ~IFC_MULTISEG; in iflib_if_ioctl()
4516 err = if_setmtu(ifp, ifr->ifr_mtu); in iflib_if_ioctl()
4528 if ((if_getflags(ifp) ^ ctx->ifc_if_flags) & in iflib_if_ioctl()
4539 ctx->ifc_if_flags = if_getflags(ifp); in iflib_if_ioctl()
4559 err = ifmedia_ioctl(ifp, ifr, ctx->ifc_mediap, command); in iflib_if_ioctl()
4587 mask = ifr->ifr_reqcap ^ oldmask; in iflib_if_ioctl()
4588 mask &= ctx->ifc_softc_ctx.isc_capabilities | IFCAP_MEXTPG; in iflib_if_ioctl()
4601 setmask |= ctx->ifc_softc_ctx.isc_capabilities & in iflib_if_ioctl()
4615 ctx->ifc_softc_ctx.isc_capenable ^= setmask; in iflib_if_ioctl()
4676 /* Re-init to load the changes, if required */ in iflib_vlan_register()
4698 /* Re-init to load the changes, if required */ in iflib_vlan_unregister()
4728 if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC) in iflib_device_probe()
4736 if (sctx->isc_parse_devinfo != NULL) in iflib_device_probe()
4737 sctx->isc_parse_devinfo(&pci_device_id, &pci_subvendor_id, &pci_subdevice_id, &pci_rev_id); in iflib_device_probe()
4739 ent = sctx->isc_vendor_info; in iflib_device_probe()
4740 while (ent->pvi_vendor_id != 0) { in iflib_device_probe()
4741 if (pci_vendor_id != ent->pvi_vendor_id) { in iflib_device_probe()
4745 if ((pci_device_id == ent->pvi_device_id) && in iflib_device_probe()
4746 ((pci_subvendor_id == ent->pvi_subvendor_id) || in iflib_device_probe()
4747 (ent->pvi_subvendor_id == 0)) && in iflib_device_probe()
4748 ((pci_subdevice_id == ent->pvi_subdevice_id) || in iflib_device_probe()
4749 (ent->pvi_subdevice_id == 0)) && in iflib_device_probe()
4750 ((pci_rev_id == ent->pvi_rev_id) || in iflib_device_probe()
4751 (ent->pvi_rev_id == 0))) { in iflib_device_probe()
4752 device_set_desc_copy(dev, ent->pvi_name); in iflib_device_probe()
4754 * ever stops re-probing on best match because the sctx in iflib_device_probe()
4780 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_reset_qvalues()
4781 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_reset_qvalues()
4782 device_t dev = ctx->ifc_dev; in iflib_reset_qvalues()
4785 if (ctx->ifc_sysctl_ntxqs != 0) in iflib_reset_qvalues()
4786 scctx->isc_ntxqsets = ctx->ifc_sysctl_ntxqs; in iflib_reset_qvalues()
4787 if (ctx->ifc_sysctl_nrxqs != 0) in iflib_reset_qvalues()
4788 scctx->isc_nrxqsets = ctx->ifc_sysctl_nrxqs; in iflib_reset_qvalues()
4790 for (i = 0; i < sctx->isc_ntxqs; i++) { in iflib_reset_qvalues()
4791 if (ctx->ifc_sysctl_ntxds[i] != 0) in iflib_reset_qvalues()
4792 scctx->isc_ntxd[i] = ctx->ifc_sysctl_ntxds[i]; in iflib_reset_qvalues()
4794 scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i]; in iflib_reset_qvalues()
4797 for (i = 0; i < sctx->isc_nrxqs; i++) { in iflib_reset_qvalues()
4798 if (ctx->ifc_sysctl_nrxds[i] != 0) in iflib_reset_qvalues()
4799 scctx->isc_nrxd[i] = ctx->ifc_sysctl_nrxds[i]; in iflib_reset_qvalues()
4801 scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i]; in iflib_reset_qvalues()
4804 for (i = 0; i < sctx->isc_nrxqs; i++) { in iflib_reset_qvalues()
4805 if (scctx->isc_nrxd[i] < sctx->isc_nrxd_min[i]) { in iflib_reset_qvalues()
4806 device_printf(dev, "nrxd%d: %d less than nrxd_min %d - resetting to min\n", in iflib_reset_qvalues()
4807 i, scctx->isc_nrxd[i], sctx->isc_nrxd_min[i]); in iflib_reset_qvalues()
4808 scctx->isc_nrxd[i] = sctx->isc_nrxd_min[i]; in iflib_reset_qvalues()
4810 if (scctx->isc_nrxd[i] > sctx->isc_nrxd_max[i]) { in iflib_reset_qvalues()
4811 device_printf(dev, "nrxd%d: %d greater than nrxd_max %d - resetting to max\n", in iflib_reset_qvalues()
4812 i, scctx->isc_nrxd[i], sctx->isc_nrxd_max[i]); in iflib_reset_qvalues()
4813 scctx->isc_nrxd[i] = sctx->isc_nrxd_max[i]; in iflib_reset_qvalues()
4815 if (!powerof2(scctx->isc_nrxd[i])) { in iflib_reset_qvalues()
4816 device_printf(dev, "nrxd%d: %d is not a power of 2 - using default value of %d\n", in iflib_reset_qvalues()
4817 i, scctx->isc_nrxd[i], sctx->isc_nrxd_default[i]); in iflib_reset_qvalues()
4818 scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i]; in iflib_reset_qvalues()
4822 for (i = 0; i < sctx->isc_ntxqs; i++) { in iflib_reset_qvalues()
4823 if (scctx->isc_ntxd[i] < sctx->isc_ntxd_min[i]) { in iflib_reset_qvalues()
4824 device_printf(dev, "ntxd%d: %d less than ntxd_min %d - resetting to min\n", in iflib_reset_qvalues()
4825 i, scctx->isc_ntxd[i], sctx->isc_ntxd_min[i]); in iflib_reset_qvalues()
4826 scctx->isc_ntxd[i] = sctx->isc_ntxd_min[i]; in iflib_reset_qvalues()
4828 if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) { in iflib_reset_qvalues()
4829 device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n", in iflib_reset_qvalues()
4830 i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]); in iflib_reset_qvalues()
4831 scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i]; in iflib_reset_qvalues()
4833 if (!powerof2(scctx->isc_ntxd[i])) { in iflib_reset_qvalues()
4834 device_printf(dev, "ntxd%d: %d is not a power of 2 - using default value of %d\n", in iflib_reset_qvalues()
4835 i, scctx->isc_ntxd[i], sctx->isc_ntxd_default[i]); in iflib_reset_qvalues()
4836 scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i]; in iflib_reset_qvalues()
4852 pa.pa_headname = if_name(ctx->ifc_ifp); in iflib_add_pfil()
4855 for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { in iflib_add_pfil()
4856 rxq->pfil = pfil; in iflib_add_pfil()
4867 rxq = ctx->ifc_rxqs; in iflib_rem_pfil()
4868 pfil = rxq->pfil; in iflib_rem_pfil()
4870 rxq->pfil = NULL; in iflib_rem_pfil()
4877 * Advance forward by n members of the cpuset ctx->ifc_cpus starting from
4887 MPASS(CPU_ISSET(cpuid, &ctx->ifc_cpus)); in cpuid_advance()
4890 MPASS(!CPU_EMPTY(&ctx->ifc_cpus)); in cpuid_advance()
4892 first_valid = CPU_FFS(&ctx->ifc_cpus) - 1; in cpuid_advance()
4893 last_valid = CPU_FLS(&ctx->ifc_cpus) - 1; in cpuid_advance()
4894 n = n % CPU_COUNT(&ctx->ifc_cpus); in cpuid_advance()
4900 } while (!CPU_ISSET(cpuid, &ctx->ifc_cpus)); in cpuid_advance()
4901 n--; in cpuid_advance()
4915 if (grp->cg_children == 0) in find_child_with_core()
4916 return (-1); in find_child_with_core()
4918 MPASS(grp->cg_child); in find_child_with_core()
4919 for (i = 0; i < grp->cg_children; i++) { in find_child_with_core()
4920 if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask)) in find_child_with_core()
4924 return (-1); in find_child_with_core()
4929 * Find an L2 neighbor of the given CPU or return -1 if none found. This
4941 return (-1); in find_l2_neighbor()
4947 while ((i = find_child_with_core(cpu, grp)) != -1) { in find_l2_neighbor()
4953 if (grp->cg_child[i].cg_count <= 1) in find_l2_neighbor()
4954 return (-1); in find_l2_neighbor()
4955 grp = &grp->cg_child[i]; in find_l2_neighbor()
4959 if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE) in find_l2_neighbor()
4960 return (-1); in find_l2_neighbor()
4967 if (CPU_ISSET(i, &grp->cg_mask) && i != cpu) in find_l2_neighbor()
4972 return (-1); in find_l2_neighbor()
4980 return (-1); in find_l2_neighbor()
4986 * ---------------------
4993 * ---------- --------- ------ ------------------------------------------------
4994 * - - X RX and TX queues mapped to consecutive physical
4997 * - X X RX and TX queues mapped to consecutive cores
5000 * X - X RX and TX queues mapped to consecutive physical
5006 * - n/a - RX and TX queues mapped to consecutive cores of
5009 * X n/a - RX and TX queues mapped to consecutive cores of
5016 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in get_cpuid_for_queue()
5019 if (ctx->ifc_sysctl_separate_txrx) { in get_cpuid_for_queue()
5029 if (ctx->ifc_sysctl_use_logical_cores && in get_cpuid_for_queue()
5030 ctx->ifc_cpus_are_physical_cores && in get_cpuid_for_queue()
5031 is_tx && qid < scctx->isc_nrxqsets) { in get_cpuid_for_queue()
5037 if (l2_neighbor != -1) { in get_cpuid_for_queue()
5042 * consecutive-after-RX assignment scheme. in get_cpuid_for_queue()
5052 core_index = scctx->isc_nrxqsets + qid; in get_cpuid_for_queue()
5065 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in get_ctx_core_offset()
5069 unsigned int base_cpuid = ctx->ifc_sysctl_core_offset; in get_ctx_core_offset()
5074 first_valid = CPU_FFS(&ctx->ifc_cpus) - 1; in get_ctx_core_offset()
5075 last_valid = CPU_FLS(&ctx->ifc_cpus) - 1; in get_ctx_core_offset()
5079 * Align the user-chosen base CPU ID to the next valid CPU in get_ctx_core_offset()
5084 * zero-based reference frame, and so we shift the given in get_ctx_core_offset()
5092 /* shift from zero-based to first_valid-based */ in get_ctx_core_offset()
5095 base_cpuid = (base_cpuid - first_valid) % in get_ctx_core_offset()
5096 (last_valid - first_valid + 1); in get_ctx_core_offset()
5098 if (!CPU_ISSET(base_cpuid, &ctx->ifc_cpus)) { in get_ctx_core_offset()
5106 while (!CPU_ISSET(base_cpuid, &ctx->ifc_cpus)) in get_ctx_core_offset()
5123 * neighbors of CPUs that RX queues have been mapped to - in this in get_ctx_core_offset()
5130 for (i = 0; i < scctx->isc_ntxqsets; i++) in get_ctx_core_offset()
5133 for (i = 0; i < scctx->isc_nrxqsets; i++) in get_ctx_core_offset()
5136 CPU_AND(&assigned_cpus, &assigned_cpus, &ctx->ifc_cpus); in get_ctx_core_offset()
5141 if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) { in get_ctx_core_offset()
5142 base_cpuid = op->next_cpuid; in get_ctx_core_offset()
5143 op->next_cpuid = cpuid_advance(ctx, op->next_cpuid, in get_ctx_core_offset()
5145 MPASS(op->refcount < UINT_MAX); in get_ctx_core_offset()
5146 op->refcount++; in get_ctx_core_offset()
5155 device_printf(ctx->ifc_dev, in get_ctx_core_offset()
5158 op->next_cpuid = cpuid_advance(ctx, base_cpuid, in get_ctx_core_offset()
5160 op->refcount = 1; in get_ctx_core_offset()
5161 CPU_COPY(&ctx->ifc_cpus, &op->set); in get_ctx_core_offset()
5177 if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) { in unref_ctx_core_offset()
5178 MPASS(op->refcount > 0); in unref_ctx_core_offset()
5179 op->refcount--; in unref_ctx_core_offset()
5180 if (op->refcount == 0) { in unref_ctx_core_offset()
5204 sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK | M_ZERO); in iflib_device_register()
5206 ctx->ifc_flags |= IFC_SC_ALLOCATED; in iflib_device_register()
5209 ctx->ifc_sctx = sctx; in iflib_device_register()
5210 ctx->ifc_dev = dev; in iflib_device_register()
5211 ctx->ifc_softc = sc; in iflib_device_register()
5219 scctx = &ctx->ifc_softc_ctx; in iflib_device_register()
5220 ifp = ctx->ifc_ifp; in iflib_device_register()
5230 ctx->ifc_txrx = *scctx->isc_txrx; in iflib_device_register()
5232 MPASS(scctx->isc_dma_width <= flsll(BUS_SPACE_MAXADDR)); in iflib_device_register()
5234 if (sctx->isc_flags & IFLIB_DRIVER_MEDIA) in iflib_device_register()
5235 ctx->ifc_mediap = scctx->isc_media; in iflib_device_register()
5238 if (scctx->isc_capabilities & IFCAP_TXCSUM) in iflib_device_register()
5239 MPASS(scctx->isc_tx_csum_flags); in iflib_device_register()
5243 scctx->isc_capabilities | IFCAP_HWSTATS | IFCAP_MEXTPG); in iflib_device_register()
5245 scctx->isc_capenable | IFCAP_HWSTATS | IFCAP_MEXTPG); in iflib_device_register()
5247 …if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_n… in iflib_device_register()
5248 scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; in iflib_device_register()
5249 …if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_n… in iflib_device_register()
5250 scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; in iflib_device_register()
5255 /* XXX change for per-queue sizes */ in iflib_device_register()
5259 if (scctx->isc_tx_nsegments > num_txd / MAX_SINGLE_PACKET_FRACTION) in iflib_device_register()
5260 scctx->isc_tx_nsegments = max(1, num_txd / in iflib_device_register()
5262 if (scctx->isc_tx_tso_segments_max > num_txd / in iflib_device_register()
5264 scctx->isc_tx_tso_segments_max = max(1, in iflib_device_register()
5267 /* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */ in iflib_device_register()
5273 if_sethwtsomax(ifp, min(scctx->isc_tx_tso_size_max, in iflib_device_register()
5278 * add another mbuf and, thus, the requirement for another DMA in iflib_device_register()
5283 if_sethwtsomaxsegcount(ifp, scctx->isc_tx_tso_segments_max - 3); in iflib_device_register()
5284 if_sethwtsomaxsegsize(ifp, scctx->isc_tx_tso_segsize_max); in iflib_device_register()
5286 if (scctx->isc_rss_table_size == 0) in iflib_device_register()
5287 scctx->isc_rss_table_size = 64; in iflib_device_register()
5288 scctx->isc_rss_table_mask = scctx->isc_rss_table_size - 1; in iflib_device_register()
5290 GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx); in iflib_device_register()
5292 taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx, in iflib_device_register()
5296 if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) { in iflib_device_register()
5298 CPU_COPY(&all_cpus, &ctx->ifc_cpus); in iflib_device_register()
5299 ctx->ifc_cpus_are_physical_cores = false; in iflib_device_register()
5301 ctx->ifc_cpus_are_physical_cores = true; in iflib_device_register()
5302 MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0); in iflib_device_register()
5305 * Now set up MSI or MSI-X, should return us the number of supported in iflib_device_register()
5308 if (sctx->isc_flags & IFLIB_SKIP_MSIX) { in iflib_device_register()
5309 msix = scctx->isc_vectors; in iflib_device_register()
5310 } else if (scctx->isc_msix_bar != 0) in iflib_device_register()
5317 scctx->isc_vectors = 1; in iflib_device_register()
5318 scctx->isc_ntxqsets = 1; in iflib_device_register()
5319 scctx->isc_nrxqsets = 1; in iflib_device_register()
5320 scctx->isc_intr = IFLIB_INTR_LEGACY; in iflib_device_register()
5335 ctx->ifc_sysctl_core_offset = get_ctx_core_offset(ctx); in iflib_device_register()
5339 * When using MSI-X, ensure that ifdi_{r,t}x_queue_intr_enable in iflib_device_register()
5343 kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL, in iflib_device_register()
5345 if (kobj_method == &kobj_desc->deflt) { in iflib_device_register()
5347 "MSI-X requires ifdi_rx_queue_intr_enable method"); in iflib_device_register()
5352 kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL, in iflib_device_register()
5354 if (kobj_method == &kobj_desc->deflt) { in iflib_device_register()
5356 "MSI-X requires ifdi_tx_queue_intr_enable method"); in iflib_device_register()
5362 * Assign the MSI-X vectors. in iflib_device_register()
5372 } else if (scctx->isc_intr != IFLIB_INTR_MSIX) { in iflib_device_register()
5374 if (scctx->isc_intr == IFLIB_INTR_MSI) { in iflib_device_register()
5378 if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) { in iflib_device_register()
5384 "Cannot use iflib with only 1 MSI-X interrupt!\n"); in iflib_device_register()
5390 * It prevents a double-locking panic with iflib_media_status when in iflib_device_register()
5394 ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet); in iflib_device_register()
5411 device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err); in iflib_device_register()
5416 DEBUGNET_SET(ctx->ifc_ifp, iflib); in iflib_device_register()
5418 if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); in iflib_device_register()
5421 ctx->ifc_flags |= IFC_INIT_DONE; in iflib_device_register()
5428 ether_ifdetach(ctx->ifc_ifp); in iflib_device_register()
5442 device_set_softc(ctx->ifc_dev, NULL); in iflib_device_register()
5443 if (ctx->ifc_flags & IFC_SC_ALLOCATED) in iflib_device_register()
5444 free(ctx->ifc_softc, M_IFLIB); in iflib_device_register()
5455 if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC) in iflib_device_attach()
5466 if_t ifp = ctx->ifc_ifp; in iflib_device_deregister()
5467 device_t dev = ctx->ifc_dev; in iflib_device_deregister()
5476 device_printf(dev, "SR-IOV in use; detach first.\n"); in iflib_device_deregister()
5482 ctx->ifc_flags |= IFC_IN_DETACH; in iflib_device_deregister()
5496 if (ctx->ifc_led_dev != NULL) in iflib_device_deregister()
5497 led_destroy(ctx->ifc_led_dev); in iflib_device_deregister()
5508 /* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/ in iflib_device_deregister()
5515 device_set_softc(ctx->ifc_dev, NULL); in iflib_device_deregister()
5516 if (ctx->ifc_flags & IFC_SC_ALLOCATED) in iflib_device_deregister()
5517 free(ctx->ifc_softc, M_IFLIB); in iflib_device_deregister()
5533 for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { in iflib_tqg_detach()
5534 callout_drain(&txq->ift_timer); in iflib_tqg_detach()
5536 callout_drain(&txq->ift_netmap_timer); in iflib_tqg_detach()
5538 if (txq->ift_task.gt_uniq != NULL) in iflib_tqg_detach()
5539 taskqgroup_detach(tqg, &txq->ift_task); in iflib_tqg_detach()
5541 for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { in iflib_tqg_detach()
5542 if (rxq->ifr_task.gt_uniq != NULL) in iflib_tqg_detach()
5543 taskqgroup_detach(tqg, &rxq->ifr_task); in iflib_tqg_detach()
5546 if (ctx->ifc_admin_task.gt_uniq != NULL) in iflib_tqg_detach()
5547 taskqgroup_detach(tqg, &ctx->ifc_admin_task); in iflib_tqg_detach()
5548 if (ctx->ifc_vflr_task.gt_uniq != NULL) in iflib_tqg_detach()
5549 taskqgroup_detach(tqg, &ctx->ifc_vflr_task); in iflib_tqg_detach()
5556 if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { in iflib_free_intr_mem()
5557 iflib_irq_free(ctx, &ctx->ifc_legacy_irq); in iflib_free_intr_mem()
5559 if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { in iflib_free_intr_mem()
5560 pci_release_msi(ctx->ifc_dev); in iflib_free_intr_mem()
5562 if (ctx->ifc_msix_mem != NULL) { in iflib_free_intr_mem()
5563 bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, in iflib_free_intr_mem()
5564 rman_get_rid(ctx->ifc_msix_mem), ctx->ifc_msix_mem); in iflib_free_intr_mem()
5565 ctx->ifc_msix_mem = NULL; in iflib_free_intr_mem()
5604 iflib_txq_t txq = ctx->ifc_txqs; in iflib_device_resume()
5659 * - Start a fast taskqueue thread for each core
5660 * - Start a taskqueue for control operations
5700 MPASS(sctx->isc_tx_maxsize); in _iflib_assert()
5701 MPASS(sctx->isc_tx_maxsegsize); in _iflib_assert()
5703 MPASS(sctx->isc_rx_maxsize); in _iflib_assert()
5704 MPASS(sctx->isc_rx_nsegments); in _iflib_assert()
5705 MPASS(sctx->isc_rx_maxsegsize); in _iflib_assert()
5707 MPASS(sctx->isc_nrxqs >= 1 && sctx->isc_nrxqs <= 8); in _iflib_assert()
5708 for (i = 0; i < sctx->isc_nrxqs; i++) { in _iflib_assert()
5709 MPASS(sctx->isc_nrxd_min[i]); in _iflib_assert()
5710 MPASS(powerof2(sctx->isc_nrxd_min[i])); in _iflib_assert()
5711 MPASS(sctx->isc_nrxd_max[i]); in _iflib_assert()
5712 MPASS(powerof2(sctx->isc_nrxd_max[i])); in _iflib_assert()
5713 MPASS(sctx->isc_nrxd_default[i]); in _iflib_assert()
5714 MPASS(powerof2(sctx->isc_nrxd_default[i])); in _iflib_assert()
5717 MPASS(sctx->isc_ntxqs >= 1 && sctx->isc_ntxqs <= 8); in _iflib_assert()
5718 for (i = 0; i < sctx->isc_ntxqs; i++) { in _iflib_assert()
5719 MPASS(sctx->isc_ntxd_min[i]); in _iflib_assert()
5720 MPASS(powerof2(sctx->isc_ntxd_min[i])); in _iflib_assert()
5721 MPASS(sctx->isc_ntxd_max[i]); in _iflib_assert()
5722 MPASS(powerof2(sctx->isc_ntxd_max[i])); in _iflib_assert()
5723 MPASS(sctx->isc_ntxd_default[i]); in _iflib_assert()
5724 MPASS(powerof2(sctx->isc_ntxd_default[i])); in _iflib_assert()
5732 MPASS(scctx->isc_txrx->ift_txd_encap); in _iflib_pre_assert()
5733 MPASS(scctx->isc_txrx->ift_txd_flush); in _iflib_pre_assert()
5734 MPASS(scctx->isc_txrx->ift_txd_credits_update); in _iflib_pre_assert()
5735 MPASS(scctx->isc_txrx->ift_rxd_available); in _iflib_pre_assert()
5736 MPASS(scctx->isc_txrx->ift_rxd_pkt_get); in _iflib_pre_assert()
5737 MPASS(scctx->isc_txrx->ift_rxd_refill); in _iflib_pre_assert()
5738 MPASS(scctx->isc_txrx->ift_rxd_flush); in _iflib_pre_assert()
5744 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_register()
5745 driver_t *driver = sctx->isc_driver; in iflib_register()
5746 device_t dev = ctx->ifc_dev; in iflib_register()
5752 STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev)); in iflib_register()
5753 ifp = ctx->ifc_ifp = if_alloc_dev(IFT_ETHER, dev); in iflib_register()
5775 ctx->ifc_vlan_attach_event = in iflib_register()
5778 ctx->ifc_vlan_detach_event = in iflib_register()
5782 if ((sctx->isc_flags & IFLIB_DRIVER_MEDIA) == 0) { in iflib_register()
5783 ctx->ifc_mediap = &ctx->ifc_media; in iflib_register()
5784 ifmedia_init(ctx->ifc_mediap, IFM_IMASK, in iflib_register()
5794 if (ctx->ifc_vlan_attach_event != NULL) { in iflib_unregister_vlan_handlers()
5795 EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); in iflib_unregister_vlan_handlers()
5796 ctx->ifc_vlan_attach_event = NULL; in iflib_unregister_vlan_handlers()
5798 if (ctx->ifc_vlan_detach_event != NULL) { in iflib_unregister_vlan_handlers()
5799 EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); in iflib_unregister_vlan_handlers()
5800 ctx->ifc_vlan_detach_event = NULL; in iflib_unregister_vlan_handlers()
5808 if_t ifp = ctx->ifc_ifp; in iflib_deregister()
5811 ifmedia_removeall(&ctx->ifc_media); in iflib_deregister()
5824 /* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/ in iflib_deregister()
5831 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_queues_alloc()
5832 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_queues_alloc()
5833 device_t dev = ctx->ifc_dev; in iflib_queues_alloc()
5834 int nrxqsets = scctx->isc_nrxqsets; in iflib_queues_alloc()
5835 int ntxqsets = scctx->isc_ntxqsets; in iflib_queues_alloc()
5841 uint32_t *rxqsizes = scctx->isc_rxqsizes; in iflib_queues_alloc()
5842 uint32_t *txqsizes = scctx->isc_txqsizes; in iflib_queues_alloc()
5843 uint8_t nrxqs = sctx->isc_nrxqs; in iflib_queues_alloc()
5844 uint8_t ntxqs = sctx->isc_ntxqs; in iflib_queues_alloc()
5845 int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1; in iflib_queues_alloc()
5846 int fl_offset = (sctx->isc_flags & IFLIB_HAS_RXCQ ? 1 : 0); in iflib_queues_alloc()
5856 if (!(ctx->ifc_txqs = in iflib_queues_alloc()
5865 if (!(ctx->ifc_rxqs = in iflib_queues_alloc()
5873 txq = ctx->ifc_txqs; in iflib_queues_alloc()
5874 rxq = ctx->ifc_rxqs; in iflib_queues_alloc()
5885 "Unable to allocate TX DMA info memory\n"); in iflib_queues_alloc()
5889 txq->ift_ifdi = ifdip; in iflib_queues_alloc()
5897 txq->ift_txd_size[j] = scctx->isc_txd_size[j]; in iflib_queues_alloc()
5898 bzero((void *)ifdip->idi_vaddr, txqsizes[j]); in iflib_queues_alloc()
5900 txq->ift_ctx = ctx; in iflib_queues_alloc()
5901 txq->ift_id = i; in iflib_queues_alloc()
5902 if (sctx->isc_flags & IFLIB_HAS_TXCQ) { in iflib_queues_alloc()
5903 txq->ift_br_offset = 1; in iflib_queues_alloc()
5905 txq->ift_br_offset = 0; in iflib_queues_alloc()
5915 snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:TX(%d):callout", in iflib_queues_alloc()
5916 device_get_nameunit(dev), txq->ift_id); in iflib_queues_alloc()
5917 mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); in iflib_queues_alloc()
5918 callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); in iflib_queues_alloc()
5919 txq->ift_timer.c_cpu = cpu; in iflib_queues_alloc()
5921 callout_init_mtx(&txq->ift_netmap_timer, &txq->ift_mtx, 0); in iflib_queues_alloc()
5922 txq->ift_netmap_timer.c_cpu = cpu; in iflib_queues_alloc()
5925 err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, in iflib_queues_alloc()
5936 callout_init(&rxq->ifr_watchdog, 1); in iflib_queues_alloc()
5941 "Unable to allocate RX DMA info memory\n"); in iflib_queues_alloc()
5946 rxq->ifr_ifdi = ifdip; in iflib_queues_alloc()
5948 rxq->ifr_ntxqirq = 1; in iflib_queues_alloc()
5949 rxq->ifr_txqid[0] = i; in iflib_queues_alloc()
5957 bzero((void *)ifdip->idi_vaddr, rxqsizes[j]); in iflib_queues_alloc()
5959 rxq->ifr_ctx = ctx; in iflib_queues_alloc()
5960 rxq->ifr_id = i; in iflib_queues_alloc()
5961 rxq->ifr_fl_offset = fl_offset; in iflib_queues_alloc()
5962 rxq->ifr_nfl = nfree_lists; in iflib_queues_alloc()
5969 rxq->ifr_fl = fl; in iflib_queues_alloc()
5973 fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; in iflib_queues_alloc()
5974 fl[j].ifl_rxd_size = scctx->isc_rxd_size[j]; in iflib_queues_alloc()
5984 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_queues_alloc()
5985 fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, in iflib_queues_alloc()
5993 iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi; in iflib_queues_alloc()
5996 vaddrs[i * ntxqs + j] = di->idi_vaddr; in iflib_queues_alloc()
5997 paddrs[i * ntxqs + j] = di->idi_paddr; in iflib_queues_alloc()
6001 device_printf(ctx->ifc_dev, in iflib_queues_alloc()
6015 iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi; in iflib_queues_alloc()
6018 vaddrs[i * nrxqs + j] = di->idi_vaddr; in iflib_queues_alloc()
6019 paddrs[i * nrxqs + j] = di->idi_paddr; in iflib_queues_alloc()
6023 device_printf(ctx->ifc_dev, in iflib_queues_alloc()
6039 if (ctx->ifc_rxqs != NULL) in iflib_queues_alloc()
6040 free(ctx->ifc_rxqs, M_IFLIB); in iflib_queues_alloc()
6041 ctx->ifc_rxqs = NULL; in iflib_queues_alloc()
6042 if (ctx->ifc_txqs != NULL) in iflib_queues_alloc()
6043 free(ctx->ifc_txqs, M_IFLIB); in iflib_queues_alloc()
6044 ctx->ifc_txqs = NULL; in iflib_queues_alloc()
6052 iflib_txq_t txq = ctx->ifc_txqs; in iflib_tx_structures_setup()
6064 iflib_txq_t txq = ctx->ifc_txqs; in iflib_tx_structures_free()
6065 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_tx_structures_free()
6069 for (j = 0; j < sctx->isc_ntxqs; j++) in iflib_tx_structures_free()
6070 iflib_dma_free(&txq->ift_ifdi[j]); in iflib_tx_structures_free()
6073 free(ctx->ifc_txqs, M_IFLIB); in iflib_tx_structures_free()
6074 ctx->ifc_txqs = NULL; in iflib_tx_structures_free()
6085 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_rx_structures_setup()
6091 for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) { in iflib_rx_structures_setup()
6093 err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp, in iflib_rx_structures_setup()
6095 ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset])); in iflib_rx_structures_setup()
6097 device_printf(ctx->ifc_dev, in iflib_rx_structures_setup()
6102 IFDI_RXQ_SETUP(ctx, rxq->ifr_id); in iflib_rx_structures_setup()
6112 rxq = ctx->ifc_rxqs; in iflib_rx_structures_setup()
6114 tcp_lro_free(&rxq->ifr_lc); in iflib_rx_structures_setup()
6128 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_rx_structures_free()
6129 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_rx_structures_free()
6132 for (i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) { in iflib_rx_structures_free()
6133 for (j = 0; j < sctx->isc_nrxqs; j++) in iflib_rx_structures_free()
6134 iflib_dma_free(&rxq->ifr_ifdi[j]); in iflib_rx_structures_free()
6137 tcp_lro_free(&rxq->ifr_lc); in iflib_rx_structures_free()
6140 free(ctx->ifc_rxqs, M_IFLIB); in iflib_rx_structures_free()
6141 ctx->ifc_rxqs = NULL; in iflib_rx_structures_free()
6154 device_printf(ctx->ifc_dev, "iflib_tx_structures_setup failed: %d\n", err); in iflib_qset_structures_setup()
6159 device_printf(ctx->ifc_dev, "iflib_rx_structures_setup failed: %d\n", err); in iflib_qset_structures_setup()
6182 dev = ctx->ifc_dev; in iflib_irq_set_affinity()
6183 base_cpuid = ctx->ifc_sysctl_core_offset; in iflib_irq_set_affinity()
6186 irq ? irq->ii_res : NULL, name); in iflib_irq_set_affinity()
6192 if (cpuid > ctx->ifc_cpuid_highest) in iflib_irq_set_affinity()
6193 ctx->ifc_cpuid_highest = cpuid; in iflib_irq_set_affinity()
6217 iflib_filter_info_t info; in iflib_irq_alloc_generic_subctx() local
6227 dev = ctx->ifc_dev; in iflib_irq_alloc_generic_subctx()
6228 subdev = subctx->ifc_dev; in iflib_irq_alloc_generic_subctx()
6232 q = &subctx->ifc_rxqs[qid]; in iflib_irq_alloc_generic_subctx()
6233 info = &subctx->ifc_rxqs[qid].ifr_filter_info; in iflib_irq_alloc_generic_subctx()
6234 gtask = &subctx->ifc_rxqs[qid].ifr_task; in iflib_irq_alloc_generic_subctx()
6246 info->ifi_filter = filter; in iflib_irq_alloc_generic_subctx()
6247 info->ifi_filter_arg = filter_arg; in iflib_irq_alloc_generic_subctx()
6248 info->ifi_task = gtask; in iflib_irq_alloc_generic_subctx()
6249 info->ifi_ctx = q; in iflib_irq_alloc_generic_subctx()
6254 err = _iflib_irq_alloc(ctx, irq, rid, intr_fast, NULL, info, name); in iflib_irq_alloc_generic_subctx()
6261 if (tqrid != -1) { in iflib_irq_alloc_generic_subctx()
6267 taskqgroup_attach(tqg, gtask, q, dev, irq->ii_res, name); in iflib_irq_alloc_generic_subctx()
6281 iflib_filter_info_t info; in iflib_irq_alloc_generic() local
6287 info = &ctx->ifc_filter_info; in iflib_irq_alloc_generic()
6293 q = &ctx->ifc_txqs[qid]; in iflib_irq_alloc_generic()
6294 info = &ctx->ifc_txqs[qid].ift_filter_info; in iflib_irq_alloc_generic()
6295 gtask = &ctx->ifc_txqs[qid].ift_task; in iflib_irq_alloc_generic()
6300 ctx->ifc_flags |= IFC_NETMAP_TX_IRQ; in iflib_irq_alloc_generic()
6303 q = &ctx->ifc_rxqs[qid]; in iflib_irq_alloc_generic()
6304 info = &ctx->ifc_rxqs[qid].ifr_filter_info; in iflib_irq_alloc_generic()
6305 gtask = &ctx->ifc_rxqs[qid].ifr_task; in iflib_irq_alloc_generic()
6312 q = &ctx->ifc_rxqs[qid]; in iflib_irq_alloc_generic()
6313 info = &ctx->ifc_rxqs[qid].ifr_filter_info; in iflib_irq_alloc_generic()
6314 gtask = &ctx->ifc_rxqs[qid].ifr_task; in iflib_irq_alloc_generic()
6322 tqrid = -1; in iflib_irq_alloc_generic()
6323 info = &ctx->ifc_filter_info; in iflib_irq_alloc_generic()
6324 gtask = &ctx->ifc_admin_task; in iflib_irq_alloc_generic()
6330 device_printf(ctx->ifc_dev, "%s: unknown net intr type\n", in iflib_irq_alloc_generic()
6335 info->ifi_filter = filter; in iflib_irq_alloc_generic()
6336 info->ifi_filter_arg = filter_arg; in iflib_irq_alloc_generic()
6337 info->ifi_task = gtask; in iflib_irq_alloc_generic()
6338 info->ifi_ctx = q; in iflib_irq_alloc_generic()
6340 dev = ctx->ifc_dev; in iflib_irq_alloc_generic()
6341 err = _iflib_irq_alloc(ctx, irq, rid, intr_fast, NULL, info, name); in iflib_irq_alloc_generic()
6349 if (tqrid != -1) { in iflib_irq_alloc_generic()
6355 taskqgroup_attach(tqg, gtask, q, dev, irq->ii_res, name); in iflib_irq_alloc_generic()
6374 q = &ctx->ifc_txqs[qid]; in iflib_softirq_alloc_generic()
6375 gtask = &ctx->ifc_txqs[qid].ift_task; in iflib_softirq_alloc_generic()
6381 q = &ctx->ifc_rxqs[qid]; in iflib_softirq_alloc_generic()
6382 gtask = &ctx->ifc_rxqs[qid].ifr_task; in iflib_softirq_alloc_generic()
6389 gtask = &ctx->ifc_vflr_task; in iflib_softirq_alloc_generic()
6399 dev = ctx->ifc_dev; in iflib_softirq_alloc_generic()
6400 taskqgroup_attach(tqg, gtask, q, dev, irq ? irq->ii_res : NULL, in iflib_softirq_alloc_generic()
6409 if (irq->ii_tag) in iflib_irq_free()
6410 bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); in iflib_irq_free()
6412 if (irq->ii_res) in iflib_irq_free()
6413 bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, in iflib_irq_free()
6414 rman_get_rid(irq->ii_res), irq->ii_res); in iflib_irq_free()
6420 iflib_txq_t txq = ctx->ifc_txqs; in iflib_legacy_setup()
6421 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_legacy_setup()
6422 if_irq_t irq = &ctx->ifc_legacy_irq; in iflib_legacy_setup()
6423 iflib_filter_info_t info; in iflib_legacy_setup() local
6430 info = &rxq->ifr_filter_info; in iflib_legacy_setup()
6431 gtask = &rxq->ifr_task; in iflib_legacy_setup()
6433 rx_only = (ctx->ifc_sctx->isc_flags & IFLIB_SINGLE_IRQ_RX_ONLY) != 0; in iflib_legacy_setup()
6435 ctx->ifc_flags |= IFC_LEGACY; in iflib_legacy_setup()
6436 info->ifi_filter = filter; in iflib_legacy_setup()
6437 info->ifi_filter_arg = filter_arg; in iflib_legacy_setup()
6438 info->ifi_task = gtask; in iflib_legacy_setup()
6439 info->ifi_ctx = rxq; in iflib_legacy_setup()
6441 dev = ctx->ifc_dev; in iflib_legacy_setup()
6444 iflib_fast_intr_rxtx, NULL, info, name); in iflib_legacy_setup()
6448 res = irq->ii_res; in iflib_legacy_setup()
6451 GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq); in iflib_legacy_setup()
6452 taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, dev, res, in iflib_legacy_setup()
6461 ctx->ifc_led_dev = led_create(iflib_led_func, ctx, in iflib_led_create()
6462 device_get_nameunit(ctx->ifc_dev)); in iflib_led_create()
6469 GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task); in iflib_tx_intr_deferred()
6476 GROUPTASK_ENQUEUE(&ctx->ifc_rxqs[rxqid].ifr_task); in iflib_rx_intr_deferred()
6483 MPASS(ctx->ifc_admin_task.gt_taskqueue != NULL); in iflib_admin_intr_deferred()
6484 GROUPTASK_ENQUEUE(&ctx->ifc_admin_task); in iflib_admin_intr_deferred()
6491 GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task); in iflib_iov_intr_deferred()
6522 if_t ifp = ctx->ifc_ifp; in iflib_link_state_change()
6523 iflib_txq_t txq = ctx->ifc_txqs; in iflib_link_state_change()
6528 ctx->ifc_flags |= IFC_PREFETCH; in iflib_link_state_change()
6532 if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) { in iflib_link_state_change()
6533 for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++) in iflib_link_state_change()
6534 txq->ift_qstatus = IFLIB_QUEUE_IDLE; in iflib_link_state_change()
6536 ctx->ifc_link_state = link_state; in iflib_link_state_change()
6545 int credits_pre = txq->ift_cidx_processed; in iflib_tx_credits_update()
6548 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, in iflib_tx_credits_update()
6550 if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0) in iflib_tx_credits_update()
6553 txq->ift_processed += credits; in iflib_tx_credits_update()
6554 txq->ift_cidx_processed += credits; in iflib_tx_credits_update()
6556 MPASS(credits_pre + credits == txq->ift_cidx_processed); in iflib_tx_credits_update()
6557 if (txq->ift_cidx_processed >= txq->ift_size) in iflib_tx_credits_update()
6558 txq->ift_cidx_processed -= txq->ift_size; in iflib_tx_credits_update()
6568 for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++) in iflib_rxd_avail()
6569 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_rxd_avail()
6571 return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx, in iflib_rxd_avail()
6577 const char *description, if_int_delay_info_t info, in iflib_add_int_delay_sysctl() argument
6580 info->iidi_ctx = ctx; in iflib_add_int_delay_sysctl()
6581 info->iidi_offset = offset; in iflib_add_int_delay_sysctl()
6582 info->iidi_value = value; in iflib_add_int_delay_sysctl()
6583 SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev), in iflib_add_int_delay_sysctl()
6584 SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)), in iflib_add_int_delay_sysctl()
6586 info, 0, iflib_sysctl_int_delay, "I", description); in iflib_add_int_delay_sysctl()
6593 return (&ctx->ifc_ctx_sx); in iflib_ctx_lock_get()
6599 device_t dev = ctx->ifc_dev; in iflib_msix_init()
6600 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_msix_init()
6601 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_msix_init()
6605 iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs; in iflib_msix_init()
6606 iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs; in iflib_msix_init()
6610 imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets)); in iflib_msix_init()
6613 if (scctx->isc_disable_msix) in iflib_msix_init()
6616 /* First try MSI-X */ in iflib_msix_init()
6619 device_printf(dev, "MSI-X not supported or disabled\n"); in iflib_msix_init()
6623 bar = ctx->ifc_softc_ctx.isc_msix_bar; in iflib_msix_init()
6625 * bar == -1 => "trust me I know what I'm doing" in iflib_msix_init()
6629 * allows shoddy garbage to use MSI-X in this framework. in iflib_msix_init()
6631 if (bar != -1) { in iflib_msix_init()
6632 ctx->ifc_msix_mem = bus_alloc_resource_any(dev, in iflib_msix_init()
6634 if (ctx->ifc_msix_mem == NULL) { in iflib_msix_init()
6635 device_printf(dev, "Unable to map MSI-X table\n"); in iflib_msix_init()
6640 admincnt = sctx->isc_admin_intrcnt; in iflib_msix_init()
6643 queuemsgs = min(msgs - admincnt, 1); in iflib_msix_init()
6645 queuemsgs = msgs - admincnt; in iflib_msix_init()
6652 queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues); in iflib_msix_init()
6656 CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt); in iflib_msix_init()
6662 if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queuemsgs - admincnt) in iflib_msix_init()
6667 if (rx_queues > scctx->isc_nrxqsets) in iflib_msix_init()
6668 rx_queues = scctx->isc_nrxqsets; in iflib_msix_init()
6678 if (tx_queues > scctx->isc_ntxqsets) in iflib_msix_init()
6679 tx_queues = scctx->isc_ntxqsets; in iflib_msix_init()
6681 if (ctx->ifc_sysctl_qs_eq_override == 0) { in iflib_msix_init()
6695 "insufficient number of MSI-X vectors " in iflib_msix_init()
6706 "Unable to allocate sufficient MSI-X vectors " in iflib_msix_init()
6709 if (bar != -1) { in iflib_msix_init()
6711 ctx->ifc_msix_mem); in iflib_msix_init()
6712 ctx->ifc_msix_mem = NULL; in iflib_msix_init()
6716 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", in iflib_msix_init()
6718 scctx->isc_vectors = vectors; in iflib_msix_init()
6719 scctx->isc_nrxqsets = rx_queues; in iflib_msix_init()
6720 scctx->isc_ntxqsets = tx_queues; in iflib_msix_init()
6721 scctx->isc_intr = IFLIB_INTR_MSIX; in iflib_msix_init()
6726 "failed to allocate %d MSI-X vectors, err: %d\n", vectors, in iflib_msix_init()
6728 if (bar != -1) { in iflib_msix_init()
6730 ctx->ifc_msix_mem); in iflib_msix_init()
6731 ctx->ifc_msix_mem = NULL; in iflib_msix_init()
6737 scctx->isc_nrxqsets = 1; in iflib_msix_init()
6738 scctx->isc_ntxqsets = 1; in iflib_msix_init()
6739 scctx->isc_vectors = vectors; in iflib_msix_init()
6742 scctx->isc_intr = IFLIB_INTR_MSI; in iflib_msix_init()
6744 scctx->isc_vectors = 1; in iflib_msix_init()
6746 scctx->isc_intr = IFLIB_INTR_LEGACY; in iflib_msix_init()
6758 uint16_t *state = ((uint16_t *)oidp->oid_arg1); in mp_ring_state_handler()
6799 ndesc = ctx->ifc_sysctl_ntxds; in mp_ndesc_handler()
6800 if (ctx->ifc_sctx) in mp_ndesc_handler()
6801 nqs = ctx->ifc_sctx->isc_ntxqs; in mp_ndesc_handler()
6804 ndesc = ctx->ifc_sysctl_nrxds; in mp_ndesc_handler()
6805 if (ctx->ifc_sctx) in mp_ndesc_handler()
6806 nqs = ctx->ifc_sctx->isc_nrxqs; in mp_ndesc_handler()
6824 if (rc || req->newptr == NULL) in mp_ndesc_handler()
6846 ctx->ifc_sysctl_node = node = SYSCTL_ADD_NODE(ctx_list, child, in iflib_add_device_sysctl_pre()
6852 CTLFLAG_RD, ctx->ifc_sctx->isc_driver_version, "driver version"); in iflib_add_device_sysctl_pre()
6855 CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0, in iflib_add_device_sysctl_pre()
6858 CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0, in iflib_add_device_sysctl_pre()
6861 CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0, in iflib_add_device_sysctl_pre()
6864 CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, in iflib_add_device_sysctl_pre()
6865 "disable MSI-X (default 0)"); in iflib_add_device_sysctl_pre()
6867 CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, "set the RX budget"); in iflib_add_device_sysctl_pre()
6869 CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, in iflib_add_device_sysctl_pre()
6871 ctx->ifc_sysctl_core_offset = CORE_OFFSET_UNSPECIFIED; in iflib_add_device_sysctl_pre()
6873 CTLFLAG_RDTUN, &ctx->ifc_sysctl_core_offset, 0, in iflib_add_device_sysctl_pre()
6876 CTLFLAG_RDTUN, &ctx->ifc_sysctl_separate_txrx, 0, in iflib_add_device_sysctl_pre()
6879 CTLFLAG_RDTUN, &ctx->ifc_sysctl_use_logical_cores, 0, in iflib_add_device_sysctl_pre()
6882 CTLFLAG_RDTUN, &ctx->ifc_sysctl_extra_msix_vectors, 0, in iflib_add_device_sysctl_pre()
6883 …"attempt to reserve the given number of extra MSI-X vectors during driver load for the creation of… in iflib_add_device_sysctl_pre()
6885 CTLFLAG_RDTUN, &ctx->ifc_softc_ctx.isc_vectors, 0, in iflib_add_device_sysctl_pre()
6886 "total # of MSI-X vectors allocated by driver"); in iflib_add_device_sysctl_pre()
6888 /* XXX change for per-queue sizes */ in iflib_add_device_sysctl_pre()
6902 if_shared_ctx_t sctx = ctx->ifc_sctx; in iflib_add_device_sysctl_post()
6903 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; in iflib_add_device_sysctl_post()
6917 node = ctx->ifc_sysctl_node; in iflib_add_device_sysctl_post()
6920 if (scctx->isc_ntxqsets > 100) in iflib_add_device_sysctl_post()
6922 else if (scctx->isc_ntxqsets > 10) in iflib_add_device_sysctl_post()
6926 for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) { in iflib_add_device_sysctl_post()
6932 CTLFLAG_RD, &txq->ift_task.gt_cpu, 0, in iflib_add_device_sysctl_post()
6936 CTLFLAG_RD, &txq->ift_dequeued, "total mbufs freed"); in iflib_add_device_sysctl_post()
6938 CTLFLAG_RD, &txq->ift_enqueued, "total mbufs enqueued"); in iflib_add_device_sysctl_post()
6941 CTLFLAG_RD, &txq->ift_mbuf_defrag, in iflib_add_device_sysctl_post()
6944 CTLFLAG_RD, &txq->ift_pullups, in iflib_add_device_sysctl_post()
6948 &txq->ift_mbuf_defrag_failed, "# of times m_defrag failed"); in iflib_add_device_sysctl_post()
6950 "no_desc_avail", CTLFLAG_RD, &txq->ift_no_desc_avail, in iflib_add_device_sysctl_post()
6953 "tx_map_failed", CTLFLAG_RD, &txq->ift_map_failed, in iflib_add_device_sysctl_post()
6954 "# of times DMA map failed"); in iflib_add_device_sysctl_post()
6956 "txd_encap_efbig", CTLFLAG_RD, &txq->ift_txd_encap_efbig, in iflib_add_device_sysctl_post()
6959 "no_tx_dma_setup", CTLFLAG_RD, &txq->ift_no_tx_dma_setup, in iflib_add_device_sysctl_post()
6962 CTLFLAG_RD, &txq->ift_pidx, 1, "Producer Index"); in iflib_add_device_sysctl_post()
6964 CTLFLAG_RD, &txq->ift_cidx, 1, "Consumer Index"); in iflib_add_device_sysctl_post()
6966 "txq_cidx_processed", CTLFLAG_RD, &txq->ift_cidx_processed, in iflib_add_device_sysctl_post()
6969 CTLFLAG_RD, &txq->ift_in_use, 1, "descriptors in use"); in iflib_add_device_sysctl_post()
6971 "txq_processed", CTLFLAG_RD, &txq->ift_processed, in iflib_add_device_sysctl_post()
6974 CTLFLAG_RD, &txq->ift_cleaned, "total cleaned"); in iflib_add_device_sysctl_post()
6977 __DEVOLATILE(uint64_t *, &txq->ift_br->state), 0, in iflib_add_device_sysctl_post()
6980 "r_enqueues", CTLFLAG_RD, &txq->ift_br->enqueues, in iflib_add_device_sysctl_post()
6983 "r_drops", CTLFLAG_RD, &txq->ift_br->drops, in iflib_add_device_sysctl_post()
6986 "r_starts", CTLFLAG_RD, &txq->ift_br->starts, in iflib_add_device_sysctl_post()
6989 "r_stalls", CTLFLAG_RD, &txq->ift_br->stalls, in iflib_add_device_sysctl_post()
6992 "r_restarts", CTLFLAG_RD, &txq->ift_br->restarts, in iflib_add_device_sysctl_post()
6995 "r_abdications", CTLFLAG_RD, &txq->ift_br->abdications, in iflib_add_device_sysctl_post()
6999 if (scctx->isc_nrxqsets > 100) in iflib_add_device_sysctl_post()
7001 else if (scctx->isc_nrxqsets > 10) in iflib_add_device_sysctl_post()
7005 for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_add_device_sysctl_post()
7011 CTLFLAG_RD, &rxq->ifr_task.gt_cpu, 0, in iflib_add_device_sysctl_post()
7013 if (sctx->isc_flags & IFLIB_HAS_RXCQ) { in iflib_add_device_sysctl_post()
7015 "rxq_cq_cidx", CTLFLAG_RD, &rxq->ifr_cq_cidx, 1, in iflib_add_device_sysctl_post()
7019 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_add_device_sysctl_post()
7026 CTLFLAG_RD, &fl->ifl_pidx, 1, "Producer Index"); in iflib_add_device_sysctl_post()
7028 CTLFLAG_RD, &fl->ifl_cidx, 1, "Consumer Index"); in iflib_add_device_sysctl_post()
7030 CTLFLAG_RD, &fl->ifl_credits, 1, in iflib_add_device_sysctl_post()
7033 CTLFLAG_RD, &fl->ifl_buf_size, 1, "buffer size"); in iflib_add_device_sysctl_post()
7036 "fl_m_enqueued", CTLFLAG_RD, &fl->ifl_m_enqueued, in iflib_add_device_sysctl_post()
7039 "fl_m_dequeued", CTLFLAG_RD, &fl->ifl_m_dequeued, in iflib_add_device_sysctl_post()
7042 "fl_cl_enqueued", CTLFLAG_RD, &fl->ifl_cl_enqueued, in iflib_add_device_sysctl_post()
7045 "fl_cl_dequeued", CTLFLAG_RD, &fl->ifl_cl_dequeued, in iflib_add_device_sysctl_post()
7058 ctx->ifc_flags |= IFC_DO_RESET; in iflib_request_reset()
7068 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { in iflib_fixup_rx()
7069 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); in iflib_fixup_rx()
7070 m->m_data += ETHER_HDR_LEN; in iflib_fixup_rx()
7078 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); in iflib_fixup_rx()
7079 m->m_data += ETHER_HDR_LEN; in iflib_fixup_rx()
7080 m->m_len -= ETHER_HDR_LEN; in iflib_fixup_rx()
7081 n->m_len = ETHER_HDR_LEN; in iflib_fixup_rx()
7083 n->m_next = m; in iflib_fixup_rx()
7098 *ncl = ctx->ifc_rxqs[0].ifr_fl->ifl_size; in iflib_debugnet_init()
7099 *clsize = ctx->ifc_rxqs[0].ifr_fl->ifl_buf_size; in iflib_debugnet_init()
7113 scctx = &ctx->ifc_softc_ctx; in iflib_debugnet_event()
7117 for (i = 0; i < scctx->isc_nrxqsets; i++) { in iflib_debugnet_event()
7118 rxq = &ctx->ifc_rxqs[i]; in iflib_debugnet_event()
7119 for (j = 0; j < rxq->ifr_nfl; j++) { in iflib_debugnet_event()
7120 fl = rxq->ifr_fl; in iflib_debugnet_event()
7121 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_debugnet_event()
7143 txq = &ctx->ifc_txqs[0]; in iflib_debugnet_transmit()
7160 scctx = &ctx->ifc_softc_ctx; in iflib_debugnet_poll()
7166 txq = &ctx->ifc_txqs[0]; in iflib_debugnet_poll()
7170 for (i = 0; i < scctx->isc_nrxqsets; i++) in iflib_debugnet_poll()
7171 (void)iflib_rxeof(&ctx->ifc_rxqs[i], 16 /* XXX */); in iflib_debugnet_poll()