Lines Matching refs:rxq
141 static void iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid);
702 static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget);
769 static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init);
843 iflib_rxq_t rxq = &ctx->ifc_rxqs[0]; in iflib_netmap_config() local
844 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_config()
859 netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init) in netmap_fl_refill() argument
866 if_ctx_t ctx = rxq->ifr_ctx; in netmap_fl_refill()
867 iflib_fl_t fl = &rxq->ifr_fl[0]; in netmap_fl_refill()
901 iru_init(&iru, rxq, 0 /* flid */); in netmap_fl_refill()
965 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, in netmap_fl_refill()
1182 iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id]; in iflib_netmap_rxsync() local
1183 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_rxsync()
1222 cidxp = &rxq->ifr_cq_cidx; in iflib_netmap_rxsync()
1226 rxq->ifr_id, *cidxp, USHRT_MAX); in iflib_netmap_rxsync()
1233 ri.iri_frags = rxq->ifr_frags; in iflib_netmap_rxsync()
1287 netmap_fl_refill(rxq, kring, false); in iflib_netmap_rxsync()
1358 iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq) in iflib_netmap_rxq_init() argument
1364 slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0); in iflib_netmap_rxq_init()
1367 kring = na->rx_rings[rxq->ifr_id]; in iflib_netmap_rxq_init()
1368 netmap_fl_refill(rxq, kring, true); in iflib_netmap_rxq_init()
1389 #define iflib_netmap_rxq_init(ctx, rxq) (0) argument
1426 iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid) in iru_init() argument
1430 fl = &rxq->ifr_fl[flid]; in iru_init()
1433 iru->iru_qsidx = rxq->ifr_id; in iru_init()
1589 iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; in iflib_fast_intr_rxtx() local
1603 ctx = rxq->ifr_ctx; in iflib_fast_intr_rxtx()
1607 MPASS(rxq->ifr_ntxqirq); in iflib_fast_intr_rxtx()
1608 for (i = 0; i < rxq->ifr_ntxqirq; i++) { in iflib_fast_intr_rxtx()
1609 txqid = rxq->ifr_txqid[i]; in iflib_fast_intr_rxtx()
1623 cidx = rxq->ifr_cq_cidx; in iflib_fast_intr_rxtx()
1625 cidx = rxq->ifr_fl[0].ifl_cidx; in iflib_fast_intr_rxtx()
1626 if (iflib_rxd_avail(ctx, rxq, cidx, 1)) in iflib_fast_intr_rxtx()
1632 IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); in iflib_fast_intr_rxtx()
1949 iflib_rxsd_alloc(iflib_rxq_t rxq) in iflib_rxsd_alloc() argument
1951 if_ctx_t ctx = rxq->ifr_ctx; in iflib_rxsd_alloc()
1960 MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0); in iflib_rxsd_alloc()
1964 fl = rxq->ifr_fl; in iflib_rxsd_alloc()
1965 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { in iflib_rxsd_alloc()
1966 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ in iflib_rxsd_alloc()
1989 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
1999 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
2009 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { in iflib_rxsd_alloc()
2020 …(bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOW… in iflib_rxsd_alloc()
2026 for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { in iflib_rxsd_alloc()
2300 iflib_rxq_t rxq = fl->ifl_rxq; in iflib_fl_setup() local
2301 if_ctx_t ctx = rxq->ifr_ctx; in iflib_fl_setup()
2312 qidx = rxq->ifr_fl_offset + fl->ifl_id; in iflib_fl_setup()
2340 MPASS(rxq != NULL); in iflib_fl_setup()
2353 iflib_rx_sds_free(iflib_rxq_t rxq) in iflib_rx_sds_free() argument
2358 if (rxq->ifr_fl != NULL) { in iflib_rx_sds_free()
2359 for (i = 0; i < rxq->ifr_nfl; i++) { in iflib_rx_sds_free()
2360 fl = &rxq->ifr_fl[i]; in iflib_rx_sds_free()
2390 free(rxq->ifr_fl, M_IFLIB); in iflib_rx_sds_free()
2391 rxq->ifr_fl = NULL; in iflib_rx_sds_free()
2392 free(rxq->ifr_ifdi, M_IFLIB); in iflib_rx_sds_free()
2393 rxq->ifr_ifdi = NULL; in iflib_rx_sds_free()
2394 rxq->ifr_cq_cidx = 0; in iflib_rx_sds_free()
2491 iflib_rxq_t rxq; in iflib_init_locked() local
2538 for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_init_locked()
2539 if (iflib_netmap_rxq_init(ctx, rxq) > 0) { in iflib_init_locked()
2543 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_init_locked()
2592 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_stop() local
2646 for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_stop()
2647 if (rxq->ifr_task.gt_taskqueue != NULL) in iflib_stop()
2648 gtaskqueue_drain(rxq->ifr_task.gt_taskqueue, in iflib_stop()
2649 &rxq->ifr_task.gt_task); in iflib_stop()
2651 rxq->ifr_cq_cidx = 0; in iflib_stop()
2652 for (j = 0, di = rxq->ifr_ifdi; j < sctx->isc_nrxqs; j++, di++) in iflib_stop()
2655 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_stop()
2702 rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, bool unload, if_rxsd_t sd, in rxd_frag_to_sd() argument
2714 fl = &rxq->ifr_fl[flid]; in rxd_frag_to_sd()
2721 if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) in rxd_frag_to_sd()
2729 if (rxq->pfil != NULL && PFIL_HOOKED_IN(rxq->pfil) && pf_rv != NULL && in rxd_frag_to_sd()
2734 *pf_rv = pfil_mem_in(rxq->pfil, payload, len, ri->iri_ifp, &m); in rxd_frag_to_sd()
2779 assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd, int *pf_rv) in assemble_segments() argument
2792 m = rxd_frag_to_sd(rxq, &ri->iri_frags[i], !consumed, sd, in assemble_segments()
2846 iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri) in iflib_rxd_pkt_get() argument
2856 m = rxd_frag_to_sd(rxq, &ri->iri_frags[0], false, &sd, in iflib_rxd_pkt_get()
2872 m = assemble_segments(rxq, ri, &sd, &pf_rv); in iflib_rxd_pkt_get()
2940 iflib_rxq_t rxq = context; in _task_fn_rx_watchdog() local
2942 GROUPTASK_ENQUEUE(&rxq->ifr_task); in _task_fn_rx_watchdog()
2946 iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) in iflib_rxeof() argument
2949 if_ctx_t ctx = rxq->ifr_ctx; in iflib_rxeof()
2975 cidxp = &rxq->ifr_cq_cidx; in iflib_rxeof()
2977 cidxp = &rxq->ifr_fl[0].ifl_cidx; in iflib_rxeof()
2978 if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) { in iflib_rxeof()
2979 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2996 ri.iri_qsidx = rxq->ifr_id; in iflib_rxeof()
2999 ri.iri_frags = rxq->ifr_frags; in iflib_rxeof()
3010 while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) in iflib_rxeof()
3011 rxq->ifr_cq_cidx -= scctx->isc_nrxd[0]; in iflib_rxeof()
3020 m = iflib_rxd_pkt_get(rxq, &ri); in iflib_rxeof()
3024 avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget_left); in iflib_rxeof()
3039 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
3044 iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding); in iflib_rxeof()
3066 if (lro_possible && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0) in iflib_rxeof()
3095 tcp_lro_flush_all(&rxq->ifr_lc); in iflib_rxeof()
3097 if (avail != 0 || iflib_rxd_avail(ctx, rxq, *cidxp, 1) != 0) in iflib_rxeof()
4133 iflib_rxq_t rxq = context; in _task_fn_rx() local
4134 if_ctx_t ctx = rxq->ifr_ctx; in _task_fn_rx()
4143 rxq->ifr_cpu_exec_count[curcpu]++; in _task_fn_rx()
4149 nmirq = netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work); in _task_fn_rx()
4158 more = iflib_rxeof(rxq, budget); in _task_fn_rx()
4166 IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); in _task_fn_rx()
4173 GROUPTASK_ENQUEUE(&rxq->ifr_task); in _task_fn_rx()
4175 callout_reset_curcpu(&rxq->ifr_watchdog, 1, &_task_fn_rx_watchdog, rxq); in _task_fn_rx()
4849 iflib_rxq_t rxq; in iflib_add_pfil() local
4858 for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { in iflib_add_pfil()
4859 rxq->pfil = pfil; in iflib_add_pfil()
4867 iflib_rxq_t rxq; in iflib_rem_pfil() local
4870 rxq = ctx->ifc_rxqs; in iflib_rem_pfil()
4871 pfil = rxq->pfil; in iflib_rem_pfil()
4872 for (i = 0; i < NRXQSETS(ctx); i++, rxq++) { in iflib_rem_pfil()
4873 rxq->pfil = NULL; in iflib_rem_pfil()
5550 iflib_rxq_t rxq; in iflib_tqg_detach() local
5564 for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { in iflib_tqg_detach()
5565 if (rxq->ifr_task.gt_uniq != NULL) in iflib_tqg_detach()
5566 taskqgroup_detach(tqg, &rxq->ifr_task); in iflib_tqg_detach()
5855 iflib_rxq_t rxq; in iflib_queues_alloc() local
5892 rxq = ctx->ifc_rxqs; in iflib_queues_alloc()
5952 for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) { in iflib_queues_alloc()
5954 callout_init(&rxq->ifr_watchdog, 1); in iflib_queues_alloc()
5964 rxq->ifr_ifdi = ifdip; in iflib_queues_alloc()
5966 rxq->ifr_ntxqirq = 1; in iflib_queues_alloc()
5967 rxq->ifr_txqid[0] = i; in iflib_queues_alloc()
5977 rxq->ifr_ctx = ctx; in iflib_queues_alloc()
5978 rxq->ifr_id = i; in iflib_queues_alloc()
5979 rxq->ifr_fl_offset = fl_offset; in iflib_queues_alloc()
5980 rxq->ifr_nfl = nfree_lists; in iflib_queues_alloc()
5987 rxq->ifr_fl = fl; in iflib_queues_alloc()
5989 fl[j].ifl_rxq = rxq; in iflib_queues_alloc()
5991 fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; in iflib_queues_alloc()
5995 if (iflib_rxsd_alloc(rxq)) { in iflib_queues_alloc()
6002 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_queues_alloc()
6103 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_rx_structures_setup() local
6109 for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) { in iflib_rx_structures_setup()
6111 err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp, in iflib_rx_structures_setup()
6113 ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset])); in iflib_rx_structures_setup()
6120 IFDI_RXQ_SETUP(ctx, rxq->ifr_id); in iflib_rx_structures_setup()
6130 rxq = ctx->ifc_rxqs; in iflib_rx_structures_setup()
6131 for (i = 0; i < q; ++i, rxq++) { in iflib_rx_structures_setup()
6132 tcp_lro_free(&rxq->ifr_lc); in iflib_rx_structures_setup()
6146 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_rx_structures_free() local
6150 for (i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) { in iflib_rx_structures_free()
6152 iflib_dma_free(&rxq->ifr_ifdi[j]); in iflib_rx_structures_free()
6153 iflib_rx_sds_free(rxq); in iflib_rx_structures_free()
6155 tcp_lro_free(&rxq->ifr_lc); in iflib_rx_structures_free()
6433 iflib_rxq_t rxq = ctx->ifc_rxqs; in iflib_legacy_setup() local
6442 info = &rxq->ifr_filter_info; in iflib_legacy_setup()
6443 gtask = &rxq->ifr_task; in iflib_legacy_setup()
6451 info->ifi_ctx = rxq; in iflib_legacy_setup()
6459 NET_GROUPTASK_INIT(gtask, 0, _task_fn_rx, rxq); in iflib_legacy_setup()
6461 taskqgroup_attach(qgroup_if_io_tqg, gtask, rxq, dev, res, name); in iflib_legacy_setup()
6569 iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget) in iflib_rxd_avail() argument
6574 for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++) in iflib_rxd_avail()
6577 return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx, in iflib_rxd_avail()
6915 iflib_rxq_t rxq; in iflib_add_device_sysctl_post() local
7011 for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) { in iflib_add_device_sysctl_post()
7017 CTLFLAG_RD, &rxq->ifr_task.gt_cpu, 0, in iflib_add_device_sysctl_post()
7021 "rxq_cq_cidx", CTLFLAG_RD, &rxq->ifr_cq_cidx, 1, in iflib_add_device_sysctl_post()
7025 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_add_device_sysctl_post()
7115 iflib_rxq_t rxq; in iflib_debugnet_event() local
7124 rxq = &ctx->ifc_rxqs[i]; in iflib_debugnet_event()
7125 for (j = 0; j < rxq->ifr_nfl; j++) { in iflib_debugnet_event()
7126 fl = rxq->ifr_fl; in iflib_debugnet_event()