Lines Matching refs:fl
844 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_config() local
850 info->rx_buf_maxsize = fl->ifl_buf_size; in iflib_netmap_config()
867 iflib_fl_t fl = &rxq->ifr_fl[0]; in netmap_fl_refill() local
902 map = fl->ifl_sds.ifsd_map; in netmap_fl_refill()
903 nic_i = fl->ifl_pidx; in netmap_fl_refill()
931 fl->ifl_bus_addrs[i] = paddr + in netmap_fl_refill()
933 fl->ifl_rxd_idxs[i] = nic_i; in netmap_fl_refill()
936 netmap_load_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
940 netmap_reload_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
943 bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i], in netmap_fl_refill()
955 fl->ifl_pidx = nic_i; in netmap_fl_refill()
963 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in netmap_fl_refill()
965 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, in netmap_fl_refill()
1183 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_rxsync() local
1192 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_netmap_rxsync()
1224 cidxp = &fl->ifl_cidx; in iflib_netmap_rxsync()
1228 nic_i = fl->ifl_cidx; in iflib_netmap_rxsync()
1256 bus_dmamap_sync(fl->ifl_buf_tag, in iflib_netmap_rxsync()
1257 fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); in iflib_netmap_rxsync()
1259 fl->ifl_cidx = nic_i = nm_next(nic_i, lim); in iflib_netmap_rxsync()
1428 iflib_fl_t fl; in iru_init() local
1430 fl = &rxq->ifr_fl[flid]; in iru_init()
1431 iru->iru_paddrs = fl->ifl_bus_addrs; in iru_init()
1432 iru->iru_idxs = fl->ifl_rxd_idxs; in iru_init()
1434 iru->iru_buf_size = fl->ifl_buf_size; in iru_init()
1435 iru->iru_flidx = fl->ifl_id; in iru_init()
1955 iflib_fl_t fl; in iflib_rxsd_alloc() local
1964 fl = rxq->ifr_fl; in iflib_rxsd_alloc()
1965 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { in iflib_rxsd_alloc()
1966 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ in iflib_rxsd_alloc()
1979 &fl->ifl_buf_tag); in iflib_rxsd_alloc()
1987 if (!(fl->ifl_sds.ifsd_m = in iflib_rxsd_alloc()
1997 if (!(fl->ifl_sds.ifsd_cl = in iflib_rxsd_alloc()
2007 if (!(fl->ifl_sds.ifsd_ba = in iflib_rxsd_alloc()
2019 if (!(fl->ifl_sds.ifsd_map = in iflib_rxsd_alloc()
2027 err = bus_dmamap_create(fl->ifl_buf_tag, 0, in iflib_rxsd_alloc()
2028 &fl->ifl_sds.ifsd_map[i]); in iflib_rxsd_alloc()
2073 iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) in iflib_fl_refill() argument
2085 MPASS(count <= fl->ifl_size - fl->ifl_credits - 1); in iflib_fl_refill()
2087 sd_m = fl->ifl_sds.ifsd_m; in iflib_fl_refill()
2088 sd_map = fl->ifl_sds.ifsd_map; in iflib_fl_refill()
2089 sd_cl = fl->ifl_sds.ifsd_cl; in iflib_fl_refill()
2090 sd_ba = fl->ifl_sds.ifsd_ba; in iflib_fl_refill()
2091 pidx = fl->ifl_pidx; in iflib_fl_refill()
2093 frag_idx = fl->ifl_fragidx; in iflib_fl_refill()
2094 credits = fl->ifl_credits; in iflib_fl_refill()
2099 MPASS(credits + n <= fl->ifl_size); in iflib_fl_refill()
2101 if (pidx < fl->ifl_cidx) in iflib_fl_refill()
2102 MPASS(pidx + n <= fl->ifl_cidx); in iflib_fl_refill()
2103 if (pidx == fl->ifl_cidx && (credits < fl->ifl_size)) in iflib_fl_refill()
2104 MPASS(fl->ifl_gen == 0); in iflib_fl_refill()
2105 if (pidx > fl->ifl_cidx) in iflib_fl_refill()
2106 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx); in iflib_fl_refill()
2111 iru_init(&iru, fl->ifl_rxq, fl->ifl_id); in iflib_fl_refill()
2120 bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, in iflib_fl_refill()
2123 bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); in iflib_fl_refill()
2126 cl = uma_zalloc(fl->ifl_zone, M_NOWAIT); in iflib_fl_refill()
2132 err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2133 cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, in iflib_fl_refill()
2136 uma_zfree(fl->ifl_zone, cl); in iflib_fl_refill()
2143 fl->ifl_cl_enqueued++; in iflib_fl_refill()
2148 bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2157 bit_set(fl->ifl_rx_bitmap, frag_idx); in iflib_fl_refill()
2159 fl->ifl_m_enqueued++; in iflib_fl_refill()
2163 fl->ifl_rxd_idxs[i] = frag_idx; in iflib_fl_refill()
2164 fl->ifl_bus_addrs[i] = bus_addr; in iflib_fl_refill()
2167 MPASS(credits <= fl->ifl_size); in iflib_fl_refill()
2168 if (++idx == fl->ifl_size) { in iflib_fl_refill()
2170 fl->ifl_gen = 1; in iflib_fl_refill()
2178 fl->ifl_pidx = idx; in iflib_fl_refill()
2179 fl->ifl_credits = credits; in iflib_fl_refill()
2190 fl->ifl_pidx = idx; in iflib_fl_refill()
2191 fl->ifl_credits = credits; in iflib_fl_refill()
2194 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_refill()
2196 ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, in iflib_fl_refill()
2197 fl->ifl_id, fl->ifl_pidx); in iflib_fl_refill()
2198 if (__predict_true(bit_test(fl->ifl_rx_bitmap, frag_idx))) { in iflib_fl_refill()
2199 fl->ifl_fragidx = frag_idx + 1; in iflib_fl_refill()
2200 if (fl->ifl_fragidx == fl->ifl_size) in iflib_fl_refill()
2201 fl->ifl_fragidx = 0; in iflib_fl_refill()
2203 fl->ifl_fragidx = frag_idx; in iflib_fl_refill()
2211 iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl) in iflib_fl_refill_all() argument
2224 int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; in iflib_fl_refill_all()
2226 …int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - … in iflib_fl_refill_all()
2229 MPASS(fl->ifl_credits <= fl->ifl_size); in iflib_fl_refill_all()
2233 return (iflib_fl_refill(ctx, fl, reclaimable)); in iflib_fl_refill_all()
2249 iflib_fl_bufs_free(iflib_fl_t fl) in iflib_fl_bufs_free() argument
2251 iflib_dma_info_t idi = fl->ifl_ifdi; in iflib_fl_bufs_free()
2255 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2256 struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; in iflib_fl_bufs_free()
2257 caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; in iflib_fl_bufs_free()
2260 sd_map = fl->ifl_sds.ifsd_map[i]; in iflib_fl_bufs_free()
2261 bus_dmamap_sync(fl->ifl_buf_tag, sd_map, in iflib_fl_bufs_free()
2263 bus_dmamap_unload(fl->ifl_buf_tag, sd_map); in iflib_fl_bufs_free()
2264 uma_zfree(fl->ifl_zone, *sd_cl); in iflib_fl_bufs_free()
2275 fl->ifl_m_dequeued++; in iflib_fl_bufs_free()
2276 fl->ifl_cl_dequeued++; in iflib_fl_bufs_free()
2280 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2281 MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); in iflib_fl_bufs_free()
2282 MPASS(fl->ifl_sds.ifsd_m[i] == NULL); in iflib_fl_bufs_free()
2288 fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; in iflib_fl_bufs_free()
2298 iflib_fl_setup(iflib_fl_t fl) in iflib_fl_setup() argument
2300 iflib_rxq_t rxq = fl->ifl_rxq; in iflib_fl_setup()
2305 bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1); in iflib_fl_setup()
2309 iflib_fl_bufs_free(fl); in iflib_fl_setup()
2311 MPASS(fl->ifl_credits == 0); in iflib_fl_setup()
2312 qidx = rxq->ifr_fl_offset + fl->ifl_id; in iflib_fl_setup()
2314 fl->ifl_buf_size = scctx->isc_rxd_buf_size[qidx]; in iflib_fl_setup()
2316 fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz; in iflib_fl_setup()
2321 fl->ifl_buf_size = iflib_get_mbuf_size_for(fl->ifl_buf_size); in iflib_fl_setup()
2322 if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size) in iflib_fl_setup()
2323 ctx->ifc_max_fl_buf_size = fl->ifl_buf_size; in iflib_fl_setup()
2324 fl->ifl_cltype = m_gettype(fl->ifl_buf_size); in iflib_fl_setup()
2325 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_fl_setup()
2333 MPASS(fl->ifl_size > 0); in iflib_fl_setup()
2334 (void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size - 1)); in iflib_fl_setup()
2335 if (min(128, fl->ifl_size - 1) != fl->ifl_credits) in iflib_fl_setup()
2341 MPASS(fl->ifl_ifdi != NULL); in iflib_fl_setup()
2342 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_setup()
2355 iflib_fl_t fl; in iflib_rx_sds_free() local
2360 fl = &rxq->ifr_fl[i]; in iflib_rx_sds_free()
2361 if (fl->ifl_buf_tag != NULL) { in iflib_rx_sds_free()
2362 if (fl->ifl_sds.ifsd_map != NULL) { in iflib_rx_sds_free()
2363 for (j = 0; j < fl->ifl_size; j++) { in iflib_rx_sds_free()
2365 fl->ifl_buf_tag, in iflib_rx_sds_free()
2366 fl->ifl_sds.ifsd_map[j], in iflib_rx_sds_free()
2369 fl->ifl_buf_tag, in iflib_rx_sds_free()
2370 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2372 fl->ifl_buf_tag, in iflib_rx_sds_free()
2373 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2376 bus_dma_tag_destroy(fl->ifl_buf_tag); in iflib_rx_sds_free()
2377 fl->ifl_buf_tag = NULL; in iflib_rx_sds_free()
2379 free(fl->ifl_sds.ifsd_m, M_IFLIB); in iflib_rx_sds_free()
2380 free(fl->ifl_sds.ifsd_cl, M_IFLIB); in iflib_rx_sds_free()
2381 free(fl->ifl_sds.ifsd_ba, M_IFLIB); in iflib_rx_sds_free()
2382 free(fl->ifl_sds.ifsd_map, M_IFLIB); in iflib_rx_sds_free()
2383 free(fl->ifl_rx_bitmap, M_IFLIB); in iflib_rx_sds_free()
2384 fl->ifl_sds.ifsd_m = NULL; in iflib_rx_sds_free()
2385 fl->ifl_sds.ifsd_cl = NULL; in iflib_rx_sds_free()
2386 fl->ifl_sds.ifsd_ba = NULL; in iflib_rx_sds_free()
2387 fl->ifl_sds.ifsd_map = NULL; in iflib_rx_sds_free()
2388 fl->ifl_rx_bitmap = NULL; in iflib_rx_sds_free()
2489 iflib_fl_t fl; in iflib_init_locked() local
2543 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_init_locked()
2544 if (iflib_fl_setup(fl)) { in iflib_init_locked()
2596 iflib_fl_t fl; in iflib_stop() local
2655 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_stop()
2656 iflib_fl_bufs_free(fl); in iflib_stop()
2661 calc_next_rxd(iflib_fl_t fl, int cidx) in calc_next_rxd() argument
2667 nrxd = fl->ifl_size; in calc_next_rxd()
2668 size = fl->ifl_rxd_size; in calc_next_rxd()
2669 start = fl->ifl_ifdi->idi_vaddr; in calc_next_rxd()
2680 prefetch_pkts(iflib_fl_t fl, int cidx) in prefetch_pkts() argument
2683 int nrxd = fl->ifl_size; in prefetch_pkts()
2687 prefetch(&fl->ifl_sds.ifsd_m[nextptr]); in prefetch_pkts()
2688 prefetch(&fl->ifl_sds.ifsd_cl[nextptr]); in prefetch_pkts()
2689 next_rxd = calc_next_rxd(fl, cidx); in prefetch_pkts()
2691 prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd - 1)]); in prefetch_pkts()
2692 prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd - 1)]); in prefetch_pkts()
2693 prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd - 1)]); in prefetch_pkts()
2694 prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd - 1)]); in prefetch_pkts()
2695 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd - 1)]); in prefetch_pkts()
2696 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd - 1)]); in prefetch_pkts()
2697 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd - 1)]); in prefetch_pkts()
2698 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd - 1)]); in prefetch_pkts()
2706 iflib_fl_t fl; in rxd_frag_to_sd() local
2714 fl = &rxq->ifr_fl[flid]; in rxd_frag_to_sd()
2715 sd->ifsd_fl = fl; in rxd_frag_to_sd()
2716 sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; in rxd_frag_to_sd()
2717 fl->ifl_credits--; in rxd_frag_to_sd()
2719 fl->ifl_m_dequeued++; in rxd_frag_to_sd()
2722 prefetch_pkts(fl, cidx); in rxd_frag_to_sd()
2723 next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size - 1); in rxd_frag_to_sd()
2724 prefetch(&fl->ifl_sds.ifsd_map[next]); in rxd_frag_to_sd()
2725 map = fl->ifl_sds.ifsd_map[cidx]; in rxd_frag_to_sd()
2727 bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); in rxd_frag_to_sd()
2756 m = fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2757 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2763 m = fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2764 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2770 bus_dmamap_unload(fl->ifl_buf_tag, map); in rxd_frag_to_sd()
2771 fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size - 1); in rxd_frag_to_sd()
2772 if (__predict_false(fl->ifl_cidx == 0)) in rxd_frag_to_sd()
2773 fl->ifl_gen = 0; in rxd_frag_to_sd()
2774 bit_clear(fl->ifl_rx_bitmap, cidx); in rxd_frag_to_sd()
2956 iflib_fl_t fl; in iflib_rxeof() local
2979 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2980 retval |= iflib_fl_refill_all(ctx, fl); in iflib_rxeof()
3039 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
3040 retval |= iflib_fl_refill_all(ctx, fl); in iflib_rxeof()
5856 iflib_fl_t fl = NULL; in iflib_queues_alloc() local
5981 if (!(fl = in iflib_queues_alloc()
5987 rxq->ifr_fl = fl; in iflib_queues_alloc()
5989 fl[j].ifl_rxq = rxq; in iflib_queues_alloc()
5990 fl[j].ifl_id = j; in iflib_queues_alloc()
5991 fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; in iflib_queues_alloc()
5992 fl[j].ifl_rxd_size = scctx->isc_rxd_size[j]; in iflib_queues_alloc()
6002 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_queues_alloc()
6003 fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, in iflib_queues_alloc()
6571 iflib_fl_t fl; in iflib_rxd_avail() local
6574 for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++) in iflib_rxd_avail()
6575 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_rxd_avail()
6913 iflib_fl_t fl; in iflib_add_device_sysctl_post() local
7025 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_add_device_sysctl_post()
7032 CTLFLAG_RD, &fl->ifl_pidx, 1, "Producer Index"); in iflib_add_device_sysctl_post()
7034 CTLFLAG_RD, &fl->ifl_cidx, 1, "Consumer Index"); in iflib_add_device_sysctl_post()
7036 CTLFLAG_RD, &fl->ifl_credits, 1, in iflib_add_device_sysctl_post()
7039 CTLFLAG_RD, &fl->ifl_buf_size, 1, "buffer size"); in iflib_add_device_sysctl_post()
7042 "fl_m_enqueued", CTLFLAG_RD, &fl->ifl_m_enqueued, in iflib_add_device_sysctl_post()
7045 "fl_m_dequeued", CTLFLAG_RD, &fl->ifl_m_dequeued, in iflib_add_device_sysctl_post()
7048 "fl_cl_enqueued", CTLFLAG_RD, &fl->ifl_cl_enqueued, in iflib_add_device_sysctl_post()
7051 "fl_cl_dequeued", CTLFLAG_RD, &fl->ifl_cl_dequeued, in iflib_add_device_sysctl_post()
7114 iflib_fl_t fl; in iflib_debugnet_event() local
7126 fl = rxq->ifr_fl; in iflib_debugnet_event()
7127 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_debugnet_event()