Lines Matching refs:kring
808 static int netmap_txsync_to_host(struct netmap_kring *kring, int flags);
809 static int netmap_rxsync_from_host(struct netmap_kring *kring, int flags);
812 netmap_default_bufcfg(struct netmap_kring *kring, uint64_t target) in netmap_default_bufcfg() argument
814 kring->hwbuf_len = target; in netmap_default_bufcfg()
815 kring->buf_align = 0; /* no alignment */ in netmap_default_bufcfg()
848 struct netmap_kring *kring; in netmap_krings_create() local
876 kring = (struct netmap_kring *)((char *)na->tailroom + tailroom); in netmap_krings_create()
878 na->tx_rings[i] = kring; in netmap_krings_create()
879 kring++; in netmap_krings_create()
889 kring = NMR(na, t)[i]; in netmap_krings_create()
890 bzero(kring, sizeof(*kring)); in netmap_krings_create()
891 kring->notify_na = na; in netmap_krings_create()
892 kring->ring_id = i; in netmap_krings_create()
893 kring->tx = t; in netmap_krings_create()
894 kring->nkr_num_slots = ndesc; in netmap_krings_create()
895 kring->nr_mode = NKR_NETMAP_OFF; in netmap_krings_create()
896 kring->nr_pending_mode = NKR_NETMAP_OFF; in netmap_krings_create()
898 kring->nm_sync = (t == NR_TX ? na->nm_txsync : na->nm_rxsync); in netmap_krings_create()
899 kring->nm_bufcfg = na->nm_bufcfg; in netmap_krings_create()
900 if (kring->nm_bufcfg == NULL) in netmap_krings_create()
901 kring->nm_bufcfg = netmap_default_bufcfg; in netmap_krings_create()
904 kring->nr_kflags |= NKR_FAKERING; in netmap_krings_create()
905 kring->nm_sync = (t == NR_TX ? in netmap_krings_create()
908 kring->nm_bufcfg = netmap_default_bufcfg; in netmap_krings_create()
910 kring->nm_notify = na->nm_notify; in netmap_krings_create()
911 kring->rhead = kring->rcur = kring->nr_hwcur = 0; in netmap_krings_create()
915 kring->rtail = kring->nr_hwtail = (t == NR_TX ? ndesc - 1 : 0); in netmap_krings_create()
916 snprintf(kring->name, sizeof(kring->name) - 1, "%s %s%d", na->name, in netmap_krings_create()
919 kring->name, kring->rhead, kring->rcur, kring->rtail); in netmap_krings_create()
920 err = nm_os_selinfo_init(&kring->si, kring->name); in netmap_krings_create()
925 mtx_init(&kring->q_lock, (t == NR_TX ? "nm_txq_lock" : "nm_rxq_lock"), NULL, MTX_DEF); in netmap_krings_create()
926 kring->na = na; /* setting this field marks the mutex as initialized */ in netmap_krings_create()
944 struct netmap_kring **kring = na->tx_rings; in netmap_krings_delete() local
957 for ( ; kring != na->tailroom; kring++) { in netmap_krings_delete()
958 if ((*kring)->na != NULL) in netmap_krings_delete()
959 mtx_destroy(&(*kring)->q_lock); in netmap_krings_delete()
960 nm_os_selinfo_uninit(&(*kring)->si); in netmap_krings_delete()
1014 struct netmap_kring *kring; in netmap_update_hostrings_mode() local
1020 kring = NMR(na, t)[i]; in netmap_update_hostrings_mode()
1021 kring->nr_mode = kring->nr_pending_mode; in netmap_update_hostrings_mode()
1230 netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force) in netmap_grab_packets() argument
1232 u_int const lim = kring->nkr_num_slots - 1; in netmap_grab_packets()
1233 u_int const head = kring->rhead; in netmap_grab_packets()
1235 struct netmap_adapter *na = kring->na; in netmap_grab_packets()
1237 for (n = kring->nr_hwcur; n != head; n = nm_next(n, lim)) { in netmap_grab_packets()
1239 struct netmap_slot *slot = &kring->ring->slot[n]; in netmap_grab_packets()
1258 _nm_may_forward(struct netmap_kring *kring) in _nm_may_forward() argument
1260 return ((netmap_fwd || kring->ring->flags & NR_FORWARD) && in _nm_may_forward()
1261 kring->na->na_flags & NAF_HOST_RINGS && in _nm_may_forward()
1262 kring->tx == NR_RX); in _nm_may_forward()
1266 nm_may_forward_up(struct netmap_kring *kring) in nm_may_forward_up() argument
1268 return _nm_may_forward(kring) && in nm_may_forward_up()
1269 kring->ring_id != kring->na->num_rx_rings; in nm_may_forward_up()
1273 nm_may_forward_down(struct netmap_kring *kring, int sync_flags) in nm_may_forward_down() argument
1275 return _nm_may_forward(kring) && in nm_may_forward_down()
1277 kring->ring_id == kring->na->num_rx_rings; in nm_may_forward_down()
1295 struct netmap_kring *kring = na->rx_rings[na->num_rx_rings]; in netmap_sw_to_nic() local
1296 struct netmap_slot *rxslot = kring->ring->slot; in netmap_sw_to_nic()
1297 u_int i, rxcur = kring->nr_hwcur; in netmap_sw_to_nic()
1298 u_int const head = kring->rhead; in netmap_sw_to_nic()
1299 u_int const src_lim = kring->nkr_num_slots - 1; in netmap_sw_to_nic()
1346 netmap_txsync_to_host(struct netmap_kring *kring, int flags) in netmap_txsync_to_host() argument
1348 struct netmap_adapter *na = kring->na; in netmap_txsync_to_host()
1349 u_int const lim = kring->nkr_num_slots - 1; in netmap_txsync_to_host()
1350 u_int const head = kring->rhead; in netmap_txsync_to_host()
1357 netmap_grab_packets(kring, &q, 1 /* force */); in netmap_txsync_to_host()
1359 kring->nr_hwcur = head; in netmap_txsync_to_host()
1360 kring->nr_hwtail = head + lim; in netmap_txsync_to_host()
1361 if (kring->nr_hwtail > lim) in netmap_txsync_to_host()
1362 kring->nr_hwtail -= lim + 1; in netmap_txsync_to_host()
1379 netmap_rxsync_from_host(struct netmap_kring *kring, int flags) in netmap_rxsync_from_host() argument
1381 struct netmap_adapter *na = kring->na; in netmap_rxsync_from_host()
1382 struct netmap_ring *ring = kring->ring; in netmap_rxsync_from_host()
1384 u_int const lim = kring->nkr_num_slots - 1; in netmap_rxsync_from_host()
1385 u_int const head = kring->rhead; in netmap_rxsync_from_host()
1387 struct mbq *q = &kring->rx_queue, fq; in netmap_rxsync_from_host()
1399 nm_i = kring->nr_hwtail; in netmap_rxsync_from_host()
1400 stop_i = nm_prev(kring->nr_hwcur, lim); in netmap_rxsync_from_host()
1415 kring->nr_hwtail = nm_i; in netmap_rxsync_from_host()
1421 nm_i = kring->nr_hwcur; in netmap_rxsync_from_host()
1423 if (nm_may_forward_down(kring, flags)) { in netmap_rxsync_from_host()
1426 kring->nr_kflags |= NR_FORWARD; in netmap_rxsync_from_host()
1430 kring->nr_hwcur = head; in netmap_rxsync_from_host()
1688 kring->name, \
1690 kring->rhead, kring->rcur, kring->rtail, \
1691 kring->nr_hwcur, kring->nr_hwtail); \
1692 return kring->nkr_num_slots; \
1710 nm_txsync_prologue(struct netmap_kring *kring, struct netmap_ring *ring) in nm_txsync_prologue() argument
1714 u_int n = kring->nkr_num_slots; in nm_txsync_prologue()
1717 kring->name, in nm_txsync_prologue()
1718 kring->nr_hwcur, kring->nr_hwtail, in nm_txsync_prologue()
1721 NM_FAIL_ON(kring->nr_hwcur >= n || kring->rhead >= n || in nm_txsync_prologue()
1722 kring->rtail >= n || kring->nr_hwtail >= n); in nm_txsync_prologue()
1733 if (kring->rtail >= kring->rhead) { in nm_txsync_prologue()
1735 NM_FAIL_ON(head < kring->rhead || head > kring->rtail); in nm_txsync_prologue()
1737 NM_FAIL_ON(cur < head || cur > kring->rtail); in nm_txsync_prologue()
1740 NM_FAIL_ON(head > kring->rtail && head < kring->rhead); in nm_txsync_prologue()
1743 if (head <= kring->rtail) { in nm_txsync_prologue()
1745 NM_FAIL_ON(cur < head || cur > kring->rtail); in nm_txsync_prologue()
1748 NM_FAIL_ON(cur > kring->rtail && cur < head); in nm_txsync_prologue()
1751 if (ring->tail != kring->rtail) { in nm_txsync_prologue()
1752 nm_prlim(5, "%s tail overwritten was %d need %d", kring->name, in nm_txsync_prologue()
1753 ring->tail, kring->rtail); in nm_txsync_prologue()
1754 ring->tail = kring->rtail; in nm_txsync_prologue()
1756 kring->rhead = head; in nm_txsync_prologue()
1757 kring->rcur = cur; in nm_txsync_prologue()
1774 nm_rxsync_prologue(struct netmap_kring *kring, struct netmap_ring *ring) in nm_rxsync_prologue() argument
1776 uint32_t const n = kring->nkr_num_slots; in nm_rxsync_prologue()
1780 kring->name, in nm_rxsync_prologue()
1781 kring->nr_hwcur, kring->nr_hwtail, in nm_rxsync_prologue()
1790 cur = kring->rcur = NM_ACCESS_ONCE(ring->cur); in nm_rxsync_prologue()
1791 head = kring->rhead = NM_ACCESS_ONCE(ring->head); in nm_rxsync_prologue()
1793 NM_FAIL_ON(kring->nr_hwcur >= n || kring->nr_hwtail >= n); in nm_rxsync_prologue()
1796 if (kring->nr_hwtail >= kring->nr_hwcur) { in nm_rxsync_prologue()
1798 NM_FAIL_ON(head < kring->nr_hwcur || head > kring->nr_hwtail); in nm_rxsync_prologue()
1800 NM_FAIL_ON(cur < head || cur > kring->nr_hwtail); in nm_rxsync_prologue()
1803 NM_FAIL_ON(head < kring->nr_hwcur && head > kring->nr_hwtail); in nm_rxsync_prologue()
1805 if (head <= kring->nr_hwtail) { in nm_rxsync_prologue()
1807 NM_FAIL_ON(cur < head || cur > kring->nr_hwtail); in nm_rxsync_prologue()
1810 NM_FAIL_ON(cur < head && cur > kring->nr_hwtail); in nm_rxsync_prologue()
1813 if (ring->tail != kring->rtail) { in nm_rxsync_prologue()
1815 kring->name, in nm_rxsync_prologue()
1816 ring->tail, kring->rtail); in nm_rxsync_prologue()
1817 ring->tail = kring->rtail; in nm_rxsync_prologue()
1836 netmap_ring_reinit(struct netmap_kring *kring) in netmap_ring_reinit() argument
1838 struct netmap_ring *ring = kring->ring; in netmap_ring_reinit()
1839 u_int i, lim = kring->nkr_num_slots - 1; in netmap_ring_reinit()
1843 nm_prlim(10, "called for %s", kring->name); in netmap_ring_reinit()
1845 kring->rhead = ring->head; in netmap_ring_reinit()
1846 kring->rcur = ring->cur; in netmap_ring_reinit()
1847 kring->rtail = ring->tail; in netmap_ring_reinit()
1858 if (idx < 2 || idx >= kring->na->na_lut.objtotal) { in netmap_ring_reinit()
1862 } else if (len > NETMAP_BUF_SIZE(kring->na)) { in netmap_ring_reinit()
1870 kring->name, in netmap_ring_reinit()
1871 ring->cur, kring->nr_hwcur, in netmap_ring_reinit()
1872 ring->tail, kring->nr_hwtail); in netmap_ring_reinit()
1873 ring->head = kring->rhead = kring->nr_hwcur; in netmap_ring_reinit()
1874 ring->cur = kring->rcur = kring->nr_hwcur; in netmap_ring_reinit()
1875 ring->tail = kring->rtail = kring->nr_hwtail; in netmap_ring_reinit()
2055 struct netmap_kring *kring; in netmap_krings_get() local
2071 foreach_selected_ring(priv, t, i, kring) { in netmap_krings_get()
2072 if ((kring->nr_kflags & NKR_EXCLUSIVE) || in netmap_krings_get()
2073 (kring->users && excl)) in netmap_krings_get()
2075 nm_prdis("ring %s busy", kring->name); in netmap_krings_get()
2083 foreach_selected_ring(priv, t, i, kring) { in netmap_krings_get()
2084 kring->users++; in netmap_krings_get()
2086 kring->nr_kflags |= NKR_EXCLUSIVE; in netmap_krings_get()
2087 kring->nr_pending_mode = NKR_NETMAP_ON; in netmap_krings_get()
2101 struct netmap_kring *kring; in netmap_krings_put() local
2112 foreach_selected_ring(priv, t, i, kring) { in netmap_krings_put()
2114 kring->nr_kflags &= ~NKR_EXCLUSIVE; in netmap_krings_put()
2115 kring->users--; in netmap_krings_put()
2116 if (kring->users == 0) in netmap_krings_put()
2117 kring->nr_pending_mode = NKR_NETMAP_OFF; in netmap_krings_put()
2203 struct netmap_kring *kring = in netmap_csb_validate() local
2213 CSB_WRITE(csb_atok, head, kring->rhead); in netmap_csb_validate()
2214 CSB_WRITE(csb_atok, cur, kring->rcur); in netmap_csb_validate()
2217 CSB_WRITE(csb_ktoa, hwcur, kring->nr_hwcur); in netmap_csb_validate()
2218 CSB_WRITE(csb_ktoa, hwtail, kring->nr_hwtail); in netmap_csb_validate()
2222 "hwcur %u, hwtail %u", kring->name, in netmap_csb_validate()
2223 kring->rhead, kring->rcur, kring->nr_hwcur, in netmap_csb_validate()
2224 kring->nr_hwtail); in netmap_csb_validate()
2285 struct netmap_kring *kring; in netmap_offsets_init() local
2352 foreach_selected_ring(priv, t, i, kring) { in netmap_offsets_init()
2353 struct netmap_kring *kring = NMR(na, t)[i]; in netmap_offsets_init() local
2354 struct netmap_ring *ring = kring->ring; in netmap_offsets_init()
2360 if (kring->offset_mask) { in netmap_offsets_init()
2361 if ((kring->offset_mask & mask) != mask || in netmap_offsets_init()
2362 kring->offset_max < max_offset) { in netmap_offsets_init()
2367 kring->name, in netmap_offsets_init()
2368 (unsigned long long)kring->offset_mask, in netmap_offsets_init()
2369 (unsigned long long)kring->offset_max); in netmap_offsets_init()
2373 mask = kring->offset_mask; in netmap_offsets_init()
2374 max_offset = kring->offset_max; in netmap_offsets_init()
2376 kring->offset_mask = mask; in netmap_offsets_init()
2378 kring->offset_max = max_offset; in netmap_offsets_init()
2379 kring->offset_gap = min_gap; in netmap_offsets_init()
2388 if (!initial_offset || kring->users > 1) in netmap_offsets_init()
2391 for (j = 0; j < kring->nkr_num_slots; j++) { in netmap_offsets_init()
2394 nm_write_offset(kring, slot, initial_offset); in netmap_offsets_init()
2418 struct netmap_kring *kring; in netmap_compute_buf_len() local
2424 foreach_selected_ring(priv, t, i, kring) { in netmap_compute_buf_len()
2428 if (kring->users > 1) in netmap_compute_buf_len()
2442 target = NETMAP_BUF_SIZE(kring->na) - in netmap_compute_buf_len()
2443 kring->offset_max; in netmap_compute_buf_len()
2454 if (!kring->offset_gap) in netmap_compute_buf_len()
2455 kring->offset_gap = in netmap_compute_buf_len()
2456 NETMAP_BUF_SIZE(kring->na); in netmap_compute_buf_len()
2458 if (kring->offset_gap < target) in netmap_compute_buf_len()
2459 target = kring->offset_gap; in netmap_compute_buf_len()
2460 error = kring->nm_bufcfg(kring, target); in netmap_compute_buf_len()
2464 *(uint64_t *)(uintptr_t)&kring->ring->buf_align = kring->buf_align; in netmap_compute_buf_len()
2466 if (mtu && t == NR_RX && kring->hwbuf_len < mtu) { in netmap_compute_buf_len()
2479 kring->name, mtu, in netmap_compute_buf_len()
2480 (unsigned long long)kring->hwbuf_len); in netmap_compute_buf_len()
2699 nm_sync_finalize(struct netmap_kring *kring) in nm_sync_finalize() argument
2706 kring->ring->tail = kring->rtail = kring->nr_hwtail; in nm_sync_finalize()
2709 kring->name, kring->nr_hwcur, kring->nr_hwtail, in nm_sync_finalize()
2710 kring->rhead, kring->rcur, kring->rtail); in nm_sync_finalize()
3186 struct netmap_kring *kring = krings[i]; in netmap_ioctl() local
3187 struct netmap_ring *ring = kring->ring; in netmap_ioctl()
3189 if (unlikely(nm_kr_tryget(kring, 1, &error))) { in netmap_ioctl()
3198 kring->nr_hwcur); in netmap_ioctl()
3199 if (nm_txsync_prologue(kring, ring) >= kring->nkr_num_slots) { in netmap_ioctl()
3200 netmap_ring_reinit(kring); in netmap_ioctl()
3201 } else if (kring->nm_sync(kring, sync_flags | NAF_FORCE_RECLAIM) == 0) { in netmap_ioctl()
3202 nm_sync_finalize(kring); in netmap_ioctl()
3207 kring->nr_hwcur); in netmap_ioctl()
3209 if (nm_rxsync_prologue(kring, ring) >= kring->nkr_num_slots) { in netmap_ioctl()
3210 netmap_ring_reinit(kring); in netmap_ioctl()
3212 if (nm_may_forward_up(kring)) { in netmap_ioctl()
3214 netmap_grab_packets(kring, &q, netmap_fwd); in netmap_ioctl()
3216 if (kring->nm_sync(kring, sync_flags | NAF_FORCE_READ) == 0) { in netmap_ioctl()
3217 nm_sync_finalize(kring); in netmap_ioctl()
3221 nm_kr_put(kring); in netmap_ioctl()
3657 struct netmap_kring *kring; in netmap_poll() local
3729 kring = NMR(na, t)[i]; in netmap_poll()
3730 if (kring->ring->cur != kring->ring->tail) { in netmap_poll()
3744 kring = NMR(na, t)[i]; in netmap_poll()
3745 if (kring->ring->cur == kring->ring->tail in netmap_poll()
3746 || kring->rhead != kring->ring->head) { in netmap_poll()
3785 kring = na->tx_rings[i]; in netmap_poll()
3786 ring = kring->ring; in netmap_poll()
3794 if (!send_down && !want_tx && ring->head == kring->nr_hwcur) in netmap_poll()
3797 if (nm_kr_tryget(kring, 1, &revents)) in netmap_poll()
3800 if (nm_txsync_prologue(kring, ring) >= kring->nkr_num_slots) { in netmap_poll()
3801 netmap_ring_reinit(kring); in netmap_poll()
3804 if (kring->nm_sync(kring, sync_flags)) in netmap_poll()
3807 nm_sync_finalize(kring); in netmap_poll()
3816 found = kring->rcur != kring->rtail; in netmap_poll()
3817 nm_kr_put(kring); in netmap_poll()
3822 kring->nm_notify(kring, 0); in netmap_poll()
3847 kring = na->rx_rings[i]; in netmap_poll()
3848 ring = kring->ring; in netmap_poll()
3850 if (unlikely(nm_kr_tryget(kring, 1, &revents))) in netmap_poll()
3853 if (nm_rxsync_prologue(kring, ring) >= kring->nkr_num_slots) { in netmap_poll()
3854 netmap_ring_reinit(kring); in netmap_poll()
3863 if (nm_may_forward_up(kring)) { in netmap_poll()
3864 netmap_grab_packets(kring, &q, netmap_fwd); in netmap_poll()
3870 kring->nr_kflags &= ~NR_FORWARD; in netmap_poll()
3871 if (kring->nm_sync(kring, sync_flags)) in netmap_poll()
3874 nm_sync_finalize(kring); in netmap_poll()
3875 send_down |= (kring->nr_kflags & NR_FORWARD); in netmap_poll()
3877 found = kring->rcur != kring->rtail; in netmap_poll()
3878 nm_kr_put(kring); in netmap_poll()
3883 kring->nm_notify(kring, 0); in netmap_poll()
3926 struct netmap_kring *kring = NMR(na, t)[i]; in nma_intr_enable() local
3927 int on = !(kring->nr_kflags & NKR_NOINTR); in nma_intr_enable()
3933 kring->nr_kflags &= ~NKR_NOINTR; in nma_intr_enable()
3935 kring->nr_kflags |= NKR_NOINTR; in nma_intr_enable()
3960 netmap_notify(struct netmap_kring *kring, int flags) in netmap_notify() argument
3962 struct netmap_adapter *na = kring->notify_na; in netmap_notify()
3963 enum txrx t = kring->tx; in netmap_notify()
3965 nm_os_selwakeup(&kring->si); in netmap_notify()
4263 struct netmap_kring *kring, *tx_kring; in netmap_transmit() local
4275 kring = NMR(na, NR_RX)[nma_get_nrings(na, NR_RX) + i]; in netmap_transmit()
4297 q = &kring->rx_queue; in netmap_transmit()
4330 busy = kring->nr_hwtail - kring->nr_hwcur; in netmap_transmit()
4332 busy += kring->nkr_num_slots; in netmap_transmit()
4333 if (busy + mbq_len(q) >= kring->nkr_num_slots - 1) { in netmap_transmit()
4335 kring->nr_hwcur, kring->nr_hwtail, mbq_len(q)); in netmap_transmit()
4351 kring->nm_notify(kring, 0); in netmap_transmit()
4377 struct netmap_kring *kring; in netmap_reset() local
4388 kring = na->tx_rings[n]; in netmap_reset()
4397 new_hwofs = kring->rhead; in netmap_reset()
4398 new_hwtail = nm_prev(kring->rhead, kring->nkr_num_slots - 1); in netmap_reset()
4402 kring = na->rx_rings[n]; in netmap_reset()
4411 new_hwofs = kring->nr_hwtail; in netmap_reset()
4412 new_hwtail = kring->nr_hwtail; in netmap_reset()
4414 if (kring->nr_pending_mode == NKR_NETMAP_OFF) { in netmap_reset()
4415 kring->nr_mode = NKR_NETMAP_OFF; in netmap_reset()
4419 nm_prinf("%s, hc %u->%u, ht %u->%u, ho %u->%u", kring->name, in netmap_reset()
4420 kring->nr_hwcur, kring->rhead, in netmap_reset()
4421 kring->nr_hwtail, new_hwtail, in netmap_reset()
4422 kring->nkr_hwofs, new_hwofs); in netmap_reset()
4424 kring->nr_hwcur = kring->rhead; in netmap_reset()
4425 kring->nr_hwtail = new_hwtail; in netmap_reset()
4426 kring->nkr_hwofs = new_hwofs; in netmap_reset()
4433 kring->nr_mode = NKR_NETMAP_ON; in netmap_reset()
4434 kring->nm_notify(kring, 0); in netmap_reset()
4435 return kring->ring->slot; in netmap_reset()
4456 struct netmap_kring *kring; in netmap_common_irq() local
4468 kring = NMR(na, t)[q]; in netmap_common_irq()
4470 if (kring->nr_mode == NKR_NETMAP_OFF) { in netmap_common_irq()
4475 kring->nr_kflags |= NKR_PENDINTR; // XXX atomic ? in netmap_common_irq()
4479 return kring->nm_notify(kring, 0); in netmap_common_irq()
4564 struct netmap_kring *kring = NMR(na, t)[i]; in netmap_krings_mode_commit() local
4566 if (onoff && nm_kring_pending_on(kring)) in netmap_krings_mode_commit()
4567 kring->nr_mode = NKR_NETMAP_ON; in netmap_krings_mode_commit()
4568 else if (!onoff && nm_kring_pending_off(kring)) in netmap_krings_mode_commit()
4569 kring->nr_mode = NKR_NETMAP_OFF; in netmap_krings_mode_commit()