Lines Matching +full:txrx +full:-

2  * Copyright (C) 2014-2016 Giuseppe Lettieri
48 * Monitors can be either zero-copy or copy-based.
52 * - For tx traffic, this is when the application sends them, before they are
55 * - For rx traffic, this is when they are received by the adapter, before
60 * Zero-copy monitors only see the frames after they have been consumed:
62 * - For tx traffic, this is after the slots containing the frames have been
66 * - For rx traffic, this is after the consumer on the monitored adapter
70 * Several copy or zero-copy monitors may be active on any ring.
134 return na->nm_register == netmap_zmon_reg; in nm_is_zmon()
143 nm_prlim(1, "%s %x", kring->name, flags); in netmap_monitor_txsync()
157 (struct netmap_monitor_adapter *)kring->na; in netmap_monitor_rxsync()
158 if (unlikely(mna->priv.np_na == NULL)) { in netmap_monitor_rxsync()
162 nm_prdis("%s %x", kring->name, flags); in netmap_monitor_rxsync()
163 kring->nr_hwcur = kring->rhead; in netmap_monitor_rxsync()
174 enum txrx t; in netmap_monitor_krings_create()
184 kring->nm_sync = t == NR_TX ? netmap_monitor_txsync : in netmap_monitor_krings_create()
200 nm_txrx2flag(enum txrx t) in nm_txrx2flag()
212 if (n <= kring->max_monitors) in nm_monitor_alloc()
216 old_len = sizeof(struct netmap_kring *)*kring->max_monitors; in nm_monitor_alloc()
218 nm = nm_os_realloc(kring->monitors, len, old_len); in nm_monitor_alloc()
222 kring->monitors = nm; in nm_monitor_alloc()
223 kring->max_monitors = n; in nm_monitor_alloc()
232 if (kring->monitors) { in nm_monitor_dealloc()
233 if (kring->n_monitors > 0) { in nm_monitor_dealloc()
235 kring->name, kring->n_monitors); in nm_monitor_dealloc()
237 nm_os_free(kring->monitors); in nm_monitor_dealloc()
238 kring->monitors = NULL; in nm_monitor_dealloc()
239 kring->max_monitors = 0; in nm_monitor_dealloc()
240 kring->n_monitors = 0; in nm_monitor_dealloc()
248 return kring->n_monitors == 0 && in nm_monitor_none()
249 kring->zmon_list[NR_TX].next == NULL && in nm_monitor_none()
250 kring->zmon_list[NR_RX].next == NULL; in nm_monitor_none()
274 nm_prdis("intercept callbacks on %s", kring->name); in nm_monitor_intercept_callbacks()
275 kring->mon_sync = kring->nm_sync != NULL ? in nm_monitor_intercept_callbacks()
276 kring->nm_sync : nm_monitor_dummycb; in nm_monitor_intercept_callbacks()
277 kring->mon_notify = kring->nm_notify; in nm_monitor_intercept_callbacks()
278 if (kring->tx == NR_TX) { in nm_monitor_intercept_callbacks()
279 kring->nm_sync = netmap_monitor_parent_txsync; in nm_monitor_intercept_callbacks()
281 kring->nm_sync = netmap_monitor_parent_rxsync; in nm_monitor_intercept_callbacks()
282 kring->nm_notify = netmap_monitor_parent_notify; in nm_monitor_intercept_callbacks()
283 kring->mon_tail = kring->nr_hwtail; in nm_monitor_intercept_callbacks()
290 nm_prdis("restoring callbacks on %s", kring->name); in nm_monitor_restore_callbacks()
291 kring->nm_sync = kring->mon_sync; in nm_monitor_restore_callbacks()
292 kring->mon_sync = NULL; in nm_monitor_restore_callbacks()
293 if (kring->tx == NR_RX) { in nm_monitor_restore_callbacks()
294 kring->nm_notify = kring->mon_notify; in nm_monitor_restore_callbacks()
296 kring->mon_notify = NULL; in nm_monitor_restore_callbacks()
300 nm_zmon_list_head(struct netmap_kring *mkring, enum txrx t) in nm_zmon_list_head()
302 struct netmap_adapter *na = mkring->na; in nm_zmon_list_head()
304 struct netmap_zmon_list *z = &kring->zmon_list[t]; in nm_zmon_list_head()
306 while (nm_is_zmon(na) && z->prev != NULL) { in nm_zmon_list_head()
307 kring = z->prev; in nm_zmon_list_head()
308 na = kring->na; in nm_zmon_list_head()
309 z = &kring->zmon_list[t]; in nm_zmon_list_head()
321 enum txrx t = kring->tx; in netmap_monitor_add()
322 struct netmap_zmon_list *z = &kring->zmon_list[t]; in netmap_monitor_add()
323 struct netmap_zmon_list *mz = &mkring->zmon_list[t]; in netmap_monitor_add()
326 /* a zero-copy monitor which is not the first in the list in netmap_monitor_add()
329 if (zmon && z->prev != NULL) in netmap_monitor_add()
330 ikring = z->prev; /* tail of the list */ in netmap_monitor_add()
337 nm_prdis("%s: intercept callbacks on %s", mkring->name, ikring->name); in netmap_monitor_add()
343 ikring->zmon_list[t].next = mkring; in netmap_monitor_add()
344 z->prev = mkring; /* new tail */ in netmap_monitor_add()
345 mz->prev = ikring; in netmap_monitor_add()
346 mz->next = NULL; in netmap_monitor_add()
349 * or another zero-copy monitor) in netmap_monitor_add()
351 netmap_adapter_get(ikring->na); in netmap_monitor_add()
354 error = nm_monitor_alloc(kring, kring->n_monitors + 1); in netmap_monitor_add()
357 kring->monitors[kring->n_monitors] = mkring; in netmap_monitor_add()
358 mkring->mon_pos[kring->tx] = kring->n_monitors; in netmap_monitor_add()
359 kring->n_monitors++; in netmap_monitor_add()
371 netmap_monitor_del(struct netmap_kring *mkring, struct netmap_kring *kring, enum txrx t) in netmap_monitor_del()
373 int zmon = nm_is_zmon(mkring->na); in netmap_monitor_del()
374 struct netmap_zmon_list *mz = &mkring->zmon_list[t]; in netmap_monitor_del()
381 ikring = mz->prev; in netmap_monitor_del()
393 if (mz->next != NULL) { in netmap_monitor_del()
394 mz->next->zmon_list[t].prev = mz->prev; in netmap_monitor_del()
399 if (mz->prev != NULL) in netmap_monitor_del()
400 netmap_adapter_get(mz->prev->na); in netmap_monitor_del()
401 netmap_adapter_put(mkring->na); in netmap_monitor_del()
406 kring->zmon_list[t].prev = in netmap_monitor_del()
407 (mz->prev != kring ? mz->prev : NULL); in netmap_monitor_del()
409 if (mz->prev != NULL) { in netmap_monitor_del()
410 netmap_adapter_put(mz->prev->na); in netmap_monitor_del()
411 mz->prev->zmon_list[t].next = mz->next; in netmap_monitor_del()
413 mz->prev = NULL; in netmap_monitor_del()
414 mz->next = NULL; in netmap_monitor_del()
417 uint32_t mon_pos = mkring->mon_pos[kring->tx]; in netmap_monitor_del()
418 kring->n_monitors--; in netmap_monitor_del()
419 if (mon_pos != kring->n_monitors) { in netmap_monitor_del()
420 kring->monitors[mon_pos] = in netmap_monitor_del()
421 kring->monitors[kring->n_monitors]; in netmap_monitor_del()
422 kring->monitors[mon_pos]->mon_pos[kring->tx] = mon_pos; in netmap_monitor_del()
424 kring->monitors[kring->n_monitors] = NULL; in netmap_monitor_del()
425 if (kring->n_monitors == 0) { in netmap_monitor_del()
443 * We do this by setting their mna->priv.np_na to NULL.
450 enum txrx t; in netmap_monitor_stop()
457 struct netmap_zmon_list *z = &kring->zmon_list[t]; in netmap_monitor_stop()
463 for (j = 0; j < kring->n_monitors; j++) { in netmap_monitor_stop()
465 kring->monitors[j]; in netmap_monitor_stop()
467 (struct netmap_monitor_adapter *)mkring->na; in netmap_monitor_stop()
469 if (mna->priv.np_na != NULL) { in netmap_monitor_stop()
470 netmap_adapter_put(mna->priv.np_na); in netmap_monitor_stop()
471 mna->priv.np_na = NULL; in netmap_monitor_stop()
473 kring->monitors[j] = NULL; in netmap_monitor_stop()
475 kring->n_monitors = 0; in netmap_monitor_stop()
481 for (zkring = z->next; zkring != NULL; in netmap_monitor_stop()
482 zkring = zkring->zmon_list[t].next) in netmap_monitor_stop()
485 (struct netmap_monitor_adapter *)zkring->na; in netmap_monitor_stop()
487 netmap_adapter_put(next->priv.np_na); /* nop if null */ in netmap_monitor_stop()
488 next->priv.np_na = NULL; in netmap_monitor_stop()
490 netmap_adapter_put(zkring->zmon_list[t].prev->na); in netmap_monitor_stop()
493 if (z->next != NULL) in netmap_monitor_stop()
494 z->next->zmon_list[t].prev = NULL; in netmap_monitor_stop()
495 z->next = NULL; in netmap_monitor_stop()
496 z->prev = NULL; in netmap_monitor_stop()
513 struct netmap_priv_d *priv = &mna->priv; in netmap_monitor_reg_common()
514 struct netmap_adapter *pna = priv->np_na; in netmap_monitor_reg_common()
517 enum txrx t, s; in netmap_monitor_reg_common()
523 nm_prerr("%s: parent left netmap mode", na->name); in netmap_monitor_reg_common()
531 mkring->nr_mode = NKR_NETMAP_ON; in netmap_monitor_reg_common()
537 if (mna->flags & nm_txrx2flag(s)) { in netmap_monitor_reg_common()
544 na->na_flags |= NAF_NETMAP_ON; in netmap_monitor_reg_common()
546 if (na->active_fds == 0) in netmap_monitor_reg_common()
547 na->na_flags &= ~NAF_NETMAP_ON; in netmap_monitor_reg_common()
553 mkring->nr_mode = NKR_NETMAP_OFF; in netmap_monitor_reg_common()
565 if (mna->flags & nm_txrx2flag(s)) { in netmap_monitor_reg_common()
578 * functions specific for zero-copy monitors
583 * Common function for both zero-copy tx and rx nm_sync()
587 netmap_zmon_parent_sync(struct netmap_kring *kring, int flags, enum txrx tx) in netmap_zmon_parent_sync()
589 struct netmap_kring *mkring = kring->zmon_list[tx].next; in netmap_zmon_parent_sync()
590 struct netmap_ring *ring = kring->ring, *mring; in netmap_zmon_parent_sync()
594 u_int lim = kring->nkr_num_slots - 1, in netmap_zmon_parent_sync()
595 mlim; // = mkring->nkr_num_slots - 1; in netmap_zmon_parent_sync()
596 uint16_t txmon = kring->tx == NR_TX ? NS_TXMON : 0; in netmap_zmon_parent_sync()
599 nm_prlim(5, "NULL monitor on %s", kring->name); in netmap_zmon_parent_sync()
602 mring = mkring->ring; in netmap_zmon_parent_sync()
603 mlim = mkring->nkr_num_slots - 1; in netmap_zmon_parent_sync()
607 beg = kring->nr_hwtail + 1; in netmap_zmon_parent_sync()
608 error = kring->mon_sync(kring, flags); in netmap_zmon_parent_sync()
611 end = kring->nr_hwtail + 1; in netmap_zmon_parent_sync()
613 beg = kring->nr_hwcur; in netmap_zmon_parent_sync()
614 end = kring->rhead; in netmap_zmon_parent_sync()
617 rel_slots = end - beg; in netmap_zmon_parent_sync()
619 rel_slots += kring->nkr_num_slots; in netmap_zmon_parent_sync()
632 mtx_lock(&mkring->q_lock); in netmap_zmon_parent_sync()
634 i = mkring->nr_hwtail; in netmap_zmon_parent_sync()
635 busy = i - mkring->nr_hwcur; in netmap_zmon_parent_sync()
637 busy += mkring->nkr_num_slots; in netmap_zmon_parent_sync()
638 free_slots = mlim - busy; in netmap_zmon_parent_sync()
645 beg += (rel_slots - free_slots); in netmap_zmon_parent_sync()
648 if (unlikely(beg >= kring->nkr_num_slots)) in netmap_zmon_parent_sync()
649 beg -= kring->nkr_num_slots; in netmap_zmon_parent_sync()
652 for ( ; rel_slots; rel_slots--) { in netmap_zmon_parent_sync()
653 struct netmap_slot *s = &ring->slot[beg]; in netmap_zmon_parent_sync()
654 struct netmap_slot *ms = &mring->slot[i]; in netmap_zmon_parent_sync()
657 tmp = ms->buf_idx; in netmap_zmon_parent_sync()
658 ms->buf_idx = s->buf_idx; in netmap_zmon_parent_sync()
659 s->buf_idx = tmp; in netmap_zmon_parent_sync()
662 tmp = ms->len; in netmap_zmon_parent_sync()
663 ms->len = s->len; in netmap_zmon_parent_sync()
664 s->len = tmp; in netmap_zmon_parent_sync()
666 ms->flags = (s->flags & ~NS_TXMON) | txmon; in netmap_zmon_parent_sync()
667 s->flags |= NS_BUF_CHANGED; in netmap_zmon_parent_sync()
674 mkring->nr_hwtail = i; in netmap_zmon_parent_sync()
677 mtx_unlock(&mkring->q_lock); in netmap_zmon_parent_sync()
681 mkring->nm_notify(mkring, 0); in netmap_zmon_parent_sync()
686 error = kring->mon_sync(kring, flags); in netmap_zmon_parent_sync()
717 struct netmap_priv_d *priv = &mna->priv; in netmap_zmon_dtor()
718 struct netmap_adapter *pna = priv->np_na; in netmap_zmon_dtor()
733 uint16_t txmon = kring->tx == NR_TX ? NS_TXMON : 0; in netmap_monitor_parent_sync()
735 for (j = 0; j < kring->n_monitors; j++) { in netmap_monitor_parent_sync()
736 struct netmap_kring *mkring = kring->monitors[j]; in netmap_monitor_parent_sync()
739 u_int lim = kring->nkr_num_slots - 1; in netmap_monitor_parent_sync()
740 struct netmap_ring *ring = kring->ring, *mring = mkring->ring; in netmap_monitor_parent_sync()
742 mlim = mkring->nkr_num_slots - 1; in netmap_monitor_parent_sync()
748 mtx_lock(&mkring->q_lock); in netmap_monitor_parent_sync()
750 i = mkring->nr_hwtail; in netmap_monitor_parent_sync()
751 busy = i - mkring->nr_hwcur; in netmap_monitor_parent_sync()
753 busy += mkring->nkr_num_slots; in netmap_monitor_parent_sync()
754 free_slots = mlim - busy; in netmap_monitor_parent_sync()
763 beg += (m - free_slots); in netmap_monitor_parent_sync()
764 if (beg >= kring->nkr_num_slots) in netmap_monitor_parent_sync()
765 beg -= kring->nkr_num_slots; in netmap_monitor_parent_sync()
769 for ( ; m; m--) { in netmap_monitor_parent_sync()
770 struct netmap_slot *s = &ring->slot[beg]; in netmap_monitor_parent_sync()
771 struct netmap_slot *ms = &mring->slot[i]; in netmap_monitor_parent_sync()
772 u_int copy_len = s->len; in netmap_monitor_parent_sync()
776 max_len = NETMAP_BUF_SIZE(mkring->na) - nm_get_offset(mkring, ms); in netmap_monitor_parent_sync()
778 nm_prlim(5, "%s->%s: truncating %d to %d", kring->name, in netmap_monitor_parent_sync()
779 mkring->name, copy_len, max_len); in netmap_monitor_parent_sync()
784 ms->len = copy_len; in netmap_monitor_parent_sync()
785 ms->flags = (s->flags & ~NS_TXMON) | txmon; in netmap_monitor_parent_sync()
792 mkring->nr_hwtail = i; in netmap_monitor_parent_sync()
794 mtx_unlock(&mkring->q_lock); in netmap_monitor_parent_sync()
798 mkring->nm_notify(mkring, 0); in netmap_monitor_parent_sync()
811 if (kring->n_monitors > 0) { in netmap_monitor_parent_txsync()
812 first_new = kring->nr_hwcur; in netmap_monitor_parent_txsync()
813 new_slots = kring->rhead - first_new; in netmap_monitor_parent_txsync()
815 new_slots += kring->nkr_num_slots; in netmap_monitor_parent_txsync()
819 if (kring->zmon_list[NR_TX].next != NULL) { in netmap_monitor_parent_txsync()
822 return kring->mon_sync(kring, flags); in netmap_monitor_parent_txsync()
833 if (kring->zmon_list[NR_RX].next != NULL) { in netmap_monitor_parent_rxsync()
836 error = kring->mon_sync(kring, flags); in netmap_monitor_parent_rxsync()
840 if (kring->n_monitors > 0) { in netmap_monitor_parent_rxsync()
841 first_new = kring->mon_tail; in netmap_monitor_parent_rxsync()
842 new_slots = kring->nr_hwtail - first_new; in netmap_monitor_parent_rxsync()
844 new_slots += kring->nkr_num_slots; in netmap_monitor_parent_rxsync()
847 kring->mon_tail = kring->nr_hwtail; in netmap_monitor_parent_rxsync()
857 nm_prdis(5, "%s %x", kring->name, flags); in netmap_monitor_parent_notify()
866 if (kring->n_monitors > 0) { in netmap_monitor_parent_notify()
873 notify = kring->nm_notify; in netmap_monitor_parent_notify()
875 notify = kring->mon_notify; in netmap_monitor_parent_notify()
893 struct netmap_priv_d *priv = &mna->priv; in netmap_monitor_dtor()
894 struct netmap_adapter *pna = priv->np_na; in netmap_monitor_dtor()
905 struct nmreq_register *req = (struct nmreq_register *)(uintptr_t)hdr->nr_body; in netmap_get_monitor_na()
911 int zcopy = (req->nr_flags & NR_ZCOPY_MON); in netmap_get_monitor_na()
914 req->nr_flags |= (NR_MONITOR_TX | NR_MONITOR_RX); in netmap_get_monitor_na()
916 if ((req->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX)) == 0) { in netmap_get_monitor_na()
922 nm_prdis("flags %lx", req->nr_flags); in netmap_get_monitor_na()
931 hdr->nr_body = (uintptr_t)&preq; in netmap_get_monitor_na()
933 hdr->nr_body = (uintptr_t)req; in netmap_get_monitor_na()
938 nm_prdis("found parent: %s", pna->name); in netmap_get_monitor_na()
943 * by intercepting its nm_register callback (2014-03-16) in netmap_get_monitor_na()
945 nm_prerr("%s not in netmap mode", pna->name); in netmap_get_monitor_na()
955 mna->priv.np_na = pna; in netmap_get_monitor_na()
958 error = netmap_interp_ringid(&mna->priv, hdr); in netmap_get_monitor_na()
963 snprintf(mna->up.name, sizeof(mna->up.name), "%s/%s%s%s#%lu", pna->name, in netmap_get_monitor_na()
965 (req->nr_flags & NR_MONITOR_RX) ? "r" : "", in netmap_get_monitor_na()
966 (req->nr_flags & NR_MONITOR_TX) ? "t" : "", in netmap_get_monitor_na()
967 pna->monitor_id++); in netmap_get_monitor_na()
970 mna->up.na_flags |= (pna->na_flags & NAF_HOST_RINGS) & ~NAF_OFFSETS; in netmap_get_monitor_na()
972 mna->up.na_flags |= NAF_OFFSETS; in netmap_get_monitor_na()
973 /* a do-nothing txsync: monitors cannot be used to inject packets */ in netmap_get_monitor_na()
974 mna->up.nm_txsync = netmap_monitor_txsync; in netmap_get_monitor_na()
975 mna->up.nm_rxsync = netmap_monitor_rxsync; in netmap_get_monitor_na()
976 mna->up.nm_krings_create = netmap_monitor_krings_create; in netmap_get_monitor_na()
977 mna->up.nm_krings_delete = netmap_monitor_krings_delete; in netmap_get_monitor_na()
978 mna->up.num_tx_rings = 1; // XXX what should we do here with chained zmons? in netmap_get_monitor_na()
982 mna->up.num_rx_rings = pna->num_rx_rings; in netmap_get_monitor_na()
983 if (pna->num_tx_rings > pna->num_rx_rings) in netmap_get_monitor_na()
984 mna->up.num_rx_rings = pna->num_tx_rings; in netmap_get_monitor_na()
989 mna->up.num_tx_desc = req->nr_tx_slots; in netmap_get_monitor_na()
990 nm_bound_var(&mna->up.num_tx_desc, pna->num_tx_desc, in netmap_get_monitor_na()
992 mna->up.num_rx_desc = req->nr_rx_slots; in netmap_get_monitor_na()
993 nm_bound_var(&mna->up.num_rx_desc, pna->num_rx_desc, in netmap_get_monitor_na()
996 mna->up.nm_register = netmap_zmon_reg; in netmap_get_monitor_na()
997 mna->up.nm_dtor = netmap_zmon_dtor; in netmap_get_monitor_na()
1001 mna->up.nm_mem = netmap_mem_get(pna->nm_mem); in netmap_get_monitor_na()
1003 mna->up.na_flags |= NAF_MEM_OWNER; in netmap_get_monitor_na()
1005 mna->up.nm_register = netmap_monitor_reg; in netmap_get_monitor_na()
1006 mna->up.nm_dtor = netmap_monitor_dtor; in netmap_get_monitor_na()
1007 mna->up.nm_mem = netmap_mem_private_new( in netmap_get_monitor_na()
1008 mna->up.num_tx_rings, in netmap_get_monitor_na()
1009 mna->up.num_tx_desc, in netmap_get_monitor_na()
1010 mna->up.num_rx_rings, in netmap_get_monitor_na()
1011 mna->up.num_rx_desc, in netmap_get_monitor_na()
1015 if (mna->up.nm_mem == NULL) in netmap_get_monitor_na()
1019 error = netmap_attach_common(&mna->up); in netmap_get_monitor_na()
1026 mna->flags = (req->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX | NR_ZCOPY_MON)); in netmap_get_monitor_na()
1028 *na = &mna->up; in netmap_get_monitor_na()
1041 netmap_mem_put(mna->up.nm_mem); in netmap_get_monitor_na()