141 netmap_monitor_txsync(struct netmap_kring *kring, int flags)  in netmap_monitor_txsync()  argument
143 	nm_prlim(1, "%s %x", kring->name, flags);  in netmap_monitor_txsync()
154 netmap_monitor_rxsync(struct netmap_kring *kring, int flags)  in netmap_monitor_rxsync()  argument
157 		(struct netmap_monitor_adapter *)kring->na;  in netmap_monitor_rxsync()
162 	nm_prdis("%s %x", kring->name, flags);  in netmap_monitor_rxsync()
163 	kring->nr_hwcur = kring->rhead;  in netmap_monitor_rxsync()
183 			struct netmap_kring *kring = NMR(na, t)[first + i];  in netmap_monitor_krings_create()  local
184 			kring->nm_sync = t == NR_TX ? netmap_monitor_txsync :  in netmap_monitor_krings_create()
207 nm_monitor_alloc(struct netmap_kring *kring, u_int n)  in nm_monitor_alloc()  argument
212 	if (n <= kring->max_monitors)  in nm_monitor_alloc()
216 	old_len = sizeof(struct netmap_kring *)*kring->max_monitors;  in nm_monitor_alloc()
218 	nm = nm_os_realloc(kring->monitors, len, old_len);  in nm_monitor_alloc()
222 	kring->monitors = nm;  in nm_monitor_alloc()
223 	kring->max_monitors = n;  in nm_monitor_alloc()
230 nm_monitor_dealloc(struct netmap_kring *kring)  in nm_monitor_dealloc()  argument
232 	if (kring->monitors) {  in nm_monitor_dealloc()
233 		if (kring->n_monitors > 0) {  in nm_monitor_dealloc()
235 			    kring->name, kring->n_monitors);  in nm_monitor_dealloc()
237 		nm_os_free(kring->monitors);  in nm_monitor_dealloc()
238 		kring->monitors = NULL;  in nm_monitor_dealloc()
239 		kring->max_monitors = 0;  in nm_monitor_dealloc()
240 		kring->n_monitors = 0;  in nm_monitor_dealloc()
246 nm_monitor_none(struct netmap_kring *kring)  in nm_monitor_none()  argument
248 	return kring->n_monitors == 0 &&  in nm_monitor_none()
249 		kring->zmon_list[NR_TX].next == NULL &&  in nm_monitor_none()
250 		kring->zmon_list[NR_RX].next == NULL;  in nm_monitor_none()
264 nm_monitor_dummycb(struct netmap_kring *kring, int flags)  in nm_monitor_dummycb()  argument
266 	(void)kring;  in nm_monitor_dummycb()
272 nm_monitor_intercept_callbacks(struct netmap_kring *kring)  in nm_monitor_intercept_callbacks()  argument
274 	nm_prdis("intercept callbacks on %s", kring->name);  in nm_monitor_intercept_callbacks()
275 	kring->mon_sync = kring->nm_sync != NULL ?  in nm_monitor_intercept_callbacks()
276 		kring->nm_sync : nm_monitor_dummycb;  in nm_monitor_intercept_callbacks()
277 	kring->mon_notify = kring->nm_notify;  in nm_monitor_intercept_callbacks()
278 	if (kring->tx == NR_TX) {  in nm_monitor_intercept_callbacks()
279 		kring->nm_sync = netmap_monitor_parent_txsync;  in nm_monitor_intercept_callbacks()
281 		kring->nm_sync = netmap_monitor_parent_rxsync;  in nm_monitor_intercept_callbacks()
282 		kring->nm_notify = netmap_monitor_parent_notify;  in nm_monitor_intercept_callbacks()
283 		kring->mon_tail = kring->nr_hwtail;  in nm_monitor_intercept_callbacks()
288 nm_monitor_restore_callbacks(struct netmap_kring *kring)  in nm_monitor_restore_callbacks()  argument
290 	nm_prdis("restoring callbacks on %s", kring->name);  in nm_monitor_restore_callbacks()
291 	kring->nm_sync = kring->mon_sync;  in nm_monitor_restore_callbacks()
292 	kring->mon_sync = NULL;  in nm_monitor_restore_callbacks()
293 	if (kring->tx == NR_RX) {  in nm_monitor_restore_callbacks()
294 		kring->nm_notify = kring->mon_notify;  in nm_monitor_restore_callbacks()
296 	kring->mon_notify = NULL;  in nm_monitor_restore_callbacks()
303 	struct netmap_kring *kring = mkring;  in nm_zmon_list_head()  local
304 	struct netmap_zmon_list *z = &kring->zmon_list[t];  in nm_zmon_list_head()
307 		kring = z->prev;  in nm_zmon_list_head()
308 		na = kring->na;  in nm_zmon_list_head()
309 		z = &kring->zmon_list[t];  in nm_zmon_list_head()
311 	return nm_is_zmon(na) ? NULL : kring;  in nm_zmon_list_head()
318 netmap_monitor_add(struct netmap_kring *mkring, struct netmap_kring *kring, int zmon)  in netmap_monitor_add()  argument
321 	enum txrx t = kring->tx;  in netmap_monitor_add()
322 	struct netmap_zmon_list *z = &kring->zmon_list[t];  in netmap_monitor_add()
324 	struct netmap_kring *ikring = kring;  in netmap_monitor_add()
333 	nm_kr_stop(kring, NM_KR_LOCKED);  in netmap_monitor_add()
354 		error = nm_monitor_alloc(kring, kring->n_monitors + 1);  in netmap_monitor_add()
357 		kring->monitors[kring->n_monitors] = mkring;  in netmap_monitor_add()
358 		mkring->mon_pos[kring->tx] = kring->n_monitors;  in netmap_monitor_add()
359 		kring->n_monitors++;  in netmap_monitor_add()
363 	nm_kr_start(kring);  in netmap_monitor_add()
371 netmap_monitor_del(struct netmap_kring *mkring, struct netmap_kring *kring, enum txrx t)  in netmap_monitor_del()  argument
375 	struct netmap_kring *ikring = kring;  in netmap_monitor_del()
380 		kring = nm_zmon_list_head(mkring, t);  in netmap_monitor_del()
388 	if (kring != NULL)  in netmap_monitor_del()
389 		nm_kr_stop(kring, NM_KR_LOCKED);  in netmap_monitor_del()
402 		} else if (kring != NULL) {  in netmap_monitor_del()
406 			kring->zmon_list[t].prev =  in netmap_monitor_del()
407 				(mz->prev != kring ? mz->prev : NULL);  in netmap_monitor_del()
417 		uint32_t mon_pos = mkring->mon_pos[kring->tx];  in netmap_monitor_del()
418 		kring->n_monitors--;  in netmap_monitor_del()
419 		if (mon_pos != kring->n_monitors) {  in netmap_monitor_del()
420 			kring->monitors[mon_pos] =  in netmap_monitor_del()
421 				kring->monitors[kring->n_monitors];  in netmap_monitor_del()
422 			kring->monitors[mon_pos]->mon_pos[kring->tx] = mon_pos;  in netmap_monitor_del()
424 		kring->monitors[kring->n_monitors] = NULL;  in netmap_monitor_del()
425 		if (kring->n_monitors == 0) {  in netmap_monitor_del()
426 			nm_monitor_dealloc(kring);  in netmap_monitor_del()
435 	if (kring != NULL)  in netmap_monitor_del()
436 		nm_kr_start(kring);  in netmap_monitor_del()
456 			struct netmap_kring *kring = NMR(na, t)[i];  in netmap_monitor_stop()  local
457 			struct netmap_zmon_list *z = &kring->zmon_list[t];  in netmap_monitor_stop()
460 			if (nm_monitor_none(kring))  in netmap_monitor_stop()
463 			for (j = 0; j < kring->n_monitors; j++) {  in netmap_monitor_stop()
465 					kring->monitors[j];  in netmap_monitor_stop()
473 				kring->monitors[j] = NULL;  in netmap_monitor_stop()
475 			kring->n_monitors = 0;  in netmap_monitor_stop()
476 			nm_monitor_dealloc(kring);  in netmap_monitor_stop()
499 			nm_monitor_restore_callbacks(kring);  in netmap_monitor_stop()
515 	struct netmap_kring *kring, *mkring;  in netmap_monitor_reg_common()  local
538 						kring = NMR(pna, s)[i];  in netmap_monitor_reg_common()
539 						netmap_monitor_add(mkring, kring, zmon);  in netmap_monitor_reg_common()
566 						kring = NMR(pna, s)[i];  in netmap_monitor_reg_common()
567 						netmap_monitor_del(mkring, kring, s);  in netmap_monitor_reg_common()
587 netmap_zmon_parent_sync(struct netmap_kring *kring, int flags, enum txrx tx)  in netmap_zmon_parent_sync()  argument
589 	struct netmap_kring *mkring = kring->zmon_list[tx].next;  in netmap_zmon_parent_sync()
590 	struct netmap_ring *ring = kring->ring, *mring;  in netmap_zmon_parent_sync()
594 	u_int lim = kring->nkr_num_slots - 1,  in netmap_zmon_parent_sync()
596 	uint16_t txmon = kring->tx == NR_TX ? NS_TXMON : 0;  in netmap_zmon_parent_sync()
599 		nm_prlim(5, "NULL monitor on %s", kring->name);  in netmap_zmon_parent_sync()
607 		beg = kring->nr_hwtail + 1;  in netmap_zmon_parent_sync()
608 		error = kring->mon_sync(kring, flags);  in netmap_zmon_parent_sync()
611 		end = kring->nr_hwtail + 1;  in netmap_zmon_parent_sync()
613 		beg = kring->nr_hwcur;  in netmap_zmon_parent_sync()
614 		end = kring->rhead;  in netmap_zmon_parent_sync()
619 		rel_slots += kring->nkr_num_slots;  in netmap_zmon_parent_sync()
648 	if (unlikely(beg >= kring->nkr_num_slots))  in netmap_zmon_parent_sync()
649 		beg -= kring->nkr_num_slots;  in netmap_zmon_parent_sync()
686 		error = kring->mon_sync(kring, flags);  in netmap_zmon_parent_sync()
693 netmap_zmon_parent_txsync(struct netmap_kring *kring, int flags)  in netmap_zmon_parent_txsync()  argument
695 	return netmap_zmon_parent_sync(kring, flags, NR_TX);  in netmap_zmon_parent_txsync()
700 netmap_zmon_parent_rxsync(struct netmap_kring *kring, int flags)  in netmap_zmon_parent_rxsync()  argument
702 	return netmap_zmon_parent_sync(kring, flags, NR_RX);  in netmap_zmon_parent_rxsync()
730 netmap_monitor_parent_sync(struct netmap_kring *kring, u_int first_new, int new_slots)  in netmap_monitor_parent_sync()  argument
733 	uint16_t txmon = kring->tx == NR_TX ? NS_TXMON : 0;  in netmap_monitor_parent_sync()
735 	for (j = 0; j < kring->n_monitors; j++) {  in netmap_monitor_parent_sync()
736 		struct netmap_kring *mkring = kring->monitors[j];  in netmap_monitor_parent_sync()
739 		u_int lim = kring->nkr_num_slots - 1;  in netmap_monitor_parent_sync()
740 		struct netmap_ring *ring = kring->ring, *mring = mkring->ring;  in netmap_monitor_parent_sync()
764 			if (beg >= kring->nkr_num_slots)  in netmap_monitor_parent_sync()
765 				beg -= kring->nkr_num_slots;  in netmap_monitor_parent_sync()
773 			char *src = NMB_O(kring, s),  in netmap_monitor_parent_sync()
778 				nm_prlim(5, "%s->%s: truncating %d to %d", kring->name,  in netmap_monitor_parent_sync()
805 netmap_monitor_parent_txsync(struct netmap_kring *kring, int flags)  in netmap_monitor_parent_txsync()  argument
811 	if (kring->n_monitors > 0) {  in netmap_monitor_parent_txsync()
812 		first_new = kring->nr_hwcur;  in netmap_monitor_parent_txsync()
813 		new_slots = kring->rhead - first_new;  in netmap_monitor_parent_txsync()
815 			new_slots += kring->nkr_num_slots;  in netmap_monitor_parent_txsync()
817 			netmap_monitor_parent_sync(kring, first_new, new_slots);  in netmap_monitor_parent_txsync()
819 	if (kring->zmon_list[NR_TX].next != NULL) {  in netmap_monitor_parent_txsync()
820 		return netmap_zmon_parent_txsync(kring, flags);  in netmap_monitor_parent_txsync()
822 	return kring->mon_sync(kring, flags);  in netmap_monitor_parent_txsync()
827 netmap_monitor_parent_rxsync(struct netmap_kring *kring, int flags)  in netmap_monitor_parent_rxsync()  argument
833 	if (kring->zmon_list[NR_RX].next != NULL) {  in netmap_monitor_parent_rxsync()
834 		error = netmap_zmon_parent_rxsync(kring, flags);  in netmap_monitor_parent_rxsync()
836 		error =  kring->mon_sync(kring, flags);  in netmap_monitor_parent_rxsync()
840 	if (kring->n_monitors > 0) {  in netmap_monitor_parent_rxsync()
841 		first_new = kring->mon_tail;  in netmap_monitor_parent_rxsync()
842 		new_slots = kring->nr_hwtail - first_new;  in netmap_monitor_parent_rxsync()
844 			new_slots += kring->nkr_num_slots;  in netmap_monitor_parent_rxsync()
846 			netmap_monitor_parent_sync(kring, first_new, new_slots);  in netmap_monitor_parent_rxsync()
847 		kring->mon_tail = kring->nr_hwtail;  in netmap_monitor_parent_rxsync()
854 netmap_monitor_parent_notify(struct netmap_kring *kring, int flags)  in netmap_monitor_parent_notify()  argument
857 	nm_prdis(5, "%s %x", kring->name, flags);  in netmap_monitor_parent_notify()
862 	if (nm_kr_tryget(kring, 0, NULL)) {  in netmap_monitor_parent_notify()
866 	if (kring->n_monitors > 0) {  in netmap_monitor_parent_notify()
867 		netmap_monitor_parent_rxsync(kring, NAF_FORCE_READ);  in netmap_monitor_parent_notify()
869 	if (nm_monitor_none(kring)) {  in netmap_monitor_parent_notify()
873 		notify = kring->nm_notify;  in netmap_monitor_parent_notify()
875 		notify = kring->mon_notify;  in netmap_monitor_parent_notify()
877 	nm_kr_put(kring);  in netmap_monitor_parent_notify()
878 	return notify(kring, flags);  in netmap_monitor_parent_notify()