xref: /freebsd/sys/dev/netmap/netmap_kern.h (revision e40139ff33b48b56a24c808b166b04b8ee6f5b21)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo
5  * Copyright (C) 2013-2016 Universita` di Pisa
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *   1. Redistributions of source code must retain the above copyright
12  *      notice, this list of conditions and the following disclaimer.
13  *   2. Redistributions in binary form must reproduce the above copyright
14  *      notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 /*
31  * $FreeBSD$
32  *
33  * The header contains the definitions of constants and function
34  * prototypes used only in kernelspace.
35  */
36 
37 #ifndef _NET_NETMAP_KERN_H_
38 #define _NET_NETMAP_KERN_H_
39 
40 #if defined(linux)
41 
42 #if defined(CONFIG_NETMAP_EXTMEM)
43 #define WITH_EXTMEM
44 #endif
45 #if  defined(CONFIG_NETMAP_VALE)
46 #define WITH_VALE
47 #endif
48 #if defined(CONFIG_NETMAP_PIPE)
49 #define WITH_PIPES
50 #endif
51 #if defined(CONFIG_NETMAP_MONITOR)
52 #define WITH_MONITOR
53 #endif
54 #if defined(CONFIG_NETMAP_GENERIC)
55 #define WITH_GENERIC
56 #endif
57 #if defined(CONFIG_NETMAP_PTNETMAP)
58 #define WITH_PTNETMAP
59 #endif
60 #if defined(CONFIG_NETMAP_SINK)
61 #define WITH_SINK
62 #endif
63 #if defined(CONFIG_NETMAP_NULL)
64 #define WITH_NMNULL
65 #endif
66 
67 #elif defined (_WIN32)
68 #define WITH_VALE	// comment out to disable VALE support
69 #define WITH_PIPES
70 #define WITH_MONITOR
71 #define WITH_GENERIC
72 #define WITH_NMNULL
73 
74 #else	/* neither linux nor windows */
75 #define WITH_VALE	// comment out to disable VALE support
76 #define WITH_PIPES
77 #define WITH_MONITOR
78 #define WITH_GENERIC
79 #define WITH_EXTMEM
80 #define WITH_NMNULL
81 #endif
82 
83 #if defined(__FreeBSD__)
84 #include <sys/selinfo.h>
85 
86 #define likely(x)	__builtin_expect((long)!!(x), 1L)
87 #define unlikely(x)	__builtin_expect((long)!!(x), 0L)
88 #define __user
89 
90 #define	NM_LOCK_T	struct mtx	/* low level spinlock, used to protect queues */
91 
92 #define NM_MTX_T	struct sx	/* OS-specific mutex (sleepable) */
93 #define NM_MTX_INIT(m)		sx_init(&(m), #m)
94 #define NM_MTX_DESTROY(m)	sx_destroy(&(m))
95 #define NM_MTX_LOCK(m)		sx_xlock(&(m))
96 #define NM_MTX_SPINLOCK(m)	while (!sx_try_xlock(&(m))) ;
97 #define NM_MTX_UNLOCK(m)	sx_xunlock(&(m))
98 #define NM_MTX_ASSERT(m)	sx_assert(&(m), SA_XLOCKED)
99 
100 #define	NM_SELINFO_T	struct nm_selinfo
101 #define NM_SELRECORD_T	struct thread
102 #define	MBUF_LEN(m)	((m)->m_pkthdr.len)
103 #define MBUF_TXQ(m)	((m)->m_pkthdr.flowid)
104 #define MBUF_TRANSMIT(na, ifp, m)	((na)->if_transmit(ifp, m))
105 #define	GEN_TX_MBUF_IFP(m)	((m)->m_pkthdr.rcvif)
106 
107 #define NM_ATOMIC_T	volatile int /* required by atomic/bitops.h */
108 /* atomic operations */
109 #include <machine/atomic.h>
110 #define NM_ATOMIC_TEST_AND_SET(p)       (!atomic_cmpset_acq_int((p), 0, 1))
111 #define NM_ATOMIC_CLEAR(p)              atomic_store_rel_int((p), 0)
112 
113 #if __FreeBSD_version >= 1100030
114 #define	WNA(_ifp)	(_ifp)->if_netmap
115 #else /* older FreeBSD */
116 #define	WNA(_ifp)	(_ifp)->if_pspare[0]
117 #endif /* older FreeBSD */
118 
119 #if __FreeBSD_version >= 1100005
120 struct netmap_adapter *netmap_getna(if_t ifp);
121 #endif
122 
123 #if __FreeBSD_version >= 1100027
124 #define MBUF_REFCNT(m)		((m)->m_ext.ext_count)
125 #define SET_MBUF_REFCNT(m, x)   (m)->m_ext.ext_count = x
126 #else
127 #define MBUF_REFCNT(m)		((m)->m_ext.ref_cnt ? *((m)->m_ext.ref_cnt) : -1)
128 #define SET_MBUF_REFCNT(m, x)   *((m)->m_ext.ref_cnt) = x
129 #endif
130 
131 #define MBUF_QUEUED(m)		1
132 
133 struct nm_selinfo {
134 	/* Support for select(2) and poll(2). */
135 	struct selinfo si;
136 	/* Support for kqueue(9). See comments in netmap_freebsd.c */
137 	struct taskqueue *ntfytq;
138 	struct task ntfytask;
139 	struct mtx m;
140 	char mtxname[32];
141 	int kqueue_users;
142 };
143 
144 
145 struct hrtimer {
146     /* Not used in FreeBSD. */
147 };
148 
149 #define NM_BNS_GET(b)
150 #define NM_BNS_PUT(b)
151 
152 #elif defined (linux)
153 
154 #define	NM_LOCK_T	safe_spinlock_t	// see bsd_glue.h
155 #define	NM_SELINFO_T	wait_queue_head_t
156 #define	MBUF_LEN(m)	((m)->len)
157 #define MBUF_TRANSMIT(na, ifp, m)							\
158 	({										\
159 		/* Avoid infinite recursion with generic. */				\
160 		m->priority = NM_MAGIC_PRIORITY_TX;					\
161 		(((struct net_device_ops *)(na)->if_transmit)->ndo_start_xmit(m, ifp));	\
162 		0;									\
163 	})
164 
165 /* See explanation in nm_os_generic_xmit_frame. */
166 #define	GEN_TX_MBUF_IFP(m)	((struct ifnet *)skb_shinfo(m)->destructor_arg)
167 
168 #define NM_ATOMIC_T	volatile long unsigned int
169 
170 #define NM_MTX_T	struct mutex	/* OS-specific sleepable lock */
171 #define NM_MTX_INIT(m)	mutex_init(&(m))
172 #define NM_MTX_DESTROY(m)	do { (void)(m); } while (0)
173 #define NM_MTX_LOCK(m)		mutex_lock(&(m))
174 #define NM_MTX_UNLOCK(m)	mutex_unlock(&(m))
175 #define NM_MTX_ASSERT(m)	mutex_is_locked(&(m))
176 
177 #ifndef DEV_NETMAP
178 #define DEV_NETMAP
179 #endif /* DEV_NETMAP */
180 
181 #elif defined (__APPLE__)
182 
183 #warning apple support is incomplete.
184 #define likely(x)	__builtin_expect(!!(x), 1)
185 #define unlikely(x)	__builtin_expect(!!(x), 0)
186 #define	NM_LOCK_T	IOLock *
187 #define	NM_SELINFO_T	struct selinfo
188 #define	MBUF_LEN(m)	((m)->m_pkthdr.len)
189 
190 #elif defined (_WIN32)
191 #include "../../../WINDOWS/win_glue.h"
192 
193 #define NM_SELRECORD_T		IO_STACK_LOCATION
194 #define NM_SELINFO_T		win_SELINFO		// see win_glue.h
195 #define NM_LOCK_T		win_spinlock_t	// see win_glue.h
196 #define NM_MTX_T		KGUARDED_MUTEX	/* OS-specific mutex (sleepable) */
197 
198 #define NM_MTX_INIT(m)		KeInitializeGuardedMutex(&m);
199 #define NM_MTX_DESTROY(m)	do { (void)(m); } while (0)
200 #define NM_MTX_LOCK(m)		KeAcquireGuardedMutex(&(m))
201 #define NM_MTX_UNLOCK(m)	KeReleaseGuardedMutex(&(m))
202 #define NM_MTX_ASSERT(m)	assert(&m.Count>0)
203 
204 //These linknames are for the NDIS driver
205 #define NETMAP_NDIS_LINKNAME_STRING             L"\\DosDevices\\NMAPNDIS"
206 #define NETMAP_NDIS_NTDEVICE_STRING             L"\\Device\\NMAPNDIS"
207 
208 //Definition of internal driver-to-driver ioctl codes
209 #define NETMAP_KERNEL_XCHANGE_POINTERS		_IO('i', 180)
210 #define NETMAP_KERNEL_SEND_SHUTDOWN_SIGNAL	_IO_direct('i', 195)
211 
212 typedef struct hrtimer{
213 	KTIMER timer;
214 	BOOLEAN active;
215 	KDPC deferred_proc;
216 };
217 
218 /* MSVC does not have likely/unlikely support */
219 #ifdef _MSC_VER
220 #define likely(x)	(x)
221 #define unlikely(x)	(x)
222 #else
223 #define likely(x)	__builtin_expect((long)!!(x), 1L)
224 #define unlikely(x)	__builtin_expect((long)!!(x), 0L)
225 #endif //_MSC_VER
226 
227 #else
228 
229 #error unsupported platform
230 
231 #endif /* end - platform-specific code */
232 
233 #ifndef _WIN32 /* support for emulated sysctl */
234 #define SYSBEGIN(x)
235 #define SYSEND
236 #endif /* _WIN32 */
237 
238 #define NM_ACCESS_ONCE(x)	(*(volatile __typeof__(x) *)&(x))
239 
240 #define	NMG_LOCK_T		NM_MTX_T
241 #define	NMG_LOCK_INIT()		NM_MTX_INIT(netmap_global_lock)
242 #define	NMG_LOCK_DESTROY()	NM_MTX_DESTROY(netmap_global_lock)
243 #define	NMG_LOCK()		NM_MTX_LOCK(netmap_global_lock)
244 #define	NMG_UNLOCK()		NM_MTX_UNLOCK(netmap_global_lock)
245 #define	NMG_LOCK_ASSERT()	NM_MTX_ASSERT(netmap_global_lock)
246 
247 #if defined(__FreeBSD__)
248 #define nm_prerr_int	printf
249 #define nm_prinf_int	printf
250 #elif defined (_WIN32)
251 #define nm_prerr_int	DbgPrint
252 #define nm_prinf_int	DbgPrint
253 #elif defined(linux)
254 #define nm_prerr_int(fmt, arg...)    printk(KERN_ERR fmt, ##arg)
255 #define nm_prinf_int(fmt, arg...)    printk(KERN_INFO fmt, ##arg)
256 #endif
257 
258 #define nm_prinf(format, ...)					\
259 	do {							\
260 		struct timeval __xxts;				\
261 		microtime(&__xxts);				\
262 		nm_prinf_int("%03d.%06d [%4d] %-25s " format "\n",\
263 		(int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec,	\
264 		__LINE__, __FUNCTION__, ##__VA_ARGS__);		\
265 	} while (0)
266 
267 #define nm_prerr(format, ...)					\
268 	do {							\
269 		struct timeval __xxts;				\
270 		microtime(&__xxts);				\
271 		nm_prerr_int("%03d.%06d [%4d] %-25s " format "\n",\
272 		(int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec,	\
273 		__LINE__, __FUNCTION__, ##__VA_ARGS__);		\
274 	} while (0)
275 
276 /* Disabled printf (used to be nm_prdis). */
277 #define nm_prdis(format, ...)
278 
279 /* Rate limited, lps indicates how many per second. */
280 #define nm_prlim(lps, format, ...)				\
281 	do {							\
282 		static int t0, __cnt;				\
283 		if (t0 != time_second) {			\
284 			t0 = time_second;			\
285 			__cnt = 0;				\
286 		}						\
287 		if (__cnt++ < lps)				\
288 			nm_prinf(format, ##__VA_ARGS__);	\
289 	} while (0)
290 
291 struct netmap_adapter;
292 struct nm_bdg_fwd;
293 struct nm_bridge;
294 struct netmap_priv_d;
295 struct nm_bdg_args;
296 
297 /* os-specific NM_SELINFO_T initialzation/destruction functions */
298 int nm_os_selinfo_init(NM_SELINFO_T *, const char *name);
299 void nm_os_selinfo_uninit(NM_SELINFO_T *);
300 
301 const char *nm_dump_buf(char *p, int len, int lim, char *dst);
302 
303 void nm_os_selwakeup(NM_SELINFO_T *si);
304 void nm_os_selrecord(NM_SELRECORD_T *sr, NM_SELINFO_T *si);
305 
306 int nm_os_ifnet_init(void);
307 void nm_os_ifnet_fini(void);
308 void nm_os_ifnet_lock(void);
309 void nm_os_ifnet_unlock(void);
310 
311 unsigned nm_os_ifnet_mtu(struct ifnet *ifp);
312 
313 void nm_os_get_module(void);
314 void nm_os_put_module(void);
315 
316 void netmap_make_zombie(struct ifnet *);
317 void netmap_undo_zombie(struct ifnet *);
318 
319 /* os independent alloc/realloc/free */
320 void *nm_os_malloc(size_t);
321 void *nm_os_vmalloc(size_t);
322 void *nm_os_realloc(void *, size_t new_size, size_t old_size);
323 void nm_os_free(void *);
324 void nm_os_vfree(void *);
325 
326 /* os specific attach/detach enter/exit-netmap-mode routines */
327 void nm_os_onattach(struct ifnet *);
328 void nm_os_ondetach(struct ifnet *);
329 void nm_os_onenter(struct ifnet *);
330 void nm_os_onexit(struct ifnet *);
331 
332 /* passes a packet up to the host stack.
333  * If the packet is sent (or dropped) immediately it returns NULL,
334  * otherwise it links the packet to prev and returns m.
335  * In this case, a final call with m=NULL and prev != NULL will send up
336  * the entire chain to the host stack.
337  */
338 void *nm_os_send_up(struct ifnet *, struct mbuf *m, struct mbuf *prev);
339 
340 int nm_os_mbuf_has_seg_offld(struct mbuf *m);
341 int nm_os_mbuf_has_csum_offld(struct mbuf *m);
342 
343 #include "netmap_mbq.h"
344 
345 extern NMG_LOCK_T	netmap_global_lock;
346 
347 enum txrx { NR_RX = 0, NR_TX = 1, NR_TXRX };
348 
349 static __inline const char*
350 nm_txrx2str(enum txrx t)
351 {
352 	return (t== NR_RX ? "RX" : "TX");
353 }
354 
355 static __inline enum txrx
356 nm_txrx_swap(enum txrx t)
357 {
358 	return (t== NR_RX ? NR_TX : NR_RX);
359 }
360 
361 #define for_rx_tx(t)	for ((t) = 0; (t) < NR_TXRX; (t)++)
362 
363 #ifdef WITH_MONITOR
364 struct netmap_zmon_list {
365 	struct netmap_kring *next;
366 	struct netmap_kring *prev;
367 };
368 #endif /* WITH_MONITOR */
369 
370 /*
371  * private, kernel view of a ring. Keeps track of the status of
372  * a ring across system calls.
373  *
374  *	nr_hwcur	index of the next buffer to refill.
375  *			It corresponds to ring->head
376  *			at the time the system call returns.
377  *
378  *	nr_hwtail	index of the first buffer owned by the kernel.
379  *			On RX, hwcur->hwtail are receive buffers
380  *			not yet released. hwcur is advanced following
381  *			ring->head, hwtail is advanced on incoming packets,
382  *			and a wakeup is generated when hwtail passes ring->cur
383  *			    On TX, hwcur->rcur have been filled by the sender
384  *			but not sent yet to the NIC; rcur->hwtail are available
385  *			for new transmissions, and hwtail->hwcur-1 are pending
386  *			transmissions not yet acknowledged.
387  *
388  * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots.
389  * This is so that, on a reset, buffers owned by userspace are not
390  * modified by the kernel. In particular:
391  * RX rings: the next empty buffer (hwtail + hwofs) coincides with
392  * 	the next empty buffer as known by the hardware (next_to_check or so).
393  * TX rings: hwcur + hwofs coincides with next_to_send
394  *
395  * The following fields are used to implement lock-free copy of packets
396  * from input to output ports in VALE switch:
397  *	nkr_hwlease	buffer after the last one being copied.
398  *			A writer in nm_bdg_flush reserves N buffers
399  *			from nr_hwlease, advances it, then does the
400  *			copy outside the lock.
401  *			In RX rings (used for VALE ports),
402  *			nkr_hwtail <= nkr_hwlease < nkr_hwcur+N-1
403  *			In TX rings (used for NIC or host stack ports)
404  *			nkr_hwcur <= nkr_hwlease < nkr_hwtail
405  *	nkr_leases	array of nkr_num_slots where writers can report
406  *			completion of their block. NR_NOSLOT (~0) indicates
407  *			that the writer has not finished yet
408  *	nkr_lease_idx	index of next free slot in nr_leases, to be assigned
409  *
410  * The kring is manipulated by txsync/rxsync and generic netmap function.
411  *
412  * Concurrent rxsync or txsync on the same ring are prevented through
413  * by nm_kr_(try)lock() which in turn uses nr_busy. This is all we need
414  * for NIC rings, and for TX rings attached to the host stack.
415  *
416  * RX rings attached to the host stack use an mbq (rx_queue) on both
417  * rxsync_from_host() and netmap_transmit(). The mbq is protected
418  * by its internal lock.
419  *
420  * RX rings attached to the VALE switch are accessed by both senders
421  * and receiver. They are protected through the q_lock on the RX ring.
422  */
423 struct netmap_kring {
424 	struct netmap_ring	*ring;
425 
426 	uint32_t	nr_hwcur;  /* should be nr_hwhead */
427 	uint32_t	nr_hwtail;
428 
429 	/*
430 	 * Copies of values in user rings, so we do not need to look
431 	 * at the ring (which could be modified). These are set in the
432 	 * *sync_prologue()/finalize() routines.
433 	 */
434 	uint32_t	rhead;
435 	uint32_t	rcur;
436 	uint32_t	rtail;
437 
438 	uint32_t	nr_kflags;	/* private driver flags */
439 #define NKR_PENDINTR	0x1		// Pending interrupt.
440 #define NKR_EXCLUSIVE	0x2		/* exclusive binding */
441 #define NKR_FORWARD	0x4		/* (host ring only) there are
442 					   packets to forward
443 					 */
444 #define NKR_NEEDRING	0x8		/* ring needed even if users==0
445 					 * (used internally by pipes and
446 					 *  by ptnetmap host ports)
447 					 */
448 #define NKR_NOINTR      0x10            /* don't use interrupts on this ring */
449 #define NKR_FAKERING	0x20		/* don't allocate/free buffers */
450 
451 	uint32_t	nr_mode;
452 	uint32_t	nr_pending_mode;
453 #define NKR_NETMAP_OFF	0x0
454 #define NKR_NETMAP_ON	0x1
455 
456 	uint32_t	nkr_num_slots;
457 
458 	/*
459 	 * On a NIC reset, the NIC ring indexes may be reset but the
460 	 * indexes in the netmap rings remain the same. nkr_hwofs
461 	 * keeps track of the offset between the two.
462 	 */
463 	int32_t		nkr_hwofs;
464 
465 	/* last_reclaim is opaque marker to help reduce the frequency
466 	 * of operations such as reclaiming tx buffers. A possible use
467 	 * is set it to ticks and do the reclaim only once per tick.
468 	 */
469 	uint64_t	last_reclaim;
470 
471 
472 	NM_SELINFO_T	si;		/* poll/select wait queue */
473 	NM_LOCK_T	q_lock;		/* protects kring and ring. */
474 	NM_ATOMIC_T	nr_busy;	/* prevent concurrent syscalls */
475 
476 	/* the adapter the owns this kring */
477 	struct netmap_adapter *na;
478 
479 	/* the adapter that wants to be notified when this kring has
480 	 * new slots avaialable. This is usually the same as the above,
481 	 * but wrappers may let it point to themselves
482 	 */
483 	struct netmap_adapter *notify_na;
484 
485 	/* The following fields are for VALE switch support */
486 	struct nm_bdg_fwd *nkr_ft;
487 	uint32_t	*nkr_leases;
488 #define NR_NOSLOT	((uint32_t)~0)	/* used in nkr_*lease* */
489 	uint32_t	nkr_hwlease;
490 	uint32_t	nkr_lease_idx;
491 
492 	/* while nkr_stopped is set, no new [tr]xsync operations can
493 	 * be started on this kring.
494 	 * This is used by netmap_disable_all_rings()
495 	 * to find a synchronization point where critical data
496 	 * structures pointed to by the kring can be added or removed
497 	 */
498 	volatile int nkr_stopped;
499 
500 	/* Support for adapters without native netmap support.
501 	 * On tx rings we preallocate an array of tx buffers
502 	 * (same size as the netmap ring), on rx rings we
503 	 * store incoming mbufs in a queue that is drained by
504 	 * a rxsync.
505 	 */
506 	struct mbuf	**tx_pool;
507 	struct mbuf	*tx_event;	/* TX event used as a notification */
508 	NM_LOCK_T	tx_event_lock;	/* protects the tx_event mbuf */
509 	struct mbq	rx_queue;       /* intercepted rx mbufs. */
510 
511 	uint32_t	users;		/* existing bindings for this ring */
512 
513 	uint32_t	ring_id;	/* kring identifier */
514 	enum txrx	tx;		/* kind of ring (tx or rx) */
515 	char name[64];			/* diagnostic */
516 
517 	/* [tx]sync callback for this kring.
518 	 * The default nm_kring_create callback (netmap_krings_create)
519 	 * sets the nm_sync callback of each hardware tx(rx) kring to
520 	 * the corresponding nm_txsync(nm_rxsync) taken from the
521 	 * netmap_adapter; moreover, it sets the sync callback
522 	 * of the host tx(rx) ring to netmap_txsync_to_host
523 	 * (netmap_rxsync_from_host).
524 	 *
525 	 * Overrides: the above configuration is not changed by
526 	 * any of the nm_krings_create callbacks.
527 	 */
528 	int (*nm_sync)(struct netmap_kring *kring, int flags);
529 	int (*nm_notify)(struct netmap_kring *kring, int flags);
530 
531 #ifdef WITH_PIPES
532 	struct netmap_kring *pipe;	/* if this is a pipe ring,
533 					 * pointer to the other end
534 					 */
535 	uint32_t pipe_tail;		/* hwtail updated by the other end */
536 #endif /* WITH_PIPES */
537 
538 	int (*save_notify)(struct netmap_kring *kring, int flags);
539 
540 #ifdef WITH_MONITOR
541 	/* array of krings that are monitoring this kring */
542 	struct netmap_kring **monitors;
543 	uint32_t max_monitors; /* current size of the monitors array */
544 	uint32_t n_monitors;	/* next unused entry in the monitor array */
545 	uint32_t mon_pos[NR_TXRX]; /* index of this ring in the monitored ring array */
546 	uint32_t mon_tail;  /* last seen slot on rx */
547 
548 	/* circular list of zero-copy monitors */
549 	struct netmap_zmon_list zmon_list[NR_TXRX];
550 
551 	/*
552 	 * Monitors work by intercepting the sync and notify callbacks of the
553 	 * monitored krings. This is implemented by replacing the pointers
554 	 * above and saving the previous ones in mon_* pointers below
555 	 */
556 	int (*mon_sync)(struct netmap_kring *kring, int flags);
557 	int (*mon_notify)(struct netmap_kring *kring, int flags);
558 
559 #endif
560 }
561 #ifdef _WIN32
562 __declspec(align(64));
563 #else
564 __attribute__((__aligned__(64)));
565 #endif
566 
567 /* return 1 iff the kring needs to be turned on */
568 static inline int
569 nm_kring_pending_on(struct netmap_kring *kring)
570 {
571 	return kring->nr_pending_mode == NKR_NETMAP_ON &&
572 	       kring->nr_mode == NKR_NETMAP_OFF;
573 }
574 
575 /* return 1 iff the kring needs to be turned off */
576 static inline int
577 nm_kring_pending_off(struct netmap_kring *kring)
578 {
579 	return kring->nr_pending_mode == NKR_NETMAP_OFF &&
580 	       kring->nr_mode == NKR_NETMAP_ON;
581 }
582 
583 /* return the next index, with wraparound */
584 static inline uint32_t
585 nm_next(uint32_t i, uint32_t lim)
586 {
587 	return unlikely (i == lim) ? 0 : i + 1;
588 }
589 
590 
591 /* return the previous index, with wraparound */
592 static inline uint32_t
593 nm_prev(uint32_t i, uint32_t lim)
594 {
595 	return unlikely (i == 0) ? lim : i - 1;
596 }
597 
598 
599 /*
600  *
601  * Here is the layout for the Rx and Tx rings.
602 
603        RxRING                            TxRING
604 
605       +-----------------+            +-----------------+
606       |                 |            |                 |
607       |      free       |            |      free       |
608       +-----------------+            +-----------------+
609 head->| owned by user   |<-hwcur     | not sent to nic |<-hwcur
610       |                 |            | yet             |
611       +-----------------+            |                 |
612  cur->| available to    |            |                 |
613       | user, not read  |            +-----------------+
614       | yet             |       cur->| (being          |
615       |                 |            |  prepared)      |
616       |                 |            |                 |
617       +-----------------+            +     ------      +
618 tail->|                 |<-hwtail    |                 |<-hwlease
619       | (being          | ...        |                 | ...
620       |  prepared)      | ...        |                 | ...
621       +-----------------+ ...        |                 | ...
622       |                 |<-hwlease   +-----------------+
623       |                 |      tail->|                 |<-hwtail
624       |                 |            |                 |
625       |                 |            |                 |
626       |                 |            |                 |
627       +-----------------+            +-----------------+
628 
629  * The cur/tail (user view) and hwcur/hwtail (kernel view)
630  * are used in the normal operation of the card.
631  *
632  * When a ring is the output of a switch port (Rx ring for
633  * a VALE port, Tx ring for the host stack or NIC), slots
634  * are reserved in blocks through 'hwlease' which points
635  * to the next unused slot.
636  * On an Rx ring, hwlease is always after hwtail,
637  * and completions cause hwtail to advance.
638  * On a Tx ring, hwlease is always between cur and hwtail,
639  * and completions cause cur to advance.
640  *
641  * nm_kr_space() returns the maximum number of slots that
642  * can be assigned.
643  * nm_kr_lease() reserves the required number of buffers,
644  *    advances nkr_hwlease and also returns an entry in
645  *    a circular array where completions should be reported.
646  */
647 
648 struct lut_entry;
649 #ifdef __FreeBSD__
650 #define plut_entry lut_entry
651 #endif
652 
653 struct netmap_lut {
654 	struct lut_entry *lut;
655 	struct plut_entry *plut;
656 	uint32_t objtotal;	/* max buffer index */
657 	uint32_t objsize;	/* buffer size */
658 };
659 
660 struct netmap_vp_adapter; // forward
661 struct nm_bridge;
662 
663 /* Struct to be filled by nm_config callbacks. */
664 struct nm_config_info {
665 	unsigned num_tx_rings;
666 	unsigned num_rx_rings;
667 	unsigned num_tx_descs;
668 	unsigned num_rx_descs;
669 	unsigned rx_buf_maxsize;
670 };
671 
672 /*
673  * default type for the magic field.
674  * May be overriden in glue code.
675  */
676 #ifndef NM_OS_MAGIC
677 #define NM_OS_MAGIC uint32_t
678 #endif /* !NM_OS_MAGIC */
679 
680 /*
681  * The "struct netmap_adapter" extends the "struct adapter"
682  * (or equivalent) device descriptor.
683  * It contains all base fields needed to support netmap operation.
684  * There are in fact different types of netmap adapters
685  * (native, generic, VALE switch...) so a netmap_adapter is
686  * just the first field in the derived type.
687  */
688 struct netmap_adapter {
689 	/*
690 	 * On linux we do not have a good way to tell if an interface
691 	 * is netmap-capable. So we always use the following trick:
692 	 * NA(ifp) points here, and the first entry (which hopefully
693 	 * always exists and is at least 32 bits) contains a magic
694 	 * value which we can use to detect that the interface is good.
695 	 */
696 	NM_OS_MAGIC magic;
697 	uint32_t na_flags;	/* enabled, and other flags */
698 #define NAF_SKIP_INTR	1	/* use the regular interrupt handler.
699 				 * useful during initialization
700 				 */
701 #define NAF_SW_ONLY	2	/* forward packets only to sw adapter */
702 #define NAF_BDG_MAYSLEEP 4	/* the bridge is allowed to sleep when
703 				 * forwarding packets coming from this
704 				 * interface
705 				 */
706 #define NAF_MEM_OWNER	8	/* the adapter uses its own memory area
707 				 * that cannot be changed
708 				 */
709 #define NAF_NATIVE      16      /* the adapter is native.
710 				 * Virtual ports (non persistent vale ports,
711 				 * pipes, monitors...) should never use
712 				 * this flag.
713 				 */
714 #define	NAF_NETMAP_ON	32	/* netmap is active (either native or
715 				 * emulated). Where possible (e.g. FreeBSD)
716 				 * IFCAP_NETMAP also mirrors this flag.
717 				 */
718 #define NAF_HOST_RINGS  64	/* the adapter supports the host rings */
719 #define NAF_FORCE_NATIVE 128	/* the adapter is always NATIVE */
720 /* free */
721 #define NAF_MOREFRAG	512	/* the adapter supports NS_MOREFRAG */
722 #define NAF_ZOMBIE	(1U<<30) /* the nic driver has been unloaded */
723 #define	NAF_BUSY	(1U<<31) /* the adapter is used internally and
724 				  * cannot be registered from userspace
725 				  */
726 	int active_fds; /* number of user-space descriptors using this
727 			 interface, which is equal to the number of
728 			 struct netmap_if objs in the mapped region. */
729 
730 	u_int num_rx_rings; /* number of adapter receive rings */
731 	u_int num_tx_rings; /* number of adapter transmit rings */
732 	u_int num_host_rx_rings; /* number of host receive rings */
733 	u_int num_host_tx_rings; /* number of host transmit rings */
734 
735 	u_int num_tx_desc;  /* number of descriptor in each queue */
736 	u_int num_rx_desc;
737 
738 	/* tx_rings and rx_rings are private but allocated as a
739 	 * contiguous chunk of memory. Each array has N+K entries,
740 	 * N for the hardware rings and K for the host rings.
741 	 */
742 	struct netmap_kring **tx_rings; /* array of TX rings. */
743 	struct netmap_kring **rx_rings; /* array of RX rings. */
744 
745 	void *tailroom;		       /* space below the rings array */
746 				       /* (used for leases) */
747 
748 
749 	NM_SELINFO_T si[NR_TXRX];	/* global wait queues */
750 
751 	/* count users of the global wait queues */
752 	int si_users[NR_TXRX];
753 
754 	void *pdev; /* used to store pci device */
755 
756 	/* copy of if_qflush and if_transmit pointers, to intercept
757 	 * packets from the network stack when netmap is active.
758 	 */
759 	int     (*if_transmit)(struct ifnet *, struct mbuf *);
760 
761 	/* copy of if_input for netmap_send_up() */
762 	void     (*if_input)(struct ifnet *, struct mbuf *);
763 
764 	/* Back reference to the parent ifnet struct. Used for
765 	 * hardware ports (emulated netmap included). */
766 	struct ifnet *ifp; /* adapter is ifp->if_softc */
767 
768 	/*---- callbacks for this netmap adapter -----*/
769 	/*
770 	 * nm_dtor() is the cleanup routine called when destroying
771 	 *	the adapter.
772 	 *	Called with NMG_LOCK held.
773 	 *
774 	 * nm_register() is called on NIOCREGIF and close() to enter
775 	 *	or exit netmap mode on the NIC
776 	 *	Called with NNG_LOCK held.
777 	 *
778 	 * nm_txsync() pushes packets to the underlying hw/switch
779 	 *
780 	 * nm_rxsync() collects packets from the underlying hw/switch
781 	 *
782 	 * nm_config() returns configuration information from the OS
783 	 *	Called with NMG_LOCK held.
784 	 *
785 	 * nm_krings_create() create and init the tx_rings and
786 	 * 	rx_rings arrays of kring structures. In particular,
787 	 * 	set the nm_sync callbacks for each ring.
788 	 * 	There is no need to also allocate the corresponding
789 	 * 	netmap_rings, since netmap_mem_rings_create() will always
790 	 * 	be called to provide the missing ones.
791 	 *	Called with NNG_LOCK held.
792 	 *
793 	 * nm_krings_delete() cleanup and delete the tx_rings and rx_rings
794 	 * 	arrays
795 	 *	Called with NMG_LOCK held.
796 	 *
797 	 * nm_notify() is used to act after data have become available
798 	 * 	(or the stopped state of the ring has changed)
799 	 *	For hw devices this is typically a selwakeup(),
800 	 *	but for NIC/host ports attached to a switch (or vice-versa)
801 	 *	we also need to invoke the 'txsync' code downstream.
802 	 *      This callback pointer is actually used only to initialize
803 	 *      kring->nm_notify.
804 	 *      Return values are the same as for netmap_rx_irq().
805 	 */
806 	void (*nm_dtor)(struct netmap_adapter *);
807 
808 	int (*nm_register)(struct netmap_adapter *, int onoff);
809 	void (*nm_intr)(struct netmap_adapter *, int onoff);
810 
811 	int (*nm_txsync)(struct netmap_kring *kring, int flags);
812 	int (*nm_rxsync)(struct netmap_kring *kring, int flags);
813 	int (*nm_notify)(struct netmap_kring *kring, int flags);
814 #define NAF_FORCE_READ      1
815 #define NAF_FORCE_RECLAIM   2
816 #define NAF_CAN_FORWARD_DOWN 4
817 	/* return configuration information */
818 	int (*nm_config)(struct netmap_adapter *, struct nm_config_info *info);
819 	int (*nm_krings_create)(struct netmap_adapter *);
820 	void (*nm_krings_delete)(struct netmap_adapter *);
821 	/*
822 	 * nm_bdg_attach() initializes the na_vp field to point
823 	 *      to an adapter that can be attached to a VALE switch. If the
824 	 *      current adapter is already a VALE port, na_vp is simply a cast;
825 	 *      otherwise, na_vp points to a netmap_bwrap_adapter.
826 	 *      If applicable, this callback also initializes na_hostvp,
827 	 *      that can be used to connect the adapter host rings to the
828 	 *      switch.
829 	 *      Called with NMG_LOCK held.
830 	 *
831 	 * nm_bdg_ctl() is called on the actual attach/detach to/from
832 	 *      to/from the switch, to perform adapter-specific
833 	 *      initializations
834 	 *      Called with NMG_LOCK held.
835 	 */
836 	int (*nm_bdg_attach)(const char *bdg_name, struct netmap_adapter *,
837 			struct nm_bridge *);
838 	int (*nm_bdg_ctl)(struct nmreq_header *, struct netmap_adapter *);
839 
840 	/* adapter used to attach this adapter to a VALE switch (if any) */
841 	struct netmap_vp_adapter *na_vp;
842 	/* adapter used to attach the host rings of this adapter
843 	 * to a VALE switch (if any) */
844 	struct netmap_vp_adapter *na_hostvp;
845 
846 	/* standard refcount to control the lifetime of the adapter
847 	 * (it should be equal to the lifetime of the corresponding ifp)
848 	 */
849 	int na_refcount;
850 
851 	/* memory allocator (opaque)
852 	 * We also cache a pointer to the lut_entry for translating
853 	 * buffer addresses, the total number of buffers and the buffer size.
854 	 */
855  	struct netmap_mem_d *nm_mem;
856 	struct netmap_mem_d *nm_mem_prev;
857 	struct netmap_lut na_lut;
858 
859 	/* additional information attached to this adapter
860 	 * by other netmap subsystems. Currently used by
861 	 * bwrap, LINUX/v1000 and ptnetmap
862 	 */
863 	void *na_private;
864 
865 	/* array of pipes that have this adapter as a parent */
866 	struct netmap_pipe_adapter **na_pipes;
867 	int na_next_pipe;	/* next free slot in the array */
868 	int na_max_pipes;	/* size of the array */
869 
870 	/* Offset of ethernet header for each packet. */
871 	u_int virt_hdr_len;
872 
873 	/* Max number of bytes that the NIC can store in the buffer
874 	 * referenced by each RX descriptor. This translates to the maximum
875 	 * bytes that a single netmap slot can reference. Larger packets
876 	 * require NS_MOREFRAG support. */
877 	unsigned rx_buf_maxsize;
878 
879 	char name[NETMAP_REQ_IFNAMSIZ]; /* used at least by pipes */
880 
881 #ifdef WITH_MONITOR
882 	unsigned long	monitor_id;	/* debugging */
883 #endif
884 };
885 
886 static __inline u_int
887 nma_get_ndesc(struct netmap_adapter *na, enum txrx t)
888 {
889 	return (t == NR_TX ? na->num_tx_desc : na->num_rx_desc);
890 }
891 
892 static __inline void
893 nma_set_ndesc(struct netmap_adapter *na, enum txrx t, u_int v)
894 {
895 	if (t == NR_TX)
896 		na->num_tx_desc = v;
897 	else
898 		na->num_rx_desc = v;
899 }
900 
901 static __inline u_int
902 nma_get_nrings(struct netmap_adapter *na, enum txrx t)
903 {
904 	return (t == NR_TX ? na->num_tx_rings : na->num_rx_rings);
905 }
906 
907 static __inline u_int
908 nma_get_host_nrings(struct netmap_adapter *na, enum txrx t)
909 {
910 	return (t == NR_TX ? na->num_host_tx_rings : na->num_host_rx_rings);
911 }
912 
913 static __inline void
914 nma_set_nrings(struct netmap_adapter *na, enum txrx t, u_int v)
915 {
916 	if (t == NR_TX)
917 		na->num_tx_rings = v;
918 	else
919 		na->num_rx_rings = v;
920 }
921 
922 static __inline void
923 nma_set_host_nrings(struct netmap_adapter *na, enum txrx t, u_int v)
924 {
925 	if (t == NR_TX)
926 		na->num_host_tx_rings = v;
927 	else
928 		na->num_host_rx_rings = v;
929 }
930 
931 static __inline struct netmap_kring**
932 NMR(struct netmap_adapter *na, enum txrx t)
933 {
934 	return (t == NR_TX ? na->tx_rings : na->rx_rings);
935 }
936 
937 int nma_intr_enable(struct netmap_adapter *na, int onoff);
938 
939 /*
940  * If the NIC is owned by the kernel
941  * (i.e., bridge), neither another bridge nor user can use it;
942  * if the NIC is owned by a user, only users can share it.
943  * Evaluation must be done under NMG_LOCK().
944  */
945 #define NETMAP_OWNED_BY_KERN(na)	((na)->na_flags & NAF_BUSY)
946 #define NETMAP_OWNED_BY_ANY(na) \
947 	(NETMAP_OWNED_BY_KERN(na) || ((na)->active_fds > 0))
948 
949 /*
950  * derived netmap adapters for various types of ports
951  */
952 struct netmap_vp_adapter {	/* VALE software port */
953 	struct netmap_adapter up;
954 
955 	/*
956 	 * Bridge support:
957 	 *
958 	 * bdg_port is the port number used in the bridge;
959 	 * na_bdg points to the bridge this NA is attached to.
960 	 */
961 	int bdg_port;
962 	struct nm_bridge *na_bdg;
963 	int retry;
964 	int autodelete; /* remove the ifp on last reference */
965 
966 	/* Maximum Frame Size, used in bdg_mismatch_datapath() */
967 	u_int mfs;
968 	/* Last source MAC on this port */
969 	uint64_t last_smac;
970 };
971 
972 
973 struct netmap_hw_adapter {	/* physical device */
974 	struct netmap_adapter up;
975 
976 #ifdef linux
977 	struct net_device_ops nm_ndo;
978 	struct ethtool_ops    nm_eto;
979 #endif
980 	const struct ethtool_ops*   save_ethtool;
981 
982 	int (*nm_hw_register)(struct netmap_adapter *, int onoff);
983 };
984 
985 #ifdef WITH_GENERIC
986 /* Mitigation support. */
987 struct nm_generic_mit {
988 	struct hrtimer mit_timer;
989 	int mit_pending;
990 	int mit_ring_idx;  /* index of the ring being mitigated */
991 	struct netmap_adapter *mit_na;  /* backpointer */
992 };
993 
994 struct netmap_generic_adapter {	/* emulated device */
995 	struct netmap_hw_adapter up;
996 
997 	/* Pointer to a previously used netmap adapter. */
998 	struct netmap_adapter *prev;
999 
1000 	/* Emulated netmap adapters support:
1001 	 *  - save_if_input saves the if_input hook (FreeBSD);
1002 	 *  - mit implements rx interrupt mitigation;
1003 	 */
1004 	void (*save_if_input)(struct ifnet *, struct mbuf *);
1005 
1006 	struct nm_generic_mit *mit;
1007 #ifdef linux
1008         netdev_tx_t (*save_start_xmit)(struct mbuf *, struct ifnet *);
1009 #endif
1010 	/* Is the adapter able to use multiple RX slots to scatter
1011 	 * each packet pushed up by the driver? */
1012 	int rxsg;
1013 
1014 	/* Is the transmission path controlled by a netmap-aware
1015 	 * device queue (i.e. qdisc on linux)? */
1016 	int txqdisc;
1017 };
1018 #endif  /* WITH_GENERIC */
1019 
1020 static __inline u_int
1021 netmap_real_rings(struct netmap_adapter *na, enum txrx t)
1022 {
1023 	return nma_get_nrings(na, t) +
1024 		!!(na->na_flags & NAF_HOST_RINGS) * nma_get_host_nrings(na, t);
1025 }
1026 
1027 /* account for fake rings */
1028 static __inline u_int
1029 netmap_all_rings(struct netmap_adapter *na, enum txrx t)
1030 {
1031 	return max(nma_get_nrings(na, t) + 1, netmap_real_rings(na, t));
1032 }
1033 
1034 int netmap_default_bdg_attach(const char *name, struct netmap_adapter *na,
1035 		struct nm_bridge *);
1036 struct nm_bdg_polling_state;
1037 /*
1038  * Bridge wrapper for non VALE ports attached to a VALE switch.
1039  *
1040  * The real device must already have its own netmap adapter (hwna).
1041  * The bridge wrapper and the hwna adapter share the same set of
1042  * netmap rings and buffers, but they have two separate sets of
1043  * krings descriptors, with tx/rx meanings swapped:
1044  *
1045  *                                  netmap
1046  *           bwrap     krings       rings      krings      hwna
1047  *         +------+   +------+     +-----+    +------+   +------+
1048  *         |tx_rings->|      |\   /|     |----|      |<-tx_rings|
1049  *         |      |   +------+ \ / +-----+    +------+   |      |
1050  *         |      |             X                        |      |
1051  *         |      |            / \                       |      |
1052  *         |      |   +------+/   \+-----+    +------+   |      |
1053  *         |rx_rings->|      |     |     |----|      |<-rx_rings|
1054  *         |      |   +------+     +-----+    +------+   |      |
1055  *         +------+                                      +------+
1056  *
1057  * - packets coming from the bridge go to the brwap rx rings,
1058  *   which are also the hwna tx rings.  The bwrap notify callback
1059  *   will then complete the hwna tx (see netmap_bwrap_notify).
1060  *
1061  * - packets coming from the outside go to the hwna rx rings,
1062  *   which are also the bwrap tx rings.  The (overwritten) hwna
1063  *   notify method will then complete the bridge tx
1064  *   (see netmap_bwrap_intr_notify).
1065  *
1066  *   The bridge wrapper may optionally connect the hwna 'host' rings
1067  *   to the bridge. This is done by using a second port in the
1068  *   bridge and connecting it to the 'host' netmap_vp_adapter
1069  *   contained in the netmap_bwrap_adapter. The brwap host adapter
1070  *   cross-links the hwna host rings in the same way as shown above.
1071  *
1072  * - packets coming from the bridge and directed to the host stack
1073  *   are handled by the bwrap host notify callback
1074  *   (see netmap_bwrap_host_notify)
1075  *
1076  * - packets coming from the host stack are still handled by the
1077  *   overwritten hwna notify callback (netmap_bwrap_intr_notify),
1078  *   but are diverted to the host adapter depending on the ring number.
1079  *
1080  */
1081 struct netmap_bwrap_adapter {
1082 	struct netmap_vp_adapter up;
1083 	struct netmap_vp_adapter host;  /* for host rings */
1084 	struct netmap_adapter *hwna;	/* the underlying device */
1085 
1086 	/*
1087 	 * When we attach a physical interface to the bridge, we
1088 	 * allow the controlling process to terminate, so we need
1089 	 * a place to store the n_detmap_priv_d data structure.
1090 	 * This is only done when physical interfaces
1091 	 * are attached to a bridge.
1092 	 */
1093 	struct netmap_priv_d *na_kpriv;
1094 	struct nm_bdg_polling_state *na_polling_state;
1095 	/* we overwrite the hwna->na_vp pointer, so we save
1096 	 * here its original value, to be restored at detach
1097 	 */
1098 	struct netmap_vp_adapter *saved_na_vp;
1099 };
1100 int nm_bdg_polling(struct nmreq_header *hdr);
1101 
1102 #ifdef WITH_VALE
1103 int netmap_vale_attach(struct nmreq_header *hdr, void *auth_token);
1104 int netmap_vale_detach(struct nmreq_header *hdr, void *auth_token);
1105 int netmap_vale_list(struct nmreq_header *hdr);
1106 int netmap_vi_create(struct nmreq_header *hdr, int);
1107 int nm_vi_create(struct nmreq_header *);
1108 int nm_vi_destroy(const char *name);
1109 #else /* !WITH_VALE */
1110 #define netmap_vi_create(hdr, a) (EOPNOTSUPP)
1111 #endif /* WITH_VALE */
1112 
1113 #ifdef WITH_PIPES
1114 
1115 #define NM_MAXPIPES 	64	/* max number of pipes per adapter */
1116 
1117 struct netmap_pipe_adapter {
1118 	/* pipe identifier is up.name */
1119 	struct netmap_adapter up;
1120 
1121 #define NM_PIPE_ROLE_MASTER	0x1
1122 #define NM_PIPE_ROLE_SLAVE	0x2
1123 	int role;	/* either NM_PIPE_ROLE_MASTER or NM_PIPE_ROLE_SLAVE */
1124 
1125 	struct netmap_adapter *parent; /* adapter that owns the memory */
1126 	struct netmap_pipe_adapter *peer; /* the other end of the pipe */
1127 	int peer_ref;		/* 1 iff we are holding a ref to the peer */
1128 	struct ifnet *parent_ifp;	/* maybe null */
1129 
1130 	u_int parent_slot; /* index in the parent pipe array */
1131 };
1132 
1133 #endif /* WITH_PIPES */
1134 
1135 #ifdef WITH_NMNULL
1136 struct netmap_null_adapter {
1137 	struct netmap_adapter up;
1138 };
1139 #endif /* WITH_NMNULL */
1140 
1141 
1142 /* return slots reserved to rx clients; used in drivers */
1143 static inline uint32_t
1144 nm_kr_rxspace(struct netmap_kring *k)
1145 {
1146 	int space = k->nr_hwtail - k->nr_hwcur;
1147 	if (space < 0)
1148 		space += k->nkr_num_slots;
1149 	nm_prdis("preserving %d rx slots %d -> %d", space, k->nr_hwcur, k->nr_hwtail);
1150 
1151 	return space;
1152 }
1153 
1154 /* return slots reserved to tx clients */
1155 #define nm_kr_txspace(_k) nm_kr_rxspace(_k)
1156 
1157 
1158 /* True if no space in the tx ring, only valid after txsync_prologue */
1159 static inline int
1160 nm_kr_txempty(struct netmap_kring *kring)
1161 {
1162 	return kring->rhead == kring->nr_hwtail;
1163 }
1164 
1165 /* True if no more completed slots in the rx ring, only valid after
1166  * rxsync_prologue */
1167 #define nm_kr_rxempty(_k)	nm_kr_txempty(_k)
1168 
1169 /* True if the application needs to wait for more space on the ring
1170  * (more received packets or more free tx slots).
1171  * Only valid after *xsync_prologue. */
1172 static inline int
1173 nm_kr_wouldblock(struct netmap_kring *kring)
1174 {
1175 	return kring->rcur == kring->nr_hwtail;
1176 }
1177 
1178 /*
1179  * protect against multiple threads using the same ring.
1180  * also check that the ring has not been stopped or locked
1181  */
1182 #define NM_KR_BUSY	1	/* some other thread is syncing the ring */
1183 #define NM_KR_STOPPED	2	/* unbounded stop (ifconfig down or driver unload) */
1184 #define NM_KR_LOCKED	3	/* bounded, brief stop for mutual exclusion */
1185 
1186 
1187 /* release the previously acquired right to use the *sync() methods of the ring */
1188 static __inline void nm_kr_put(struct netmap_kring *kr)
1189 {
1190 	NM_ATOMIC_CLEAR(&kr->nr_busy);
1191 }
1192 
1193 
1194 /* true if the ifp that backed the adapter has disappeared (e.g., the
1195  * driver has been unloaded)
1196  */
1197 static inline int nm_iszombie(struct netmap_adapter *na);
1198 
1199 /* try to obtain exclusive right to issue the *sync() operations on the ring.
1200  * The right is obtained and must be later relinquished via nm_kr_put() if and
1201  * only if nm_kr_tryget() returns 0.
1202  * If can_sleep is 1 there are only two other possible outcomes:
1203  * - the function returns NM_KR_BUSY
1204  * - the function returns NM_KR_STOPPED and sets the POLLERR bit in *perr
1205  *   (if non-null)
1206  * In both cases the caller will typically skip the ring, possibly collecting
1207  * errors along the way.
1208  * If the calling context does not allow sleeping, the caller must pass 0 in can_sleep.
1209  * In the latter case, the function may also return NM_KR_LOCKED and leave *perr
1210  * untouched: ideally, the caller should try again at a later time.
1211  */
1212 static __inline int nm_kr_tryget(struct netmap_kring *kr, int can_sleep, int *perr)
1213 {
1214 	int busy = 1, stopped;
1215 	/* check a first time without taking the lock
1216 	 * to avoid starvation for nm_kr_get()
1217 	 */
1218 retry:
1219 	stopped = kr->nkr_stopped;
1220 	if (unlikely(stopped)) {
1221 		goto stop;
1222 	}
1223 	busy = NM_ATOMIC_TEST_AND_SET(&kr->nr_busy);
1224 	/* we should not return NM_KR_BUSY if the ring was
1225 	 * actually stopped, so check another time after
1226 	 * the barrier provided by the atomic operation
1227 	 */
1228 	stopped = kr->nkr_stopped;
1229 	if (unlikely(stopped)) {
1230 		goto stop;
1231 	}
1232 
1233 	if (unlikely(nm_iszombie(kr->na))) {
1234 		stopped = NM_KR_STOPPED;
1235 		goto stop;
1236 	}
1237 
1238 	return unlikely(busy) ? NM_KR_BUSY : 0;
1239 
1240 stop:
1241 	if (!busy)
1242 		nm_kr_put(kr);
1243 	if (stopped == NM_KR_STOPPED) {
1244 /* if POLLERR is defined we want to use it to simplify netmap_poll().
1245  * Otherwise, any non-zero value will do.
1246  */
1247 #ifdef POLLERR
1248 #define NM_POLLERR POLLERR
1249 #else
1250 #define NM_POLLERR 1
1251 #endif /* POLLERR */
1252 		if (perr)
1253 			*perr |= NM_POLLERR;
1254 #undef NM_POLLERR
1255 	} else if (can_sleep) {
1256 		tsleep(kr, 0, "NM_KR_TRYGET", 4);
1257 		goto retry;
1258 	}
1259 	return stopped;
1260 }
1261 
1262 /* put the ring in the 'stopped' state and wait for the current user (if any) to
1263  * notice. stopped must be either NM_KR_STOPPED or NM_KR_LOCKED
1264  */
1265 static __inline void nm_kr_stop(struct netmap_kring *kr, int stopped)
1266 {
1267 	kr->nkr_stopped = stopped;
1268 	while (NM_ATOMIC_TEST_AND_SET(&kr->nr_busy))
1269 		tsleep(kr, 0, "NM_KR_GET", 4);
1270 }
1271 
1272 /* restart a ring after a stop */
1273 static __inline void nm_kr_start(struct netmap_kring *kr)
1274 {
1275 	kr->nkr_stopped = 0;
1276 	nm_kr_put(kr);
1277 }
1278 
1279 
1280 /*
1281  * The following functions are used by individual drivers to
1282  * support netmap operation.
1283  *
1284  * netmap_attach() initializes a struct netmap_adapter, allocating the
1285  * 	struct netmap_ring's and the struct selinfo.
1286  *
1287  * netmap_detach() frees the memory allocated by netmap_attach().
1288  *
1289  * netmap_transmit() replaces the if_transmit routine of the interface,
1290  *	and is used to intercept packets coming from the stack.
1291  *
1292  * netmap_load_map/netmap_reload_map are helper routines to set/reset
1293  *	the dmamap for a packet buffer
1294  *
1295  * netmap_reset() is a helper routine to be called in the hw driver
1296  *	when reinitializing a ring. It should not be called by
1297  *	virtual ports (vale, pipes, monitor)
1298  */
1299 int netmap_attach(struct netmap_adapter *);
1300 int netmap_attach_ext(struct netmap_adapter *, size_t size, int override_reg);
1301 void netmap_detach(struct ifnet *);
1302 int netmap_transmit(struct ifnet *, struct mbuf *);
1303 struct netmap_slot *netmap_reset(struct netmap_adapter *na,
1304 	enum txrx tx, u_int n, u_int new_cur);
1305 int netmap_ring_reinit(struct netmap_kring *);
1306 int netmap_rings_config_get(struct netmap_adapter *, struct nm_config_info *);
1307 
1308 /* Return codes for netmap_*x_irq. */
1309 enum {
1310 	/* Driver should do normal interrupt processing, e.g. because
1311 	 * the interface is not in netmap mode. */
1312 	NM_IRQ_PASS = 0,
1313 	/* Port is in netmap mode, and the interrupt work has been
1314 	 * completed. The driver does not have to notify netmap
1315 	 * again before the next interrupt. */
1316 	NM_IRQ_COMPLETED = -1,
1317 	/* Port is in netmap mode, but the interrupt work has not been
1318 	 * completed. The driver has to make sure netmap will be
1319 	 * notified again soon, even if no more interrupts come (e.g.
1320 	 * on Linux the driver should not call napi_complete()). */
1321 	NM_IRQ_RESCHED = -2,
1322 };
1323 
1324 /* default functions to handle rx/tx interrupts */
1325 int netmap_rx_irq(struct ifnet *, u_int, u_int *);
1326 #define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL)
1327 int netmap_common_irq(struct netmap_adapter *, u_int, u_int *work_done);
1328 
1329 
1330 #ifdef WITH_VALE
1331 /* functions used by external modules to interface with VALE */
1332 #define netmap_vp_to_ifp(_vp)	((_vp)->up.ifp)
1333 #define netmap_ifp_to_vp(_ifp)	(NA(_ifp)->na_vp)
1334 #define netmap_ifp_to_host_vp(_ifp) (NA(_ifp)->na_hostvp)
1335 #define netmap_bdg_idx(_vp)	((_vp)->bdg_port)
1336 const char *netmap_bdg_name(struct netmap_vp_adapter *);
1337 #else /* !WITH_VALE */
1338 #define netmap_vp_to_ifp(_vp)	NULL
1339 #define netmap_ifp_to_vp(_ifp)	NULL
1340 #define netmap_ifp_to_host_vp(_ifp) NULL
1341 #define netmap_bdg_idx(_vp)	-1
1342 #endif /* WITH_VALE */
1343 
1344 static inline int
1345 nm_netmap_on(struct netmap_adapter *na)
1346 {
1347 	return na && na->na_flags & NAF_NETMAP_ON;
1348 }
1349 
1350 static inline int
1351 nm_native_on(struct netmap_adapter *na)
1352 {
1353 	return nm_netmap_on(na) && (na->na_flags & NAF_NATIVE);
1354 }
1355 
1356 static inline int
1357 nm_iszombie(struct netmap_adapter *na)
1358 {
1359 	return na == NULL || (na->na_flags & NAF_ZOMBIE);
1360 }
1361 
1362 static inline void
1363 nm_update_hostrings_mode(struct netmap_adapter *na)
1364 {
1365 	/* Process nr_mode and nr_pending_mode for host rings. */
1366 	na->tx_rings[na->num_tx_rings]->nr_mode =
1367 		na->tx_rings[na->num_tx_rings]->nr_pending_mode;
1368 	na->rx_rings[na->num_rx_rings]->nr_mode =
1369 		na->rx_rings[na->num_rx_rings]->nr_pending_mode;
1370 }
1371 
1372 void nm_set_native_flags(struct netmap_adapter *);
1373 void nm_clear_native_flags(struct netmap_adapter *);
1374 
1375 void netmap_krings_mode_commit(struct netmap_adapter *na, int onoff);
1376 
1377 /*
1378  * nm_*sync_prologue() functions are used in ioctl/poll and ptnetmap
1379  * kthreads.
1380  * We need netmap_ring* parameter, because in ptnetmap it is decoupled
1381  * from host kring.
1382  * The user-space ring pointers (head/cur/tail) are shared through
1383  * CSB between host and guest.
1384  */
1385 
1386 /*
1387  * validates parameters in the ring/kring, returns a value for head
1388  * If any error, returns ring_size to force a reinit.
1389  */
1390 uint32_t nm_txsync_prologue(struct netmap_kring *, struct netmap_ring *);
1391 
1392 
1393 /*
1394  * validates parameters in the ring/kring, returns a value for head
1395  * If any error, returns ring_size lim to force a reinit.
1396  */
1397 uint32_t nm_rxsync_prologue(struct netmap_kring *, struct netmap_ring *);
1398 
1399 
1400 /* check/fix address and len in tx rings */
1401 #if 1 /* debug version */
1402 #define	NM_CHECK_ADDR_LEN(_na, _a, _l)	do {				\
1403 	if (_a == NETMAP_BUF_BASE(_na) || _l > NETMAP_BUF_SIZE(_na)) {	\
1404 		nm_prlim(5, "bad addr/len ring %d slot %d idx %d len %d",	\
1405 			kring->ring_id, nm_i, slot->buf_idx, len);	\
1406 		if (_l > NETMAP_BUF_SIZE(_na))				\
1407 			_l = NETMAP_BUF_SIZE(_na);			\
1408 	} } while (0)
1409 #else /* no debug version */
1410 #define	NM_CHECK_ADDR_LEN(_na, _a, _l)	do {				\
1411 		if (_l > NETMAP_BUF_SIZE(_na))				\
1412 			_l = NETMAP_BUF_SIZE(_na);			\
1413 	} while (0)
1414 #endif
1415 
1416 
1417 /*---------------------------------------------------------------*/
1418 /*
1419  * Support routines used by netmap subsystems
1420  * (native drivers, VALE, generic, pipes, monitors, ...)
1421  */
1422 
1423 
1424 /* common routine for all functions that create a netmap adapter. It performs
1425  * two main tasks:
1426  * - if the na points to an ifp, mark the ifp as netmap capable
1427  *   using na as its native adapter;
1428  * - provide defaults for the setup callbacks and the memory allocator
1429  */
1430 int netmap_attach_common(struct netmap_adapter *);
1431 /* fill priv->np_[tr]xq{first,last} using the ringid and flags information
1432  * coming from a struct nmreq_register
1433  */
1434 int netmap_interp_ringid(struct netmap_priv_d *priv, uint32_t nr_mode,
1435 			uint16_t nr_ringid, uint64_t nr_flags);
1436 /* update the ring parameters (number and size of tx and rx rings).
1437  * It calls the nm_config callback, if available.
1438  */
1439 int netmap_update_config(struct netmap_adapter *na);
1440 /* create and initialize the common fields of the krings array.
1441  * using the information that must be already available in the na.
1442  * tailroom can be used to request the allocation of additional
1443  * tailroom bytes after the krings array. This is used by
1444  * netmap_vp_adapter's (i.e., VALE ports) to make room for
1445  * leasing-related data structures
1446  */
1447 int netmap_krings_create(struct netmap_adapter *na, u_int tailroom);
1448 /* deletes the kring array of the adapter. The array must have
1449  * been created using netmap_krings_create
1450  */
1451 void netmap_krings_delete(struct netmap_adapter *na);
1452 
1453 int netmap_hw_krings_create(struct netmap_adapter *na);
1454 void netmap_hw_krings_delete(struct netmap_adapter *na);
1455 
1456 /* set the stopped/enabled status of ring
1457  * When stopping, they also wait for all current activity on the ring to
1458  * terminate. The status change is then notified using the na nm_notify
1459  * callback.
1460  */
1461 void netmap_set_ring(struct netmap_adapter *, u_int ring_id, enum txrx, int stopped);
1462 /* set the stopped/enabled status of all rings of the adapter. */
1463 void netmap_set_all_rings(struct netmap_adapter *, int stopped);
1464 /* convenience wrappers for netmap_set_all_rings */
1465 void netmap_disable_all_rings(struct ifnet *);
1466 void netmap_enable_all_rings(struct ifnet *);
1467 
1468 int netmap_buf_size_validate(const struct netmap_adapter *na, unsigned mtu);
1469 int netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
1470 		uint32_t nr_mode, uint16_t nr_ringid, uint64_t nr_flags);
1471 void netmap_do_unregif(struct netmap_priv_d *priv);
1472 
1473 u_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg);
1474 int netmap_get_na(struct nmreq_header *hdr, struct netmap_adapter **na,
1475 		struct ifnet **ifp, struct netmap_mem_d *nmd, int create);
1476 void netmap_unget_na(struct netmap_adapter *na, struct ifnet *ifp);
1477 int netmap_get_hw_na(struct ifnet *ifp,
1478 		struct netmap_mem_d *nmd, struct netmap_adapter **na);
1479 
1480 #ifdef WITH_VALE
1481 uint32_t netmap_vale_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring,
1482 		struct netmap_vp_adapter *, void *private_data);
1483 
1484 /* these are redefined in case of no VALE support */
1485 int netmap_get_vale_na(struct nmreq_header *hdr, struct netmap_adapter **na,
1486 		struct netmap_mem_d *nmd, int create);
1487 void *netmap_vale_create(const char *bdg_name, int *return_status);
1488 int netmap_vale_destroy(const char *bdg_name, void *auth_token);
1489 
1490 #else /* !WITH_VALE */
1491 #define netmap_bdg_learning(_1, _2, _3, _4)	0
1492 #define	netmap_get_vale_na(_1, _2, _3, _4)	0
1493 #define netmap_bdg_create(_1, _2)	NULL
1494 #define netmap_bdg_destroy(_1, _2)	0
1495 #endif /* !WITH_VALE */
1496 
1497 #ifdef WITH_PIPES
1498 /* max number of pipes per device */
1499 #define NM_MAXPIPES	64	/* XXX this should probably be a sysctl */
1500 void netmap_pipe_dealloc(struct netmap_adapter *);
1501 int netmap_get_pipe_na(struct nmreq_header *hdr, struct netmap_adapter **na,
1502 			struct netmap_mem_d *nmd, int create);
1503 #else /* !WITH_PIPES */
1504 #define NM_MAXPIPES	0
1505 #define netmap_pipe_alloc(_1, _2) 	0
1506 #define netmap_pipe_dealloc(_1)
1507 #define netmap_get_pipe_na(hdr, _2, _3, _4)	\
1508 	((strchr(hdr->nr_name, '{') != NULL || strchr(hdr->nr_name, '}') != NULL) ? EOPNOTSUPP : 0)
1509 #endif
1510 
1511 #ifdef WITH_MONITOR
1512 int netmap_get_monitor_na(struct nmreq_header *hdr, struct netmap_adapter **na,
1513 		struct netmap_mem_d *nmd, int create);
1514 void netmap_monitor_stop(struct netmap_adapter *na);
1515 #else
1516 #define netmap_get_monitor_na(hdr, _2, _3, _4) \
1517 	(((struct nmreq_register *)(uintptr_t)hdr->nr_body)->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX) ? EOPNOTSUPP : 0)
1518 #endif
1519 
1520 #ifdef WITH_NMNULL
1521 int netmap_get_null_na(struct nmreq_header *hdr, struct netmap_adapter **na,
1522 		struct netmap_mem_d *nmd, int create);
1523 #else /* !WITH_NMNULL */
1524 #define netmap_get_null_na(hdr, _2, _3, _4) \
1525 	(((struct nmreq_register *)(uintptr_t)hdr->nr_body)->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX) ? EOPNOTSUPP : 0)
1526 #endif /* WITH_NMNULL */
1527 
1528 #ifdef CONFIG_NET_NS
1529 struct net *netmap_bns_get(void);
1530 void netmap_bns_put(struct net *);
1531 void netmap_bns_getbridges(struct nm_bridge **, u_int *);
1532 #else
1533 extern struct nm_bridge *nm_bridges;
1534 #define netmap_bns_get()
1535 #define netmap_bns_put(_1)
1536 #define netmap_bns_getbridges(b, n) \
1537 	do { *b = nm_bridges; *n = NM_BRIDGES; } while (0)
1538 #endif
1539 
1540 /* Various prototypes */
1541 int netmap_poll(struct netmap_priv_d *, int events, NM_SELRECORD_T *td);
1542 int netmap_init(void);
1543 void netmap_fini(void);
1544 int netmap_get_memory(struct netmap_priv_d* p);
1545 void netmap_dtor(void *data);
1546 
1547 int netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data,
1548 		struct thread *, int nr_body_is_user);
1549 int netmap_ioctl_legacy(struct netmap_priv_d *priv, u_long cmd, caddr_t data,
1550 			struct thread *td);
1551 size_t nmreq_size_by_type(uint16_t nr_reqtype);
1552 
1553 /* netmap_adapter creation/destruction */
1554 
1555 // #define NM_DEBUG_PUTGET 1
1556 
1557 #ifdef NM_DEBUG_PUTGET
1558 
1559 #define NM_DBG(f) __##f
1560 
1561 void __netmap_adapter_get(struct netmap_adapter *na);
1562 
1563 #define netmap_adapter_get(na) 				\
1564 	do {						\
1565 		struct netmap_adapter *__na = na;	\
1566 		nm_prinf("getting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount);	\
1567 		__netmap_adapter_get(__na);		\
1568 	} while (0)
1569 
1570 int __netmap_adapter_put(struct netmap_adapter *na);
1571 
1572 #define netmap_adapter_put(na)				\
1573 	({						\
1574 		struct netmap_adapter *__na = na;	\
1575 		nm_prinf("putting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount);	\
1576 		__netmap_adapter_put(__na);		\
1577 	})
1578 
1579 #else /* !NM_DEBUG_PUTGET */
1580 
1581 #define NM_DBG(f) f
1582 void netmap_adapter_get(struct netmap_adapter *na);
1583 int netmap_adapter_put(struct netmap_adapter *na);
1584 
1585 #endif /* !NM_DEBUG_PUTGET */
1586 
1587 
1588 /*
1589  * module variables
1590  */
1591 #define NETMAP_BUF_BASE(_na)	((_na)->na_lut.lut[0].vaddr)
1592 #define NETMAP_BUF_SIZE(_na)	((_na)->na_lut.objsize)
1593 extern int netmap_no_pendintr;
1594 extern int netmap_mitigate;
1595 extern int netmap_verbose;
1596 #ifdef CONFIG_NETMAP_DEBUG
1597 extern int netmap_debug;		/* for debugging */
1598 #else /* !CONFIG_NETMAP_DEBUG */
1599 #define netmap_debug (0)
1600 #endif /* !CONFIG_NETMAP_DEBUG */
1601 enum {                                  /* debug flags */
1602 	NM_DEBUG_ON = 1,		/* generic debug messsages */
1603 	NM_DEBUG_HOST = 0x2,            /* debug host stack */
1604 	NM_DEBUG_RXSYNC = 0x10,         /* debug on rxsync/txsync */
1605 	NM_DEBUG_TXSYNC = 0x20,
1606 	NM_DEBUG_RXINTR = 0x100,        /* debug on rx/tx intr (driver) */
1607 	NM_DEBUG_TXINTR = 0x200,
1608 	NM_DEBUG_NIC_RXSYNC = 0x1000,   /* debug on rx/tx intr (driver) */
1609 	NM_DEBUG_NIC_TXSYNC = 0x2000,
1610 	NM_DEBUG_MEM = 0x4000,		/* verbose memory allocations/deallocations */
1611 	NM_DEBUG_VALE = 0x8000,		/* debug messages from memory allocators */
1612 	NM_DEBUG_BDG = NM_DEBUG_VALE,
1613 };
1614 
1615 extern int netmap_txsync_retry;
1616 extern int netmap_flags;
1617 extern int netmap_generic_hwcsum;
1618 extern int netmap_generic_mit;
1619 extern int netmap_generic_ringsize;
1620 extern int netmap_generic_rings;
1621 #ifdef linux
1622 extern int netmap_generic_txqdisc;
1623 #endif
1624 
1625 /*
1626  * NA returns a pointer to the struct netmap adapter from the ifp.
1627  * WNA is os-specific and must be defined in glue code.
1628  */
1629 #define	NA(_ifp)	((struct netmap_adapter *)WNA(_ifp))
1630 
1631 /*
1632  * we provide a default implementation of NM_ATTACH_NA/NM_DETACH_NA
1633  * based on the WNA field.
1634  * Glue code may override this by defining its own NM_ATTACH_NA
1635  */
1636 #ifndef NM_ATTACH_NA
1637 /*
1638  * On old versions of FreeBSD, NA(ifp) is a pspare. On linux we
1639  * overload another pointer in the netdev.
1640  *
1641  * We check if NA(ifp) is set and its first element has a related
1642  * magic value. The capenable is within the struct netmap_adapter.
1643  */
1644 #define	NETMAP_MAGIC	0x52697a7a
1645 
1646 #define NM_NA_VALID(ifp)	(NA(ifp) &&		\
1647 	((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC )
1648 
1649 #define	NM_ATTACH_NA(ifp, na) do {					\
1650 	WNA(ifp) = na;							\
1651 	if (NA(ifp))							\
1652 		NA(ifp)->magic = 					\
1653 			((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC;	\
1654 } while(0)
1655 #define NM_RESTORE_NA(ifp, na) 	WNA(ifp) = na;
1656 
1657 #define NM_DETACH_NA(ifp)	do { WNA(ifp) = NULL; } while (0)
1658 #define NM_NA_CLASH(ifp)	(NA(ifp) && !NM_NA_VALID(ifp))
1659 #endif /* !NM_ATTACH_NA */
1660 
1661 
1662 #define NM_IS_NATIVE(ifp)	(NM_NA_VALID(ifp) && NA(ifp)->nm_dtor == netmap_hw_dtor)
1663 
1664 #if defined(__FreeBSD__)
1665 
1666 /* Assigns the device IOMMU domain to an allocator.
1667  * Returns -ENOMEM in case the domain is different */
1668 #define nm_iommu_group_id(dev) (0)
1669 
1670 /* Callback invoked by the dma machinery after a successful dmamap_load */
1671 static void netmap_dmamap_cb(__unused void *arg,
1672     __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error)
1673 {
1674 }
1675 
1676 /* bus_dmamap_load wrapper: call aforementioned function if map != NULL.
1677  * XXX can we do it without a callback ?
1678  */
1679 static inline int
1680 netmap_load_map(struct netmap_adapter *na,
1681 	bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
1682 {
1683 	if (map)
1684 		bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na),
1685 		    netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
1686 	return 0;
1687 }
1688 
1689 static inline void
1690 netmap_unload_map(struct netmap_adapter *na,
1691         bus_dma_tag_t tag, bus_dmamap_t map)
1692 {
1693 	if (map)
1694 		bus_dmamap_unload(tag, map);
1695 }
1696 
1697 #define netmap_sync_map(na, tag, map, sz, t)
1698 
1699 /* update the map when a buffer changes. */
1700 static inline void
1701 netmap_reload_map(struct netmap_adapter *na,
1702 	bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
1703 {
1704 	if (map) {
1705 		bus_dmamap_unload(tag, map);
1706 		bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na),
1707 		    netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
1708 	}
1709 }
1710 
1711 #elif defined(_WIN32)
1712 
1713 #else /* linux */
1714 
1715 int nm_iommu_group_id(bus_dma_tag_t dev);
1716 #include <linux/dma-mapping.h>
1717 
1718 /*
1719  * on linux we need
1720  *	dma_map_single(&pdev->dev, virt_addr, len, direction)
1721  *	dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction)
1722  */
1723 #if 0
1724 	struct e1000_buffer *buffer_info =  &tx_ring->buffer_info[l];
1725 	/* set time_stamp *before* dma to help avoid a possible race */
1726 	buffer_info->time_stamp = jiffies;
1727 	buffer_info->mapped_as_page = false;
1728 	buffer_info->length = len;
1729 	//buffer_info->next_to_watch = l;
1730 	/* reload dma map */
1731 	dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1732 			NETMAP_BUF_SIZE, DMA_TO_DEVICE);
1733 	buffer_info->dma = dma_map_single(&adapter->pdev->dev,
1734 			addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE);
1735 
1736 	if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) {
1737 		nm_prerr("dma mapping error");
1738 		/* goto dma_error; See e1000_put_txbuf() */
1739 		/* XXX reset */
1740 	}
1741 	tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX
1742 
1743 #endif
1744 
1745 static inline int
1746 netmap_load_map(struct netmap_adapter *na,
1747 	bus_dma_tag_t tag, bus_dmamap_t map, void *buf, u_int size)
1748 {
1749 	if (map) {
1750 		*map = dma_map_single(na->pdev, buf, size,
1751 				      DMA_BIDIRECTIONAL);
1752 		if (dma_mapping_error(na->pdev, *map)) {
1753 			*map = 0;
1754 			return ENOMEM;
1755 		}
1756 	}
1757 	return 0;
1758 }
1759 
1760 static inline void
1761 netmap_unload_map(struct netmap_adapter *na,
1762 	bus_dma_tag_t tag, bus_dmamap_t map, u_int sz)
1763 {
1764 	if (*map) {
1765 		dma_unmap_single(na->pdev, *map, sz,
1766 				 DMA_BIDIRECTIONAL);
1767 	}
1768 }
1769 
1770 #ifdef NETMAP_LINUX_HAVE_DMASYNC
1771 static inline void
1772 netmap_sync_map_cpu(struct netmap_adapter *na,
1773 	bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t)
1774 {
1775 	if (*map) {
1776 		dma_sync_single_for_cpu(na->pdev, *map, sz,
1777 			(t == NR_TX ? DMA_TO_DEVICE : DMA_FROM_DEVICE));
1778 	}
1779 }
1780 
1781 static inline void
1782 netmap_sync_map_dev(struct netmap_adapter *na,
1783 	bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t)
1784 {
1785 	if (*map) {
1786 		dma_sync_single_for_device(na->pdev, *map, sz,
1787 			(t == NR_TX ? DMA_TO_DEVICE : DMA_FROM_DEVICE));
1788 	}
1789 }
1790 
1791 static inline void
1792 netmap_reload_map(struct netmap_adapter *na,
1793 	bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
1794 {
1795 	u_int sz = NETMAP_BUF_SIZE(na);
1796 
1797 	if (*map) {
1798 		dma_unmap_single(na->pdev, *map, sz,
1799 				DMA_BIDIRECTIONAL);
1800 	}
1801 
1802 	*map = dma_map_single(na->pdev, buf, sz,
1803 				DMA_BIDIRECTIONAL);
1804 }
1805 #else /* !NETMAP_LINUX_HAVE_DMASYNC */
1806 #define netmap_sync_map_cpu(na, tag, map, sz, t)
1807 #define netmap_sync_map_dev(na, tag, map, sz, t)
1808 #endif /* NETMAP_LINUX_HAVE_DMASYNC */
1809 
1810 #endif /* linux */
1811 
1812 
1813 /*
1814  * functions to map NIC to KRING indexes (n2k) and vice versa (k2n)
1815  */
1816 static inline int
1817 netmap_idx_n2k(struct netmap_kring *kr, int idx)
1818 {
1819 	int n = kr->nkr_num_slots;
1820 
1821 	if (likely(kr->nkr_hwofs == 0)) {
1822 		return idx;
1823 	}
1824 
1825 	idx += kr->nkr_hwofs;
1826 	if (idx < 0)
1827 		return idx + n;
1828 	else if (idx < n)
1829 		return idx;
1830 	else
1831 		return idx - n;
1832 }
1833 
1834 
1835 static inline int
1836 netmap_idx_k2n(struct netmap_kring *kr, int idx)
1837 {
1838 	int n = kr->nkr_num_slots;
1839 
1840 	if (likely(kr->nkr_hwofs == 0)) {
1841 		return idx;
1842 	}
1843 
1844 	idx -= kr->nkr_hwofs;
1845 	if (idx < 0)
1846 		return idx + n;
1847 	else if (idx < n)
1848 		return idx;
1849 	else
1850 		return idx - n;
1851 }
1852 
1853 
1854 /* Entries of the look-up table. */
1855 #ifdef __FreeBSD__
1856 struct lut_entry {
1857 	void *vaddr;		/* virtual address. */
1858 	vm_paddr_t paddr;	/* physical address. */
1859 };
1860 #else /* linux & _WIN32 */
1861 /* dma-mapping in linux can assign a buffer a different address
1862  * depending on the device, so we need to have a separate
1863  * physical-address look-up table for each na.
1864  * We can still share the vaddrs, though, therefore we split
1865  * the lut_entry structure.
1866  */
1867 struct lut_entry {
1868 	void *vaddr;		/* virtual address. */
1869 };
1870 
1871 struct plut_entry {
1872 	vm_paddr_t paddr;	/* physical address. */
1873 };
1874 #endif /* linux & _WIN32 */
1875 
1876 struct netmap_obj_pool;
1877 
1878 /*
1879  * NMB return the virtual address of a buffer (buffer 0 on bad index)
1880  * PNMB also fills the physical address
1881  */
1882 static inline void *
1883 NMB(struct netmap_adapter *na, struct netmap_slot *slot)
1884 {
1885 	struct lut_entry *lut = na->na_lut.lut;
1886 	uint32_t i = slot->buf_idx;
1887 	return (unlikely(i >= na->na_lut.objtotal)) ?
1888 		lut[0].vaddr : lut[i].vaddr;
1889 }
1890 
1891 static inline void *
1892 PNMB(struct netmap_adapter *na, struct netmap_slot *slot, uint64_t *pp)
1893 {
1894 	uint32_t i = slot->buf_idx;
1895 	struct lut_entry *lut = na->na_lut.lut;
1896 	struct plut_entry *plut = na->na_lut.plut;
1897 	void *ret = (i >= na->na_lut.objtotal) ? lut[0].vaddr : lut[i].vaddr;
1898 
1899 #ifdef _WIN32
1900 	*pp = (i >= na->na_lut.objtotal) ? (uint64_t)plut[0].paddr.QuadPart : (uint64_t)plut[i].paddr.QuadPart;
1901 #else
1902 	*pp = (i >= na->na_lut.objtotal) ? plut[0].paddr : plut[i].paddr;
1903 #endif
1904 	return ret;
1905 }
1906 
1907 
1908 /*
1909  * Structure associated to each netmap file descriptor.
1910  * It is created on open and left unbound (np_nifp == NULL).
1911  * A successful NIOCREGIF will set np_nifp and the first few fields;
1912  * this is protected by a global lock (NMG_LOCK) due to low contention.
1913  *
1914  * np_refs counts the number of references to the structure: one for the fd,
1915  * plus (on FreeBSD) one for each active mmap which we track ourselves
1916  * (linux automatically tracks them, but FreeBSD does not).
1917  * np_refs is protected by NMG_LOCK.
1918  *
1919  * Read access to the structure is lock free, because ni_nifp once set
1920  * can only go to 0 when nobody is using the entry anymore. Readers
1921  * must check that np_nifp != NULL before using the other fields.
1922  */
1923 struct netmap_priv_d {
1924 	struct netmap_if * volatile np_nifp;	/* netmap if descriptor. */
1925 
1926 	struct netmap_adapter	*np_na;
1927 	struct ifnet	*np_ifp;
1928 	uint32_t	np_flags;	/* from the ioctl */
1929 	u_int		np_qfirst[NR_TXRX],
1930 			np_qlast[NR_TXRX]; /* range of tx/rx rings to scan */
1931 	uint16_t	np_txpoll;
1932 	uint16_t        np_kloop_state;	/* use with NMG_LOCK held */
1933 #define NM_SYNC_KLOOP_RUNNING	(1 << 0)
1934 #define NM_SYNC_KLOOP_STOPPING	(1 << 1)
1935 	int             np_sync_flags; /* to be passed to nm_sync */
1936 
1937 	int		np_refs;	/* use with NMG_LOCK held */
1938 
1939 	/* pointers to the selinfo to be used for selrecord.
1940 	 * Either the local or the global one depending on the
1941 	 * number of rings.
1942 	 */
1943 	NM_SELINFO_T *np_si[NR_TXRX];
1944 
1945 	/* In the optional CSB mode, the user must specify the start address
1946 	 * of two arrays of Communication Status Block (CSB) entries, for the
1947 	 * two directions (kernel read application write, and kernel write
1948 	 * application read).
1949 	 * The number of entries must agree with the number of rings bound to
1950 	 * the netmap file descriptor. The entries corresponding to the TX
1951 	 * rings are laid out before the ones corresponding to the RX rings.
1952 	 *
1953 	 * Array of CSB entries for application --> kernel communication
1954 	 * (N entries). */
1955 	struct nm_csb_atok	*np_csb_atok_base;
1956 	/* Array of CSB entries for kernel --> application communication
1957 	 * (N entries). */
1958 	struct nm_csb_ktoa	*np_csb_ktoa_base;
1959 
1960 #ifdef linux
1961 	struct file	*np_filp;  /* used by sync kloop */
1962 #endif /* linux */
1963 };
1964 
1965 struct netmap_priv_d *netmap_priv_new(void);
1966 void netmap_priv_delete(struct netmap_priv_d *);
1967 
1968 static inline int nm_kring_pending(struct netmap_priv_d *np)
1969 {
1970 	struct netmap_adapter *na = np->np_na;
1971 	enum txrx t;
1972 	int i;
1973 
1974 	for_rx_tx(t) {
1975 		for (i = np->np_qfirst[t]; i < np->np_qlast[t]; i++) {
1976 			struct netmap_kring *kring = NMR(na, t)[i];
1977 			if (kring->nr_mode != kring->nr_pending_mode) {
1978 				return 1;
1979 			}
1980 		}
1981 	}
1982 	return 0;
1983 }
1984 
1985 /* call with NMG_LOCK held */
1986 static __inline int
1987 nm_si_user(struct netmap_priv_d *priv, enum txrx t)
1988 {
1989 	return (priv->np_na != NULL &&
1990 		(priv->np_qlast[t] - priv->np_qfirst[t] > 1));
1991 }
1992 
1993 #ifdef WITH_PIPES
1994 int netmap_pipe_txsync(struct netmap_kring *txkring, int flags);
1995 int netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags);
1996 int netmap_pipe_krings_create_both(struct netmap_adapter *na,
1997 				  struct netmap_adapter *ona);
1998 void netmap_pipe_krings_delete_both(struct netmap_adapter *na,
1999 				    struct netmap_adapter *ona);
2000 int netmap_pipe_reg_both(struct netmap_adapter *na,
2001 			 struct netmap_adapter *ona);
2002 #endif /* WITH_PIPES */
2003 
2004 #ifdef WITH_MONITOR
2005 
2006 struct netmap_monitor_adapter {
2007 	struct netmap_adapter up;
2008 
2009 	struct netmap_priv_d priv;
2010 	uint32_t flags;
2011 };
2012 
2013 #endif /* WITH_MONITOR */
2014 
2015 
2016 #ifdef WITH_GENERIC
2017 /*
2018  * generic netmap emulation for devices that do not have
2019  * native netmap support.
2020  */
2021 int generic_netmap_attach(struct ifnet *ifp);
2022 int generic_rx_handler(struct ifnet *ifp, struct mbuf *m);;
2023 
2024 int nm_os_catch_rx(struct netmap_generic_adapter *gna, int intercept);
2025 int nm_os_catch_tx(struct netmap_generic_adapter *gna, int intercept);
2026 
2027 int na_is_generic(struct netmap_adapter *na);
2028 
2029 /*
2030  * the generic transmit routine is passed a structure to optionally
2031  * build a queue of descriptors, in an OS-specific way.
2032  * The payload is at addr, if non-null, and the routine should send or queue
2033  * the packet, returning 0 if successful, 1 on failure.
2034  *
2035  * At the end, if head is non-null, there will be an additional call
2036  * to the function with addr = NULL; this should tell the OS-specific
2037  * routine to send the queue and free any resources. Failure is ignored.
2038  */
2039 struct nm_os_gen_arg {
2040 	struct ifnet *ifp;
2041 	void *m;	/* os-specific mbuf-like object */
2042 	void *head, *tail; /* tailq, if the OS-specific routine needs to build one */
2043 	void *addr;	/* payload of current packet */
2044 	u_int len;	/* packet length */
2045 	u_int ring_nr;	/* packet length */
2046 	u_int qevent;   /* in txqdisc mode, place an event on this mbuf */
2047 };
2048 
2049 int nm_os_generic_xmit_frame(struct nm_os_gen_arg *);
2050 int nm_os_generic_find_num_desc(struct ifnet *ifp, u_int *tx, u_int *rx);
2051 void nm_os_generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq);
2052 void nm_os_generic_set_features(struct netmap_generic_adapter *gna);
2053 
2054 static inline struct ifnet*
2055 netmap_generic_getifp(struct netmap_generic_adapter *gna)
2056 {
2057         if (gna->prev)
2058             return gna->prev->ifp;
2059 
2060         return gna->up.up.ifp;
2061 }
2062 
2063 void netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done);
2064 
2065 //#define RATE_GENERIC  /* Enables communication statistics for generic. */
2066 #ifdef RATE_GENERIC
2067 void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi);
2068 #else
2069 #define generic_rate(txp, txs, txi, rxp, rxs, rxi)
2070 #endif
2071 
2072 /*
2073  * netmap_mitigation API. This is used by the generic adapter
2074  * to reduce the number of interrupt requests/selwakeup
2075  * to clients on incoming packets.
2076  */
2077 void nm_os_mitigation_init(struct nm_generic_mit *mit, int idx,
2078                                 struct netmap_adapter *na);
2079 void nm_os_mitigation_start(struct nm_generic_mit *mit);
2080 void nm_os_mitigation_restart(struct nm_generic_mit *mit);
2081 int nm_os_mitigation_active(struct nm_generic_mit *mit);
2082 void nm_os_mitigation_cleanup(struct nm_generic_mit *mit);
2083 #else /* !WITH_GENERIC */
2084 #define generic_netmap_attach(ifp)	(EOPNOTSUPP)
2085 #define na_is_generic(na)		(0)
2086 #endif /* WITH_GENERIC */
2087 
2088 /* Shared declarations for the VALE switch. */
2089 
2090 /*
2091  * Each transmit queue accumulates a batch of packets into
2092  * a structure before forwarding. Packets to the same
2093  * destination are put in a list using ft_next as a link field.
2094  * ft_frags and ft_next are valid only on the first fragment.
2095  */
2096 struct nm_bdg_fwd {	/* forwarding entry for a bridge */
2097 	void *ft_buf;		/* netmap or indirect buffer */
2098 	uint8_t ft_frags;	/* how many fragments (only on 1st frag) */
2099 	uint16_t ft_offset;	/* dst port (unused) */
2100 	uint16_t ft_flags;	/* flags, e.g. indirect */
2101 	uint16_t ft_len;	/* src fragment len */
2102 	uint16_t ft_next;	/* next packet to same destination */
2103 };
2104 
2105 /* struct 'virtio_net_hdr' from linux. */
2106 struct nm_vnet_hdr {
2107 #define VIRTIO_NET_HDR_F_NEEDS_CSUM     1	/* Use csum_start, csum_offset */
2108 #define VIRTIO_NET_HDR_F_DATA_VALID    2	/* Csum is valid */
2109     uint8_t flags;
2110 #define VIRTIO_NET_HDR_GSO_NONE         0       /* Not a GSO frame */
2111 #define VIRTIO_NET_HDR_GSO_TCPV4        1       /* GSO frame, IPv4 TCP (TSO) */
2112 #define VIRTIO_NET_HDR_GSO_UDP          3       /* GSO frame, IPv4 UDP (UFO) */
2113 #define VIRTIO_NET_HDR_GSO_TCPV6        4       /* GSO frame, IPv6 TCP */
2114 #define VIRTIO_NET_HDR_GSO_ECN          0x80    /* TCP has ECN set */
2115     uint8_t gso_type;
2116     uint16_t hdr_len;
2117     uint16_t gso_size;
2118     uint16_t csum_start;
2119     uint16_t csum_offset;
2120 };
2121 
2122 #define WORST_CASE_GSO_HEADER	(14+40+60)  /* IPv6 + TCP */
2123 
2124 /* Private definitions for IPv4, IPv6, UDP and TCP headers. */
2125 
2126 struct nm_iphdr {
2127 	uint8_t		version_ihl;
2128 	uint8_t		tos;
2129 	uint16_t	tot_len;
2130 	uint16_t	id;
2131 	uint16_t	frag_off;
2132 	uint8_t		ttl;
2133 	uint8_t		protocol;
2134 	uint16_t	check;
2135 	uint32_t	saddr;
2136 	uint32_t	daddr;
2137 	/*The options start here. */
2138 };
2139 
2140 struct nm_tcphdr {
2141 	uint16_t	source;
2142 	uint16_t	dest;
2143 	uint32_t	seq;
2144 	uint32_t	ack_seq;
2145 	uint8_t		doff;  /* Data offset + Reserved */
2146 	uint8_t		flags;
2147 	uint16_t	window;
2148 	uint16_t	check;
2149 	uint16_t	urg_ptr;
2150 };
2151 
2152 struct nm_udphdr {
2153 	uint16_t	source;
2154 	uint16_t	dest;
2155 	uint16_t	len;
2156 	uint16_t	check;
2157 };
2158 
2159 struct nm_ipv6hdr {
2160 	uint8_t		priority_version;
2161 	uint8_t		flow_lbl[3];
2162 
2163 	uint16_t	payload_len;
2164 	uint8_t		nexthdr;
2165 	uint8_t		hop_limit;
2166 
2167 	uint8_t		saddr[16];
2168 	uint8_t		daddr[16];
2169 };
2170 
2171 /* Type used to store a checksum (in host byte order) that hasn't been
2172  * folded yet.
2173  */
2174 #define rawsum_t uint32_t
2175 
2176 rawsum_t nm_os_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum);
2177 uint16_t nm_os_csum_ipv4(struct nm_iphdr *iph);
2178 void nm_os_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data,
2179 		      size_t datalen, uint16_t *check);
2180 void nm_os_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data,
2181 		      size_t datalen, uint16_t *check);
2182 uint16_t nm_os_csum_fold(rawsum_t cur_sum);
2183 
2184 void bdg_mismatch_datapath(struct netmap_vp_adapter *na,
2185 			   struct netmap_vp_adapter *dst_na,
2186 			   const struct nm_bdg_fwd *ft_p,
2187 			   struct netmap_ring *dst_ring,
2188 			   u_int *j, u_int lim, u_int *howmany);
2189 
2190 /* persistent virtual port routines */
2191 int nm_os_vi_persist(const char *, struct ifnet **);
2192 void nm_os_vi_detach(struct ifnet *);
2193 void nm_os_vi_init_index(void);
2194 
2195 /*
2196  * kernel thread routines
2197  */
2198 struct nm_kctx; /* OS-specific kernel context - opaque */
2199 typedef void (*nm_kctx_worker_fn_t)(void *data);
2200 
2201 /* kthread configuration */
2202 struct nm_kctx_cfg {
2203 	long			type;		/* kthread type/identifier */
2204 	nm_kctx_worker_fn_t	worker_fn;	/* worker function */
2205 	void			*worker_private;/* worker parameter */
2206 	int			attach_user;	/* attach kthread to user process */
2207 };
2208 /* kthread configuration */
2209 struct nm_kctx *nm_os_kctx_create(struct nm_kctx_cfg *cfg,
2210 					void *opaque);
2211 int nm_os_kctx_worker_start(struct nm_kctx *);
2212 void nm_os_kctx_worker_stop(struct nm_kctx *);
2213 void nm_os_kctx_destroy(struct nm_kctx *);
2214 void nm_os_kctx_worker_setaff(struct nm_kctx *, int);
2215 u_int nm_os_ncpus(void);
2216 
2217 int netmap_sync_kloop(struct netmap_priv_d *priv,
2218 		      struct nmreq_header *hdr);
2219 int netmap_sync_kloop_stop(struct netmap_priv_d *priv);
2220 
2221 #ifdef WITH_PTNETMAP
2222 /* ptnetmap guest routines */
2223 
2224 /*
2225  * ptnetmap_memdev routines used to talk with ptnetmap_memdev device driver
2226  */
2227 struct ptnetmap_memdev;
2228 int nm_os_pt_memdev_iomap(struct ptnetmap_memdev *, vm_paddr_t *, void **,
2229                           uint64_t *);
2230 void nm_os_pt_memdev_iounmap(struct ptnetmap_memdev *);
2231 uint32_t nm_os_pt_memdev_ioread(struct ptnetmap_memdev *, unsigned int);
2232 
2233 /*
2234  * netmap adapter for guest ptnetmap ports
2235  */
2236 struct netmap_pt_guest_adapter {
2237         /* The netmap adapter to be used by netmap applications.
2238 	 * This field must be the first, to allow upcast. */
2239 	struct netmap_hw_adapter hwup;
2240 
2241         /* The netmap adapter to be used by the driver. */
2242         struct netmap_hw_adapter dr;
2243 
2244 	/* Reference counter to track users of backend netmap port: the
2245 	 * network stack and netmap clients.
2246 	 * Used to decide when we need (de)allocate krings/rings and
2247 	 * start (stop) ptnetmap kthreads. */
2248 	int backend_users;
2249 
2250 };
2251 
2252 int netmap_pt_guest_attach(struct netmap_adapter *na,
2253 			unsigned int nifp_offset,
2254 			unsigned int memid);
2255 bool netmap_pt_guest_txsync(struct nm_csb_atok *atok,
2256 			struct nm_csb_ktoa *ktoa,
2257 			struct netmap_kring *kring, int flags);
2258 bool netmap_pt_guest_rxsync(struct nm_csb_atok *atok,
2259 			struct nm_csb_ktoa *ktoa,
2260 			struct netmap_kring *kring, int flags);
2261 int ptnet_nm_krings_create(struct netmap_adapter *na);
2262 void ptnet_nm_krings_delete(struct netmap_adapter *na);
2263 void ptnet_nm_dtor(struct netmap_adapter *na);
2264 
2265 /* Helper function wrapping nm_sync_kloop_appl_read(). */
2266 static inline void
2267 ptnet_sync_tail(struct nm_csb_ktoa *ktoa, struct netmap_kring *kring)
2268 {
2269 	struct netmap_ring *ring = kring->ring;
2270 
2271 	/* Update hwcur and hwtail as known by the host. */
2272         nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail, &kring->nr_hwcur);
2273 
2274 	/* nm_sync_finalize */
2275 	ring->tail = kring->rtail = kring->nr_hwtail;
2276 }
2277 #endif /* WITH_PTNETMAP */
2278 
2279 #ifdef __FreeBSD__
2280 /*
2281  * FreeBSD mbuf allocator/deallocator in emulation mode:
2282  */
2283 #if __FreeBSD_version < 1100000
2284 
2285 /*
2286  * For older versions of FreeBSD:
2287  *
2288  * We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE
2289  * so that the destructor, if invoked, will not free the packet.
2290  * In principle we should set the destructor only on demand,
2291  * but since there might be a race we better do it on allocation.
2292  * As a consequence, we also need to set the destructor or we
2293  * would leak buffers.
2294  */
2295 
2296 /* mbuf destructor, also need to change the type to EXT_EXTREF,
2297  * add an M_NOFREE flag, and then clear the flag and
2298  * chain into uma_zfree(zone_pack, mf)
2299  * (or reinstall the buffer ?)
2300  */
2301 #define SET_MBUF_DESTRUCTOR(m, fn)	do {		\
2302 	(m)->m_ext.ext_free = (void *)fn;	\
2303 	(m)->m_ext.ext_type = EXT_EXTREF;	\
2304 } while (0)
2305 
2306 static int
2307 void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2)
2308 {
2309 	/* restore original mbuf */
2310 	m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1;
2311 	m->m_ext.ext_arg1 = NULL;
2312 	m->m_ext.ext_type = EXT_PACKET;
2313 	m->m_ext.ext_free = NULL;
2314 	if (MBUF_REFCNT(m) == 0)
2315 		SET_MBUF_REFCNT(m, 1);
2316 	uma_zfree(zone_pack, m);
2317 
2318 	return 0;
2319 }
2320 
2321 static inline struct mbuf *
2322 nm_os_get_mbuf(struct ifnet *ifp, int len)
2323 {
2324 	struct mbuf *m;
2325 
2326 	(void)ifp;
2327 	m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
2328 	if (m) {
2329 		/* m_getcl() (mb_ctor_mbuf) has an assert that checks that
2330 		 * M_NOFREE flag is not specified as third argument,
2331 		 * so we have to set M_NOFREE after m_getcl(). */
2332 		m->m_flags |= M_NOFREE;
2333 		m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save
2334 		m->m_ext.ext_free = (void *)void_mbuf_dtor;
2335 		m->m_ext.ext_type = EXT_EXTREF;
2336 		nm_prdis(5, "create m %p refcnt %d", m, MBUF_REFCNT(m));
2337 	}
2338 	return m;
2339 }
2340 
2341 #else /* __FreeBSD_version >= 1100000 */
2342 
2343 /*
2344  * Newer versions of FreeBSD, using a straightforward scheme.
2345  *
2346  * We allocate mbufs with m_gethdr(), since the mbuf header is needed
2347  * by the driver. We also attach a customly-provided external storage,
2348  * which in this case is a netmap buffer. When calling m_extadd(), however
2349  * we pass a NULL address, since the real address (and length) will be
2350  * filled in by nm_os_generic_xmit_frame() right before calling
2351  * if_transmit().
2352  *
2353  * The dtor function does nothing, however we need it since mb_free_ext()
2354  * has a KASSERT(), checking that the mbuf dtor function is not NULL.
2355  */
2356 
2357 #if __FreeBSD_version <= 1200050
2358 static void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { }
2359 #else  /* __FreeBSD_version >= 1200051 */
2360 /* The arg1 and arg2 pointers argument were removed by r324446, which
2361  * in included since version 1200051. */
2362 static void void_mbuf_dtor(struct mbuf *m) { }
2363 #endif /* __FreeBSD_version >= 1200051 */
2364 
2365 #define SET_MBUF_DESTRUCTOR(m, fn)	do {		\
2366 	(m)->m_ext.ext_free = (fn != NULL) ?		\
2367 	    (void *)fn : (void *)void_mbuf_dtor;	\
2368 } while (0)
2369 
2370 static inline struct mbuf *
2371 nm_os_get_mbuf(struct ifnet *ifp, int len)
2372 {
2373 	struct mbuf *m;
2374 
2375 	(void)ifp;
2376 	(void)len;
2377 
2378 	m = m_gethdr(M_NOWAIT, MT_DATA);
2379 	if (m == NULL) {
2380 		return m;
2381 	}
2382 
2383 	m_extadd(m, NULL /* buf */, 0 /* size */, void_mbuf_dtor,
2384 		 NULL, NULL, 0, EXT_NET_DRV);
2385 
2386 	return m;
2387 }
2388 
2389 #endif /* __FreeBSD_version >= 1100000 */
2390 #endif /* __FreeBSD__ */
2391 
2392 struct nmreq_option * nmreq_getoption(struct nmreq_header *, uint16_t);
2393 
2394 int netmap_init_bridges(void);
2395 void netmap_uninit_bridges(void);
2396 
2397 /* Functions to read and write CSB fields from the kernel. */
2398 #if defined (linux)
2399 #define CSB_READ(csb, field, r) (get_user(r, &csb->field))
2400 #define CSB_WRITE(csb, field, v) (put_user(v, &csb->field))
2401 #else  /* ! linux */
2402 #define CSB_READ(csb, field, r) (r = fuword32(&csb->field))
2403 #define CSB_WRITE(csb, field, v) (suword32(&csb->field, v))
2404 #endif /* ! linux */
2405 
2406 #endif /* _NET_NETMAP_KERN_H_ */
2407