xref: /freebsd/sys/dev/netmap/netmap.c (revision 8d20be1e22095c27faf8fe8b2f0d089739cc742e)
1 /*
2  * Copyright (C) 2011-2013 Matteo Landi, Luigi Rizzo. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  *   1. Redistributions of source code must retain the above copyright
8  *      notice, this list of conditions and the following disclaimer.
9  *   2. Redistributions in binary form must reproduce the above copyright
10  *      notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  */
25 
26 #define NM_BRIDGE
27 
28 /*
29  * This module supports memory mapped access to network devices,
30  * see netmap(4).
31  *
32  * The module uses a large, memory pool allocated by the kernel
33  * and accessible as mmapped memory by multiple userspace threads/processes.
34  * The memory pool contains packet buffers and "netmap rings",
35  * i.e. user-accessible copies of the interface's queues.
36  *
37  * Access to the network card works like this:
38  * 1. a process/thread issues one or more open() on /dev/netmap, to create
39  *    select()able file descriptor on which events are reported.
40  * 2. on each descriptor, the process issues an ioctl() to identify
41  *    the interface that should report events to the file descriptor.
42  * 3. on each descriptor, the process issues an mmap() request to
43  *    map the shared memory region within the process' address space.
44  *    The list of interesting queues is indicated by a location in
45  *    the shared memory region.
46  * 4. using the functions in the netmap(4) userspace API, a process
47  *    can look up the occupation state of a queue, access memory buffers,
48  *    and retrieve received packets or enqueue packets to transmit.
49  * 5. using some ioctl()s the process can synchronize the userspace view
50  *    of the queue with the actual status in the kernel. This includes both
51  *    receiving the notification of new packets, and transmitting new
52  *    packets on the output interface.
53  * 6. select() or poll() can be used to wait for events on individual
54  *    transmit or receive queues (or all queues for a given interface).
55  */
56 
57 #ifdef linux
58 #include "bsd_glue.h"
59 static netdev_tx_t linux_netmap_start(struct sk_buff *skb, struct net_device *dev);
60 #endif /* linux */
61 
62 #ifdef __APPLE__
63 #include "osx_glue.h"
64 #endif /* __APPLE__ */
65 
66 #ifdef __FreeBSD__
67 #include <sys/cdefs.h> /* prerequisite */
68 __FBSDID("$FreeBSD$");
69 
70 #include <sys/types.h>
71 #include <sys/module.h>
72 #include <sys/errno.h>
73 #include <sys/param.h>	/* defines used in kernel.h */
74 #include <sys/jail.h>
75 #include <sys/kernel.h>	/* types used in module initialization */
76 #include <sys/conf.h>	/* cdevsw struct */
77 #include <sys/uio.h>	/* uio struct */
78 #include <sys/sockio.h>
79 #include <sys/socketvar.h>	/* struct socket */
80 #include <sys/malloc.h>
81 #include <sys/mman.h>	/* PROT_EXEC */
82 #include <sys/poll.h>
83 #include <sys/proc.h>
84 #include <sys/rwlock.h>
85 #include <vm/vm.h>	/* vtophys */
86 #include <vm/pmap.h>	/* vtophys */
87 #include <sys/socket.h> /* sockaddrs */
88 #include <machine/bus.h>
89 #include <sys/selinfo.h>
90 #include <sys/sysctl.h>
91 #include <net/if.h>
92 #include <net/if_var.h>
93 #include <net/bpf.h>		/* BIOCIMMEDIATE */
94 #include <net/vnet.h>
95 #include <machine/bus.h>	/* bus_dmamap_* */
96 
97 MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map");
98 #endif /* __FreeBSD__ */
99 
100 #include <net/netmap.h>
101 #include <dev/netmap/netmap_kern.h>
102 
103 /* XXX the following variables must be deprecated and included in nm_mem */
104 u_int netmap_total_buffers;
105 u_int netmap_buf_size;
106 char *netmap_buffer_base;	/* address of an invalid buffer */
107 
108 /* user-controlled variables */
109 int netmap_verbose;
110 
111 static int netmap_no_timestamp; /* don't timestamp on rxsync */
112 
113 SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args");
114 SYSCTL_INT(_dev_netmap, OID_AUTO, verbose,
115     CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode");
116 SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp,
117     CTLFLAG_RW, &netmap_no_timestamp, 0, "no_timestamp");
118 int netmap_mitigate = 1;
119 SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, "");
120 int netmap_no_pendintr = 1;
121 SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr,
122     CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets.");
123 int netmap_txsync_retry = 2;
124 SYSCTL_INT(_dev_netmap, OID_AUTO, txsync_retry, CTLFLAG_RW,
125     &netmap_txsync_retry, 0 , "Number of txsync loops in bridge's flush.");
126 
127 int netmap_drop = 0;	/* debugging */
128 int netmap_flags = 0;	/* debug flags */
129 int netmap_fwd = 0;	/* force transparent mode */
130 
131 SYSCTL_INT(_dev_netmap, OID_AUTO, drop, CTLFLAG_RW, &netmap_drop, 0 , "");
132 SYSCTL_INT(_dev_netmap, OID_AUTO, flags, CTLFLAG_RW, &netmap_flags, 0 , "");
133 SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0 , "");
134 
135 #ifdef NM_BRIDGE /* support for netmap virtual switch, called VALE */
136 
137 /*
138  * system parameters (most of them in netmap_kern.h)
139  * NM_NAME	prefix for switch port names, default "vale"
140  * NM_MAXPORTS	number of ports
141  * NM_BRIDGES	max number of switches in the system.
142  *	XXX should become a sysctl or tunable
143  *
144  * Switch ports are named valeX:Y where X is the switch name and Y
145  * is the port. If Y matches a physical interface name, the port is
146  * connected to a physical device.
147  *
148  * Unlike physical interfaces, switch ports use their own memory region
149  * for rings and buffers.
150  * The virtual interfaces use per-queue lock instead of core lock.
151  * In the tx loop, we aggregate traffic in batches to make all operations
152  * faster. The batch size is NM_BDG_BATCH
153  */
154 #define NM_BDG_MAXRINGS		16	/* XXX unclear how many. */
155 #define NM_BRIDGE_RINGSIZE	1024	/* in the device */
156 #define NM_BDG_HASH		1024	/* forwarding table entries */
157 #define NM_BDG_BATCH		1024	/* entries in the forwarding buffer */
158 #define	NM_BRIDGES		8	/* number of bridges */
159 
160 
161 int netmap_bridge = NM_BDG_BATCH; /* bridge batch size */
162 SYSCTL_INT(_dev_netmap, OID_AUTO, bridge, CTLFLAG_RW, &netmap_bridge, 0 , "");
163 
164 #ifdef linux
165 
166 #define	refcount_acquire(_a)	atomic_add(1, (atomic_t *)_a)
167 #define	refcount_release(_a)	atomic_dec_and_test((atomic_t *)_a)
168 
169 #else /* !linux */
170 
171 #ifdef __FreeBSD__
172 #include <sys/endian.h>
173 #include <sys/refcount.h>
174 #endif /* __FreeBSD__ */
175 
176 #define prefetch(x)	__builtin_prefetch(x)
177 
178 #endif /* !linux */
179 
180 /*
181  * These are used to handle reference counters for bridge ports.
182  */
183 #define	ADD_BDG_REF(ifp)	refcount_acquire(&NA(ifp)->na_bdg_refcount)
184 #define	DROP_BDG_REF(ifp)	refcount_release(&NA(ifp)->na_bdg_refcount)
185 
186 static void bdg_netmap_attach(struct netmap_adapter *);
187 static int bdg_netmap_reg(struct ifnet *ifp, int onoff);
188 static int kern_netmap_regif(struct nmreq *nmr);
189 
190 /* per-tx-queue entry */
191 struct nm_bdg_fwd {	/* forwarding entry for a bridge */
192 	void *ft_buf;
193 	uint16_t _ft_dst;	/* dst port, unused */
194 	uint16_t ft_flags;	/* flags, e.g. indirect */
195 	uint16_t ft_len;	/* src len */
196 	uint16_t ft_next;	/* next packet to same destination */
197 };
198 
199 /* We need to build a list of buffers going to each destination.
200  * Each buffer is in one entry of struct nm_bdg_fwd, we use ft_next
201  * to build the list, and struct nm_bdg_q below for the queue.
202  * The structure should compact because potentially we have a lot
203  * of destinations.
204  */
205 struct nm_bdg_q {
206 	uint16_t bq_head;
207 	uint16_t bq_tail;
208 };
209 
210 struct nm_hash_ent {
211 	uint64_t	mac;	/* the top 2 bytes are the epoch */
212 	uint64_t	ports;
213 };
214 
215 /*
216  * Interfaces for a bridge are all in bdg_ports[].
217  * The array has fixed size, an empty entry does not terminate
218  * the search. But lookups only occur on attach/detach so we
219  * don't mind if they are slow.
220  *
221  * The bridge is non blocking on the transmit ports.
222  *
223  * bdg_lock protects accesses to the bdg_ports array.
224  * This is a rw lock (or equivalent).
225  */
226 struct nm_bridge {
227 	int namelen;	/* 0 means free */
228 
229 	/* XXX what is the proper alignment/layout ? */
230 	NM_RWLOCK_T bdg_lock;	/* protects bdg_ports */
231 	struct netmap_adapter *bdg_ports[NM_BDG_MAXPORTS];
232 
233 	char basename[IFNAMSIZ];
234 	/*
235 	 * The function to decide the destination port.
236 	 * It returns either of an index of the destination port,
237 	 * NM_BDG_BROADCAST to broadcast this packet, or NM_BDG_NOPORT not to
238 	 * forward this packet.  ring_nr is the source ring index, and the
239 	 * function may overwrite this value to forward this packet to a
240 	 * different ring index.
241 	 * This function must be set by netmap_bdgctl().
242 	 */
243 	bdg_lookup_fn_t nm_bdg_lookup;
244 
245 	/* the forwarding table, MAC+ports */
246 	struct nm_hash_ent ht[NM_BDG_HASH];
247 };
248 
249 struct nm_bridge nm_bridges[NM_BRIDGES];
250 NM_LOCK_T	netmap_bridge_mutex;
251 
252 /* other OS will have these macros defined in their own glue code. */
253 
254 #ifdef __FreeBSD__
255 #define BDG_LOCK()		mtx_lock(&netmap_bridge_mutex)
256 #define BDG_UNLOCK()		mtx_unlock(&netmap_bridge_mutex)
257 #define BDG_WLOCK(b)		rw_wlock(&(b)->bdg_lock)
258 #define BDG_WUNLOCK(b)		rw_wunlock(&(b)->bdg_lock)
259 #define BDG_RLOCK(b)		rw_rlock(&(b)->bdg_lock)
260 #define BDG_RUNLOCK(b)		rw_runlock(&(b)->bdg_lock)
261 
262 /* set/get variables. OS-specific macros may wrap these
263  * assignments into read/write lock or similar
264  */
265 #define BDG_SET_VAR(lval, p)	(lval = p)
266 #define BDG_GET_VAR(lval)	(lval)
267 #define BDG_FREE(p)		free(p, M_DEVBUF)
268 #endif /* __FreeBSD__ */
269 
270 static __inline int
271 nma_is_vp(struct netmap_adapter *na)
272 {
273 	return na->nm_register == bdg_netmap_reg;
274 }
275 static __inline int
276 nma_is_host(struct netmap_adapter *na)
277 {
278 	return na->nm_register == NULL;
279 }
280 static __inline int
281 nma_is_hw(struct netmap_adapter *na)
282 {
283 	/* In case of sw adapter, nm_register is NULL */
284 	return !nma_is_vp(na) && !nma_is_host(na);
285 }
286 
287 /*
288  * Regarding holding a NIC, if the NIC is owned by the kernel
289  * (i.e., bridge), neither another bridge nor user can use it;
290  * if the NIC is owned by a user, only users can share it.
291  * Evaluation must be done under NMA_LOCK().
292  */
293 #define NETMAP_OWNED_BY_KERN(ifp)	(!nma_is_vp(NA(ifp)) && NA(ifp)->na_bdg)
294 #define NETMAP_OWNED_BY_ANY(ifp) \
295 	(NETMAP_OWNED_BY_KERN(ifp) || (NA(ifp)->refcount > 0))
296 
297 /*
298  * NA(ifp)->bdg_port	port index
299  */
300 
301 // XXX only for multiples of 64 bytes, non overlapped.
302 static inline void
303 pkt_copy(void *_src, void *_dst, int l)
304 {
305         uint64_t *src = _src;
306         uint64_t *dst = _dst;
307         if (unlikely(l >= 1024)) {
308                 bcopy(src, dst, l);
309                 return;
310         }
311         for (; likely(l > 0); l-=64) {
312                 *dst++ = *src++;
313                 *dst++ = *src++;
314                 *dst++ = *src++;
315                 *dst++ = *src++;
316                 *dst++ = *src++;
317                 *dst++ = *src++;
318                 *dst++ = *src++;
319                 *dst++ = *src++;
320         }
321 }
322 
323 
324 /*
325  * locate a bridge among the existing ones.
326  * a ':' in the name terminates the bridge name. Otherwise, just NM_NAME.
327  * We assume that this is called with a name of at least NM_NAME chars.
328  */
329 static struct nm_bridge *
330 nm_find_bridge(const char *name, int create)
331 {
332 	int i, l, namelen;
333 	struct nm_bridge *b = NULL;
334 
335 	namelen = strlen(NM_NAME);	/* base length */
336 	l = strlen(name);		/* actual length */
337 	for (i = namelen + 1; i < l; i++) {
338 		if (name[i] == ':') {
339 			namelen = i;
340 			break;
341 		}
342 	}
343 	if (namelen >= IFNAMSIZ)
344 		namelen = IFNAMSIZ;
345 	ND("--- prefix is '%.*s' ---", namelen, name);
346 
347 	BDG_LOCK();
348 	/* lookup the name, remember empty slot if there is one */
349 	for (i = 0; i < NM_BRIDGES; i++) {
350 		struct nm_bridge *x = nm_bridges + i;
351 
352 		if (x->namelen == 0) {
353 			if (create && b == NULL)
354 				b = x;	/* record empty slot */
355 		} else if (x->namelen != namelen) {
356 			continue;
357 		} else if (strncmp(name, x->basename, namelen) == 0) {
358 			ND("found '%.*s' at %d", namelen, name, i);
359 			b = x;
360 			break;
361 		}
362 	}
363 	if (i == NM_BRIDGES && b) { /* name not found, can create entry */
364 		strncpy(b->basename, name, namelen);
365 		b->namelen = namelen;
366 		/* set the default function */
367 		b->nm_bdg_lookup = netmap_bdg_learning;
368 		/* reset the MAC address table */
369 		bzero(b->ht, sizeof(struct nm_hash_ent) * NM_BDG_HASH);
370 	}
371 	BDG_UNLOCK();
372 	return b;
373 }
374 
375 
376 /*
377  * Free the forwarding tables for rings attached to switch ports.
378  */
379 static void
380 nm_free_bdgfwd(struct netmap_adapter *na)
381 {
382 	int nrings, i;
383 	struct netmap_kring *kring;
384 
385 	nrings = nma_is_vp(na) ? na->num_tx_rings : na->num_rx_rings;
386 	kring = nma_is_vp(na) ? na->tx_rings : na->rx_rings;
387 	for (i = 0; i < nrings; i++) {
388 		if (kring[i].nkr_ft) {
389 			free(kring[i].nkr_ft, M_DEVBUF);
390 			kring[i].nkr_ft = NULL; /* protect from freeing twice */
391 		}
392 	}
393 	if (nma_is_hw(na))
394 		nm_free_bdgfwd(SWNA(na->ifp));
395 }
396 
397 
398 /*
399  * Allocate the forwarding tables for the rings attached to the bridge ports.
400  */
401 static int
402 nm_alloc_bdgfwd(struct netmap_adapter *na)
403 {
404 	int nrings, l, i, num_dstq;
405 	struct netmap_kring *kring;
406 
407 	/* all port:rings + broadcast */
408 	num_dstq = NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1;
409 	l = sizeof(struct nm_bdg_fwd) * NM_BDG_BATCH;
410 	l += sizeof(struct nm_bdg_q) * num_dstq;
411 	l += sizeof(uint16_t) * NM_BDG_BATCH;
412 
413 	nrings = nma_is_vp(na) ? na->num_tx_rings : na->num_rx_rings;
414 	kring = nma_is_vp(na) ? na->tx_rings : na->rx_rings;
415 	for (i = 0; i < nrings; i++) {
416 		struct nm_bdg_fwd *ft;
417 		struct nm_bdg_q *dstq;
418 		int j;
419 
420 		ft = malloc(l, M_DEVBUF, M_NOWAIT | M_ZERO);
421 		if (!ft) {
422 			nm_free_bdgfwd(na);
423 			return ENOMEM;
424 		}
425 		dstq = (struct nm_bdg_q *)(ft + NM_BDG_BATCH);
426 		for (j = 0; j < num_dstq; j++)
427 			dstq[j].bq_head = dstq[j].bq_tail = NM_BDG_BATCH;
428 		kring[i].nkr_ft = ft;
429 	}
430 	if (nma_is_hw(na))
431 		nm_alloc_bdgfwd(SWNA(na->ifp));
432 	return 0;
433 }
434 
435 #endif /* NM_BRIDGE */
436 
437 
438 /*
439  * Fetch configuration from the device, to cope with dynamic
440  * reconfigurations after loading the module.
441  */
442 static int
443 netmap_update_config(struct netmap_adapter *na)
444 {
445 	struct ifnet *ifp = na->ifp;
446 	u_int txr, txd, rxr, rxd;
447 
448 	txr = txd = rxr = rxd = 0;
449 	if (na->nm_config) {
450 		na->nm_config(ifp, &txr, &txd, &rxr, &rxd);
451 	} else {
452 		/* take whatever we had at init time */
453 		txr = na->num_tx_rings;
454 		txd = na->num_tx_desc;
455 		rxr = na->num_rx_rings;
456 		rxd = na->num_rx_desc;
457 	}
458 
459 	if (na->num_tx_rings == txr && na->num_tx_desc == txd &&
460 	    na->num_rx_rings == rxr && na->num_rx_desc == rxd)
461 		return 0; /* nothing changed */
462 	if (netmap_verbose || na->refcount > 0) {
463 		D("stored config %s: txring %d x %d, rxring %d x %d",
464 			ifp->if_xname,
465 			na->num_tx_rings, na->num_tx_desc,
466 			na->num_rx_rings, na->num_rx_desc);
467 		D("new config %s: txring %d x %d, rxring %d x %d",
468 			ifp->if_xname, txr, txd, rxr, rxd);
469 	}
470 	if (na->refcount == 0) {
471 		D("configuration changed (but fine)");
472 		na->num_tx_rings = txr;
473 		na->num_tx_desc = txd;
474 		na->num_rx_rings = rxr;
475 		na->num_rx_desc = rxd;
476 		return 0;
477 	}
478 	D("configuration changed while active, this is bad...");
479 	return 1;
480 }
481 
482 /*------------- memory allocator -----------------*/
483 #include "netmap_mem2.c"
484 /*------------ end of memory allocator ----------*/
485 
486 
487 /* Structure associated to each thread which registered an interface.
488  *
489  * The first 4 fields of this structure are written by NIOCREGIF and
490  * read by poll() and NIOC?XSYNC.
491  * There is low contention among writers (actually, a correct user program
492  * should have no contention among writers) and among writers and readers,
493  * so we use a single global lock to protect the structure initialization.
494  * Since initialization involves the allocation of memory, we reuse the memory
495  * allocator lock.
496  * Read access to the structure is lock free. Readers must check that
497  * np_nifp is not NULL before using the other fields.
498  * If np_nifp is NULL initialization has not been performed, so they should
499  * return an error to userlevel.
500  *
501  * The ref_done field is used to regulate access to the refcount in the
502  * memory allocator. The refcount must be incremented at most once for
503  * each open("/dev/netmap"). The increment is performed by the first
504  * function that calls netmap_get_memory() (currently called by
505  * mmap(), NIOCGINFO and NIOCREGIF).
506  * If the refcount is incremented, it is then decremented when the
507  * private structure is destroyed.
508  */
509 struct netmap_priv_d {
510 	struct netmap_if * volatile np_nifp;	/* netmap interface descriptor. */
511 
512 	struct ifnet	*np_ifp;	/* device for which we hold a reference */
513 	int		np_ringid;	/* from the ioctl */
514 	u_int		np_qfirst, np_qlast;	/* range of rings to scan */
515 	uint16_t	np_txpoll;
516 
517 	unsigned long	ref_done;	/* use with NMA_LOCK held */
518 };
519 
520 
521 static int
522 netmap_get_memory(struct netmap_priv_d* p)
523 {
524 	int error = 0;
525 	NMA_LOCK();
526 	if (!p->ref_done) {
527 		error = netmap_memory_finalize();
528 		if (!error)
529 			p->ref_done = 1;
530 	}
531 	NMA_UNLOCK();
532 	return error;
533 }
534 
535 /*
536  * File descriptor's private data destructor.
537  *
538  * Call nm_register(ifp,0) to stop netmap mode on the interface and
539  * revert to normal operation. We expect that np_ifp has not gone.
540  */
541 /* call with NMA_LOCK held */
542 static void
543 netmap_dtor_locked(void *data)
544 {
545 	struct netmap_priv_d *priv = data;
546 	struct ifnet *ifp = priv->np_ifp;
547 	struct netmap_adapter *na = NA(ifp);
548 	struct netmap_if *nifp = priv->np_nifp;
549 
550 	na->refcount--;
551 	if (na->refcount <= 0) {	/* last instance */
552 		u_int i, j, lim;
553 
554 		if (netmap_verbose)
555 			D("deleting last instance for %s", ifp->if_xname);
556 		/*
557 		 * (TO CHECK) This function is only called
558 		 * when the last reference to this file descriptor goes
559 		 * away. This means we cannot have any pending poll()
560 		 * or interrupt routine operating on the structure.
561 		 */
562 		na->nm_register(ifp, 0); /* off, clear IFCAP_NETMAP */
563 		/* Wake up any sleeping threads. netmap_poll will
564 		 * then return POLLERR
565 		 */
566 		for (i = 0; i < na->num_tx_rings + 1; i++)
567 			selwakeuppri(&na->tx_rings[i].si, PI_NET);
568 		for (i = 0; i < na->num_rx_rings + 1; i++)
569 			selwakeuppri(&na->rx_rings[i].si, PI_NET);
570 		selwakeuppri(&na->tx_si, PI_NET);
571 		selwakeuppri(&na->rx_si, PI_NET);
572 #ifdef NM_BRIDGE
573 		nm_free_bdgfwd(na);
574 #endif /* NM_BRIDGE */
575 		/* release all buffers */
576 		for (i = 0; i < na->num_tx_rings + 1; i++) {
577 			struct netmap_ring *ring = na->tx_rings[i].ring;
578 			lim = na->tx_rings[i].nkr_num_slots;
579 			for (j = 0; j < lim; j++)
580 				netmap_free_buf(nifp, ring->slot[j].buf_idx);
581 			/* knlist_destroy(&na->tx_rings[i].si.si_note); */
582 			mtx_destroy(&na->tx_rings[i].q_lock);
583 		}
584 		for (i = 0; i < na->num_rx_rings + 1; i++) {
585 			struct netmap_ring *ring = na->rx_rings[i].ring;
586 			lim = na->rx_rings[i].nkr_num_slots;
587 			for (j = 0; j < lim; j++)
588 				netmap_free_buf(nifp, ring->slot[j].buf_idx);
589 			/* knlist_destroy(&na->rx_rings[i].si.si_note); */
590 			mtx_destroy(&na->rx_rings[i].q_lock);
591 		}
592 		/* XXX kqueue(9) needed; these will mirror knlist_init. */
593 		/* knlist_destroy(&na->tx_si.si_note); */
594 		/* knlist_destroy(&na->rx_si.si_note); */
595 		netmap_free_rings(na);
596 		if (nma_is_hw(na))
597 			SWNA(ifp)->tx_rings = SWNA(ifp)->rx_rings = NULL;
598 	}
599 	netmap_if_free(nifp);
600 }
601 
602 
603 /* we assume netmap adapter exists */
604 static void
605 nm_if_rele(struct ifnet *ifp)
606 {
607 #ifndef NM_BRIDGE
608 	if_rele(ifp);
609 #else /* NM_BRIDGE */
610 	int i, full = 0, is_hw;
611 	struct nm_bridge *b;
612 	struct netmap_adapter *na;
613 
614 	/* I can be called not only for get_ifp()-ed references where netmap's
615 	 * capability is guaranteed, but also for non-netmap-capable NICs.
616 	 */
617 	if (!NETMAP_CAPABLE(ifp) || !NA(ifp)->na_bdg) {
618 		if_rele(ifp);
619 		return;
620 	}
621 	if (!DROP_BDG_REF(ifp))
622 		return;
623 
624 	na = NA(ifp);
625 	b = na->na_bdg;
626 	is_hw = nma_is_hw(na);
627 
628 	BDG_WLOCK(b);
629 	ND("want to disconnect %s from the bridge", ifp->if_xname);
630 	full = 0;
631 	/* remove the entry from the bridge, also check
632 	 * if there are any leftover interfaces
633 	 * XXX we should optimize this code, e.g. going directly
634 	 * to na->bdg_port, and having a counter of ports that
635 	 * are connected. But it is not in a critical path.
636 	 * In NIC's case, index of sw na is always higher than hw na
637 	 */
638 	for (i = 0; i < NM_BDG_MAXPORTS; i++) {
639 		struct netmap_adapter *tmp = BDG_GET_VAR(b->bdg_ports[i]);
640 
641 		if (tmp == na) {
642 			/* disconnect from bridge */
643 			BDG_SET_VAR(b->bdg_ports[i], NULL);
644 			na->na_bdg = NULL;
645 			if (is_hw && SWNA(ifp)->na_bdg) {
646 				/* disconnect sw adapter too */
647 				int j = SWNA(ifp)->bdg_port;
648 				BDG_SET_VAR(b->bdg_ports[j], NULL);
649 				SWNA(ifp)->na_bdg = NULL;
650 			}
651 		} else if (tmp != NULL) {
652 			full = 1;
653 		}
654 	}
655 	BDG_WUNLOCK(b);
656 	if (full == 0) {
657 		ND("marking bridge %d as free", b - nm_bridges);
658 		b->namelen = 0;
659 		b->nm_bdg_lookup = NULL;
660 	}
661 	if (na->na_bdg) { /* still attached to the bridge */
662 		D("ouch, cannot find ifp to remove");
663 	} else if (is_hw) {
664 		if_rele(ifp);
665 	} else {
666 		bzero(na, sizeof(*na));
667 		free(na, M_DEVBUF);
668 		bzero(ifp, sizeof(*ifp));
669 		free(ifp, M_DEVBUF);
670 	}
671 #endif /* NM_BRIDGE */
672 }
673 
674 static void
675 netmap_dtor(void *data)
676 {
677 	struct netmap_priv_d *priv = data;
678 	struct ifnet *ifp = priv->np_ifp;
679 
680 	NMA_LOCK();
681 	if (ifp) {
682 		struct netmap_adapter *na = NA(ifp);
683 
684 		if (na->na_bdg)
685 			BDG_WLOCK(na->na_bdg);
686 		na->nm_lock(ifp, NETMAP_REG_LOCK, 0);
687 		netmap_dtor_locked(data);
688 		na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0);
689 		if (na->na_bdg)
690 			BDG_WUNLOCK(na->na_bdg);
691 
692 		nm_if_rele(ifp); /* might also destroy *na */
693 	}
694 	if (priv->ref_done) {
695 		netmap_memory_deref();
696 	}
697 	NMA_UNLOCK();
698 	bzero(priv, sizeof(*priv));	/* XXX for safety */
699 	free(priv, M_DEVBUF);
700 }
701 
702 
703 #ifdef __FreeBSD__
704 #include <vm/vm.h>
705 #include <vm/vm_param.h>
706 #include <vm/vm_object.h>
707 #include <vm/vm_page.h>
708 #include <vm/vm_pager.h>
709 #include <vm/uma.h>
710 
711 /*
712  * In order to track whether pages are still mapped, we hook into
713  * the standard cdev_pager and intercept the constructor and
714  * destructor.
715  * XXX but then ? Do we really use the information ?
716  * Need to investigate.
717  */
718 static struct cdev_pager_ops saved_cdev_pager_ops;
719 
720 
721 static int
722 netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
723     vm_ooffset_t foff, struct ucred *cred, u_short *color)
724 {
725 	if (netmap_verbose)
726 		D("first mmap for %p", handle);
727 	return saved_cdev_pager_ops.cdev_pg_ctor(handle,
728 			size, prot, foff, cred, color);
729 }
730 
731 
732 static void
733 netmap_dev_pager_dtor(void *handle)
734 {
735 	saved_cdev_pager_ops.cdev_pg_dtor(handle);
736 	ND("ready to release memory for %p", handle);
737 }
738 
739 
740 static struct cdev_pager_ops netmap_cdev_pager_ops = {
741         .cdev_pg_ctor = netmap_dev_pager_ctor,
742         .cdev_pg_dtor = netmap_dev_pager_dtor,
743         .cdev_pg_fault = NULL,
744 };
745 
746 
747 // XXX check whether we need netmap_mmap_single _and_ netmap_mmap
748 static int
749 netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff,
750 	vm_size_t objsize,  vm_object_t *objp, int prot)
751 {
752 	vm_object_t obj;
753 
754 	ND("cdev %p foff %jd size %jd objp %p prot %d", cdev,
755 	    (intmax_t )*foff, (intmax_t )objsize, objp, prot);
756 	obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff,
757             curthread->td_ucred);
758 	ND("returns obj %p", obj);
759 	if (obj == NULL)
760 		return EINVAL;
761 	if (saved_cdev_pager_ops.cdev_pg_fault == NULL) {
762 		ND("initialize cdev_pager_ops");
763 		saved_cdev_pager_ops = *(obj->un_pager.devp.ops);
764 		netmap_cdev_pager_ops.cdev_pg_fault =
765 			saved_cdev_pager_ops.cdev_pg_fault;
766 	};
767 	obj->un_pager.devp.ops = &netmap_cdev_pager_ops;
768 	*objp = obj;
769 	return 0;
770 }
771 #endif /* __FreeBSD__ */
772 
773 
774 /*
775  * mmap(2) support for the "netmap" device.
776  *
777  * Expose all the memory previously allocated by our custom memory
778  * allocator: this way the user has only to issue a single mmap(2), and
779  * can work on all the data structures flawlessly.
780  *
781  * Return 0 on success, -1 otherwise.
782  */
783 
784 #ifdef __FreeBSD__
785 static int
786 netmap_mmap(__unused struct cdev *dev,
787 #if __FreeBSD_version < 900000
788 		vm_offset_t offset, vm_paddr_t *paddr, int nprot
789 #else
790 		vm_ooffset_t offset, vm_paddr_t *paddr, int nprot,
791 		__unused vm_memattr_t *memattr
792 #endif
793 	)
794 {
795 	int error = 0;
796 	struct netmap_priv_d *priv;
797 
798 	if (nprot & PROT_EXEC)
799 		return (-1);	// XXX -1 or EINVAL ?
800 
801 	error = devfs_get_cdevpriv((void **)&priv);
802 	if (error == EBADF) {	/* called on fault, memory is initialized */
803 		ND(5, "handling fault at ofs 0x%x", offset);
804 		error = 0;
805 	} else if (error == 0)	/* make sure memory is set */
806 		error = netmap_get_memory(priv);
807 	if (error)
808 		return (error);
809 
810 	ND("request for offset 0x%x", (uint32_t)offset);
811 	*paddr = netmap_ofstophys(offset);
812 
813 	return (*paddr ? 0 : ENOMEM);
814 }
815 
816 
817 static int
818 netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
819 {
820 	if (netmap_verbose)
821 		D("dev %p fflag 0x%x devtype %d td %p",
822 			dev, fflag, devtype, td);
823 	return 0;
824 }
825 
826 
827 static int
828 netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
829 {
830 	struct netmap_priv_d *priv;
831 	int error;
832 
833 	priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF,
834 			      M_NOWAIT | M_ZERO);
835 	if (priv == NULL)
836 		return ENOMEM;
837 
838 	error = devfs_set_cdevpriv(priv, netmap_dtor);
839 	if (error)
840 	        return error;
841 
842 	return 0;
843 }
844 #endif /* __FreeBSD__ */
845 
846 
847 /*
848  * Handlers for synchronization of the queues from/to the host.
849  * Netmap has two operating modes:
850  * - in the default mode, the rings connected to the host stack are
851  *   just another ring pair managed by userspace;
852  * - in transparent mode (XXX to be defined) incoming packets
853  *   (from the host or the NIC) are marked as NS_FORWARD upon
854  *   arrival, and the user application has a chance to reset the
855  *   flag for packets that should be dropped.
856  *   On the RXSYNC or poll(), packets in RX rings between
857  *   kring->nr_kcur and ring->cur with NS_FORWARD still set are moved
858  *   to the other side.
859  * The transfer NIC --> host is relatively easy, just encapsulate
860  * into mbufs and we are done. The host --> NIC side is slightly
861  * harder because there might not be room in the tx ring so it
862  * might take a while before releasing the buffer.
863  */
864 
865 
866 /*
867  * pass a chain of buffers to the host stack as coming from 'dst'
868  */
869 static void
870 netmap_send_up(struct ifnet *dst, struct mbuf *head)
871 {
872 	struct mbuf *m;
873 
874 	/* send packets up, outside the lock */
875 	while ((m = head) != NULL) {
876 		head = head->m_nextpkt;
877 		m->m_nextpkt = NULL;
878 		if (netmap_verbose & NM_VERB_HOST)
879 			D("sending up pkt %p size %d", m, MBUF_LEN(m));
880 		NM_SEND_UP(dst, m);
881 	}
882 }
883 
884 struct mbq {
885 	struct mbuf *head;
886 	struct mbuf *tail;
887 	int count;
888 };
889 
890 
891 /*
892  * put a copy of the buffers marked NS_FORWARD into an mbuf chain.
893  * Run from hwcur to cur - reserved
894  */
895 static void
896 netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force)
897 {
898 	/* Take packets from hwcur to cur-reserved and pass them up.
899 	 * In case of no buffers we give up. At the end of the loop,
900 	 * the queue is drained in all cases.
901 	 * XXX handle reserved
902 	 */
903 	int k = kring->ring->cur - kring->ring->reserved;
904 	u_int n, lim = kring->nkr_num_slots - 1;
905 	struct mbuf *m, *tail = q->tail;
906 
907 	if (k < 0)
908 		k = k + kring->nkr_num_slots;
909 	for (n = kring->nr_hwcur; n != k;) {
910 		struct netmap_slot *slot = &kring->ring->slot[n];
911 
912 		n = (n == lim) ? 0 : n + 1;
913 		if ((slot->flags & NS_FORWARD) == 0 && !force)
914 			continue;
915 		if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE) {
916 			D("bad pkt at %d len %d", n, slot->len);
917 			continue;
918 		}
919 		slot->flags &= ~NS_FORWARD; // XXX needed ?
920 		m = m_devget(NMB(slot), slot->len, 0, kring->na->ifp, NULL);
921 
922 		if (m == NULL)
923 			break;
924 		if (tail)
925 			tail->m_nextpkt = m;
926 		else
927 			q->head = m;
928 		tail = m;
929 		q->count++;
930 		m->m_nextpkt = NULL;
931 	}
932 	q->tail = tail;
933 }
934 
935 
936 /*
937  * called under main lock to send packets from the host to the NIC
938  * The host ring has packets from nr_hwcur to (cur - reserved)
939  * to be sent down. We scan the tx rings, which have just been
940  * flushed so nr_hwcur == cur. Pushing packets down means
941  * increment cur and decrement avail.
942  * XXX to be verified
943  */
944 static void
945 netmap_sw_to_nic(struct netmap_adapter *na)
946 {
947 	struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
948 	struct netmap_kring *k1 = &na->tx_rings[0];
949 	int i, howmany, src_lim, dst_lim;
950 
951 	howmany = kring->nr_hwavail;	/* XXX otherwise cur - reserved - nr_hwcur */
952 
953 	src_lim = kring->nkr_num_slots;
954 	for (i = 0; howmany > 0 && i < na->num_tx_rings; i++, k1++) {
955 		ND("%d packets left to ring %d (space %d)", howmany, i, k1->nr_hwavail);
956 		dst_lim = k1->nkr_num_slots;
957 		while (howmany > 0 && k1->ring->avail > 0) {
958 			struct netmap_slot *src, *dst, tmp;
959 			src = &kring->ring->slot[kring->nr_hwcur];
960 			dst = &k1->ring->slot[k1->ring->cur];
961 			tmp = *src;
962 			src->buf_idx = dst->buf_idx;
963 			src->flags = NS_BUF_CHANGED;
964 
965 			dst->buf_idx = tmp.buf_idx;
966 			dst->len = tmp.len;
967 			dst->flags = NS_BUF_CHANGED;
968 			ND("out len %d buf %d from %d to %d",
969 				dst->len, dst->buf_idx,
970 				kring->nr_hwcur, k1->ring->cur);
971 
972 			if (++kring->nr_hwcur >= src_lim)
973 				kring->nr_hwcur = 0;
974 			howmany--;
975 			kring->nr_hwavail--;
976 			if (++k1->ring->cur >= dst_lim)
977 				k1->ring->cur = 0;
978 			k1->ring->avail--;
979 		}
980 		kring->ring->cur = kring->nr_hwcur; // XXX
981 		k1++;
982 	}
983 }
984 
985 
986 /*
987  * netmap_sync_to_host() passes packets up. We are called from a
988  * system call in user process context, and the only contention
989  * can be among multiple user threads erroneously calling
990  * this routine concurrently.
991  */
992 static void
993 netmap_sync_to_host(struct netmap_adapter *na)
994 {
995 	struct netmap_kring *kring = &na->tx_rings[na->num_tx_rings];
996 	struct netmap_ring *ring = kring->ring;
997 	u_int k, lim = kring->nkr_num_slots - 1;
998 	struct mbq q = { NULL, NULL };
999 
1000 	k = ring->cur;
1001 	if (k > lim) {
1002 		netmap_ring_reinit(kring);
1003 		return;
1004 	}
1005 	// na->nm_lock(na->ifp, NETMAP_CORE_LOCK, 0);
1006 
1007 	/* Take packets from hwcur to cur and pass them up.
1008 	 * In case of no buffers we give up. At the end of the loop,
1009 	 * the queue is drained in all cases.
1010 	 */
1011 	netmap_grab_packets(kring, &q, 1);
1012 	kring->nr_hwcur = k;
1013 	kring->nr_hwavail = ring->avail = lim;
1014 	// na->nm_lock(na->ifp, NETMAP_CORE_UNLOCK, 0);
1015 
1016 	netmap_send_up(na->ifp, q.head);
1017 }
1018 
1019 
1020 /* SWNA(ifp)->txrings[0] is always NA(ifp)->txrings[NA(ifp)->num_txrings] */
1021 static int
1022 netmap_bdg_to_host(struct ifnet *ifp, u_int ring_nr, int do_lock)
1023 {
1024 	(void)ring_nr;
1025 	(void)do_lock;
1026 	netmap_sync_to_host(NA(ifp));
1027 	return 0;
1028 }
1029 
1030 
1031 /*
1032  * rxsync backend for packets coming from the host stack.
1033  * They have been put in the queue by netmap_start() so we
1034  * need to protect access to the kring using a lock.
1035  *
1036  * This routine also does the selrecord if called from the poll handler
1037  * (we know because td != NULL).
1038  *
1039  * NOTE: on linux, selrecord() is defined as a macro and uses pwait
1040  *     as an additional hidden argument.
1041  */
1042 static void
1043 netmap_sync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait)
1044 {
1045 	struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
1046 	struct netmap_ring *ring = kring->ring;
1047 	u_int j, n, lim = kring->nkr_num_slots;
1048 	u_int k = ring->cur, resvd = ring->reserved;
1049 
1050 	(void)pwait;	/* disable unused warnings */
1051 	na->nm_lock(na->ifp, NETMAP_CORE_LOCK, 0);
1052 	if (k >= lim) {
1053 		netmap_ring_reinit(kring);
1054 		return;
1055 	}
1056 	/* new packets are already set in nr_hwavail */
1057 	/* skip past packets that userspace has released */
1058 	j = kring->nr_hwcur;
1059 	if (resvd > 0) {
1060 		if (resvd + ring->avail >= lim + 1) {
1061 			D("XXX invalid reserve/avail %d %d", resvd, ring->avail);
1062 			ring->reserved = resvd = 0; // XXX panic...
1063 		}
1064 		k = (k >= resvd) ? k - resvd : k + lim - resvd;
1065         }
1066 	if (j != k) {
1067 		n = k >= j ? k - j : k + lim - j;
1068 		kring->nr_hwavail -= n;
1069 		kring->nr_hwcur = k;
1070 	}
1071 	k = ring->avail = kring->nr_hwavail - resvd;
1072 	if (k == 0 && td)
1073 		selrecord(td, &kring->si);
1074 	if (k && (netmap_verbose & NM_VERB_HOST))
1075 		D("%d pkts from stack", k);
1076 	na->nm_lock(na->ifp, NETMAP_CORE_UNLOCK, 0);
1077 }
1078 
1079 
1080 /*
1081  * get a refcounted reference to an interface.
1082  * Return ENXIO if the interface does not exist, EINVAL if netmap
1083  * is not supported by the interface.
1084  * If successful, hold a reference.
1085  *
1086  * During the NIC is attached to a bridge, reference is managed
1087  * at na->na_bdg_refcount using ADD/DROP_BDG_REF() as well as
1088  * virtual ports.  Hence, on the final DROP_BDG_REF(), the NIC
1089  * is detached from the bridge, then ifp's refcount is dropped (this
1090  * is equivalent to that ifp is destroyed in case of virtual ports.
1091  *
1092  * This function uses if_rele() when we want to prevent the NIC from
1093  * being detached from the bridge in error handling.  But once refcount
1094  * is acquired by this function, it must be released using nm_if_rele().
1095  */
1096 static int
1097 get_ifp(struct nmreq *nmr, struct ifnet **ifp)
1098 {
1099 	const char *name = nmr->nr_name;
1100 	int namelen = strlen(name);
1101 #ifdef NM_BRIDGE
1102 	struct ifnet *iter = NULL;
1103 	int no_prefix = 0;
1104 
1105 	do {
1106 		struct nm_bridge *b;
1107 		struct netmap_adapter *na;
1108 		int i, cand = -1, cand2 = -1;
1109 
1110 		if (strncmp(name, NM_NAME, sizeof(NM_NAME) - 1)) {
1111 			no_prefix = 1;
1112 			break;
1113 		}
1114 		b = nm_find_bridge(name, 1 /* create a new one if no exist */ );
1115 		if (b == NULL) {
1116 			D("no bridges available for '%s'", name);
1117 			return (ENXIO);
1118 		}
1119 		/* Now we are sure that name starts with the bridge's name */
1120 		BDG_WLOCK(b);
1121 		/* lookup in the local list of ports */
1122 		for (i = 0; i < NM_BDG_MAXPORTS; i++) {
1123 			na = BDG_GET_VAR(b->bdg_ports[i]);
1124 			if (na == NULL) {
1125 				if (cand == -1)
1126 					cand = i; /* potential insert point */
1127 				else if (cand2 == -1)
1128 					cand2 = i; /* for host stack */
1129 				continue;
1130 			}
1131 			iter = na->ifp;
1132 			/* XXX make sure the name only contains one : */
1133 			if (!strcmp(iter->if_xname, name) /* virtual port */ ||
1134 			    (namelen > b->namelen && !strcmp(iter->if_xname,
1135 			    name + b->namelen + 1)) /* NIC */) {
1136 				ADD_BDG_REF(iter);
1137 				ND("found existing interface");
1138 				BDG_WUNLOCK(b);
1139 				break;
1140 			}
1141 		}
1142 		if (i < NM_BDG_MAXPORTS) /* already unlocked */
1143 			break;
1144 		if (cand == -1) {
1145 			D("bridge full, cannot create new port");
1146 no_port:
1147 			BDG_WUNLOCK(b);
1148 			*ifp = NULL;
1149 			return EINVAL;
1150 		}
1151 		ND("create new bridge port %s", name);
1152 		/*
1153 		 * create a struct ifnet for the new port.
1154 		 * The forwarding table is attached to the kring(s).
1155 		 */
1156 		/*
1157 		 * try see if there is a matching NIC with this name
1158 		 * (after the bridge's name)
1159 		 */
1160 		iter = ifunit_ref(name + b->namelen + 1);
1161 		if (!iter) { /* this is a virtual port */
1162 			/* Create a temporary NA with arguments, then
1163 			 * bdg_netmap_attach() will allocate the real one
1164 			 * and attach it to the ifp
1165 			 */
1166 			struct netmap_adapter tmp_na;
1167 
1168 			if (nmr->nr_cmd) /* nr_cmd must be for a NIC */
1169 				goto no_port;
1170 			bzero(&tmp_na, sizeof(tmp_na));
1171 			/* bound checking */
1172 			if (nmr->nr_tx_rings < 1)
1173 				nmr->nr_tx_rings = 1;
1174 			if (nmr->nr_tx_rings > NM_BDG_MAXRINGS)
1175 				nmr->nr_tx_rings = NM_BDG_MAXRINGS;
1176 			tmp_na.num_tx_rings = nmr->nr_tx_rings;
1177 			if (nmr->nr_rx_rings < 1)
1178 				nmr->nr_rx_rings = 1;
1179 			if (nmr->nr_rx_rings > NM_BDG_MAXRINGS)
1180 				nmr->nr_rx_rings = NM_BDG_MAXRINGS;
1181 			tmp_na.num_rx_rings = nmr->nr_rx_rings;
1182 
1183 			iter = malloc(sizeof(*iter), M_DEVBUF, M_NOWAIT | M_ZERO);
1184 			if (!iter)
1185 				goto no_port;
1186 			strcpy(iter->if_xname, name);
1187 			tmp_na.ifp = iter;
1188 			/* bdg_netmap_attach creates a struct netmap_adapter */
1189 			bdg_netmap_attach(&tmp_na);
1190 		} else if (NETMAP_CAPABLE(iter)) { /* this is a NIC */
1191 			/* cannot attach the NIC that any user or another
1192 			 * bridge already holds.
1193 			 */
1194 			if (NETMAP_OWNED_BY_ANY(iter) || cand2 == -1) {
1195 ifunit_rele:
1196 				if_rele(iter); /* don't detach from bridge */
1197 				goto no_port;
1198 			}
1199 			/* bind the host stack to the bridge */
1200 			if (nmr->nr_arg1 == NETMAP_BDG_HOST) {
1201 				BDG_SET_VAR(b->bdg_ports[cand2], SWNA(iter));
1202 				SWNA(iter)->bdg_port = cand2;
1203 				SWNA(iter)->na_bdg = b;
1204 			}
1205 		} else /* not a netmap-capable NIC */
1206 			goto ifunit_rele;
1207 		na = NA(iter);
1208 		na->bdg_port = cand;
1209 		/* bind the port to the bridge (virtual ports are not active) */
1210 		BDG_SET_VAR(b->bdg_ports[cand], na);
1211 		na->na_bdg = b;
1212 		ADD_BDG_REF(iter);
1213 		BDG_WUNLOCK(b);
1214 		ND("attaching virtual bridge %p", b);
1215 	} while (0);
1216 	*ifp = iter;
1217 	if (! *ifp)
1218 #endif /* NM_BRIDGE */
1219 	*ifp = ifunit_ref(name);
1220 	if (*ifp == NULL)
1221 		return (ENXIO);
1222 	/* can do this if the capability exists and if_pspare[0]
1223 	 * points to the netmap descriptor.
1224 	 */
1225 	if (NETMAP_CAPABLE(*ifp)) {
1226 #ifdef NM_BRIDGE
1227 		/* Users cannot use the NIC attached to a bridge directly */
1228 		if (no_prefix && NETMAP_OWNED_BY_KERN(*ifp)) {
1229 			if_rele(*ifp); /* don't detach from bridge */
1230 			return EINVAL;
1231 		} else
1232 #endif /* NM_BRIDGE */
1233 		return 0;	/* valid pointer, we hold the refcount */
1234 	}
1235 	nm_if_rele(*ifp);
1236 	return EINVAL;	// not NETMAP capable
1237 }
1238 
1239 
1240 /*
1241  * Error routine called when txsync/rxsync detects an error.
1242  * Can't do much more than resetting cur = hwcur, avail = hwavail.
1243  * Return 1 on reinit.
1244  *
1245  * This routine is only called by the upper half of the kernel.
1246  * It only reads hwcur (which is changed only by the upper half, too)
1247  * and hwavail (which may be changed by the lower half, but only on
1248  * a tx ring and only to increase it, so any error will be recovered
1249  * on the next call). For the above, we don't strictly need to call
1250  * it under lock.
1251  */
1252 int
1253 netmap_ring_reinit(struct netmap_kring *kring)
1254 {
1255 	struct netmap_ring *ring = kring->ring;
1256 	u_int i, lim = kring->nkr_num_slots - 1;
1257 	int errors = 0;
1258 
1259 	RD(10, "called for %s", kring->na->ifp->if_xname);
1260 	if (ring->cur > lim)
1261 		errors++;
1262 	for (i = 0; i <= lim; i++) {
1263 		u_int idx = ring->slot[i].buf_idx;
1264 		u_int len = ring->slot[i].len;
1265 		if (idx < 2 || idx >= netmap_total_buffers) {
1266 			if (!errors++)
1267 				D("bad buffer at slot %d idx %d len %d ", i, idx, len);
1268 			ring->slot[i].buf_idx = 0;
1269 			ring->slot[i].len = 0;
1270 		} else if (len > NETMAP_BUF_SIZE) {
1271 			ring->slot[i].len = 0;
1272 			if (!errors++)
1273 				D("bad len %d at slot %d idx %d",
1274 					len, i, idx);
1275 		}
1276 	}
1277 	if (errors) {
1278 		int pos = kring - kring->na->tx_rings;
1279 		int n = kring->na->num_tx_rings + 1;
1280 
1281 		RD(10, "total %d errors", errors);
1282 		errors++;
1283 		RD(10, "%s %s[%d] reinit, cur %d -> %d avail %d -> %d",
1284 			kring->na->ifp->if_xname,
1285 			pos < n ?  "TX" : "RX", pos < n ? pos : pos - n,
1286 			ring->cur, kring->nr_hwcur,
1287 			ring->avail, kring->nr_hwavail);
1288 		ring->cur = kring->nr_hwcur;
1289 		ring->avail = kring->nr_hwavail;
1290 	}
1291 	return (errors ? 1 : 0);
1292 }
1293 
1294 
1295 /*
1296  * Set the ring ID. For devices with a single queue, a request
1297  * for all rings is the same as a single ring.
1298  */
1299 static int
1300 netmap_set_ringid(struct netmap_priv_d *priv, u_int ringid)
1301 {
1302 	struct ifnet *ifp = priv->np_ifp;
1303 	struct netmap_adapter *na = NA(ifp);
1304 	u_int i = ringid & NETMAP_RING_MASK;
1305 	/* initially (np_qfirst == np_qlast) we don't want to lock */
1306 	int need_lock = (priv->np_qfirst != priv->np_qlast);
1307 	int lim = na->num_rx_rings;
1308 
1309 	if (na->num_tx_rings > lim)
1310 		lim = na->num_tx_rings;
1311 	if ( (ringid & NETMAP_HW_RING) && i >= lim) {
1312 		D("invalid ring id %d", i);
1313 		return (EINVAL);
1314 	}
1315 	if (need_lock)
1316 		na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
1317 	priv->np_ringid = ringid;
1318 	if (ringid & NETMAP_SW_RING) {
1319 		priv->np_qfirst = NETMAP_SW_RING;
1320 		priv->np_qlast = 0;
1321 	} else if (ringid & NETMAP_HW_RING) {
1322 		priv->np_qfirst = i;
1323 		priv->np_qlast = i + 1;
1324 	} else {
1325 		priv->np_qfirst = 0;
1326 		priv->np_qlast = NETMAP_HW_RING ;
1327 	}
1328 	priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1;
1329 	if (need_lock)
1330 		na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
1331     if (netmap_verbose) {
1332 	if (ringid & NETMAP_SW_RING)
1333 		D("ringid %s set to SW RING", ifp->if_xname);
1334 	else if (ringid & NETMAP_HW_RING)
1335 		D("ringid %s set to HW RING %d", ifp->if_xname,
1336 			priv->np_qfirst);
1337 	else
1338 		D("ringid %s set to all %d HW RINGS", ifp->if_xname, lim);
1339     }
1340 	return 0;
1341 }
1342 
1343 
1344 /*
1345  * possibly move the interface to netmap-mode.
1346  * If success it returns a pointer to netmap_if, otherwise NULL.
1347  * This must be called with NMA_LOCK held.
1348  */
1349 static struct netmap_if *
1350 netmap_do_regif(struct netmap_priv_d *priv, struct ifnet *ifp,
1351 	uint16_t ringid, int *err)
1352 {
1353 	struct netmap_adapter *na = NA(ifp);
1354 	struct netmap_if *nifp = NULL;
1355 	int i, error;
1356 
1357 	if (na->na_bdg)
1358 		BDG_WLOCK(na->na_bdg);
1359 	na->nm_lock(ifp, NETMAP_REG_LOCK, 0);
1360 
1361 	/* ring configuration may have changed, fetch from the card */
1362 	netmap_update_config(na);
1363 	priv->np_ifp = ifp;     /* store the reference */
1364 	error = netmap_set_ringid(priv, ringid);
1365 	if (error)
1366 		goto out;
1367 	nifp = netmap_if_new(ifp->if_xname, na);
1368 	if (nifp == NULL) { /* allocation failed */
1369 		error = ENOMEM;
1370 	} else if (ifp->if_capenable & IFCAP_NETMAP) {
1371 		/* was already set */
1372 	} else {
1373 		/* Otherwise set the card in netmap mode
1374 		 * and make it use the shared buffers.
1375 		 */
1376 		for (i = 0 ; i < na->num_tx_rings + 1; i++)
1377 			mtx_init(&na->tx_rings[i].q_lock, "nm_txq_lock",
1378 			    MTX_NETWORK_LOCK, MTX_DEF);
1379 		for (i = 0 ; i < na->num_rx_rings + 1; i++) {
1380 			mtx_init(&na->rx_rings[i].q_lock, "nm_rxq_lock",
1381 			    MTX_NETWORK_LOCK, MTX_DEF);
1382 		}
1383 		if (nma_is_hw(na)) {
1384 			SWNA(ifp)->tx_rings = &na->tx_rings[na->num_tx_rings];
1385 			SWNA(ifp)->rx_rings = &na->rx_rings[na->num_rx_rings];
1386 		}
1387 		error = na->nm_register(ifp, 1); /* mode on */
1388 #ifdef NM_BRIDGE
1389 		if (!error)
1390 			error = nm_alloc_bdgfwd(na);
1391 #endif /* NM_BRIDGE */
1392 		if (error) {
1393 			netmap_dtor_locked(priv);
1394 			/* nifp is not yet in priv, so free it separately */
1395 			netmap_if_free(nifp);
1396 			nifp = NULL;
1397 		}
1398 
1399 	}
1400 out:
1401 	*err = error;
1402 	na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0);
1403 	if (na->na_bdg)
1404 		BDG_WUNLOCK(na->na_bdg);
1405 	return nifp;
1406 }
1407 
1408 
1409 /* Process NETMAP_BDG_ATTACH and NETMAP_BDG_DETACH */
1410 static int
1411 kern_netmap_regif(struct nmreq *nmr)
1412 {
1413 	struct ifnet *ifp;
1414 	struct netmap_if *nifp;
1415 	struct netmap_priv_d *npriv;
1416 	int error;
1417 
1418 	npriv = malloc(sizeof(*npriv), M_DEVBUF, M_NOWAIT|M_ZERO);
1419 	if (npriv == NULL)
1420 		return ENOMEM;
1421 	error = netmap_get_memory(npriv);
1422 	if (error) {
1423 free_exit:
1424 		bzero(npriv, sizeof(*npriv));
1425 		free(npriv, M_DEVBUF);
1426 		return error;
1427 	}
1428 
1429 	NMA_LOCK();
1430 	error = get_ifp(nmr, &ifp);
1431 	if (error) { /* no device, or another bridge or user owns the device */
1432 		NMA_UNLOCK();
1433 		goto free_exit;
1434 	} else if (!NETMAP_OWNED_BY_KERN(ifp)) {
1435 		/* got reference to a virtual port or direct access to a NIC.
1436 		 * perhaps specified no bridge's prefix or wrong NIC's name
1437 		 */
1438 		error = EINVAL;
1439 unref_exit:
1440 		nm_if_rele(ifp);
1441 		NMA_UNLOCK();
1442 		goto free_exit;
1443 	}
1444 
1445 	if (nmr->nr_cmd == NETMAP_BDG_DETACH) {
1446 		if (NA(ifp)->refcount == 0) { /* not registered */
1447 			error = EINVAL;
1448 			goto unref_exit;
1449 		}
1450 		NMA_UNLOCK();
1451 
1452 		netmap_dtor(NA(ifp)->na_kpriv); /* unregister */
1453 		NA(ifp)->na_kpriv = NULL;
1454 		nm_if_rele(ifp); /* detach from the bridge */
1455 		goto free_exit;
1456 	} else if (NA(ifp)->refcount > 0) { /* already registered */
1457 		error = EINVAL;
1458 		goto unref_exit;
1459 	}
1460 
1461 	nifp = netmap_do_regif(npriv, ifp, nmr->nr_ringid, &error);
1462 	if (!nifp)
1463 		goto unref_exit;
1464 	wmb(); // XXX do we need it ?
1465 	npriv->np_nifp = nifp;
1466 	NA(ifp)->na_kpriv = npriv;
1467 	NMA_UNLOCK();
1468 	D("registered %s to netmap-mode", ifp->if_xname);
1469 	return 0;
1470 }
1471 
1472 
1473 /* CORE_LOCK is not necessary */
1474 static void
1475 netmap_swlock_wrapper(struct ifnet *dev, int what, u_int queueid)
1476 {
1477 	struct netmap_adapter *na = SWNA(dev);
1478 
1479 	switch (what) {
1480 	case NETMAP_TX_LOCK:
1481 		mtx_lock(&na->tx_rings[queueid].q_lock);
1482 		break;
1483 
1484 	case NETMAP_TX_UNLOCK:
1485 		mtx_unlock(&na->tx_rings[queueid].q_lock);
1486 		break;
1487 
1488 	case NETMAP_RX_LOCK:
1489 		mtx_lock(&na->rx_rings[queueid].q_lock);
1490 		break;
1491 
1492 	case NETMAP_RX_UNLOCK:
1493 		mtx_unlock(&na->rx_rings[queueid].q_lock);
1494 		break;
1495 	}
1496 }
1497 
1498 
1499 /* Initialize necessary fields of sw adapter located in right after hw's
1500  * one.  sw adapter attaches a pair of sw rings of the netmap-mode NIC.
1501  * It is always activated and deactivated at the same tie with the hw's one.
1502  * Thus we don't need refcounting on the sw adapter.
1503  * Regardless of NIC's feature we use separate lock so that anybody can lock
1504  * me independently from the hw adapter.
1505  * Make sure nm_register is NULL to be handled as FALSE in nma_is_hw
1506  */
1507 static void
1508 netmap_attach_sw(struct ifnet *ifp)
1509 {
1510 	struct netmap_adapter *hw_na = NA(ifp);
1511 	struct netmap_adapter *na = SWNA(ifp);
1512 
1513 	na->ifp = ifp;
1514 	na->separate_locks = 1;
1515 	na->nm_lock = netmap_swlock_wrapper;
1516 	na->num_rx_rings = na->num_tx_rings = 1;
1517 	na->num_tx_desc = hw_na->num_tx_desc;
1518 	na->num_rx_desc = hw_na->num_rx_desc;
1519 	na->nm_txsync = netmap_bdg_to_host;
1520 }
1521 
1522 
1523 /* exported to kernel callers */
1524 int
1525 netmap_bdg_ctl(struct nmreq *nmr, bdg_lookup_fn_t func)
1526 {
1527 	struct nm_bridge *b;
1528 	struct netmap_adapter *na;
1529 	struct ifnet *iter;
1530 	char *name = nmr->nr_name;
1531 	int cmd = nmr->nr_cmd, namelen = strlen(name);
1532 	int error = 0, i, j;
1533 
1534 	switch (cmd) {
1535 	case NETMAP_BDG_ATTACH:
1536 	case NETMAP_BDG_DETACH:
1537 		error = kern_netmap_regif(nmr);
1538 		break;
1539 
1540 	case NETMAP_BDG_LIST:
1541 		/* this is used to enumerate bridges and ports */
1542 		if (namelen) { /* look up indexes of bridge and port */
1543 			if (strncmp(name, NM_NAME, strlen(NM_NAME))) {
1544 				error = EINVAL;
1545 				break;
1546 			}
1547 			b = nm_find_bridge(name, 0 /* don't create */);
1548 			if (!b) {
1549 				error = ENOENT;
1550 				break;
1551 			}
1552 
1553 			BDG_RLOCK(b);
1554 			error = ENOENT;
1555 			for (i = 0; i < NM_BDG_MAXPORTS; i++) {
1556 				na = BDG_GET_VAR(b->bdg_ports[i]);
1557 				if (na == NULL)
1558 					continue;
1559 				iter = na->ifp;
1560 				/* the former and the latter identify a
1561 				 * virtual port and a NIC, respectively
1562 				 */
1563 				if (!strcmp(iter->if_xname, name) ||
1564 				    (namelen > b->namelen &&
1565 				    !strcmp(iter->if_xname,
1566 				    name + b->namelen + 1))) {
1567 					/* bridge index */
1568 					nmr->nr_arg1 = b - nm_bridges;
1569 					nmr->nr_arg2 = i; /* port index */
1570 					error = 0;
1571 					break;
1572 				}
1573 			}
1574 			BDG_RUNLOCK(b);
1575 		} else {
1576 			/* return the first non-empty entry starting from
1577 			 * bridge nr_arg1 and port nr_arg2.
1578 			 *
1579 			 * Users can detect the end of the same bridge by
1580 			 * seeing the new and old value of nr_arg1, and can
1581 			 * detect the end of all the bridge by error != 0
1582 			 */
1583 			i = nmr->nr_arg1;
1584 			j = nmr->nr_arg2;
1585 
1586 			for (error = ENOENT; error && i < NM_BRIDGES; i++) {
1587 				b = nm_bridges + i;
1588 				BDG_RLOCK(b);
1589 				for (; j < NM_BDG_MAXPORTS; j++) {
1590 					na = BDG_GET_VAR(b->bdg_ports[j]);
1591 					if (na == NULL)
1592 						continue;
1593 					iter = na->ifp;
1594 					nmr->nr_arg1 = i;
1595 					nmr->nr_arg2 = j;
1596 					strncpy(name, iter->if_xname, IFNAMSIZ);
1597 					error = 0;
1598 					break;
1599 				}
1600 				BDG_RUNLOCK(b);
1601 				j = 0; /* following bridges scan from 0 */
1602 			}
1603 		}
1604 		break;
1605 
1606 	case NETMAP_BDG_LOOKUP_REG:
1607 		/* register a lookup function to the given bridge.
1608 		 * nmr->nr_name may be just bridge's name (including ':'
1609 		 * if it is not just NM_NAME).
1610 		 */
1611 		if (!func) {
1612 			error = EINVAL;
1613 			break;
1614 		}
1615 		b = nm_find_bridge(name, 0 /* don't create */);
1616 		if (!b) {
1617 			error = EINVAL;
1618 			break;
1619 		}
1620 		BDG_WLOCK(b);
1621 		b->nm_bdg_lookup = func;
1622 		BDG_WUNLOCK(b);
1623 		break;
1624 	default:
1625 		D("invalid cmd (nmr->nr_cmd) (0x%x)", cmd);
1626 		error = EINVAL;
1627 		break;
1628 	}
1629 	return error;
1630 }
1631 
1632 
1633 /*
1634  * ioctl(2) support for the "netmap" device.
1635  *
1636  * Following a list of accepted commands:
1637  * - NIOCGINFO
1638  * - SIOCGIFADDR	just for convenience
1639  * - NIOCREGIF
1640  * - NIOCUNREGIF
1641  * - NIOCTXSYNC
1642  * - NIOCRXSYNC
1643  *
1644  * Return 0 on success, errno otherwise.
1645  */
1646 static int
1647 netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
1648 	int fflag, struct thread *td)
1649 {
1650 	struct netmap_priv_d *priv = NULL;
1651 	struct ifnet *ifp;
1652 	struct nmreq *nmr = (struct nmreq *) data;
1653 	struct netmap_adapter *na;
1654 	int error;
1655 	u_int i, lim;
1656 	struct netmap_if *nifp;
1657 
1658 	(void)dev;	/* UNUSED */
1659 	(void)fflag;	/* UNUSED */
1660 #ifdef linux
1661 #define devfs_get_cdevpriv(pp)				\
1662 	({ *(struct netmap_priv_d **)pp = ((struct file *)td)->private_data; 	\
1663 		(*pp ? 0 : ENOENT); })
1664 
1665 /* devfs_set_cdevpriv cannot fail on linux */
1666 #define devfs_set_cdevpriv(p, fn)				\
1667 	({ ((struct file *)td)->private_data = p; (p ? 0 : EINVAL); })
1668 
1669 
1670 #define devfs_clear_cdevpriv()	do {				\
1671 		netmap_dtor(priv); ((struct file *)td)->private_data = 0;	\
1672 	} while (0)
1673 #endif /* linux */
1674 
1675 	CURVNET_SET(TD_TO_VNET(td));
1676 
1677 	error = devfs_get_cdevpriv((void **)&priv);
1678 	if (error) {
1679 		CURVNET_RESTORE();
1680 		/* XXX ENOENT should be impossible, since the priv
1681 		 * is now created in the open */
1682 		return (error == ENOENT ? ENXIO : error);
1683 	}
1684 
1685 	nmr->nr_name[sizeof(nmr->nr_name) - 1] = '\0';	/* truncate name */
1686 	switch (cmd) {
1687 	case NIOCGINFO:		/* return capabilities etc */
1688 		if (nmr->nr_version != NETMAP_API) {
1689 			D("API mismatch got %d have %d",
1690 				nmr->nr_version, NETMAP_API);
1691 			nmr->nr_version = NETMAP_API;
1692 			error = EINVAL;
1693 			break;
1694 		}
1695 		if (nmr->nr_cmd == NETMAP_BDG_LIST) {
1696 			error = netmap_bdg_ctl(nmr, NULL);
1697 			break;
1698 		}
1699 		/* update configuration */
1700 		error = netmap_get_memory(priv);
1701 		ND("get_memory returned %d", error);
1702 		if (error)
1703 			break;
1704 		/* memsize is always valid */
1705 		nmr->nr_memsize = nm_mem.nm_totalsize;
1706 		nmr->nr_offset = 0;
1707 		nmr->nr_rx_slots = nmr->nr_tx_slots = 0;
1708 		if (nmr->nr_name[0] == '\0')	/* just get memory info */
1709 			break;
1710 		/* lock because get_ifp and update_config see na->refcount */
1711 		NMA_LOCK();
1712 		error = get_ifp(nmr, &ifp); /* get a refcount */
1713 		if (error) {
1714 			NMA_UNLOCK();
1715 			break;
1716 		}
1717 		na = NA(ifp); /* retrieve netmap_adapter */
1718 		netmap_update_config(na);
1719 		NMA_UNLOCK();
1720 		nmr->nr_rx_rings = na->num_rx_rings;
1721 		nmr->nr_tx_rings = na->num_tx_rings;
1722 		nmr->nr_rx_slots = na->num_rx_desc;
1723 		nmr->nr_tx_slots = na->num_tx_desc;
1724 		nm_if_rele(ifp);	/* return the refcount */
1725 		break;
1726 
1727 	case NIOCREGIF:
1728 		if (nmr->nr_version != NETMAP_API) {
1729 			nmr->nr_version = NETMAP_API;
1730 			error = EINVAL;
1731 			break;
1732 		}
1733 		/* possibly attach/detach NIC and VALE switch */
1734 		i = nmr->nr_cmd;
1735 		if (i == NETMAP_BDG_ATTACH || i == NETMAP_BDG_DETACH) {
1736 			error = netmap_bdg_ctl(nmr, NULL);
1737 			break;
1738 		} else if (i != 0) {
1739 			D("nr_cmd must be 0 not %d", i);
1740 			error = EINVAL;
1741 			break;
1742 		}
1743 
1744 		/* ensure allocators are ready */
1745 		error = netmap_get_memory(priv);
1746 		ND("get_memory returned %d", error);
1747 		if (error)
1748 			break;
1749 
1750 		/* protect access to priv from concurrent NIOCREGIF */
1751 		NMA_LOCK();
1752 		if (priv->np_ifp != NULL) {	/* thread already registered */
1753 			error = netmap_set_ringid(priv, nmr->nr_ringid);
1754 unlock_out:
1755 			NMA_UNLOCK();
1756 			break;
1757 		}
1758 		/* find the interface and a reference */
1759 		error = get_ifp(nmr, &ifp); /* keep reference */
1760 		if (error)
1761 			goto unlock_out;
1762 		else if (NETMAP_OWNED_BY_KERN(ifp)) {
1763 			nm_if_rele(ifp);
1764 			goto unlock_out;
1765 		}
1766 		nifp = netmap_do_regif(priv, ifp, nmr->nr_ringid, &error);
1767 		if (!nifp) {    /* reg. failed, release priv and ref */
1768 			nm_if_rele(ifp);        /* return the refcount */
1769 			priv->np_ifp = NULL;
1770 			priv->np_nifp = NULL;
1771 			goto unlock_out;
1772 		}
1773 
1774 		/* the following assignment is a commitment.
1775 		 * Readers (i.e., poll and *SYNC) check for
1776 		 * np_nifp != NULL without locking
1777 		 */
1778 		wmb(); /* make sure previous writes are visible to all CPUs */
1779 		priv->np_nifp = nifp;
1780 		NMA_UNLOCK();
1781 
1782 		/* return the offset of the netmap_if object */
1783 		na = NA(ifp); /* retrieve netmap adapter */
1784 		nmr->nr_rx_rings = na->num_rx_rings;
1785 		nmr->nr_tx_rings = na->num_tx_rings;
1786 		nmr->nr_rx_slots = na->num_rx_desc;
1787 		nmr->nr_tx_slots = na->num_tx_desc;
1788 		nmr->nr_memsize = nm_mem.nm_totalsize;
1789 		nmr->nr_offset = netmap_if_offset(nifp);
1790 		break;
1791 
1792 	case NIOCUNREGIF:
1793 		// XXX we have no data here ?
1794 		D("deprecated, data is %p", nmr);
1795 		error = EINVAL;
1796 		break;
1797 
1798 	case NIOCTXSYNC:
1799 	case NIOCRXSYNC:
1800 		nifp = priv->np_nifp;
1801 
1802 		if (nifp == NULL) {
1803 			error = ENXIO;
1804 			break;
1805 		}
1806 		rmb(); /* make sure following reads are not from cache */
1807 
1808 
1809 		ifp = priv->np_ifp;	/* we have a reference */
1810 
1811 		if (ifp == NULL) {
1812 			D("Internal error: nifp != NULL && ifp == NULL");
1813 			error = ENXIO;
1814 			break;
1815 		}
1816 
1817 		na = NA(ifp); /* retrieve netmap adapter */
1818 		if (priv->np_qfirst == NETMAP_SW_RING) { /* host rings */
1819 			if (cmd == NIOCTXSYNC)
1820 				netmap_sync_to_host(na);
1821 			else
1822 				netmap_sync_from_host(na, NULL, NULL);
1823 			break;
1824 		}
1825 		/* find the last ring to scan */
1826 		lim = priv->np_qlast;
1827 		if (lim == NETMAP_HW_RING)
1828 			lim = (cmd == NIOCTXSYNC) ?
1829 			    na->num_tx_rings : na->num_rx_rings;
1830 
1831 		for (i = priv->np_qfirst; i < lim; i++) {
1832 			if (cmd == NIOCTXSYNC) {
1833 				struct netmap_kring *kring = &na->tx_rings[i];
1834 				if (netmap_verbose & NM_VERB_TXSYNC)
1835 					D("pre txsync ring %d cur %d hwcur %d",
1836 					    i, kring->ring->cur,
1837 					    kring->nr_hwcur);
1838 				na->nm_txsync(ifp, i, 1 /* do lock */);
1839 				if (netmap_verbose & NM_VERB_TXSYNC)
1840 					D("post txsync ring %d cur %d hwcur %d",
1841 					    i, kring->ring->cur,
1842 					    kring->nr_hwcur);
1843 			} else {
1844 				na->nm_rxsync(ifp, i, 1 /* do lock */);
1845 				microtime(&na->rx_rings[i].ring->ts);
1846 			}
1847 		}
1848 
1849 		break;
1850 
1851 #ifdef __FreeBSD__
1852 	case BIOCIMMEDIATE:
1853 	case BIOCGHDRCMPLT:
1854 	case BIOCSHDRCMPLT:
1855 	case BIOCSSEESENT:
1856 		D("ignore BIOCIMMEDIATE/BIOCSHDRCMPLT/BIOCSHDRCMPLT/BIOCSSEESENT");
1857 		break;
1858 
1859 	default:	/* allow device-specific ioctls */
1860 	    {
1861 		struct socket so;
1862 		bzero(&so, sizeof(so));
1863 		error = get_ifp(nmr, &ifp); /* keep reference */
1864 		if (error)
1865 			break;
1866 		so.so_vnet = ifp->if_vnet;
1867 		// so->so_proto not null.
1868 		error = ifioctl(&so, cmd, data, td);
1869 		nm_if_rele(ifp);
1870 		break;
1871 	    }
1872 
1873 #else /* linux */
1874 	default:
1875 		error = EOPNOTSUPP;
1876 #endif /* linux */
1877 	}
1878 
1879 	CURVNET_RESTORE();
1880 	return (error);
1881 }
1882 
1883 
1884 /*
1885  * select(2) and poll(2) handlers for the "netmap" device.
1886  *
1887  * Can be called for one or more queues.
1888  * Return true the event mask corresponding to ready events.
1889  * If there are no ready events, do a selrecord on either individual
1890  * selfd or on the global one.
1891  * Device-dependent parts (locking and sync of tx/rx rings)
1892  * are done through callbacks.
1893  *
1894  * On linux, arguments are really pwait, the poll table, and 'td' is struct file *
1895  * The first one is remapped to pwait as selrecord() uses the name as an
1896  * hidden argument.
1897  */
1898 static int
1899 netmap_poll(struct cdev *dev, int events, struct thread *td)
1900 {
1901 	struct netmap_priv_d *priv = NULL;
1902 	struct netmap_adapter *na;
1903 	struct ifnet *ifp;
1904 	struct netmap_kring *kring;
1905 	u_int core_lock, i, check_all, want_tx, want_rx, revents = 0;
1906 	u_int lim_tx, lim_rx, host_forwarded = 0;
1907 	struct mbq q = { NULL, NULL, 0 };
1908 	enum {NO_CL, NEED_CL, LOCKED_CL }; /* see below */
1909 	void *pwait = dev;	/* linux compatibility */
1910 
1911 	(void)pwait;
1912 
1913 	if (devfs_get_cdevpriv((void **)&priv) != 0 || priv == NULL)
1914 		return POLLERR;
1915 
1916 	if (priv->np_nifp == NULL) {
1917 		D("No if registered");
1918 		return POLLERR;
1919 	}
1920 	rmb(); /* make sure following reads are not from cache */
1921 
1922 	ifp = priv->np_ifp;
1923 	// XXX check for deleting() ?
1924 	if ( (ifp->if_capenable & IFCAP_NETMAP) == 0)
1925 		return POLLERR;
1926 
1927 	if (netmap_verbose & 0x8000)
1928 		D("device %s events 0x%x", ifp->if_xname, events);
1929 	want_tx = events & (POLLOUT | POLLWRNORM);
1930 	want_rx = events & (POLLIN | POLLRDNORM);
1931 
1932 	na = NA(ifp); /* retrieve netmap adapter */
1933 
1934 	lim_tx = na->num_tx_rings;
1935 	lim_rx = na->num_rx_rings;
1936 	/* how many queues we are scanning */
1937 	if (priv->np_qfirst == NETMAP_SW_RING) {
1938 		if (priv->np_txpoll || want_tx) {
1939 			/* push any packets up, then we are always ready */
1940 			netmap_sync_to_host(na);
1941 			revents |= want_tx;
1942 		}
1943 		if (want_rx) {
1944 			kring = &na->rx_rings[lim_rx];
1945 			if (kring->ring->avail == 0)
1946 				netmap_sync_from_host(na, td, dev);
1947 			if (kring->ring->avail > 0) {
1948 				revents |= want_rx;
1949 			}
1950 		}
1951 		return (revents);
1952 	}
1953 
1954 	/* if we are in transparent mode, check also the host rx ring */
1955 	kring = &na->rx_rings[lim_rx];
1956 	if ( (priv->np_qlast == NETMAP_HW_RING) // XXX check_all
1957 			&& want_rx
1958 			&& (netmap_fwd || kring->ring->flags & NR_FORWARD) ) {
1959 		if (kring->ring->avail == 0)
1960 			netmap_sync_from_host(na, td, dev);
1961 		if (kring->ring->avail > 0)
1962 			revents |= want_rx;
1963 	}
1964 
1965 	/*
1966 	 * check_all is set if the card has more than one queue and
1967 	 * the client is polling all of them. If true, we sleep on
1968 	 * the "global" selfd, otherwise we sleep on individual selfd
1969 	 * (we can only sleep on one of them per direction).
1970 	 * The interrupt routine in the driver should always wake on
1971 	 * the individual selfd, and also on the global one if the card
1972 	 * has more than one ring.
1973 	 *
1974 	 * If the card has only one lock, we just use that.
1975 	 * If the card has separate ring locks, we just use those
1976 	 * unless we are doing check_all, in which case the whole
1977 	 * loop is wrapped by the global lock.
1978 	 * We acquire locks only when necessary: if poll is called
1979 	 * when buffers are available, we can just return without locks.
1980 	 *
1981 	 * rxsync() is only called if we run out of buffers on a POLLIN.
1982 	 * txsync() is called if we run out of buffers on POLLOUT, or
1983 	 * there are pending packets to send. The latter can be disabled
1984 	 * passing NETMAP_NO_TX_POLL in the NIOCREG call.
1985 	 */
1986 	check_all = (priv->np_qlast == NETMAP_HW_RING) && (lim_tx > 1 || lim_rx > 1);
1987 
1988 	/*
1989 	 * core_lock indicates what to do with the core lock.
1990 	 * The core lock is used when either the card has no individual
1991 	 * locks, or it has individual locks but we are cheking all
1992 	 * rings so we need the core lock to avoid missing wakeup events.
1993 	 *
1994 	 * It has three possible states:
1995 	 * NO_CL	we don't need to use the core lock, e.g.
1996 	 *		because we are protected by individual locks.
1997 	 * NEED_CL	we need the core lock. In this case, when we
1998 	 *		call the lock routine, move to LOCKED_CL
1999 	 *		to remember to release the lock once done.
2000 	 * LOCKED_CL	core lock is set, so we need to release it.
2001 	 */
2002 	core_lock = (check_all || !na->separate_locks) ? NEED_CL : NO_CL;
2003 #ifdef NM_BRIDGE
2004 	/* the bridge uses separate locks */
2005 	if (na->nm_register == bdg_netmap_reg) {
2006 		ND("not using core lock for %s", ifp->if_xname);
2007 		core_lock = NO_CL;
2008 	}
2009 #endif /* NM_BRIDGE */
2010 	if (priv->np_qlast != NETMAP_HW_RING) {
2011 		lim_tx = lim_rx = priv->np_qlast;
2012 	}
2013 
2014 	/*
2015 	 * We start with a lock free round which is good if we have
2016 	 * data available. If this fails, then lock and call the sync
2017 	 * routines.
2018 	 */
2019 	for (i = priv->np_qfirst; want_rx && i < lim_rx; i++) {
2020 		kring = &na->rx_rings[i];
2021 		if (kring->ring->avail > 0) {
2022 			revents |= want_rx;
2023 			want_rx = 0;	/* also breaks the loop */
2024 		}
2025 	}
2026 	for (i = priv->np_qfirst; want_tx && i < lim_tx; i++) {
2027 		kring = &na->tx_rings[i];
2028 		if (kring->ring->avail > 0) {
2029 			revents |= want_tx;
2030 			want_tx = 0;	/* also breaks the loop */
2031 		}
2032 	}
2033 
2034 	/*
2035 	 * If we to push packets out (priv->np_txpoll) or want_tx is
2036 	 * still set, we do need to run the txsync calls (on all rings,
2037 	 * to avoid that the tx rings stall).
2038 	 */
2039 	if (priv->np_txpoll || want_tx) {
2040 flush_tx:
2041 		for (i = priv->np_qfirst; i < lim_tx; i++) {
2042 			kring = &na->tx_rings[i];
2043 			/*
2044 			 * Skip the current ring if want_tx == 0
2045 			 * (we have already done a successful sync on
2046 			 * a previous ring) AND kring->cur == kring->hwcur
2047 			 * (there are no pending transmissions for this ring).
2048 			 */
2049 			if (!want_tx && kring->ring->cur == kring->nr_hwcur)
2050 				continue;
2051 			if (core_lock == NEED_CL) {
2052 				na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
2053 				core_lock = LOCKED_CL;
2054 			}
2055 			if (na->separate_locks)
2056 				na->nm_lock(ifp, NETMAP_TX_LOCK, i);
2057 			if (netmap_verbose & NM_VERB_TXSYNC)
2058 				D("send %d on %s %d",
2059 					kring->ring->cur,
2060 					ifp->if_xname, i);
2061 			if (na->nm_txsync(ifp, i, 0 /* no lock */))
2062 				revents |= POLLERR;
2063 
2064 			/* Check avail/call selrecord only if called with POLLOUT */
2065 			if (want_tx) {
2066 				if (kring->ring->avail > 0) {
2067 					/* stop at the first ring. We don't risk
2068 					 * starvation.
2069 					 */
2070 					revents |= want_tx;
2071 					want_tx = 0;
2072 				} else if (!check_all)
2073 					selrecord(td, &kring->si);
2074 			}
2075 			if (na->separate_locks)
2076 				na->nm_lock(ifp, NETMAP_TX_UNLOCK, i);
2077 		}
2078 	}
2079 
2080 	/*
2081 	 * now if want_rx is still set we need to lock and rxsync.
2082 	 * Do it on all rings because otherwise we starve.
2083 	 */
2084 	if (want_rx) {
2085 		for (i = priv->np_qfirst; i < lim_rx; i++) {
2086 			kring = &na->rx_rings[i];
2087 			if (core_lock == NEED_CL) {
2088 				na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
2089 				core_lock = LOCKED_CL;
2090 			}
2091 			if (na->separate_locks)
2092 				na->nm_lock(ifp, NETMAP_RX_LOCK, i);
2093 			if (netmap_fwd ||kring->ring->flags & NR_FORWARD) {
2094 				ND(10, "forwarding some buffers up %d to %d",
2095 				    kring->nr_hwcur, kring->ring->cur);
2096 				netmap_grab_packets(kring, &q, netmap_fwd);
2097 			}
2098 
2099 			if (na->nm_rxsync(ifp, i, 0 /* no lock */))
2100 				revents |= POLLERR;
2101 			if (netmap_no_timestamp == 0 ||
2102 					kring->ring->flags & NR_TIMESTAMP) {
2103 				microtime(&kring->ring->ts);
2104 			}
2105 
2106 			if (kring->ring->avail > 0)
2107 				revents |= want_rx;
2108 			else if (!check_all)
2109 				selrecord(td, &kring->si);
2110 			if (na->separate_locks)
2111 				na->nm_lock(ifp, NETMAP_RX_UNLOCK, i);
2112 		}
2113 	}
2114 	if (check_all && revents == 0) { /* signal on the global queue */
2115 		if (want_tx)
2116 			selrecord(td, &na->tx_si);
2117 		if (want_rx)
2118 			selrecord(td, &na->rx_si);
2119 	}
2120 
2121 	/* forward host to the netmap ring */
2122 	kring = &na->rx_rings[lim_rx];
2123 	if (kring->nr_hwavail > 0)
2124 		ND("host rx %d has %d packets", lim_rx, kring->nr_hwavail);
2125 	if ( (priv->np_qlast == NETMAP_HW_RING) // XXX check_all
2126 			&& (netmap_fwd || kring->ring->flags & NR_FORWARD)
2127 			 && kring->nr_hwavail > 0 && !host_forwarded) {
2128 		if (core_lock == NEED_CL) {
2129 			na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
2130 			core_lock = LOCKED_CL;
2131 		}
2132 		netmap_sw_to_nic(na);
2133 		host_forwarded = 1; /* prevent another pass */
2134 		want_rx = 0;
2135 		goto flush_tx;
2136 	}
2137 
2138 	if (core_lock == LOCKED_CL)
2139 		na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
2140 	if (q.head)
2141 		netmap_send_up(na->ifp, q.head);
2142 
2143 	return (revents);
2144 }
2145 
2146 /*------- driver support routines ------*/
2147 
2148 
2149 /*
2150  * default lock wrapper.
2151  */
2152 static void
2153 netmap_lock_wrapper(struct ifnet *dev, int what, u_int queueid)
2154 {
2155 	struct netmap_adapter *na = NA(dev);
2156 
2157 	switch (what) {
2158 #ifdef linux	/* some system do not need lock on register */
2159 	case NETMAP_REG_LOCK:
2160 	case NETMAP_REG_UNLOCK:
2161 		break;
2162 #endif /* linux */
2163 
2164 	case NETMAP_CORE_LOCK:
2165 		mtx_lock(&na->core_lock);
2166 		break;
2167 
2168 	case NETMAP_CORE_UNLOCK:
2169 		mtx_unlock(&na->core_lock);
2170 		break;
2171 
2172 	case NETMAP_TX_LOCK:
2173 		mtx_lock(&na->tx_rings[queueid].q_lock);
2174 		break;
2175 
2176 	case NETMAP_TX_UNLOCK:
2177 		mtx_unlock(&na->tx_rings[queueid].q_lock);
2178 		break;
2179 
2180 	case NETMAP_RX_LOCK:
2181 		mtx_lock(&na->rx_rings[queueid].q_lock);
2182 		break;
2183 
2184 	case NETMAP_RX_UNLOCK:
2185 		mtx_unlock(&na->rx_rings[queueid].q_lock);
2186 		break;
2187 	}
2188 }
2189 
2190 
2191 /*
2192  * Initialize a ``netmap_adapter`` object created by driver on attach.
2193  * We allocate a block of memory with room for a struct netmap_adapter
2194  * plus two sets of N+2 struct netmap_kring (where N is the number
2195  * of hardware rings):
2196  * krings	0..N-1	are for the hardware queues.
2197  * kring	N	is for the host stack queue
2198  * kring	N+1	is only used for the selinfo for all queues.
2199  * Return 0 on success, ENOMEM otherwise.
2200  *
2201  * By default the receive and transmit adapter ring counts are both initialized
2202  * to num_queues.  na->num_tx_rings can be set for cards with different tx/rx
2203  * setups.
2204  */
2205 int
2206 netmap_attach(struct netmap_adapter *arg, int num_queues)
2207 {
2208 	struct netmap_adapter *na = NULL;
2209 	struct ifnet *ifp = arg ? arg->ifp : NULL;
2210 	int len;
2211 
2212 	if (arg == NULL || ifp == NULL)
2213 		goto fail;
2214 	len = nma_is_vp(arg) ? sizeof(*na) : sizeof(*na) * 2;
2215 	na = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO);
2216 	if (na == NULL)
2217 		goto fail;
2218 	WNA(ifp) = na;
2219 	*na = *arg; /* copy everything, trust the driver to not pass junk */
2220 	NETMAP_SET_CAPABLE(ifp);
2221 	if (na->num_tx_rings == 0)
2222 		na->num_tx_rings = num_queues;
2223 	na->num_rx_rings = num_queues;
2224 	na->refcount = na->na_single = na->na_multi = 0;
2225 	/* Core lock initialized here, others after netmap_if_new. */
2226 	mtx_init(&na->core_lock, "netmap core lock", MTX_NETWORK_LOCK, MTX_DEF);
2227 	if (na->nm_lock == NULL) {
2228 		ND("using default locks for %s", ifp->if_xname);
2229 		na->nm_lock = netmap_lock_wrapper;
2230 	}
2231 #ifdef linux
2232 	if (ifp->netdev_ops) {
2233 		ND("netdev_ops %p", ifp->netdev_ops);
2234 		/* prepare a clone of the netdev ops */
2235 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28)
2236 		na->nm_ndo.ndo_start_xmit = ifp->netdev_ops;
2237 #else
2238 		na->nm_ndo = *ifp->netdev_ops;
2239 #endif
2240 	}
2241 	na->nm_ndo.ndo_start_xmit = linux_netmap_start;
2242 #endif
2243 	if (!nma_is_vp(arg))
2244 		netmap_attach_sw(ifp);
2245 	D("success for %s", ifp->if_xname);
2246 	return 0;
2247 
2248 fail:
2249 	D("fail, arg %p ifp %p na %p", arg, ifp, na);
2250 	netmap_detach(ifp);
2251 	return (na ? EINVAL : ENOMEM);
2252 }
2253 
2254 
2255 /*
2256  * Free the allocated memory linked to the given ``netmap_adapter``
2257  * object.
2258  */
2259 void
2260 netmap_detach(struct ifnet *ifp)
2261 {
2262 	struct netmap_adapter *na = NA(ifp);
2263 
2264 	if (!na)
2265 		return;
2266 
2267 	mtx_destroy(&na->core_lock);
2268 
2269 	if (na->tx_rings) { /* XXX should not happen */
2270 		D("freeing leftover tx_rings");
2271 		free(na->tx_rings, M_DEVBUF);
2272 	}
2273 	bzero(na, sizeof(*na));
2274 	WNA(ifp) = NULL;
2275 	free(na, M_DEVBUF);
2276 }
2277 
2278 
2279 int
2280 nm_bdg_flush(struct nm_bdg_fwd *ft, int n, struct netmap_adapter *na, u_int ring_nr);
2281 
2282 /* we don't need to lock myself */
2283 static int
2284 bdg_netmap_start(struct ifnet *ifp, struct mbuf *m)
2285 {
2286 	struct netmap_adapter *na = SWNA(ifp);
2287 	struct nm_bdg_fwd *ft = na->rx_rings[0].nkr_ft;
2288 	char *buf = NMB(&na->rx_rings[0].ring->slot[0]);
2289 	u_int len = MBUF_LEN(m);
2290 
2291 	if (!na->na_bdg) /* SWNA is not configured to be attached */
2292 		return EBUSY;
2293 	m_copydata(m, 0, len, buf);
2294 	ft->ft_flags = 0;	// XXX could be indirect ?
2295 	ft->ft_len = len;
2296 	ft->ft_buf = buf;
2297 	ft->ft_next = NM_BDG_BATCH; // XXX is it needed ?
2298 	nm_bdg_flush(ft, 1, na, 0);
2299 
2300 	/* release the mbuf in either cases of success or failure. As an
2301 	 * alternative, put the mbuf in a free list and free the list
2302 	 * only when really necessary.
2303 	 */
2304 	m_freem(m);
2305 
2306 	return (0);
2307 }
2308 
2309 
2310 /*
2311  * Intercept packets from the network stack and pass them
2312  * to netmap as incoming packets on the 'software' ring.
2313  * We are not locked when called.
2314  */
2315 int
2316 netmap_start(struct ifnet *ifp, struct mbuf *m)
2317 {
2318 	struct netmap_adapter *na = NA(ifp);
2319 	struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
2320 	u_int i, len = MBUF_LEN(m);
2321 	u_int error = EBUSY, lim = kring->nkr_num_slots - 1;
2322 	struct netmap_slot *slot;
2323 
2324 	if (netmap_verbose & NM_VERB_HOST)
2325 		D("%s packet %d len %d from the stack", ifp->if_xname,
2326 			kring->nr_hwcur + kring->nr_hwavail, len);
2327 	if (len > NETMAP_BUF_SIZE) { /* too long for us */
2328 		D("%s from_host, drop packet size %d > %d", ifp->if_xname,
2329 			len, NETMAP_BUF_SIZE);
2330 		m_freem(m);
2331 		return EINVAL;
2332 	}
2333 	if (na->na_bdg)
2334 		return bdg_netmap_start(ifp, m);
2335 
2336 	na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
2337 	if (kring->nr_hwavail >= lim) {
2338 		if (netmap_verbose)
2339 			D("stack ring %s full\n", ifp->if_xname);
2340 		goto done;	/* no space */
2341 	}
2342 
2343 	/* compute the insert position */
2344 	i = kring->nr_hwcur + kring->nr_hwavail;
2345 	if (i > lim)
2346 		i -= lim + 1;
2347 	slot = &kring->ring->slot[i];
2348 	m_copydata(m, 0, len, NMB(slot));
2349 	slot->len = len;
2350 	slot->flags = kring->nkr_slot_flags;
2351 	kring->nr_hwavail++;
2352 	if (netmap_verbose  & NM_VERB_HOST)
2353 		D("wake up host ring %s %d", na->ifp->if_xname, na->num_rx_rings);
2354 	selwakeuppri(&kring->si, PI_NET);
2355 	error = 0;
2356 done:
2357 	na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
2358 
2359 	/* release the mbuf in either cases of success or failure. As an
2360 	 * alternative, put the mbuf in a free list and free the list
2361 	 * only when really necessary.
2362 	 */
2363 	m_freem(m);
2364 
2365 	return (error);
2366 }
2367 
2368 
2369 /*
2370  * netmap_reset() is called by the driver routines when reinitializing
2371  * a ring. The driver is in charge of locking to protect the kring.
2372  * If netmap mode is not set just return NULL.
2373  */
2374 struct netmap_slot *
2375 netmap_reset(struct netmap_adapter *na, enum txrx tx, int n,
2376 	u_int new_cur)
2377 {
2378 	struct netmap_kring *kring;
2379 	int new_hwofs, lim;
2380 
2381 	if (na == NULL)
2382 		return NULL;	/* no netmap support here */
2383 	if (!(na->ifp->if_capenable & IFCAP_NETMAP))
2384 		return NULL;	/* nothing to reinitialize */
2385 
2386 	if (tx == NR_TX) {
2387 		if (n >= na->num_tx_rings)
2388 			return NULL;
2389 		kring = na->tx_rings + n;
2390 		new_hwofs = kring->nr_hwcur - new_cur;
2391 	} else {
2392 		if (n >= na->num_rx_rings)
2393 			return NULL;
2394 		kring = na->rx_rings + n;
2395 		new_hwofs = kring->nr_hwcur + kring->nr_hwavail - new_cur;
2396 	}
2397 	lim = kring->nkr_num_slots - 1;
2398 	if (new_hwofs > lim)
2399 		new_hwofs -= lim + 1;
2400 
2401 	/* Alwayws set the new offset value and realign the ring. */
2402 	kring->nkr_hwofs = new_hwofs;
2403 	if (tx == NR_TX)
2404 		kring->nr_hwavail = kring->nkr_num_slots - 1;
2405 	ND(10, "new hwofs %d on %s %s[%d]",
2406 			kring->nkr_hwofs, na->ifp->if_xname,
2407 			tx == NR_TX ? "TX" : "RX", n);
2408 
2409 #if 0 // def linux
2410 	/* XXX check that the mappings are correct */
2411 	/* need ring_nr, adapter->pdev, direction */
2412 	buffer_info->dma = dma_map_single(&pdev->dev, addr, adapter->rx_buffer_len, DMA_FROM_DEVICE);
2413 	if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) {
2414 		D("error mapping rx netmap buffer %d", i);
2415 		// XXX fix error handling
2416 	}
2417 
2418 #endif /* linux */
2419 	/*
2420 	 * Wakeup on the individual and global lock
2421 	 * We do the wakeup here, but the ring is not yet reconfigured.
2422 	 * However, we are under lock so there are no races.
2423 	 */
2424 	selwakeuppri(&kring->si, PI_NET);
2425 	selwakeuppri(tx == NR_TX ? &na->tx_si : &na->rx_si, PI_NET);
2426 	return kring->ring->slot;
2427 }
2428 
2429 
2430 /* returns the next position in the ring */
2431 static int
2432 nm_bdg_preflush(struct netmap_adapter *na, u_int ring_nr,
2433 	struct netmap_kring *kring, u_int end)
2434 {
2435 	struct netmap_ring *ring = kring->ring;
2436 	struct nm_bdg_fwd *ft = kring->nkr_ft;
2437 	u_int j = kring->nr_hwcur, lim = kring->nkr_num_slots - 1;
2438 	u_int ft_i = 0;	/* start from 0 */
2439 
2440 	for (; likely(j != end); j = unlikely(j == lim) ? 0 : j+1) {
2441 		struct netmap_slot *slot = &ring->slot[j];
2442 		char *buf = NMB(slot);
2443 		int len = ft[ft_i].ft_len = slot->len;
2444 
2445 		ft[ft_i].ft_flags = slot->flags;
2446 
2447 		ND("flags is 0x%x", slot->flags);
2448 		/* this slot goes into a list so initialize the link field */
2449 		ft[ft_i].ft_next = NM_BDG_BATCH; /* equivalent to NULL */
2450 		if (unlikely(len < 14))
2451 			continue;
2452 		buf = ft[ft_i].ft_buf = (slot->flags & NS_INDIRECT) ?
2453 			*((void **)buf) : buf;
2454 		prefetch(buf);
2455 		if (unlikely(++ft_i == netmap_bridge))
2456 			ft_i = nm_bdg_flush(ft, ft_i, na, ring_nr);
2457 	}
2458 	if (ft_i)
2459 		ft_i = nm_bdg_flush(ft, ft_i, na, ring_nr);
2460 	return j;
2461 }
2462 
2463 
2464 /*
2465  * Pass packets from nic to the bridge. Must be called with
2466  * proper locks on the source interface.
2467  * Note, no user process can access this NIC so we can ignore
2468  * the info in the 'ring'.
2469  */
2470 static void
2471 netmap_nic_to_bdg(struct ifnet *ifp, u_int ring_nr)
2472 {
2473 	struct netmap_adapter *na = NA(ifp);
2474 	struct netmap_kring *kring = &na->rx_rings[ring_nr];
2475 	struct netmap_ring *ring = kring->ring;
2476 	int j, k, lim = kring->nkr_num_slots - 1;
2477 
2478 	/* fetch packets that have arrived */
2479 	na->nm_rxsync(ifp, ring_nr, 0);
2480 	/* XXX we don't count reserved, but it should be 0 */
2481 	j = kring->nr_hwcur;
2482 	k = j + kring->nr_hwavail;
2483 	if (k > lim)
2484 		k -= lim + 1;
2485 	if (k == j && netmap_verbose) {
2486 		D("how strange, interrupt with no packets on %s",
2487 			ifp->if_xname);
2488 		return;
2489 	}
2490 
2491 	j = nm_bdg_preflush(na, ring_nr, kring, k);
2492 
2493 	/* we consume everything, but we cannot update kring directly
2494 	 * because the nic may have destroyed the info in the NIC ring.
2495 	 * So we need to call rxsync again to restore it.
2496 	 */
2497 	ring->cur = j;
2498 	ring->avail = 0;
2499 	na->nm_rxsync(ifp, ring_nr, 0);
2500 	return;
2501 }
2502 
2503 
2504 /*
2505  * Default functions to handle rx/tx interrupts
2506  * we have 4 cases:
2507  * 1 ring, single lock:
2508  *	lock(core); wake(i=0); unlock(core)
2509  * N rings, single lock:
2510  *	lock(core); wake(i); wake(N+1) unlock(core)
2511  * 1 ring, separate locks: (i=0)
2512  *	lock(i); wake(i); unlock(i)
2513  * N rings, separate locks:
2514  *	lock(i); wake(i); unlock(i); lock(core) wake(N+1) unlock(core)
2515  * work_done is non-null on the RX path.
2516  *
2517  * The 'q' argument also includes flag to tell whether the queue is
2518  * already locked on enter, and whether it should remain locked on exit.
2519  * This helps adapting to different defaults in drivers and OSes.
2520  */
2521 int
2522 netmap_rx_irq(struct ifnet *ifp, int q, int *work_done)
2523 {
2524 	struct netmap_adapter *na;
2525 	struct netmap_kring *r;
2526 	NM_SELINFO_T *main_wq;
2527 	int locktype, unlocktype, nic_to_bridge, lock;
2528 
2529 	if (!(ifp->if_capenable & IFCAP_NETMAP))
2530 		return 0;
2531 
2532 	lock = q & (NETMAP_LOCKED_ENTER | NETMAP_LOCKED_EXIT);
2533 	q = q & NETMAP_RING_MASK;
2534 
2535 	ND(5, "received %s queue %d", work_done ? "RX" : "TX" , q);
2536 	na = NA(ifp);
2537 	if (na->na_flags & NAF_SKIP_INTR) {
2538 		ND("use regular interrupt");
2539 		return 0;
2540 	}
2541 
2542 	if (work_done) { /* RX path */
2543 		if (q >= na->num_rx_rings)
2544 			return 0;	// not a physical queue
2545 		r = na->rx_rings + q;
2546 		r->nr_kflags |= NKR_PENDINTR;
2547 		main_wq = (na->num_rx_rings > 1) ? &na->rx_si : NULL;
2548 		/* set a flag if the NIC is attached to a VALE switch */
2549 		nic_to_bridge = (na->na_bdg != NULL);
2550 		locktype = NETMAP_RX_LOCK;
2551 		unlocktype = NETMAP_RX_UNLOCK;
2552 	} else { /* TX path */
2553 		if (q >= na->num_tx_rings)
2554 			return 0;	// not a physical queue
2555 		r = na->tx_rings + q;
2556 		main_wq = (na->num_tx_rings > 1) ? &na->tx_si : NULL;
2557 		work_done = &q; /* dummy */
2558 		nic_to_bridge = 0;
2559 		locktype = NETMAP_TX_LOCK;
2560 		unlocktype = NETMAP_TX_UNLOCK;
2561 	}
2562 	if (na->separate_locks) {
2563 		if (!(lock & NETMAP_LOCKED_ENTER))
2564 			na->nm_lock(ifp, locktype, q);
2565 		/* If a NIC is attached to a bridge, flush packets
2566 		 * (and no need to wakeup anyone). Otherwise, wakeup
2567 		 * possible processes waiting for packets.
2568 		 */
2569 		if (nic_to_bridge)
2570 			netmap_nic_to_bdg(ifp, q);
2571 		else
2572 			selwakeuppri(&r->si, PI_NET);
2573 		na->nm_lock(ifp, unlocktype, q);
2574 		if (main_wq && !nic_to_bridge) {
2575 			na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
2576 			selwakeuppri(main_wq, PI_NET);
2577 			na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
2578 		}
2579 		/* lock the queue again if requested */
2580 		if (lock & NETMAP_LOCKED_EXIT)
2581 			na->nm_lock(ifp, locktype, q);
2582 	} else {
2583 		if (!(lock & NETMAP_LOCKED_ENTER))
2584 			na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
2585 		if (nic_to_bridge)
2586 			netmap_nic_to_bdg(ifp, q);
2587 		else {
2588 			selwakeuppri(&r->si, PI_NET);
2589 			if (main_wq)
2590 				selwakeuppri(main_wq, PI_NET);
2591 		}
2592 		if (!(lock & NETMAP_LOCKED_EXIT))
2593 			na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
2594 	}
2595 	*work_done = 1; /* do not fire napi again */
2596 	return 1;
2597 }
2598 
2599 
2600 #ifdef linux	/* linux-specific routines */
2601 
2602 
2603 /*
2604  * Remap linux arguments into the FreeBSD call.
2605  * - pwait is the poll table, passed as 'dev';
2606  *   If pwait == NULL someone else already woke up before. We can report
2607  *   events but they are filtered upstream.
2608  *   If pwait != NULL, then pwait->key contains the list of events.
2609  * - events is computed from pwait as above.
2610  * - file is passed as 'td';
2611  */
2612 static u_int
2613 linux_netmap_poll(struct file * file, struct poll_table_struct *pwait)
2614 {
2615 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,28)
2616 	int events = POLLIN | POLLOUT; /* XXX maybe... */
2617 #elif LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0)
2618 	int events = pwait ? pwait->key : POLLIN | POLLOUT;
2619 #else /* in 3.4.0 field 'key' was renamed to '_key' */
2620 	int events = pwait ? pwait->_key : POLLIN | POLLOUT;
2621 #endif
2622 	return netmap_poll((void *)pwait, events, (void *)file);
2623 }
2624 
2625 
2626 static int
2627 linux_netmap_mmap(struct file *f, struct vm_area_struct *vma)
2628 {
2629 	int lut_skip, i, j;
2630 	int user_skip = 0;
2631 	struct lut_entry *l_entry;
2632 	int error = 0;
2633 	unsigned long off, tomap;
2634 	/*
2635 	 * vma->vm_start: start of mapping user address space
2636 	 * vma->vm_end: end of the mapping user address space
2637 	 * vma->vm_pfoff: offset of first page in the device
2638 	 */
2639 
2640 	// XXX security checks
2641 
2642 	error = netmap_get_memory(f->private_data);
2643 	ND("get_memory returned %d", error);
2644 	if (error)
2645 	    return -error;
2646 
2647 	off = vma->vm_pgoff << PAGE_SHIFT; /* offset in bytes */
2648 	tomap = vma->vm_end - vma->vm_start;
2649 	for (i = 0; i < NETMAP_POOLS_NR; i++) {  /* loop through obj_pools */
2650 		const struct netmap_obj_pool *p = &nm_mem.pools[i];
2651 		/*
2652 		 * In each pool memory is allocated in clusters
2653 		 * of size _clustsize, each containing clustentries
2654 		 * entries. For each object k we already store the
2655 		 * vtophys mapping in lut[k] so we use that, scanning
2656 		 * the lut[] array in steps of clustentries,
2657 		 * and we map each cluster (not individual pages,
2658 		 * it would be overkill -- XXX slow ? 20130415).
2659 		 */
2660 
2661 		/*
2662 		 * We interpret vm_pgoff as an offset into the whole
2663 		 * netmap memory, as if all clusters where contiguous.
2664 		 */
2665 		for (lut_skip = 0, j = 0; j < p->_numclusters; j++, lut_skip += p->clustentries) {
2666 			unsigned long paddr, mapsize;
2667 			if (p->_clustsize <= off) {
2668 				off -= p->_clustsize;
2669 				continue;
2670 			}
2671 			l_entry = &p->lut[lut_skip]; /* first obj in the cluster */
2672 			paddr = l_entry->paddr + off;
2673 			mapsize = p->_clustsize - off;
2674 			off = 0;
2675 			if (mapsize > tomap)
2676 				mapsize = tomap;
2677 			ND("remap_pfn_range(%lx, %lx, %lx)",
2678 				vma->vm_start + user_skip,
2679 				paddr >> PAGE_SHIFT, mapsize);
2680 			if (remap_pfn_range(vma, vma->vm_start + user_skip,
2681 					paddr >> PAGE_SHIFT, mapsize,
2682 					vma->vm_page_prot))
2683 				return -EAGAIN; // XXX check return value
2684 			user_skip += mapsize;
2685 			tomap -= mapsize;
2686 			if (tomap == 0)
2687 				goto done;
2688 		}
2689 	}
2690 done:
2691 
2692 	return 0;
2693 }
2694 
2695 
2696 static netdev_tx_t
2697 linux_netmap_start(struct sk_buff *skb, struct net_device *dev)
2698 {
2699 	netmap_start(dev, skb);
2700 	return (NETDEV_TX_OK);
2701 }
2702 
2703 
2704 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37)	// XXX was 38
2705 #define LIN_IOCTL_NAME	.ioctl
2706 int
2707 linux_netmap_ioctl(struct inode *inode, struct file *file, u_int cmd, u_long data /* arg */)
2708 #else
2709 #define LIN_IOCTL_NAME	.unlocked_ioctl
2710 long
2711 linux_netmap_ioctl(struct file *file, u_int cmd, u_long data /* arg */)
2712 #endif
2713 {
2714 	int ret;
2715 	struct nmreq nmr;
2716 	bzero(&nmr, sizeof(nmr));
2717 
2718 	if (data && copy_from_user(&nmr, (void *)data, sizeof(nmr) ) != 0)
2719 		return -EFAULT;
2720 	ret = netmap_ioctl(NULL, cmd, (caddr_t)&nmr, 0, (void *)file);
2721 	if (data && copy_to_user((void*)data, &nmr, sizeof(nmr) ) != 0)
2722 		return -EFAULT;
2723 	return -ret;
2724 }
2725 
2726 
2727 static int
2728 netmap_release(struct inode *inode, struct file *file)
2729 {
2730 	(void)inode;	/* UNUSED */
2731 	if (file->private_data)
2732 		netmap_dtor(file->private_data);
2733 	return (0);
2734 }
2735 
2736 
2737 static int
2738 linux_netmap_open(struct inode *inode, struct file *file)
2739 {
2740 	struct netmap_priv_d *priv;
2741 	(void)inode;	/* UNUSED */
2742 
2743 	priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF,
2744 			      M_NOWAIT | M_ZERO);
2745 	if (priv == NULL)
2746 		return -ENOMEM;
2747 
2748 	file->private_data = priv;
2749 
2750 	return (0);
2751 }
2752 
2753 
2754 static struct file_operations netmap_fops = {
2755     .owner = THIS_MODULE,
2756     .open = linux_netmap_open,
2757     .mmap = linux_netmap_mmap,
2758     LIN_IOCTL_NAME = linux_netmap_ioctl,
2759     .poll = linux_netmap_poll,
2760     .release = netmap_release,
2761 };
2762 
2763 
2764 static struct miscdevice netmap_cdevsw = {	/* same name as FreeBSD */
2765 	MISC_DYNAMIC_MINOR,
2766 	"netmap",
2767 	&netmap_fops,
2768 };
2769 
2770 static int netmap_init(void);
2771 static void netmap_fini(void);
2772 
2773 
2774 /* Errors have negative values on linux */
2775 static int linux_netmap_init(void)
2776 {
2777 	return -netmap_init();
2778 }
2779 
2780 module_init(linux_netmap_init);
2781 module_exit(netmap_fini);
2782 /* export certain symbols to other modules */
2783 EXPORT_SYMBOL(netmap_attach);		// driver attach routines
2784 EXPORT_SYMBOL(netmap_detach);		// driver detach routines
2785 EXPORT_SYMBOL(netmap_ring_reinit);	// ring init on error
2786 EXPORT_SYMBOL(netmap_buffer_lut);
2787 EXPORT_SYMBOL(netmap_total_buffers);	// index check
2788 EXPORT_SYMBOL(netmap_buffer_base);
2789 EXPORT_SYMBOL(netmap_reset);		// ring init routines
2790 EXPORT_SYMBOL(netmap_buf_size);
2791 EXPORT_SYMBOL(netmap_rx_irq);		// default irq handler
2792 EXPORT_SYMBOL(netmap_no_pendintr);	// XXX mitigation - should go away
2793 EXPORT_SYMBOL(netmap_bdg_ctl);		// bridge configuration routine
2794 EXPORT_SYMBOL(netmap_bdg_learning);	// the default lookup function
2795 
2796 
2797 MODULE_AUTHOR("http://info.iet.unipi.it/~luigi/netmap/");
2798 MODULE_DESCRIPTION("The netmap packet I/O framework");
2799 MODULE_LICENSE("Dual BSD/GPL"); /* the code here is all BSD. */
2800 
2801 #else /* __FreeBSD__ */
2802 
2803 
2804 static struct cdevsw netmap_cdevsw = {
2805 	.d_version = D_VERSION,
2806 	.d_name = "netmap",
2807 	.d_open = netmap_open,
2808 	.d_mmap = netmap_mmap,
2809 	.d_mmap_single = netmap_mmap_single,
2810 	.d_ioctl = netmap_ioctl,
2811 	.d_poll = netmap_poll,
2812 	.d_close = netmap_close,
2813 };
2814 #endif /* __FreeBSD__ */
2815 
2816 #ifdef NM_BRIDGE
2817 /*
2818  *---- support for virtual bridge -----
2819  */
2820 
2821 /* ----- FreeBSD if_bridge hash function ------- */
2822 
2823 /*
2824  * The following hash function is adapted from "Hash Functions" by Bob Jenkins
2825  * ("Algorithm Alley", Dr. Dobbs Journal, September 1997).
2826  *
2827  * http://www.burtleburtle.net/bob/hash/spooky.html
2828  */
2829 #define mix(a, b, c)                                                    \
2830 do {                                                                    \
2831         a -= b; a -= c; a ^= (c >> 13);                                 \
2832         b -= c; b -= a; b ^= (a << 8);                                  \
2833         c -= a; c -= b; c ^= (b >> 13);                                 \
2834         a -= b; a -= c; a ^= (c >> 12);                                 \
2835         b -= c; b -= a; b ^= (a << 16);                                 \
2836         c -= a; c -= b; c ^= (b >> 5);                                  \
2837         a -= b; a -= c; a ^= (c >> 3);                                  \
2838         b -= c; b -= a; b ^= (a << 10);                                 \
2839         c -= a; c -= b; c ^= (b >> 15);                                 \
2840 } while (/*CONSTCOND*/0)
2841 
2842 static __inline uint32_t
2843 nm_bridge_rthash(const uint8_t *addr)
2844 {
2845         uint32_t a = 0x9e3779b9, b = 0x9e3779b9, c = 0; // hask key
2846 
2847         b += addr[5] << 8;
2848         b += addr[4];
2849         a += addr[3] << 24;
2850         a += addr[2] << 16;
2851         a += addr[1] << 8;
2852         a += addr[0];
2853 
2854         mix(a, b, c);
2855 #define BRIDGE_RTHASH_MASK	(NM_BDG_HASH-1)
2856         return (c & BRIDGE_RTHASH_MASK);
2857 }
2858 
2859 #undef mix
2860 
2861 
2862 static int
2863 bdg_netmap_reg(struct ifnet *ifp, int onoff)
2864 {
2865 	// struct nm_bridge *b = NA(ifp)->na_bdg;
2866 
2867 	/* the interface is already attached to the bridge,
2868 	 * so we only need to toggle IFCAP_NETMAP.
2869 	 * Locking is not necessary (we are already under
2870 	 * NMA_LOCK, and the port is not in use during this call).
2871 	 */
2872 	/* BDG_WLOCK(b); */
2873 	if (onoff) {
2874 		ifp->if_capenable |= IFCAP_NETMAP;
2875 	} else {
2876 		ifp->if_capenable &= ~IFCAP_NETMAP;
2877 	}
2878 	/* BDG_WUNLOCK(b); */
2879 	return 0;
2880 }
2881 
2882 
2883 /*
2884  * Lookup function for a learning bridge.
2885  * Update the hash table with the source address,
2886  * and then returns the destination port index, and the
2887  * ring in *dst_ring (at the moment, always use ring 0)
2888  */
2889 u_int
2890 netmap_bdg_learning(char *buf, u_int len, uint8_t *dst_ring,
2891 		struct netmap_adapter *na)
2892 {
2893 	struct nm_hash_ent *ht = na->na_bdg->ht;
2894 	uint32_t sh, dh;
2895 	u_int dst, mysrc = na->bdg_port;
2896 	uint64_t smac, dmac;
2897 
2898 	dmac = le64toh(*(uint64_t *)(buf)) & 0xffffffffffff;
2899 	smac = le64toh(*(uint64_t *)(buf + 4));
2900 	smac >>= 16;
2901 
2902 	/*
2903 	 * The hash is somewhat expensive, there might be some
2904 	 * worthwhile optimizations here.
2905 	 */
2906 	if ((buf[6] & 1) == 0) { /* valid src */
2907 		uint8_t *s = buf+6;
2908 		sh = nm_bridge_rthash(buf+6); // XXX hash of source
2909 		/* update source port forwarding entry */
2910 		ht[sh].mac = smac;	/* XXX expire ? */
2911 		ht[sh].ports = mysrc;
2912 		if (netmap_verbose)
2913 		    D("src %02x:%02x:%02x:%02x:%02x:%02x on port %d",
2914 			s[0], s[1], s[2], s[3], s[4], s[5], mysrc);
2915 	}
2916 	dst = NM_BDG_BROADCAST;
2917 	if ((buf[0] & 1) == 0) { /* unicast */
2918 		dh = nm_bridge_rthash(buf); // XXX hash of dst
2919 		if (ht[dh].mac == dmac) {	/* found dst */
2920 			dst = ht[dh].ports;
2921 		}
2922 		/* XXX otherwise return NM_BDG_UNKNOWN ? */
2923 	}
2924 	*dst_ring = 0;
2925 	return dst;
2926 }
2927 
2928 
2929 /*
2930  * This flush routine supports only unicast and broadcast but a large
2931  * number of ports, and lets us replace the learn and dispatch functions.
2932  */
2933 int
2934 nm_bdg_flush(struct nm_bdg_fwd *ft, int n, struct netmap_adapter *na,
2935 		u_int ring_nr)
2936 {
2937 	struct nm_bdg_q *dst_ents, *brddst;
2938 	uint16_t num_dsts = 0, *dsts;
2939 	struct nm_bridge *b = na->na_bdg;
2940 	u_int i, me = na->bdg_port;
2941 
2942 	dst_ents = (struct nm_bdg_q *)(ft + NM_BDG_BATCH);
2943 	dsts = (uint16_t *)(dst_ents + NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1);
2944 
2945 	BDG_RLOCK(b);
2946 
2947 	/* first pass: find a destination */
2948 	for (i = 0; likely(i < n); i++) {
2949 		uint8_t *buf = ft[i].ft_buf;
2950 		uint8_t dst_ring = ring_nr;
2951 		uint16_t dst_port, d_i;
2952 		struct nm_bdg_q *d;
2953 
2954 		dst_port = b->nm_bdg_lookup(buf, ft[i].ft_len, &dst_ring, na);
2955 		if (dst_port == NM_BDG_NOPORT) {
2956 			continue; /* this packet is identified to be dropped */
2957 		} else if (unlikely(dst_port > NM_BDG_MAXPORTS)) {
2958 			continue;
2959 		} else if (dst_port == NM_BDG_BROADCAST) {
2960 			dst_ring = 0; /* broadcasts always go to ring 0 */
2961 		} else if (unlikely(dst_port == me ||
2962 		    !BDG_GET_VAR(b->bdg_ports[dst_port]))) {
2963 			continue;
2964 		}
2965 
2966 		/* get a position in the scratch pad */
2967 		d_i = dst_port * NM_BDG_MAXRINGS + dst_ring;
2968 		d = dst_ents + d_i;
2969 		if (d->bq_head == NM_BDG_BATCH) { /* new destination */
2970 			d->bq_head = d->bq_tail = i;
2971 			/* remember this position to be scanned later */
2972 			if (dst_port != NM_BDG_BROADCAST)
2973 				dsts[num_dsts++] = d_i;
2974 		} else {
2975 			ft[d->bq_tail].ft_next = i;
2976 			d->bq_tail = i;
2977 		}
2978 	}
2979 
2980 	/* if there is a broadcast, set ring 0 of all ports to be scanned
2981 	 * XXX This would be optimized by recording the highest index of active
2982 	 * ports.
2983 	 */
2984 	brddst = dst_ents + NM_BDG_BROADCAST * NM_BDG_MAXRINGS;
2985 	if (brddst->bq_head != NM_BDG_BATCH) {
2986 		for (i = 0; likely(i < NM_BDG_MAXPORTS); i++) {
2987 			uint16_t d_i = i * NM_BDG_MAXRINGS;
2988 			if (unlikely(i == me) || !BDG_GET_VAR(b->bdg_ports[i]))
2989 				continue;
2990 			else if (dst_ents[d_i].bq_head == NM_BDG_BATCH)
2991 				dsts[num_dsts++] = d_i;
2992 		}
2993 	}
2994 
2995 	/* second pass: scan destinations (XXX will be modular somehow) */
2996 	for (i = 0; i < num_dsts; i++) {
2997 		struct ifnet *dst_ifp;
2998 		struct netmap_adapter *dst_na;
2999 		struct netmap_kring *kring;
3000 		struct netmap_ring *ring;
3001 		u_int dst_nr, is_vp, lim, j, sent = 0, d_i, next, brd_next;
3002 		int howmany, retry = netmap_txsync_retry;
3003 		struct nm_bdg_q *d;
3004 
3005 		d_i = dsts[i];
3006 		d = dst_ents + d_i;
3007 		dst_na = BDG_GET_VAR(b->bdg_ports[d_i/NM_BDG_MAXRINGS]);
3008 		/* protect from the lookup function returning an inactive
3009 		 * destination port
3010 		 */
3011 		if (unlikely(dst_na == NULL))
3012 			continue;
3013 		else if (dst_na->na_flags & NAF_SW_ONLY)
3014 			continue;
3015 		dst_ifp = dst_na->ifp;
3016 		/*
3017 		 * The interface may be in !netmap mode in two cases:
3018 		 * - when na is attached but not activated yet;
3019 		 * - when na is being deactivated but is still attached.
3020 		 */
3021 		if (unlikely(!(dst_ifp->if_capenable & IFCAP_NETMAP)))
3022 			continue;
3023 
3024 		/* there is at least one either unicast or broadcast packet */
3025 		brd_next = brddst->bq_head;
3026 		next = d->bq_head;
3027 
3028 		is_vp = nma_is_vp(dst_na);
3029 		dst_nr = d_i & (NM_BDG_MAXRINGS-1);
3030 		if (is_vp) { /* virtual port */
3031 			if (dst_nr >= dst_na->num_rx_rings)
3032 				dst_nr = dst_nr % dst_na->num_rx_rings;
3033 			kring = &dst_na->rx_rings[dst_nr];
3034 			ring = kring->ring;
3035 			lim = kring->nkr_num_slots - 1;
3036 			dst_na->nm_lock(dst_ifp, NETMAP_RX_LOCK, dst_nr);
3037 			j = kring->nr_hwcur + kring->nr_hwavail;
3038 			if (j > lim)
3039 				j -= kring->nkr_num_slots;
3040 			howmany = lim - kring->nr_hwavail;
3041 		} else { /* hw or sw adapter */
3042 			if (dst_nr >= dst_na->num_tx_rings)
3043 				dst_nr = dst_nr % dst_na->num_tx_rings;
3044 			kring = &dst_na->tx_rings[dst_nr];
3045 			ring = kring->ring;
3046 			lim = kring->nkr_num_slots - 1;
3047 			dst_na->nm_lock(dst_ifp, NETMAP_TX_LOCK, dst_nr);
3048 retry:
3049 			dst_na->nm_txsync(dst_ifp, dst_nr, 0);
3050 			/* see nm_bdg_flush() */
3051 			j = kring->nr_hwcur;
3052 			howmany = kring->nr_hwavail;
3053 		}
3054 		while (howmany-- > 0) {
3055 			struct netmap_slot *slot;
3056 			struct nm_bdg_fwd *ft_p;
3057 
3058 			/* our 'NULL' is always higher than valid indexes
3059 			 * so we never dereference it if the other list
3060 			 * has packets (and if both are NULL we never
3061 			 * get here).
3062 			 */
3063 			if (next < brd_next) {
3064 				ft_p = ft + next;
3065 				next = ft_p->ft_next;
3066 				ND("j %d uni %d next %d %d",
3067 					j, ft_p - ft, next, brd_next);
3068 			} else { /* insert broadcast */
3069 				ft_p = ft + brd_next;
3070 				brd_next = ft_p->ft_next;
3071 				ND("j %d brd %d next %d %d",
3072 					j, ft_p - ft, next, brd_next);
3073 			}
3074 			slot = &ring->slot[j];
3075 			ND("send %d %d bytes at %s:%d", i, ft_p->ft_len, dst_ifp->if_xname, j);
3076 		    if (ft_p->ft_flags & NS_INDIRECT) {
3077 			ND("copying from INDIRECT source");
3078 			copyin(ft_p->ft_buf, NMB(slot),
3079 				(ft_p->ft_len + 63) & ~63);
3080 		    } else {
3081 			pkt_copy(ft_p->ft_buf, NMB(slot), ft_p->ft_len);
3082 		    }
3083 			slot->len = ft_p->ft_len;
3084 			j = unlikely(j == lim) ? 0: j + 1; /* XXX to be macro-ed */
3085 			sent++;
3086 			/* are we done ? */
3087 			if (next == NM_BDG_BATCH && brd_next == NM_BDG_BATCH)
3088 				break;
3089 		}
3090 		if (netmap_verbose && (howmany < 0))
3091 			D("rx ring full on %s", dst_ifp->if_xname);
3092 		if (is_vp) {
3093 			if (sent) {
3094 				kring->nr_hwavail += sent;
3095 				selwakeuppri(&kring->si, PI_NET);
3096 			}
3097 			dst_na->nm_lock(dst_ifp, NETMAP_RX_UNLOCK, dst_nr);
3098 		} else {
3099 			if (sent) {
3100 				ring->avail -= sent;
3101 				ring->cur = j;
3102 				dst_na->nm_txsync(dst_ifp, dst_nr, 0);
3103 			}
3104 			/* retry to send more packets */
3105 			if (nma_is_hw(dst_na) && howmany < 0 && retry--)
3106 				goto retry;
3107 			dst_na->nm_lock(dst_ifp, NETMAP_TX_UNLOCK, dst_nr);
3108 		}
3109 		/* NM_BDG_BATCH means 'no packet' */
3110 		d->bq_head = d->bq_tail = NM_BDG_BATCH; /* cleanup */
3111 	}
3112 	brddst->bq_head = brddst->bq_tail = NM_BDG_BATCH; /* cleanup */
3113 	BDG_RUNLOCK(b);
3114 	return 0;
3115 }
3116 
3117 
3118 /*
3119  * main dispatch routine
3120  */
3121 static int
3122 bdg_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
3123 {
3124 	struct netmap_adapter *na = NA(ifp);
3125 	struct netmap_kring *kring = &na->tx_rings[ring_nr];
3126 	struct netmap_ring *ring = kring->ring;
3127 	int i, j, k, lim = kring->nkr_num_slots - 1;
3128 
3129 	k = ring->cur;
3130 	if (k > lim)
3131 		return netmap_ring_reinit(kring);
3132 	if (do_lock)
3133 		na->nm_lock(ifp, NETMAP_TX_LOCK, ring_nr);
3134 
3135 	if (netmap_bridge <= 0) { /* testing only */
3136 		j = k; // used all
3137 		goto done;
3138 	}
3139 	if (netmap_bridge > NM_BDG_BATCH)
3140 		netmap_bridge = NM_BDG_BATCH;
3141 
3142 	j = nm_bdg_preflush(na, ring_nr, kring, k);
3143 	i = k - j;
3144 	if (i < 0)
3145 		i += kring->nkr_num_slots;
3146 	kring->nr_hwavail = kring->nkr_num_slots - 1 - i;
3147 	if (j != k)
3148 		D("early break at %d/ %d, avail %d", j, k, kring->nr_hwavail);
3149 
3150 done:
3151 	kring->nr_hwcur = j;
3152 	ring->avail = kring->nr_hwavail;
3153 	if (do_lock)
3154 		na->nm_lock(ifp, NETMAP_TX_UNLOCK, ring_nr);
3155 
3156 	if (netmap_verbose)
3157 		D("%s ring %d lock %d", ifp->if_xname, ring_nr, do_lock);
3158 	return 0;
3159 }
3160 
3161 
3162 static int
3163 bdg_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
3164 {
3165 	struct netmap_adapter *na = NA(ifp);
3166 	struct netmap_kring *kring = &na->rx_rings[ring_nr];
3167 	struct netmap_ring *ring = kring->ring;
3168 	u_int j, lim = kring->nkr_num_slots - 1;
3169 	u_int k = ring->cur, resvd = ring->reserved;
3170 	int n;
3171 
3172 	ND("%s ring %d lock %d avail %d",
3173 		ifp->if_xname, ring_nr, do_lock, kring->nr_hwavail);
3174 
3175 	if (k > lim)
3176 		return netmap_ring_reinit(kring);
3177 	if (do_lock)
3178 		na->nm_lock(ifp, NETMAP_RX_LOCK, ring_nr);
3179 
3180 	/* skip past packets that userspace has released */
3181 	j = kring->nr_hwcur;    /* netmap ring index */
3182 	if (resvd > 0) {
3183 		if (resvd + ring->avail >= lim + 1) {
3184 			D("XXX invalid reserve/avail %d %d", resvd, ring->avail);
3185 			ring->reserved = resvd = 0; // XXX panic...
3186 		}
3187 		k = (k >= resvd) ? k - resvd : k + lim + 1 - resvd;
3188 	}
3189 
3190 	if (j != k) { /* userspace has released some packets. */
3191 		n = k - j;
3192 		if (n < 0)
3193 			n += kring->nkr_num_slots;
3194 		ND("userspace releases %d packets", n);
3195                 for (n = 0; likely(j != k); n++) {
3196                         struct netmap_slot *slot = &ring->slot[j];
3197                         void *addr = NMB(slot);
3198 
3199                         if (addr == netmap_buffer_base) { /* bad buf */
3200                                 if (do_lock)
3201                                         na->nm_lock(ifp, NETMAP_RX_UNLOCK, ring_nr);
3202                                 return netmap_ring_reinit(kring);
3203                         }
3204 			/* decrease refcount for buffer */
3205 
3206 			slot->flags &= ~NS_BUF_CHANGED;
3207                         j = unlikely(j == lim) ? 0 : j + 1;
3208                 }
3209                 kring->nr_hwavail -= n;
3210                 kring->nr_hwcur = k;
3211         }
3212         /* tell userspace that there are new packets */
3213         ring->avail = kring->nr_hwavail - resvd;
3214 
3215 	if (do_lock)
3216 		na->nm_lock(ifp, NETMAP_RX_UNLOCK, ring_nr);
3217 	return 0;
3218 }
3219 
3220 
3221 static void
3222 bdg_netmap_attach(struct netmap_adapter *arg)
3223 {
3224 	struct netmap_adapter na;
3225 
3226 	ND("attaching virtual bridge");
3227 	bzero(&na, sizeof(na));
3228 
3229 	na.ifp = arg->ifp;
3230 	na.separate_locks = 1;
3231 	na.num_tx_rings = arg->num_tx_rings;
3232 	na.num_rx_rings = arg->num_rx_rings;
3233 	na.num_tx_desc = NM_BRIDGE_RINGSIZE;
3234 	na.num_rx_desc = NM_BRIDGE_RINGSIZE;
3235 	na.nm_txsync = bdg_netmap_txsync;
3236 	na.nm_rxsync = bdg_netmap_rxsync;
3237 	na.nm_register = bdg_netmap_reg;
3238 	netmap_attach(&na, na.num_tx_rings);
3239 }
3240 
3241 #endif /* NM_BRIDGE */
3242 
3243 static struct cdev *netmap_dev; /* /dev/netmap character device. */
3244 
3245 
3246 /*
3247  * Module loader.
3248  *
3249  * Create the /dev/netmap device and initialize all global
3250  * variables.
3251  *
3252  * Return 0 on success, errno on failure.
3253  */
3254 static int
3255 netmap_init(void)
3256 {
3257 	int error;
3258 
3259 	error = netmap_memory_init();
3260 	if (error != 0) {
3261 		printf("netmap: unable to initialize the memory allocator.\n");
3262 		return (error);
3263 	}
3264 	printf("netmap: loaded module\n");
3265 	netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660,
3266 			      "netmap");
3267 
3268 #ifdef NM_BRIDGE
3269 	{
3270 	int i;
3271 	mtx_init(&netmap_bridge_mutex, "netmap_bridge_mutex",
3272 		MTX_NETWORK_LOCK, MTX_DEF);
3273 	bzero(nm_bridges, sizeof(struct nm_bridge) * NM_BRIDGES); /* safety */
3274 	for (i = 0; i < NM_BRIDGES; i++)
3275 		rw_init(&nm_bridges[i].bdg_lock, "bdg lock");
3276 	}
3277 #endif
3278 	return (error);
3279 }
3280 
3281 
3282 /*
3283  * Module unloader.
3284  *
3285  * Free all the memory, and destroy the ``/dev/netmap`` device.
3286  */
3287 static void
3288 netmap_fini(void)
3289 {
3290 	destroy_dev(netmap_dev);
3291 	netmap_memory_fini();
3292 	printf("netmap: unloaded module.\n");
3293 }
3294 
3295 
3296 #ifdef __FreeBSD__
3297 /*
3298  * Kernel entry point.
3299  *
3300  * Initialize/finalize the module and return.
3301  *
3302  * Return 0 on success, errno on failure.
3303  */
3304 static int
3305 netmap_loader(__unused struct module *module, int event, __unused void *arg)
3306 {
3307 	int error = 0;
3308 
3309 	switch (event) {
3310 	case MOD_LOAD:
3311 		error = netmap_init();
3312 		break;
3313 
3314 	case MOD_UNLOAD:
3315 		netmap_fini();
3316 		break;
3317 
3318 	default:
3319 		error = EOPNOTSUPP;
3320 		break;
3321 	}
3322 
3323 	return (error);
3324 }
3325 
3326 
3327 DEV_MODULE(netmap, netmap_loader, NULL);
3328 #endif /* __FreeBSD__ */
3329