xref: /freebsd/sys/dev/netmap/netmap_kern.h (revision 13de33a5dc2304b13d595d75d48c51793958474f)
1 /*
2  * Copyright (C) 2011-2013 Matteo Landi, Luigi Rizzo. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  *   1. Redistributions of source code must retain the above copyright
8  *      notice, this list of conditions and the following disclaimer.
9  *   2. Redistributions in binary form must reproduce the above copyright
10  *      notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  */
25 
26 /*
27  * $FreeBSD$
28  *
29  * The header contains the definitions of constants and function
30  * prototypes used only in kernelspace.
31  */
32 
33 #ifndef _NET_NETMAP_KERN_H_
34 #define _NET_NETMAP_KERN_H_
35 
36 #if defined(__FreeBSD__)
37 
38 #define likely(x)	__builtin_expect((long)!!(x), 1L)
39 #define unlikely(x)	__builtin_expect((long)!!(x), 0L)
40 
41 #define	NM_LOCK_T	struct mtx
42 #define	NM_SELINFO_T	struct selinfo
43 #define	MBUF_LEN(m)	((m)->m_pkthdr.len)
44 #define	NM_SEND_UP(ifp, m)	((ifp)->if_input)(ifp, m)
45 
46 #define NM_ATOMIC_T	volatile int
47 
48 #elif defined (linux)
49 
50 #define	NM_LOCK_T	safe_spinlock_t	// see bsd_glue.h
51 #define	NM_SELINFO_T	wait_queue_head_t
52 #define	MBUF_LEN(m)	((m)->len)
53 #define	NM_SEND_UP(ifp, m)	netif_rx(m)
54 
55 #define NM_ATOMIC_T	volatile long unsigned int
56 
57 #ifndef DEV_NETMAP
58 #define DEV_NETMAP
59 #endif /* DEV_NETMAP */
60 
61 /*
62  * IFCAP_NETMAP goes into net_device's priv_flags (if_capenable).
63  * This was 16 bits up to linux 2.6.36, so we need a 16 bit value on older
64  * platforms and tolerate the clash with IFF_DYNAMIC and IFF_BRIDGE_PORT.
65  * For the 32-bit value, 0x100000 has no clashes until at least 3.5.1
66  */
67 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37)
68 #define IFCAP_NETMAP	0x8000
69 #else
70 #define IFCAP_NETMAP	0x200000
71 #endif
72 
73 #elif defined (__APPLE__)
74 
75 #warning apple support is incomplete.
76 #define likely(x)	__builtin_expect(!!(x), 1)
77 #define unlikely(x)	__builtin_expect(!!(x), 0)
78 #define	NM_LOCK_T	IOLock *
79 #define	NM_SELINFO_T	struct selinfo
80 #define	MBUF_LEN(m)	((m)->m_pkthdr.len)
81 #define	NM_SEND_UP(ifp, m)	((ifp)->if_input)(ifp, m)
82 
83 #else
84 
85 #error unsupported platform
86 
87 #endif /* end - platform-specific code */
88 
89 #define ND(format, ...)
90 #define D(format, ...)						\
91 	do {							\
92 		struct timeval __xxts;				\
93 		microtime(&__xxts);				\
94 		printf("%03d.%06d %s [%d] " format "\n",	\
95 		(int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec,	\
96 		__FUNCTION__, __LINE__, ##__VA_ARGS__);		\
97 	} while (0)
98 
99 /* rate limited, lps indicates how many per second */
100 #define RD(lps, format, ...)					\
101 	do {							\
102 		static int t0, __cnt;				\
103 		if (t0 != time_second) {			\
104 			t0 = time_second;			\
105 			__cnt = 0;				\
106 		}						\
107 		if (__cnt++ < lps)				\
108 			D(format, ##__VA_ARGS__);		\
109 	} while (0)
110 
111 struct netmap_adapter;
112 struct nm_bdg_fwd;
113 struct nm_bridge;
114 struct netmap_priv_d;
115 
116 const char *nm_dump_buf(char *p, int len, int lim, char *dst);
117 
118 /*
119  * private, kernel view of a ring. Keeps track of the status of
120  * a ring across system calls.
121  *
122  *	nr_hwcur	index of the next buffer to refill.
123  *			It corresponds to ring->cur - ring->reserved
124  *
125  *	nr_hwavail	the number of slots "owned" by userspace.
126  *			nr_hwavail =:= ring->avail + ring->reserved
127  *
128  * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots.
129  * This is so that, on a reset, buffers owned by userspace are not
130  * modified by the kernel. In particular:
131  * RX rings: the next empty buffer (hwcur + hwavail + hwofs) coincides with
132  * 	the next empty buffer as known by the hardware (next_to_check or so).
133  * TX rings: hwcur + hwofs coincides with next_to_send
134  *
135  * Clients cannot issue concurrent syscall on a ring. The system
136  * detects this and reports an error using two flags,
137  * NKR_WBUSY and NKR_RBUSY
138  * For received packets, slot->flags is set to nkr_slot_flags
139  * so we can provide a proper initial value (e.g. set NS_FORWARD
140  * when operating in 'transparent' mode).
141  *
142  * The following fields are used to implement lock-free copy of packets
143  * from input to output ports in VALE switch:
144  *	nkr_hwlease	buffer after the last one being copied.
145  *			A writer in nm_bdg_flush reserves N buffers
146  *			from nr_hwlease, advances it, then does the
147  *			copy outside the lock.
148  *			In RX rings (used for VALE ports),
149  *			nkr_hwcur + nkr_hwavail <= nkr_hwlease < nkr_hwcur+N-1
150  *			In TX rings (used for NIC or host stack ports)
151  *			nkr_hwcur <= nkr_hwlease < nkr_hwcur+ nkr_hwavail
152  *	nkr_leases	array of nkr_num_slots where writers can report
153  *			completion of their block. NR_NOSLOT (~0) indicates
154  *			that the writer has not finished yet
155  *	nkr_lease_idx	index of next free slot in nr_leases, to be assigned
156  *
157  * The kring is manipulated by txsync/rxsync and generic netmap function.
158  * q_lock is used to arbitrate access to the kring from within the netmap
159  * code, and this and other protections guarantee that there is never
160  * more than 1 concurrent call to txsync or rxsync. So we are free
161  * to manipulate the kring from within txsync/rxsync without any extra
162  * locks.
163  */
164 struct netmap_kring {
165 	struct netmap_ring *ring;
166 	uint32_t nr_hwcur;
167 	uint32_t nr_hwavail;
168 	uint32_t nr_kflags;	/* private driver flags */
169 #define NKR_PENDINTR	0x1	// Pending interrupt.
170 	uint32_t nkr_num_slots;
171 	int32_t	nkr_hwofs;	/* offset between NIC and netmap ring */
172 
173 	uint16_t	nkr_slot_flags;	/* initial value for flags */
174 	struct netmap_adapter *na;
175 	struct nm_bdg_fwd *nkr_ft;
176 	uint32_t *nkr_leases;
177 #define NR_NOSLOT	((uint32_t)~0)
178 	uint32_t nkr_hwlease;
179 	uint32_t nkr_lease_idx;
180 
181 	NM_SELINFO_T si;	/* poll/select wait queue */
182 	NM_LOCK_T q_lock;	/* protects kring and ring. */
183 	NM_ATOMIC_T nr_busy;	/* prevent concurrent syscalls */
184 
185 	volatile int nkr_stopped;
186 } __attribute__((__aligned__(64)));
187 
188 
189 /* return the next index, with wraparound */
190 static inline uint32_t
191 nm_next(uint32_t i, uint32_t lim)
192 {
193 	return unlikely (i == lim) ? 0 : i + 1;
194 }
195 
196 /*
197  *
198  * Here is the layout for the Rx and Tx rings.
199 
200        RxRING                            TxRING
201 
202       +-----------------+            +-----------------+
203       |                 |            |                 |
204       |XXX free slot XXX|            |XXX free slot XXX|
205       +-----------------+            +-----------------+
206       |                 |<-hwcur     |                 |<-hwcur
207       | reserved    h   |            | (ready          |
208       +-----------  w  -+            |  to be          |
209  cur->|             a   |            |  sent)      h   |
210       |             v   |            +----------   w   |
211       |             a   |       cur->| (being      a   |
212       |             i   |            |  prepared)  v   |
213       | avail       l   |            |             a   |
214       +-----------------+            +  a  ------  i   +
215       |                 | ...        |  v          l   |<-hwlease
216       | (being          | ...        |  a              | ...
217       |  prepared)      | ...        |  i              | ...
218       +-----------------+ ...        |  l              | ...
219       |                 |<-hwlease   +-----------------+
220       |                 |            |                 |
221       |                 |            |                 |
222       |                 |            |                 |
223       |                 |            |                 |
224       +-----------------+            +-----------------+
225 
226  * The cur/avail (user view) and hwcur/hwavail (kernel view)
227  * are used in the normal operation of the card.
228  *
229  * When a ring is the output of a switch port (Rx ring for
230  * a VALE port, Tx ring for the host stack or NIC), slots
231  * are reserved in blocks through 'hwlease' which points
232  * to the next unused slot.
233  * On an Rx ring, hwlease is always after hwavail,
234  * and completions cause avail to advance.
235  * On a Tx ring, hwlease is always between cur and hwavail,
236  * and completions cause cur to advance.
237  *
238  * nm_kr_space() returns the maximum number of slots that
239  * can be assigned.
240  * nm_kr_lease() reserves the required number of buffers,
241  *    advances nkr_hwlease and also returns an entry in
242  *    a circular array where completions should be reported.
243  */
244 
245 
246 
247 
248 
249 /*
250  * This struct extends the 'struct adapter' (or
251  * equivalent) device descriptor. It contains all fields needed to
252  * support netmap operation.
253  */
254 struct netmap_adapter {
255 	/*
256 	 * On linux we do not have a good way to tell if an interface
257 	 * is netmap-capable. So we use the following trick:
258 	 * NA(ifp) points here, and the first entry (which hopefully
259 	 * always exists and is at least 32 bits) contains a magic
260 	 * value which we can use to detect that the interface is good.
261 	 */
262 	uint32_t magic;
263 	uint32_t na_flags;	/* future place for IFCAP_NETMAP */
264 #define NAF_SKIP_INTR	1	/* use the regular interrupt handler.
265 				 * useful during initialization
266 				 */
267 #define NAF_SW_ONLY	2	/* forward packets only to sw adapter */
268 #define NAF_BDG_MAYSLEEP 4	/* the bridge is allowed to sleep when
269 				 * forwarding packets coming from this
270 				 * interface
271 				 */
272 #define NAF_MEM_OWNER	8	/* the adapter is responsible for the
273 				 * deallocation of the memory allocator
274 				 */
275 	int refcount; /* number of user-space descriptors using this
276 			 interface, which is equal to the number of
277 			 struct netmap_if objs in the mapped region. */
278 	/*
279 	 * The selwakeup in the interrupt thread can use per-ring
280 	 * and/or global wait queues. We track how many clients
281 	 * of each type we have so we can optimize the drivers,
282 	 * and especially avoid huge contention on the locks.
283 	 */
284 	int na_single;	/* threads attached to a single hw queue */
285 	int na_multi;	/* threads attached to multiple hw queues */
286 
287 	u_int num_rx_rings; /* number of adapter receive rings */
288 	u_int num_tx_rings; /* number of adapter transmit rings */
289 
290 	u_int num_tx_desc; /* number of descriptor in each queue */
291 	u_int num_rx_desc;
292 
293 	/* tx_rings and rx_rings are private but allocated
294 	 * as a contiguous chunk of memory. Each array has
295 	 * N+1 entries, for the adapter queues and for the host queue.
296 	 */
297 	struct netmap_kring *tx_rings; /* array of TX rings. */
298 	struct netmap_kring *rx_rings; /* array of RX rings. */
299 
300 	NM_SELINFO_T tx_si, rx_si;	/* global wait queues */
301 
302 	/* copy of if_qflush and if_transmit pointers, to intercept
303 	 * packets from the network stack when netmap is active.
304 	 */
305 	int     (*if_transmit)(struct ifnet *, struct mbuf *);
306 
307 	/* references to the ifnet and device routines, used by
308 	 * the generic netmap functions.
309 	 */
310 	struct ifnet *ifp; /* adapter is ifp->if_softc */
311 
312 	NM_LOCK_T core_lock;	/* used if no device lock available */
313 
314 	int (*nm_register)(struct ifnet *, int onoff);
315 
316 	int (*nm_txsync)(struct ifnet *, u_int ring, int flags);
317 	int (*nm_rxsync)(struct ifnet *, u_int ring, int flags);
318 #define NAF_FORCE_READ    1
319 #define NAF_FORCE_RECLAIM 2
320 	/* return configuration information */
321 	int (*nm_config)(struct ifnet *, u_int *txr, u_int *txd,
322 					u_int *rxr, u_int *rxd);
323 
324 	/*
325 	 * Bridge support:
326 	 *
327 	 * bdg_port is the port number used in the bridge;
328 	 * na_bdg_refcount is a refcount used for bridge ports,
329 	 *	when it goes to 0 we can detach+free this port
330 	 *	(a bridge port is always attached if it exists;
331 	 *	it is not always registered)
332 	 * na_bdg points to the bridge this NA is attached to.
333 	 */
334 	int bdg_port;
335 	int na_bdg_refcount;
336 	struct nm_bridge *na_bdg;
337 	/* When we attach a physical interface to the bridge, we
338 	 * allow the controlling process to terminate, so we need
339 	 * a place to store the netmap_priv_d data structure.
340 	 * This is only done when physical interfaces are attached to a bridge.
341 	 */
342 	struct netmap_priv_d *na_kpriv;
343 
344 	/* memory allocator */
345  	struct netmap_mem_d *nm_mem;
346 #ifdef linux
347 	struct net_device_ops nm_ndo;
348 #endif /* linux */
349 };
350 
351 /*
352  * Available space in the ring.
353  */
354 static inline uint32_t
355 nm_kr_space(struct netmap_kring *k, int is_rx)
356 {
357 	int space;
358 
359 	if (is_rx) {
360 		int busy = k->nkr_hwlease - k->nr_hwcur;
361 		if (busy < 0)
362 			busy += k->nkr_num_slots;
363 		space = k->nkr_num_slots - 1 - busy;
364 	} else {
365 		space = k->nr_hwcur + k->nr_hwavail - k->nkr_hwlease;
366 		if (space < 0)
367 			space += k->nkr_num_slots;
368 	}
369 #if 0
370 	// sanity check
371 	if (k->nkr_hwlease >= k->nkr_num_slots ||
372 		k->nr_hwcur >= k->nkr_num_slots ||
373 		k->nr_hwavail >= k->nkr_num_slots ||
374 		busy < 0 ||
375 		busy >= k->nkr_num_slots) {
376 		D("invalid kring, cur %d avail %d lease %d lease_idx %d lim %d",			k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease,
377 			k->nkr_lease_idx, k->nkr_num_slots);
378 	}
379 #endif
380 	return space;
381 }
382 
383 
384 /* return update position */
385 static inline uint32_t
386 nm_kr_rxpos(struct netmap_kring *k)
387 {
388 	uint32_t pos = k->nr_hwcur + k->nr_hwavail;
389 	if (pos >= k->nkr_num_slots)
390 		pos -= k->nkr_num_slots;
391 #if 0
392 	if (pos >= k->nkr_num_slots ||
393 		k->nkr_hwlease >= k->nkr_num_slots ||
394 		k->nr_hwcur >= k->nkr_num_slots ||
395 		k->nr_hwavail >= k->nkr_num_slots ||
396 		k->nkr_lease_idx >= k->nkr_num_slots) {
397 		D("invalid kring, cur %d avail %d lease %d lease_idx %d lim %d",			k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease,
398 			k->nkr_lease_idx, k->nkr_num_slots);
399 	}
400 #endif
401 	return pos;
402 }
403 
404 
405 /* make a lease on the kring for N positions. return the
406  * lease index
407  */
408 static inline uint32_t
409 nm_kr_lease(struct netmap_kring *k, u_int n, int is_rx)
410 {
411 	uint32_t lim = k->nkr_num_slots - 1;
412 	uint32_t lease_idx = k->nkr_lease_idx;
413 
414 	k->nkr_leases[lease_idx] = NR_NOSLOT;
415 	k->nkr_lease_idx = nm_next(lease_idx, lim);
416 
417 	if (n > nm_kr_space(k, is_rx)) {
418 		D("invalid request for %d slots", n);
419 		panic("x");
420 	}
421 	/* XXX verify that there are n slots */
422 	k->nkr_hwlease += n;
423 	if (k->nkr_hwlease > lim)
424 		k->nkr_hwlease -= lim + 1;
425 
426 	if (k->nkr_hwlease >= k->nkr_num_slots ||
427 		k->nr_hwcur >= k->nkr_num_slots ||
428 		k->nr_hwavail >= k->nkr_num_slots ||
429 		k->nkr_lease_idx >= k->nkr_num_slots) {
430 		D("invalid kring %s, cur %d avail %d lease %d lease_idx %d lim %d",
431 			k->na->ifp->if_xname,
432 			k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease,
433 			k->nkr_lease_idx, k->nkr_num_slots);
434 	}
435 	return lease_idx;
436 }
437 
438 
439 /*
440  * XXX NETMAP_DELETING() is unused
441  *
442  * The combination of "enable" (ifp->if_capenable & IFCAP_NETMAP)
443  * and refcount gives the status of the interface, namely:
444  *
445  *	enable	refcount	Status
446  *
447  *	FALSE	0		normal operation
448  *	FALSE	!= 0		-- (impossible)
449  *	TRUE	1		netmap mode
450  *	TRUE	0		being deleted.
451  */
452 
453 #define NETMAP_DELETING(_na)  (  ((_na)->refcount == 0) &&	\
454 	( (_na)->ifp->if_capenable & IFCAP_NETMAP) )
455 
456 
457 /*
458  * The following are support routines used by individual drivers to
459  * support netmap operation.
460  *
461  * netmap_attach() initializes a struct netmap_adapter, allocating the
462  * 	struct netmap_ring's and the struct selinfo.
463  *
464  * netmap_detach() frees the memory allocated by netmap_attach().
465  *
466  * netmap_transmit() replaces the if_transmit routine of the interface,
467  *	and is used to intercept packets coming from the stack.
468  *
469  * netmap_load_map/netmap_reload_map are helper routines to set/reset
470  *	the dmamap for a packet buffer
471  *
472  * netmap_reset() is a helper routine to be called in the driver
473  *	when reinitializing a ring.
474  */
475 int netmap_attach(struct netmap_adapter *, u_int);
476 void netmap_detach(struct ifnet *);
477 int netmap_transmit(struct ifnet *, struct mbuf *);
478 enum txrx { NR_RX = 0, NR_TX = 1 };
479 struct netmap_slot *netmap_reset(struct netmap_adapter *na,
480 	enum txrx tx, u_int n, u_int new_cur);
481 int netmap_ring_reinit(struct netmap_kring *);
482 
483 u_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg);
484 
485 /*
486  * The following bridge-related interfaces are used by other kernel modules
487  * In the version that only supports unicast or broadcast, the lookup
488  * function can return 0 .. NM_BDG_MAXPORTS-1 for regular ports,
489  * NM_BDG_MAXPORTS for broadcast, NM_BDG_MAXPORTS+1 for unknown.
490  * XXX in practice "unknown" might be handled same as broadcast.
491  */
492 typedef u_int (*bdg_lookup_fn_t)(char *buf, u_int len, uint8_t *ring_nr,
493 		struct netmap_adapter *);
494 int netmap_bdg_ctl(struct nmreq *nmr, bdg_lookup_fn_t func);
495 u_int netmap_bdg_learning(char *, u_int, uint8_t *, struct netmap_adapter *);
496 #define	NM_NAME			"vale"	/* prefix for the bridge port name */
497 #define	NM_BDG_MAXPORTS		254	/* up to 32 for bitmap, 254 ok otherwise */
498 #define	NM_BDG_BROADCAST	NM_BDG_MAXPORTS
499 #define	NM_BDG_NOPORT		(NM_BDG_MAXPORTS+1)
500 
501 extern u_int netmap_buf_size;
502 #define NETMAP_BUF_SIZE	netmap_buf_size	// XXX remove
503 extern int netmap_mitigate;
504 extern int netmap_no_pendintr;
505 extern u_int netmap_total_buffers;
506 extern char *netmap_buffer_base;
507 extern int netmap_verbose;	// XXX debugging
508 enum {                                  /* verbose flags */
509 	NM_VERB_ON = 1,                 /* generic verbose */
510 	NM_VERB_HOST = 0x2,             /* verbose host stack */
511 	NM_VERB_RXSYNC = 0x10,          /* verbose on rxsync/txsync */
512 	NM_VERB_TXSYNC = 0x20,
513 	NM_VERB_RXINTR = 0x100,         /* verbose on rx/tx intr (driver) */
514 	NM_VERB_TXINTR = 0x200,
515 	NM_VERB_NIC_RXSYNC = 0x1000,    /* verbose on rx/tx intr (driver) */
516 	NM_VERB_NIC_TXSYNC = 0x2000,
517 };
518 
519 /*
520  * NA returns a pointer to the struct netmap adapter from the ifp,
521  * WNA is used to write it.
522  * SWNA() is used for the "host stack" endpoint associated
523  *	to an interface. It is allocated together with the main NA(),
524  *	as an array of two objects.
525  */
526 #ifndef WNA
527 #define	WNA(_ifp)	(_ifp)->if_pspare[0]
528 #endif
529 #define	NA(_ifp)	((struct netmap_adapter *)WNA(_ifp))
530 #define	SWNA(_ifp)	(NA(_ifp) + 1)
531 
532 /*
533  * Macros to determine if an interface is netmap capable or netmap enabled.
534  * See the magic field in struct netmap_adapter.
535  */
536 #ifdef __FreeBSD__
537 /*
538  * on FreeBSD just use if_capabilities and if_capenable.
539  */
540 #define NETMAP_CAPABLE(ifp)	(NA(ifp) &&		\
541 	(ifp)->if_capabilities & IFCAP_NETMAP )
542 
543 #define	NETMAP_SET_CAPABLE(ifp)				\
544 	(ifp)->if_capabilities |= IFCAP_NETMAP
545 
546 #else	/* linux */
547 
548 /*
549  * on linux:
550  * we check if NA(ifp) is set and its first element has a related
551  * magic value. The capenable is within the struct netmap_adapter.
552  */
553 #define	NETMAP_MAGIC	0x52697a7a
554 
555 #define NETMAP_CAPABLE(ifp)	(NA(ifp) &&		\
556 	((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC )
557 
558 #define	NETMAP_SET_CAPABLE(ifp)				\
559 	NA(ifp)->magic = ((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC
560 
561 #endif	/* linux */
562 
563 #ifdef __FreeBSD__
564 /* Callback invoked by the dma machinery after a successfull dmamap_load */
565 static void netmap_dmamap_cb(__unused void *arg,
566     __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error)
567 {
568 }
569 
570 /* bus_dmamap_load wrapper: call aforementioned function if map != NULL.
571  * XXX can we do it without a callback ?
572  */
573 static inline void
574 netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
575 {
576 	if (map)
577 		bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE,
578 		    netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
579 }
580 
581 /* update the map when a buffer changes. */
582 static inline void
583 netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
584 {
585 	if (map) {
586 		bus_dmamap_unload(tag, map);
587 		bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE,
588 		    netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
589 	}
590 }
591 #else /* linux */
592 
593 /*
594  * XXX How do we redefine these functions:
595  *
596  * on linux we need
597  *	dma_map_single(&pdev->dev, virt_addr, len, direction)
598  *	dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction
599  * The len can be implicit (on netmap it is NETMAP_BUF_SIZE)
600  * unfortunately the direction is not, so we need to change
601  * something to have a cross API
602  */
603 #define netmap_load_map(_t, _m, _b)
604 #define netmap_reload_map(_t, _m, _b)
605 #if 0
606 	struct e1000_buffer *buffer_info =  &tx_ring->buffer_info[l];
607 	/* set time_stamp *before* dma to help avoid a possible race */
608 	buffer_info->time_stamp = jiffies;
609 	buffer_info->mapped_as_page = false;
610 	buffer_info->length = len;
611 	//buffer_info->next_to_watch = l;
612 	/* reload dma map */
613 	dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
614 			NETMAP_BUF_SIZE, DMA_TO_DEVICE);
615 	buffer_info->dma = dma_map_single(&adapter->pdev->dev,
616 			addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE);
617 
618 	if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) {
619 		D("dma mapping error");
620 		/* goto dma_error; See e1000_put_txbuf() */
621 		/* XXX reset */
622 	}
623 	tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX
624 
625 #endif
626 
627 /*
628  * The bus_dmamap_sync() can be one of wmb() or rmb() depending on direction.
629  */
630 #define bus_dmamap_sync(_a, _b, _c)
631 
632 #endif /* linux */
633 
634 
635 /*
636  * functions to map NIC to KRING indexes (n2k) and vice versa (k2n)
637  */
638 static inline int
639 netmap_idx_n2k(struct netmap_kring *kr, int idx)
640 {
641 	int n = kr->nkr_num_slots;
642 	idx += kr->nkr_hwofs;
643 	if (idx < 0)
644 		return idx + n;
645 	else if (idx < n)
646 		return idx;
647 	else
648 		return idx - n;
649 }
650 
651 
652 static inline int
653 netmap_idx_k2n(struct netmap_kring *kr, int idx)
654 {
655 	int n = kr->nkr_num_slots;
656 	idx -= kr->nkr_hwofs;
657 	if (idx < 0)
658 		return idx + n;
659 	else if (idx < n)
660 		return idx;
661 	else
662 		return idx - n;
663 }
664 
665 
666 /* Entries of the look-up table. */
667 struct lut_entry {
668 	void *vaddr;		/* virtual address. */
669 	vm_paddr_t paddr;	/* physical address. */
670 };
671 
672 struct netmap_obj_pool;
673 extern struct lut_entry *netmap_buffer_lut;
674 #define NMB_VA(i)	(netmap_buffer_lut[i].vaddr)
675 #define NMB_PA(i)	(netmap_buffer_lut[i].paddr)
676 
677 /*
678  * NMB return the virtual address of a buffer (buffer 0 on bad index)
679  * PNMB also fills the physical address
680  */
681 static inline void *
682 NMB(struct netmap_slot *slot)
683 {
684 	uint32_t i = slot->buf_idx;
685 	return (unlikely(i >= netmap_total_buffers)) ?  NMB_VA(0) : NMB_VA(i);
686 }
687 
688 static inline void *
689 PNMB(struct netmap_slot *slot, uint64_t *pp)
690 {
691 	uint32_t i = slot->buf_idx;
692 	void *ret = (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i);
693 
694 	*pp = (i >= netmap_total_buffers) ? NMB_PA(0) : NMB_PA(i);
695 	return ret;
696 }
697 
698 /* default functions to handle rx/tx interrupts */
699 int netmap_rx_irq(struct ifnet *, u_int, u_int *);
700 #define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL)
701 
702 #ifdef __FreeBSD__
703 MALLOC_DECLARE(M_NETMAP);
704 #endif /* __FreeBSD__ */
705 
706 
707 void netmap_disable_all_rings(struct ifnet *);
708 void netmap_enable_all_rings(struct ifnet *);
709 
710 #endif /* _NET_NETMAP_KERN_H_ */
711