xref: /freebsd/sys/dev/netmap/netmap.c (revision 760fa2ab5d12880865aec9b7dec8f1bc59f1ab0a)
1718cf2ccSPedro F. Giffuni /*-
2718cf2ccSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3718cf2ccSPedro F. Giffuni  *
437e3a6d3SLuigi Rizzo  * Copyright (C) 2011-2014 Matteo Landi
537e3a6d3SLuigi Rizzo  * Copyright (C) 2011-2016 Luigi Rizzo
637e3a6d3SLuigi Rizzo  * Copyright (C) 2011-2016 Giuseppe Lettieri
737e3a6d3SLuigi Rizzo  * Copyright (C) 2011-2016 Vincenzo Maffione
837e3a6d3SLuigi Rizzo  * All rights reserved.
968b8534bSLuigi Rizzo  *
1068b8534bSLuigi Rizzo  * Redistribution and use in source and binary forms, with or without
1168b8534bSLuigi Rizzo  * modification, are permitted provided that the following conditions
1268b8534bSLuigi Rizzo  * are met:
1368b8534bSLuigi Rizzo  *   1. Redistributions of source code must retain the above copyright
1468b8534bSLuigi Rizzo  *      notice, this list of conditions and the following disclaimer.
1568b8534bSLuigi Rizzo  *   2. Redistributions in binary form must reproduce the above copyright
1668b8534bSLuigi Rizzo  *      notice, this list of conditions and the following disclaimer in the
1768b8534bSLuigi Rizzo  *      documentation and/or other materials provided with the distribution.
1868b8534bSLuigi Rizzo  *
1968b8534bSLuigi Rizzo  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
2068b8534bSLuigi Rizzo  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2168b8534bSLuigi Rizzo  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2268b8534bSLuigi Rizzo  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
2368b8534bSLuigi Rizzo  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2468b8534bSLuigi Rizzo  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2568b8534bSLuigi Rizzo  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2668b8534bSLuigi Rizzo  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2768b8534bSLuigi Rizzo  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2868b8534bSLuigi Rizzo  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2968b8534bSLuigi Rizzo  * SUCH DAMAGE.
3068b8534bSLuigi Rizzo  */
3168b8534bSLuigi Rizzo 
32ce3ee1e7SLuigi Rizzo 
3368b8534bSLuigi Rizzo /*
34f9790aebSLuigi Rizzo  * $FreeBSD$
35f9790aebSLuigi Rizzo  *
3668b8534bSLuigi Rizzo  * This module supports memory mapped access to network devices,
3768b8534bSLuigi Rizzo  * see netmap(4).
3868b8534bSLuigi Rizzo  *
3968b8534bSLuigi Rizzo  * The module uses a large, memory pool allocated by the kernel
4068b8534bSLuigi Rizzo  * and accessible as mmapped memory by multiple userspace threads/processes.
4168b8534bSLuigi Rizzo  * The memory pool contains packet buffers and "netmap rings",
4268b8534bSLuigi Rizzo  * i.e. user-accessible copies of the interface's queues.
4368b8534bSLuigi Rizzo  *
4468b8534bSLuigi Rizzo  * Access to the network card works like this:
4568b8534bSLuigi Rizzo  * 1. a process/thread issues one or more open() on /dev/netmap, to create
4668b8534bSLuigi Rizzo  *    select()able file descriptor on which events are reported.
4768b8534bSLuigi Rizzo  * 2. on each descriptor, the process issues an ioctl() to identify
4868b8534bSLuigi Rizzo  *    the interface that should report events to the file descriptor.
4968b8534bSLuigi Rizzo  * 3. on each descriptor, the process issues an mmap() request to
5068b8534bSLuigi Rizzo  *    map the shared memory region within the process' address space.
5168b8534bSLuigi Rizzo  *    The list of interesting queues is indicated by a location in
5268b8534bSLuigi Rizzo  *    the shared memory region.
5368b8534bSLuigi Rizzo  * 4. using the functions in the netmap(4) userspace API, a process
5468b8534bSLuigi Rizzo  *    can look up the occupation state of a queue, access memory buffers,
5568b8534bSLuigi Rizzo  *    and retrieve received packets or enqueue packets to transmit.
5668b8534bSLuigi Rizzo  * 5. using some ioctl()s the process can synchronize the userspace view
5768b8534bSLuigi Rizzo  *    of the queue with the actual status in the kernel. This includes both
5868b8534bSLuigi Rizzo  *    receiving the notification of new packets, and transmitting new
5968b8534bSLuigi Rizzo  *    packets on the output interface.
6068b8534bSLuigi Rizzo  * 6. select() or poll() can be used to wait for events on individual
6168b8534bSLuigi Rizzo  *    transmit or receive queues (or all queues for a given interface).
62ce3ee1e7SLuigi Rizzo  *
63ce3ee1e7SLuigi Rizzo 
64ce3ee1e7SLuigi Rizzo 		SYNCHRONIZATION (USER)
65ce3ee1e7SLuigi Rizzo 
66ce3ee1e7SLuigi Rizzo The netmap rings and data structures may be shared among multiple
67ce3ee1e7SLuigi Rizzo user threads or even independent processes.
68ce3ee1e7SLuigi Rizzo Any synchronization among those threads/processes is delegated
69ce3ee1e7SLuigi Rizzo to the threads themselves. Only one thread at a time can be in
70ce3ee1e7SLuigi Rizzo a system call on the same netmap ring. The OS does not enforce
71ce3ee1e7SLuigi Rizzo this and only guarantees against system crashes in case of
72ce3ee1e7SLuigi Rizzo invalid usage.
73ce3ee1e7SLuigi Rizzo 
74ce3ee1e7SLuigi Rizzo 		LOCKING (INTERNAL)
75ce3ee1e7SLuigi Rizzo 
76ce3ee1e7SLuigi Rizzo Within the kernel, access to the netmap rings is protected as follows:
77ce3ee1e7SLuigi Rizzo 
78ce3ee1e7SLuigi Rizzo - a spinlock on each ring, to handle producer/consumer races on
79ce3ee1e7SLuigi Rizzo   RX rings attached to the host stack (against multiple host
80ce3ee1e7SLuigi Rizzo   threads writing from the host stack to the same ring),
81ce3ee1e7SLuigi Rizzo   and on 'destination' rings attached to a VALE switch
82ce3ee1e7SLuigi Rizzo   (i.e. RX rings in VALE ports, and TX rings in NIC/host ports)
83ce3ee1e7SLuigi Rizzo   protecting multiple active senders for the same destination)
84ce3ee1e7SLuigi Rizzo 
85ce3ee1e7SLuigi Rizzo - an atomic variable to guarantee that there is at most one
86ce3ee1e7SLuigi Rizzo   instance of *_*xsync() on the ring at any time.
87ce3ee1e7SLuigi Rizzo   For rings connected to user file
88ce3ee1e7SLuigi Rizzo   descriptors, an atomic_test_and_set() protects this, and the
89ce3ee1e7SLuigi Rizzo   lock on the ring is not actually used.
90ce3ee1e7SLuigi Rizzo   For NIC RX rings connected to a VALE switch, an atomic_test_and_set()
91ce3ee1e7SLuigi Rizzo   is also used to prevent multiple executions (the driver might indeed
92ce3ee1e7SLuigi Rizzo   already guarantee this).
93ce3ee1e7SLuigi Rizzo   For NIC TX rings connected to a VALE switch, the lock arbitrates
94ce3ee1e7SLuigi Rizzo   access to the queue (both when allocating buffers and when pushing
95ce3ee1e7SLuigi Rizzo   them out).
96ce3ee1e7SLuigi Rizzo 
97ce3ee1e7SLuigi Rizzo - *xsync() should be protected against initializations of the card.
98ce3ee1e7SLuigi Rizzo   On FreeBSD most devices have the reset routine protected by
99ce3ee1e7SLuigi Rizzo   a RING lock (ixgbe, igb, em) or core lock (re). lem is missing
100ce3ee1e7SLuigi Rizzo   the RING protection on rx_reset(), this should be added.
101ce3ee1e7SLuigi Rizzo 
102ce3ee1e7SLuigi Rizzo   On linux there is an external lock on the tx path, which probably
103ce3ee1e7SLuigi Rizzo   also arbitrates access to the reset routine. XXX to be revised
104ce3ee1e7SLuigi Rizzo 
105ce3ee1e7SLuigi Rizzo - a per-interface core_lock protecting access from the host stack
106ce3ee1e7SLuigi Rizzo   while interfaces may be detached from netmap mode.
107ce3ee1e7SLuigi Rizzo   XXX there should be no need for this lock if we detach the interfaces
108ce3ee1e7SLuigi Rizzo   only while they are down.
109ce3ee1e7SLuigi Rizzo 
110ce3ee1e7SLuigi Rizzo 
111ce3ee1e7SLuigi Rizzo --- VALE SWITCH ---
112ce3ee1e7SLuigi Rizzo 
113ce3ee1e7SLuigi Rizzo NMG_LOCK() serializes all modifications to switches and ports.
114ce3ee1e7SLuigi Rizzo A switch cannot be deleted until all ports are gone.
115ce3ee1e7SLuigi Rizzo 
116ce3ee1e7SLuigi Rizzo For each switch, an SX lock (RWlock on linux) protects
117ce3ee1e7SLuigi Rizzo deletion of ports. When configuring or deleting a new port, the
118ce3ee1e7SLuigi Rizzo lock is acquired in exclusive mode (after holding NMG_LOCK).
119ce3ee1e7SLuigi Rizzo When forwarding, the lock is acquired in shared mode (without NMG_LOCK).
120ce3ee1e7SLuigi Rizzo The lock is held throughout the entire forwarding cycle,
121ce3ee1e7SLuigi Rizzo during which the thread may incur in a page fault.
122ce3ee1e7SLuigi Rizzo Hence it is important that sleepable shared locks are used.
123ce3ee1e7SLuigi Rizzo 
124ce3ee1e7SLuigi Rizzo On the rx ring, the per-port lock is grabbed initially to reserve
125ce3ee1e7SLuigi Rizzo a number of slot in the ring, then the lock is released,
126ce3ee1e7SLuigi Rizzo packets are copied from source to destination, and then
127ce3ee1e7SLuigi Rizzo the lock is acquired again and the receive ring is updated.
128ce3ee1e7SLuigi Rizzo (A similar thing is done on the tx ring for NIC and host stack
129ce3ee1e7SLuigi Rizzo ports attached to the switch)
130ce3ee1e7SLuigi Rizzo 
13168b8534bSLuigi Rizzo  */
13268b8534bSLuigi Rizzo 
1334bf50f18SLuigi Rizzo 
1344bf50f18SLuigi Rizzo /* --- internals ----
1354bf50f18SLuigi Rizzo  *
1364bf50f18SLuigi Rizzo  * Roadmap to the code that implements the above.
1374bf50f18SLuigi Rizzo  *
1384bf50f18SLuigi Rizzo  * > 1. a process/thread issues one or more open() on /dev/netmap, to create
1394bf50f18SLuigi Rizzo  * >    select()able file descriptor on which events are reported.
1404bf50f18SLuigi Rizzo  *
1414bf50f18SLuigi Rizzo  *  	Internally, we allocate a netmap_priv_d structure, that will be
14237e3a6d3SLuigi Rizzo  *  	initialized on ioctl(NIOCREGIF). There is one netmap_priv_d
14337e3a6d3SLuigi Rizzo  *  	structure for each open().
1444bf50f18SLuigi Rizzo  *
1454bf50f18SLuigi Rizzo  *      os-specific:
14637e3a6d3SLuigi Rizzo  *  	    FreeBSD: see netmap_open() (netmap_freebsd.c)
14737e3a6d3SLuigi Rizzo  *  	    linux:   see linux_netmap_open() (netmap_linux.c)
1484bf50f18SLuigi Rizzo  *
1494bf50f18SLuigi Rizzo  * > 2. on each descriptor, the process issues an ioctl() to identify
1504bf50f18SLuigi Rizzo  * >    the interface that should report events to the file descriptor.
1514bf50f18SLuigi Rizzo  *
1524bf50f18SLuigi Rizzo  * 	Implemented by netmap_ioctl(), NIOCREGIF case, with nmr->nr_cmd==0.
1534bf50f18SLuigi Rizzo  * 	Most important things happen in netmap_get_na() and
1544bf50f18SLuigi Rizzo  * 	netmap_do_regif(), called from there. Additional details can be
1554bf50f18SLuigi Rizzo  * 	found in the comments above those functions.
1564bf50f18SLuigi Rizzo  *
1574bf50f18SLuigi Rizzo  * 	In all cases, this action creates/takes-a-reference-to a
1584bf50f18SLuigi Rizzo  * 	netmap_*_adapter describing the port, and allocates a netmap_if
1594bf50f18SLuigi Rizzo  * 	and all necessary netmap rings, filling them with netmap buffers.
1604bf50f18SLuigi Rizzo  *
1614bf50f18SLuigi Rizzo  *      In this phase, the sync callbacks for each ring are set (these are used
1624bf50f18SLuigi Rizzo  *      in steps 5 and 6 below).  The callbacks depend on the type of adapter.
1634bf50f18SLuigi Rizzo  *      The adapter creation/initialization code puts them in the
1644bf50f18SLuigi Rizzo  * 	netmap_adapter (fields na->nm_txsync and na->nm_rxsync).  Then, they
1654bf50f18SLuigi Rizzo  * 	are copied from there to the netmap_kring's during netmap_do_regif(), by
1664bf50f18SLuigi Rizzo  * 	the nm_krings_create() callback.  All the nm_krings_create callbacks
1674bf50f18SLuigi Rizzo  * 	actually call netmap_krings_create() to perform this and the other
1684bf50f18SLuigi Rizzo  * 	common stuff. netmap_krings_create() also takes care of the host rings,
1694bf50f18SLuigi Rizzo  * 	if needed, by setting their sync callbacks appropriately.
1704bf50f18SLuigi Rizzo  *
1714bf50f18SLuigi Rizzo  * 	Additional actions depend on the kind of netmap_adapter that has been
1724bf50f18SLuigi Rizzo  * 	registered:
1734bf50f18SLuigi Rizzo  *
1744bf50f18SLuigi Rizzo  * 	- netmap_hw_adapter:  	     [netmap.c]
1754bf50f18SLuigi Rizzo  * 	     This is a system netdev/ifp with native netmap support.
1764bf50f18SLuigi Rizzo  * 	     The ifp is detached from the host stack by redirecting:
1774bf50f18SLuigi Rizzo  * 	       - transmissions (from the network stack) to netmap_transmit()
1784bf50f18SLuigi Rizzo  * 	       - receive notifications to the nm_notify() callback for
1794bf50f18SLuigi Rizzo  * 	         this adapter. The callback is normally netmap_notify(), unless
1804bf50f18SLuigi Rizzo  * 	         the ifp is attached to a bridge using bwrap, in which case it
1814bf50f18SLuigi Rizzo  * 	         is netmap_bwrap_intr_notify().
1824bf50f18SLuigi Rizzo  *
1834bf50f18SLuigi Rizzo  * 	- netmap_generic_adapter:      [netmap_generic.c]
1844bf50f18SLuigi Rizzo  * 	      A system netdev/ifp without native netmap support.
1854bf50f18SLuigi Rizzo  *
1864bf50f18SLuigi Rizzo  * 	(the decision about native/non native support is taken in
1874bf50f18SLuigi Rizzo  * 	 netmap_get_hw_na(), called by netmap_get_na())
1884bf50f18SLuigi Rizzo  *
1894bf50f18SLuigi Rizzo  * 	- netmap_vp_adapter 		[netmap_vale.c]
1904bf50f18SLuigi Rizzo  * 	      Returned by netmap_get_bdg_na().
1914bf50f18SLuigi Rizzo  * 	      This is a persistent or ephemeral VALE port. Ephemeral ports
1924bf50f18SLuigi Rizzo  * 	      are created on the fly if they don't already exist, and are
1934bf50f18SLuigi Rizzo  * 	      always attached to a bridge.
194453130d9SPedro F. Giffuni  * 	      Persistent VALE ports must must be created separately, and i
1954bf50f18SLuigi Rizzo  * 	      then attached like normal NICs. The NIOCREGIF we are examining
1964bf50f18SLuigi Rizzo  * 	      will find them only if they had previosly been created and
1974bf50f18SLuigi Rizzo  * 	      attached (see VALE_CTL below).
1984bf50f18SLuigi Rizzo  *
1994bf50f18SLuigi Rizzo  * 	- netmap_pipe_adapter 	      [netmap_pipe.c]
2004bf50f18SLuigi Rizzo  * 	      Returned by netmap_get_pipe_na().
2014bf50f18SLuigi Rizzo  * 	      Both pipe ends are created, if they didn't already exist.
2024bf50f18SLuigi Rizzo  *
2034bf50f18SLuigi Rizzo  * 	- netmap_monitor_adapter      [netmap_monitor.c]
2044bf50f18SLuigi Rizzo  * 	      Returned by netmap_get_monitor_na().
2054bf50f18SLuigi Rizzo  * 	      If successful, the nm_sync callbacks of the monitored adapter
2064bf50f18SLuigi Rizzo  * 	      will be intercepted by the returned monitor.
2074bf50f18SLuigi Rizzo  *
2084bf50f18SLuigi Rizzo  * 	- netmap_bwrap_adapter	      [netmap_vale.c]
2094bf50f18SLuigi Rizzo  * 	      Cannot be obtained in this way, see VALE_CTL below
2104bf50f18SLuigi Rizzo  *
2114bf50f18SLuigi Rizzo  *
2124bf50f18SLuigi Rizzo  * 	os-specific:
2134bf50f18SLuigi Rizzo  * 	    linux: we first go through linux_netmap_ioctl() to
2144bf50f18SLuigi Rizzo  * 	           adapt the FreeBSD interface to the linux one.
2154bf50f18SLuigi Rizzo  *
2164bf50f18SLuigi Rizzo  *
2174bf50f18SLuigi Rizzo  * > 3. on each descriptor, the process issues an mmap() request to
2184bf50f18SLuigi Rizzo  * >    map the shared memory region within the process' address space.
2194bf50f18SLuigi Rizzo  * >    The list of interesting queues is indicated by a location in
2204bf50f18SLuigi Rizzo  * >    the shared memory region.
2214bf50f18SLuigi Rizzo  *
2224bf50f18SLuigi Rizzo  *      os-specific:
2234bf50f18SLuigi Rizzo  *  	    FreeBSD: netmap_mmap_single (netmap_freebsd.c).
2244bf50f18SLuigi Rizzo  *  	    linux:   linux_netmap_mmap (netmap_linux.c).
2254bf50f18SLuigi Rizzo  *
2264bf50f18SLuigi Rizzo  * > 4. using the functions in the netmap(4) userspace API, a process
2274bf50f18SLuigi Rizzo  * >    can look up the occupation state of a queue, access memory buffers,
2284bf50f18SLuigi Rizzo  * >    and retrieve received packets or enqueue packets to transmit.
2294bf50f18SLuigi Rizzo  *
2304bf50f18SLuigi Rizzo  * 	these actions do not involve the kernel.
2314bf50f18SLuigi Rizzo  *
2324bf50f18SLuigi Rizzo  * > 5. using some ioctl()s the process can synchronize the userspace view
2334bf50f18SLuigi Rizzo  * >    of the queue with the actual status in the kernel. This includes both
2344bf50f18SLuigi Rizzo  * >    receiving the notification of new packets, and transmitting new
2354bf50f18SLuigi Rizzo  * >    packets on the output interface.
2364bf50f18SLuigi Rizzo  *
2374bf50f18SLuigi Rizzo  * 	These are implemented in netmap_ioctl(), NIOCTXSYNC and NIOCRXSYNC
2384bf50f18SLuigi Rizzo  * 	cases. They invoke the nm_sync callbacks on the netmap_kring
2394bf50f18SLuigi Rizzo  * 	structures, as initialized in step 2 and maybe later modified
2404bf50f18SLuigi Rizzo  * 	by a monitor. Monitors, however, will always call the original
2414bf50f18SLuigi Rizzo  * 	callback before doing anything else.
2424bf50f18SLuigi Rizzo  *
2434bf50f18SLuigi Rizzo  *
2444bf50f18SLuigi Rizzo  * > 6. select() or poll() can be used to wait for events on individual
2454bf50f18SLuigi Rizzo  * >    transmit or receive queues (or all queues for a given interface).
2464bf50f18SLuigi Rizzo  *
2474bf50f18SLuigi Rizzo  * 	Implemented in netmap_poll(). This will call the same nm_sync()
2484bf50f18SLuigi Rizzo  * 	callbacks as in step 5 above.
2494bf50f18SLuigi Rizzo  *
2504bf50f18SLuigi Rizzo  * 	os-specific:
2514bf50f18SLuigi Rizzo  * 		linux: we first go through linux_netmap_poll() to adapt
2524bf50f18SLuigi Rizzo  * 		       the FreeBSD interface to the linux one.
2534bf50f18SLuigi Rizzo  *
2544bf50f18SLuigi Rizzo  *
2554bf50f18SLuigi Rizzo  *  ----  VALE_CTL -----
2564bf50f18SLuigi Rizzo  *
2574bf50f18SLuigi Rizzo  *  VALE switches are controlled by issuing a NIOCREGIF with a non-null
2584bf50f18SLuigi Rizzo  *  nr_cmd in the nmreq structure. These subcommands are handled by
2594bf50f18SLuigi Rizzo  *  netmap_bdg_ctl() in netmap_vale.c. Persistent VALE ports are created
2604bf50f18SLuigi Rizzo  *  and destroyed by issuing the NETMAP_BDG_NEWIF and NETMAP_BDG_DELIF
2614bf50f18SLuigi Rizzo  *  subcommands, respectively.
2624bf50f18SLuigi Rizzo  *
2634bf50f18SLuigi Rizzo  *  Any network interface known to the system (including a persistent VALE
2644bf50f18SLuigi Rizzo  *  port) can be attached to a VALE switch by issuing the
2652ff91c17SVincenzo Maffione  *  NETMAP_REQ_VALE_ATTACH command. After the attachment, persistent VALE ports
2664bf50f18SLuigi Rizzo  *  look exactly like ephemeral VALE ports (as created in step 2 above).  The
2674bf50f18SLuigi Rizzo  *  attachment of other interfaces, instead, requires the creation of a
2684bf50f18SLuigi Rizzo  *  netmap_bwrap_adapter.  Moreover, the attached interface must be put in
2694bf50f18SLuigi Rizzo  *  netmap mode. This may require the creation of a netmap_generic_adapter if
2704bf50f18SLuigi Rizzo  *  we have no native support for the interface, or if generic adapters have
2714bf50f18SLuigi Rizzo  *  been forced by sysctl.
2724bf50f18SLuigi Rizzo  *
2734bf50f18SLuigi Rizzo  *  Both persistent VALE ports and bwraps are handled by netmap_get_bdg_na(),
2744bf50f18SLuigi Rizzo  *  called by nm_bdg_ctl_attach(), and discriminated by the nm_bdg_attach()
2754bf50f18SLuigi Rizzo  *  callback.  In the case of the bwrap, the callback creates the
2764bf50f18SLuigi Rizzo  *  netmap_bwrap_adapter.  The initialization of the bwrap is then
2774bf50f18SLuigi Rizzo  *  completed by calling netmap_do_regif() on it, in the nm_bdg_ctl()
2784bf50f18SLuigi Rizzo  *  callback (netmap_bwrap_bdg_ctl in netmap_vale.c).
2794bf50f18SLuigi Rizzo  *  A generic adapter for the wrapped ifp will be created if needed, when
2804bf50f18SLuigi Rizzo  *  netmap_get_bdg_na() calls netmap_get_hw_na().
2814bf50f18SLuigi Rizzo  *
2824bf50f18SLuigi Rizzo  *
2834bf50f18SLuigi Rizzo  *  ---- DATAPATHS -----
2844bf50f18SLuigi Rizzo  *
2854bf50f18SLuigi Rizzo  *              -= SYSTEM DEVICE WITH NATIVE SUPPORT =-
2864bf50f18SLuigi Rizzo  *
2874bf50f18SLuigi Rizzo  *    na == NA(ifp) == netmap_hw_adapter created in DEVICE_netmap_attach()
2884bf50f18SLuigi Rizzo  *
2894bf50f18SLuigi Rizzo  *    - tx from netmap userspace:
2904bf50f18SLuigi Rizzo  *	 concurrently:
2914bf50f18SLuigi Rizzo  *           1) ioctl(NIOCTXSYNC)/netmap_poll() in process context
2924bf50f18SLuigi Rizzo  *                kring->nm_sync() == DEVICE_netmap_txsync()
2934bf50f18SLuigi Rizzo  *           2) device interrupt handler
2944bf50f18SLuigi Rizzo  *                na->nm_notify()  == netmap_notify()
2954bf50f18SLuigi Rizzo  *    - rx from netmap userspace:
2964bf50f18SLuigi Rizzo  *       concurrently:
2974bf50f18SLuigi Rizzo  *           1) ioctl(NIOCRXSYNC)/netmap_poll() in process context
2984bf50f18SLuigi Rizzo  *                kring->nm_sync() == DEVICE_netmap_rxsync()
2994bf50f18SLuigi Rizzo  *           2) device interrupt handler
3004bf50f18SLuigi Rizzo  *                na->nm_notify()  == netmap_notify()
301847bf383SLuigi Rizzo  *    - rx from host stack
3024bf50f18SLuigi Rizzo  *       concurrently:
3034bf50f18SLuigi Rizzo  *           1) host stack
3044bf50f18SLuigi Rizzo  *                netmap_transmit()
3054bf50f18SLuigi Rizzo  *                  na->nm_notify  == netmap_notify()
3064bf50f18SLuigi Rizzo  *           2) ioctl(NIOCRXSYNC)/netmap_poll() in process context
30737e3a6d3SLuigi Rizzo  *                kring->nm_sync() == netmap_rxsync_from_host
3084bf50f18SLuigi Rizzo  *                  netmap_rxsync_from_host(na, NULL, NULL)
3094bf50f18SLuigi Rizzo  *    - tx to host stack
3104bf50f18SLuigi Rizzo  *           ioctl(NIOCTXSYNC)/netmap_poll() in process context
31137e3a6d3SLuigi Rizzo  *             kring->nm_sync() == netmap_txsync_to_host
3124bf50f18SLuigi Rizzo  *               netmap_txsync_to_host(na)
31337e3a6d3SLuigi Rizzo  *                 nm_os_send_up()
31437e3a6d3SLuigi Rizzo  *                   FreeBSD: na->if_input() == ether_input()
3154bf50f18SLuigi Rizzo  *                   linux: netif_rx() with NM_MAGIC_PRIORITY_RX
3164bf50f18SLuigi Rizzo  *
3174bf50f18SLuigi Rizzo  *
3184bf50f18SLuigi Rizzo  *               -= SYSTEM DEVICE WITH GENERIC SUPPORT =-
3194bf50f18SLuigi Rizzo  *
320847bf383SLuigi Rizzo  *    na == NA(ifp) == generic_netmap_adapter created in generic_netmap_attach()
321847bf383SLuigi Rizzo  *
322847bf383SLuigi Rizzo  *    - tx from netmap userspace:
323847bf383SLuigi Rizzo  *       concurrently:
324847bf383SLuigi Rizzo  *           1) ioctl(NIOCTXSYNC)/netmap_poll() in process context
325847bf383SLuigi Rizzo  *               kring->nm_sync() == generic_netmap_txsync()
32637e3a6d3SLuigi Rizzo  *                   nm_os_generic_xmit_frame()
327847bf383SLuigi Rizzo  *                       linux:   dev_queue_xmit() with NM_MAGIC_PRIORITY_TX
32837e3a6d3SLuigi Rizzo  *                           ifp->ndo_start_xmit == generic_ndo_start_xmit()
32937e3a6d3SLuigi Rizzo  *                               gna->save_start_xmit == orig. dev. start_xmit
330847bf383SLuigi Rizzo  *                       FreeBSD: na->if_transmit() == orig. dev if_transmit
331847bf383SLuigi Rizzo  *           2) generic_mbuf_destructor()
332847bf383SLuigi Rizzo  *                   na->nm_notify() == netmap_notify()
333847bf383SLuigi Rizzo  *    - rx from netmap userspace:
334847bf383SLuigi Rizzo  *           1) ioctl(NIOCRXSYNC)/netmap_poll() in process context
335847bf383SLuigi Rizzo  *               kring->nm_sync() == generic_netmap_rxsync()
336847bf383SLuigi Rizzo  *                   mbq_safe_dequeue()
337847bf383SLuigi Rizzo  *           2) device driver
338847bf383SLuigi Rizzo  *               generic_rx_handler()
339847bf383SLuigi Rizzo  *                   mbq_safe_enqueue()
340847bf383SLuigi Rizzo  *                   na->nm_notify() == netmap_notify()
34137e3a6d3SLuigi Rizzo  *    - rx from host stack
34237e3a6d3SLuigi Rizzo  *        FreeBSD: same as native
34337e3a6d3SLuigi Rizzo  *        Linux: same as native except:
344847bf383SLuigi Rizzo  *           1) host stack
34537e3a6d3SLuigi Rizzo  *               dev_queue_xmit() without NM_MAGIC_PRIORITY_TX
34637e3a6d3SLuigi Rizzo  *                   ifp->ndo_start_xmit == generic_ndo_start_xmit()
347847bf383SLuigi Rizzo  *                       netmap_transmit()
348847bf383SLuigi Rizzo  *                           na->nm_notify() == netmap_notify()
34937e3a6d3SLuigi Rizzo  *    - tx to host stack (same as native):
3504bf50f18SLuigi Rizzo  *
3514bf50f18SLuigi Rizzo  *
352847bf383SLuigi Rizzo  *                           -= VALE =-
3534bf50f18SLuigi Rizzo  *
354847bf383SLuigi Rizzo  *   INCOMING:
3554bf50f18SLuigi Rizzo  *
356847bf383SLuigi Rizzo  *      - VALE ports:
357847bf383SLuigi Rizzo  *          ioctl(NIOCTXSYNC)/netmap_poll() in process context
358847bf383SLuigi Rizzo  *              kring->nm_sync() == netmap_vp_txsync()
3594bf50f18SLuigi Rizzo  *
360847bf383SLuigi Rizzo  *      - system device with native support:
361847bf383SLuigi Rizzo  *         from cable:
362847bf383SLuigi Rizzo  *             interrupt
363847bf383SLuigi Rizzo  *                na->nm_notify() == netmap_bwrap_intr_notify(ring_nr != host ring)
364847bf383SLuigi Rizzo  *                     kring->nm_sync() == DEVICE_netmap_rxsync()
365847bf383SLuigi Rizzo  *                     netmap_vp_txsync()
366847bf383SLuigi Rizzo  *                     kring->nm_sync() == DEVICE_netmap_rxsync()
367847bf383SLuigi Rizzo  *         from host stack:
368847bf383SLuigi Rizzo  *             netmap_transmit()
369847bf383SLuigi Rizzo  *                na->nm_notify() == netmap_bwrap_intr_notify(ring_nr == host ring)
37037e3a6d3SLuigi Rizzo  *                     kring->nm_sync() == netmap_rxsync_from_host()
371847bf383SLuigi Rizzo  *                     netmap_vp_txsync()
3724bf50f18SLuigi Rizzo  *
373847bf383SLuigi Rizzo  *      - system device with generic support:
374847bf383SLuigi Rizzo  *         from device driver:
375847bf383SLuigi Rizzo  *            generic_rx_handler()
376847bf383SLuigi Rizzo  *                na->nm_notify() == netmap_bwrap_intr_notify(ring_nr != host ring)
377847bf383SLuigi Rizzo  *                     kring->nm_sync() == generic_netmap_rxsync()
378847bf383SLuigi Rizzo  *                     netmap_vp_txsync()
379847bf383SLuigi Rizzo  *                     kring->nm_sync() == generic_netmap_rxsync()
380847bf383SLuigi Rizzo  *         from host stack:
381847bf383SLuigi Rizzo  *            netmap_transmit()
382847bf383SLuigi Rizzo  *                na->nm_notify() == netmap_bwrap_intr_notify(ring_nr == host ring)
38337e3a6d3SLuigi Rizzo  *                     kring->nm_sync() == netmap_rxsync_from_host()
384847bf383SLuigi Rizzo  *                     netmap_vp_txsync()
3854bf50f18SLuigi Rizzo  *
386847bf383SLuigi Rizzo  *   (all cases) --> nm_bdg_flush()
387847bf383SLuigi Rizzo  *                      dest_na->nm_notify() == (see below)
3884bf50f18SLuigi Rizzo  *
389847bf383SLuigi Rizzo  *   OUTGOING:
3904bf50f18SLuigi Rizzo  *
391847bf383SLuigi Rizzo  *      - VALE ports:
392847bf383SLuigi Rizzo  *         concurrently:
393c3e9b4dbSLuiz Otavio O Souza  *             1) ioctl(NIOCRXSYNC)/netmap_poll() in process context
394847bf383SLuigi Rizzo  *                    kring->nm_sync() == netmap_vp_rxsync()
395847bf383SLuigi Rizzo  *             2) from nm_bdg_flush()
396847bf383SLuigi Rizzo  *                    na->nm_notify() == netmap_notify()
3974bf50f18SLuigi Rizzo  *
398847bf383SLuigi Rizzo  *      - system device with native support:
399847bf383SLuigi Rizzo  *          to cable:
400847bf383SLuigi Rizzo  *             na->nm_notify() == netmap_bwrap_notify()
401847bf383SLuigi Rizzo  *                 netmap_vp_rxsync()
402847bf383SLuigi Rizzo  *                 kring->nm_sync() == DEVICE_netmap_txsync()
403847bf383SLuigi Rizzo  *                 netmap_vp_rxsync()
404847bf383SLuigi Rizzo  *          to host stack:
405847bf383SLuigi Rizzo  *                 netmap_vp_rxsync()
40637e3a6d3SLuigi Rizzo  *                 kring->nm_sync() == netmap_txsync_to_host
407847bf383SLuigi Rizzo  *                 netmap_vp_rxsync_locked()
4084bf50f18SLuigi Rizzo  *
409847bf383SLuigi Rizzo  *      - system device with generic adapter:
410847bf383SLuigi Rizzo  *          to device driver:
411847bf383SLuigi Rizzo  *             na->nm_notify() == netmap_bwrap_notify()
412847bf383SLuigi Rizzo  *                 netmap_vp_rxsync()
413847bf383SLuigi Rizzo  *                 kring->nm_sync() == generic_netmap_txsync()
414847bf383SLuigi Rizzo  *                 netmap_vp_rxsync()
415847bf383SLuigi Rizzo  *          to host stack:
416847bf383SLuigi Rizzo  *                 netmap_vp_rxsync()
41737e3a6d3SLuigi Rizzo  *                 kring->nm_sync() == netmap_txsync_to_host
418847bf383SLuigi Rizzo  *                 netmap_vp_rxsync()
4194bf50f18SLuigi Rizzo  *
4204bf50f18SLuigi Rizzo  */
4214bf50f18SLuigi Rizzo 
422ce3ee1e7SLuigi Rizzo /*
423ce3ee1e7SLuigi Rizzo  * OS-specific code that is used only within this file.
424ce3ee1e7SLuigi Rizzo  * Other OS-specific code that must be accessed by drivers
425ce3ee1e7SLuigi Rizzo  * is present in netmap_kern.h
426ce3ee1e7SLuigi Rizzo  */
42701c7d25fSLuigi Rizzo 
428ce3ee1e7SLuigi Rizzo #if defined(__FreeBSD__)
42968b8534bSLuigi Rizzo #include <sys/cdefs.h> /* prerequisite */
43068b8534bSLuigi Rizzo #include <sys/types.h>
43168b8534bSLuigi Rizzo #include <sys/errno.h>
43268b8534bSLuigi Rizzo #include <sys/param.h>	/* defines used in kernel.h */
43368b8534bSLuigi Rizzo #include <sys/kernel.h>	/* types used in module initialization */
434f9790aebSLuigi Rizzo #include <sys/conf.h>	/* cdevsw struct, UID, GID */
43589e3fd52SLuigi Rizzo #include <sys/filio.h>	/* FIONBIO */
43668b8534bSLuigi Rizzo #include <sys/sockio.h>
43768b8534bSLuigi Rizzo #include <sys/socketvar.h>	/* struct socket */
43868b8534bSLuigi Rizzo #include <sys/malloc.h>
43968b8534bSLuigi Rizzo #include <sys/poll.h>
44089f6b863SAttilio Rao #include <sys/rwlock.h>
44168b8534bSLuigi Rizzo #include <sys/socket.h> /* sockaddrs */
44268b8534bSLuigi Rizzo #include <sys/selinfo.h>
44368b8534bSLuigi Rizzo #include <sys/sysctl.h>
444339f59c0SGleb Smirnoff #include <sys/jail.h>
445339f59c0SGleb Smirnoff #include <net/vnet.h>
44668b8534bSLuigi Rizzo #include <net/if.h>
44776039bc8SGleb Smirnoff #include <net/if_var.h>
44868b8534bSLuigi Rizzo #include <net/bpf.h>		/* BIOCIMMEDIATE */
44968b8534bSLuigi Rizzo #include <machine/bus.h>	/* bus_dmamap_* */
450ce3ee1e7SLuigi Rizzo #include <sys/endian.h>
451ce3ee1e7SLuigi Rizzo #include <sys/refcount.h>
45289a9a5b5SVincenzo Maffione #include <net/ethernet.h>	/* ETHER_BPF_MTAP */
45368b8534bSLuigi Rizzo 
45468b8534bSLuigi Rizzo 
455ce3ee1e7SLuigi Rizzo #elif defined(linux)
456ce3ee1e7SLuigi Rizzo 
457ce3ee1e7SLuigi Rizzo #include "bsd_glue.h"
458ce3ee1e7SLuigi Rizzo 
459ce3ee1e7SLuigi Rizzo #elif defined(__APPLE__)
460ce3ee1e7SLuigi Rizzo 
461ce3ee1e7SLuigi Rizzo #warning OSX support is only partial
462ce3ee1e7SLuigi Rizzo #include "osx_glue.h"
463ce3ee1e7SLuigi Rizzo 
46437e3a6d3SLuigi Rizzo #elif defined (_WIN32)
46537e3a6d3SLuigi Rizzo 
46637e3a6d3SLuigi Rizzo #include "win_glue.h"
46737e3a6d3SLuigi Rizzo 
468ce3ee1e7SLuigi Rizzo #else
469ce3ee1e7SLuigi Rizzo 
470ce3ee1e7SLuigi Rizzo #error	Unsupported platform
471ce3ee1e7SLuigi Rizzo 
472ce3ee1e7SLuigi Rizzo #endif /* unsupported */
473ce3ee1e7SLuigi Rizzo 
474ce3ee1e7SLuigi Rizzo /*
475ce3ee1e7SLuigi Rizzo  * common headers
476ce3ee1e7SLuigi Rizzo  */
4770b8ed8e0SLuigi Rizzo #include <net/netmap.h>
4780b8ed8e0SLuigi Rizzo #include <dev/netmap/netmap_kern.h>
479ce3ee1e7SLuigi Rizzo #include <dev/netmap/netmap_mem2.h>
4800b8ed8e0SLuigi Rizzo 
481ce3ee1e7SLuigi Rizzo 
4825819da83SLuigi Rizzo /* user-controlled variables */
4835819da83SLuigi Rizzo int netmap_verbose;
484b6e66be2SVincenzo Maffione #ifdef CONFIG_NETMAP_DEBUG
485b6e66be2SVincenzo Maffione int netmap_debug;
486b6e66be2SVincenzo Maffione #endif /* CONFIG_NETMAP_DEBUG */
4875819da83SLuigi Rizzo 
4885819da83SLuigi Rizzo static int netmap_no_timestamp; /* don't timestamp on rxsync */
489c85cb1a0SLuigi Rizzo int netmap_no_pendintr = 1;
490f18be576SLuigi Rizzo int netmap_txsync_retry = 2;
491c3e9b4dbSLuiz Otavio O Souza static int netmap_fwd = 0;	/* force transparent forwarding */
492f196ce38SLuigi Rizzo 
493f9790aebSLuigi Rizzo /*
494f9790aebSLuigi Rizzo  * netmap_admode selects the netmap mode to use.
495f9790aebSLuigi Rizzo  * Invalid values are reset to NETMAP_ADMODE_BEST
496f9790aebSLuigi Rizzo  */
497f9790aebSLuigi Rizzo enum {	NETMAP_ADMODE_BEST = 0,	/* use native, fallback to generic */
498f9790aebSLuigi Rizzo 	NETMAP_ADMODE_NATIVE,	/* either native or none */
499f9790aebSLuigi Rizzo 	NETMAP_ADMODE_GENERIC,	/* force generic */
500f9790aebSLuigi Rizzo 	NETMAP_ADMODE_LAST };
501f9790aebSLuigi Rizzo static int netmap_admode = NETMAP_ADMODE_BEST;
502f9790aebSLuigi Rizzo 
50337e3a6d3SLuigi Rizzo /* netmap_generic_mit controls mitigation of RX notifications for
50437e3a6d3SLuigi Rizzo  * the generic netmap adapter. The value is a time interval in
50537e3a6d3SLuigi Rizzo  * nanoseconds. */
50637e3a6d3SLuigi Rizzo int netmap_generic_mit = 100*1000;
50737e3a6d3SLuigi Rizzo 
50837e3a6d3SLuigi Rizzo /* We use by default netmap-aware qdiscs with generic netmap adapters,
50937e3a6d3SLuigi Rizzo  * even if there can be a little performance hit with hardware NICs.
51037e3a6d3SLuigi Rizzo  * However, using the qdisc is the safer approach, for two reasons:
51137e3a6d3SLuigi Rizzo  * 1) it prevents non-fifo qdiscs to break the TX notification
51237e3a6d3SLuigi Rizzo  *    scheme, which is based on mbuf destructors when txqdisc is
51337e3a6d3SLuigi Rizzo  *    not used.
51437e3a6d3SLuigi Rizzo  * 2) it makes it possible to transmit over software devices that
51537e3a6d3SLuigi Rizzo  *    change skb->dev, like bridge, veth, ...
51637e3a6d3SLuigi Rizzo  *
51737e3a6d3SLuigi Rizzo  * Anyway users looking for the best performance should
51837e3a6d3SLuigi Rizzo  * use native adapters.
51937e3a6d3SLuigi Rizzo  */
5204f80b14cSVincenzo Maffione #ifdef linux
52137e3a6d3SLuigi Rizzo int netmap_generic_txqdisc = 1;
5224f80b14cSVincenzo Maffione #endif
52337e3a6d3SLuigi Rizzo 
52437e3a6d3SLuigi Rizzo /* Default number of slots and queues for generic adapters. */
52537e3a6d3SLuigi Rizzo int netmap_generic_ringsize = 1024;
52637e3a6d3SLuigi Rizzo int netmap_generic_rings = 1;
52737e3a6d3SLuigi Rizzo 
5282a7db7a6SVincenzo Maffione /* Non-zero to enable checksum offloading in NIC drivers */
5292a7db7a6SVincenzo Maffione int netmap_generic_hwcsum = 0;
5302a7db7a6SVincenzo Maffione 
53137e3a6d3SLuigi Rizzo /* Non-zero if ptnet devices are allowed to use virtio-net headers. */
53237e3a6d3SLuigi Rizzo int ptnet_vnet_hdr = 1;
53337e3a6d3SLuigi Rizzo 
53437e3a6d3SLuigi Rizzo /*
53537e3a6d3SLuigi Rizzo  * SYSCTL calls are grouped between SYSBEGIN and SYSEND to be emulated
53637e3a6d3SLuigi Rizzo  * in some other operating systems
53737e3a6d3SLuigi Rizzo  */
53837e3a6d3SLuigi Rizzo SYSBEGIN(main_init);
53937e3a6d3SLuigi Rizzo 
54037e3a6d3SLuigi Rizzo SYSCTL_DECL(_dev_netmap);
54137e3a6d3SLuigi Rizzo SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args");
54237e3a6d3SLuigi Rizzo SYSCTL_INT(_dev_netmap, OID_AUTO, verbose,
54337e3a6d3SLuigi Rizzo 		CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode");
544b6e66be2SVincenzo Maffione #ifdef CONFIG_NETMAP_DEBUG
545b6e66be2SVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, debug,
546b6e66be2SVincenzo Maffione 		CTLFLAG_RW, &netmap_debug, 0, "Debug messages");
547b6e66be2SVincenzo Maffione #endif /* CONFIG_NETMAP_DEBUG */
54837e3a6d3SLuigi Rizzo SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp,
54937e3a6d3SLuigi Rizzo 		CTLFLAG_RW, &netmap_no_timestamp, 0, "no_timestamp");
5504f80b14cSVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr, CTLFLAG_RW, &netmap_no_pendintr,
5514f80b14cSVincenzo Maffione 		0, "Always look for new received packets.");
55237e3a6d3SLuigi Rizzo SYSCTL_INT(_dev_netmap, OID_AUTO, txsync_retry, CTLFLAG_RW,
55337e3a6d3SLuigi Rizzo 		&netmap_txsync_retry, 0, "Number of txsync loops in bridge's flush.");
554f9790aebSLuigi Rizzo 
5554f80b14cSVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0,
5564f80b14cSVincenzo Maffione 		"Force NR_FORWARD mode");
5574f80b14cSVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, admode, CTLFLAG_RW, &netmap_admode, 0,
5584f80b14cSVincenzo Maffione 		"Adapter mode. 0 selects the best option available,"
5594f80b14cSVincenzo Maffione 		"1 forces native adapter, 2 forces emulated adapter");
5602a7db7a6SVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, generic_hwcsum, CTLFLAG_RW, &netmap_generic_hwcsum,
5612a7db7a6SVincenzo Maffione 		0, "Hardware checksums. 0 to disable checksum generation by the NIC (default),"
5622a7db7a6SVincenzo Maffione 		"1 to enable checksum generation by the NIC");
5634f80b14cSVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, generic_mit, CTLFLAG_RW, &netmap_generic_mit,
5644f80b14cSVincenzo Maffione 		0, "RX notification interval in nanoseconds");
5654f80b14cSVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, generic_ringsize, CTLFLAG_RW,
5664f80b14cSVincenzo Maffione 		&netmap_generic_ringsize, 0,
5674f80b14cSVincenzo Maffione 		"Number of per-ring slots for emulated netmap mode");
5684f80b14cSVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, generic_rings, CTLFLAG_RW,
5694f80b14cSVincenzo Maffione 		&netmap_generic_rings, 0,
5704f80b14cSVincenzo Maffione 		"Number of TX/RX queues for emulated netmap adapters");
5714f80b14cSVincenzo Maffione #ifdef linux
5724f80b14cSVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, generic_txqdisc, CTLFLAG_RW,
5734f80b14cSVincenzo Maffione 		&netmap_generic_txqdisc, 0, "Use qdisc for generic adapters");
5744f80b14cSVincenzo Maffione #endif
5754f80b14cSVincenzo Maffione SYSCTL_INT(_dev_netmap, OID_AUTO, ptnet_vnet_hdr, CTLFLAG_RW, &ptnet_vnet_hdr,
5764f80b14cSVincenzo Maffione 		0, "Allow ptnet devices to use virtio-net headers");
57737e3a6d3SLuigi Rizzo 
57837e3a6d3SLuigi Rizzo SYSEND;
579f196ce38SLuigi Rizzo 
580ce3ee1e7SLuigi Rizzo NMG_LOCK_T	netmap_global_lock;
581ce3ee1e7SLuigi Rizzo 
58217885a7bSLuigi Rizzo /*
58317885a7bSLuigi Rizzo  * mark the ring as stopped, and run through the locks
58417885a7bSLuigi Rizzo  * to make sure other users get to see it.
58537e3a6d3SLuigi Rizzo  * stopped must be either NR_KR_STOPPED (for unbounded stop)
58637e3a6d3SLuigi Rizzo  * of NR_KR_LOCKED (brief stop for mutual exclusion purposes)
58717885a7bSLuigi Rizzo  */
5884bf50f18SLuigi Rizzo static void
58937e3a6d3SLuigi Rizzo netmap_disable_ring(struct netmap_kring *kr, int stopped)
590ce3ee1e7SLuigi Rizzo {
59137e3a6d3SLuigi Rizzo 	nm_kr_stop(kr, stopped);
59237e3a6d3SLuigi Rizzo 	// XXX check if nm_kr_stop is sufficient
593ce3ee1e7SLuigi Rizzo 	mtx_lock(&kr->q_lock);
594ce3ee1e7SLuigi Rizzo 	mtx_unlock(&kr->q_lock);
595ce3ee1e7SLuigi Rizzo 	nm_kr_put(kr);
596ce3ee1e7SLuigi Rizzo }
597ce3ee1e7SLuigi Rizzo 
598847bf383SLuigi Rizzo /* stop or enable a single ring */
5994bf50f18SLuigi Rizzo void
600847bf383SLuigi Rizzo netmap_set_ring(struct netmap_adapter *na, u_int ring_id, enum txrx t, int stopped)
6014bf50f18SLuigi Rizzo {
6024bf50f18SLuigi Rizzo 	if (stopped)
6032ff91c17SVincenzo Maffione 		netmap_disable_ring(NMR(na, t)[ring_id], stopped);
6044bf50f18SLuigi Rizzo 	else
6052ff91c17SVincenzo Maffione 		NMR(na, t)[ring_id]->nkr_stopped = 0;
6064bf50f18SLuigi Rizzo }
6074bf50f18SLuigi Rizzo 
608f9790aebSLuigi Rizzo 
60989cc2556SLuigi Rizzo /* stop or enable all the rings of na */
6104bf50f18SLuigi Rizzo void
6114bf50f18SLuigi Rizzo netmap_set_all_rings(struct netmap_adapter *na, int stopped)
612ce3ee1e7SLuigi Rizzo {
613ce3ee1e7SLuigi Rizzo 	int i;
614847bf383SLuigi Rizzo 	enum txrx t;
615ce3ee1e7SLuigi Rizzo 
6164bf50f18SLuigi Rizzo 	if (!nm_netmap_on(na))
617ce3ee1e7SLuigi Rizzo 		return;
618ce3ee1e7SLuigi Rizzo 
619847bf383SLuigi Rizzo 	for_rx_tx(t) {
620847bf383SLuigi Rizzo 		for (i = 0; i < netmap_real_rings(na, t); i++) {
621847bf383SLuigi Rizzo 			netmap_set_ring(na, i, t, stopped);
622ce3ee1e7SLuigi Rizzo 		}
623ce3ee1e7SLuigi Rizzo 	}
624ce3ee1e7SLuigi Rizzo }
625ce3ee1e7SLuigi Rizzo 
62689cc2556SLuigi Rizzo /*
62789cc2556SLuigi Rizzo  * Convenience function used in drivers.  Waits for current txsync()s/rxsync()s
62889cc2556SLuigi Rizzo  * to finish and prevents any new one from starting.  Call this before turning
629ddb13598SKevin Lo  * netmap mode off, or before removing the hardware rings (e.g., on module
63037e3a6d3SLuigi Rizzo  * onload).
63189cc2556SLuigi Rizzo  */
632f9790aebSLuigi Rizzo void
633f9790aebSLuigi Rizzo netmap_disable_all_rings(struct ifnet *ifp)
634f9790aebSLuigi Rizzo {
63537e3a6d3SLuigi Rizzo 	if (NM_NA_VALID(ifp)) {
63637e3a6d3SLuigi Rizzo 		netmap_set_all_rings(NA(ifp), NM_KR_STOPPED);
63737e3a6d3SLuigi Rizzo 	}
638f9790aebSLuigi Rizzo }
639f9790aebSLuigi Rizzo 
64089cc2556SLuigi Rizzo /*
64189cc2556SLuigi Rizzo  * Convenience function used in drivers.  Re-enables rxsync and txsync on the
64289cc2556SLuigi Rizzo  * adapter's rings In linux drivers, this should be placed near each
64389cc2556SLuigi Rizzo  * napi_enable().
64489cc2556SLuigi Rizzo  */
645f9790aebSLuigi Rizzo void
646f9790aebSLuigi Rizzo netmap_enable_all_rings(struct ifnet *ifp)
647f9790aebSLuigi Rizzo {
64837e3a6d3SLuigi Rizzo 	if (NM_NA_VALID(ifp)) {
6494bf50f18SLuigi Rizzo 		netmap_set_all_rings(NA(ifp), 0 /* enabled */);
650f9790aebSLuigi Rizzo 	}
65137e3a6d3SLuigi Rizzo }
652f9790aebSLuigi Rizzo 
65337e3a6d3SLuigi Rizzo void
65437e3a6d3SLuigi Rizzo netmap_make_zombie(struct ifnet *ifp)
65537e3a6d3SLuigi Rizzo {
65637e3a6d3SLuigi Rizzo 	if (NM_NA_VALID(ifp)) {
65737e3a6d3SLuigi Rizzo 		struct netmap_adapter *na = NA(ifp);
65837e3a6d3SLuigi Rizzo 		netmap_set_all_rings(na, NM_KR_LOCKED);
65937e3a6d3SLuigi Rizzo 		na->na_flags |= NAF_ZOMBIE;
66037e3a6d3SLuigi Rizzo 		netmap_set_all_rings(na, 0);
66137e3a6d3SLuigi Rizzo 	}
66237e3a6d3SLuigi Rizzo }
66337e3a6d3SLuigi Rizzo 
66437e3a6d3SLuigi Rizzo void
66537e3a6d3SLuigi Rizzo netmap_undo_zombie(struct ifnet *ifp)
66637e3a6d3SLuigi Rizzo {
66737e3a6d3SLuigi Rizzo 	if (NM_NA_VALID(ifp)) {
66837e3a6d3SLuigi Rizzo 		struct netmap_adapter *na = NA(ifp);
66937e3a6d3SLuigi Rizzo 		if (na->na_flags & NAF_ZOMBIE) {
67037e3a6d3SLuigi Rizzo 			netmap_set_all_rings(na, NM_KR_LOCKED);
67137e3a6d3SLuigi Rizzo 			na->na_flags &= ~NAF_ZOMBIE;
67237e3a6d3SLuigi Rizzo 			netmap_set_all_rings(na, 0);
67337e3a6d3SLuigi Rizzo 		}
67437e3a6d3SLuigi Rizzo 	}
67537e3a6d3SLuigi Rizzo }
676f9790aebSLuigi Rizzo 
677ce3ee1e7SLuigi Rizzo /*
678ce3ee1e7SLuigi Rizzo  * generic bound_checking function
679ce3ee1e7SLuigi Rizzo  */
680ce3ee1e7SLuigi Rizzo u_int
681ce3ee1e7SLuigi Rizzo nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg)
682ce3ee1e7SLuigi Rizzo {
683ce3ee1e7SLuigi Rizzo 	u_int oldv = *v;
684ce3ee1e7SLuigi Rizzo 	const char *op = NULL;
685ce3ee1e7SLuigi Rizzo 
686ce3ee1e7SLuigi Rizzo 	if (dflt < lo)
687ce3ee1e7SLuigi Rizzo 		dflt = lo;
688ce3ee1e7SLuigi Rizzo 	if (dflt > hi)
689ce3ee1e7SLuigi Rizzo 		dflt = hi;
690ce3ee1e7SLuigi Rizzo 	if (oldv < lo) {
691ce3ee1e7SLuigi Rizzo 		*v = dflt;
692ce3ee1e7SLuigi Rizzo 		op = "Bump";
693ce3ee1e7SLuigi Rizzo 	} else if (oldv > hi) {
694ce3ee1e7SLuigi Rizzo 		*v = hi;
695ce3ee1e7SLuigi Rizzo 		op = "Clamp";
696ce3ee1e7SLuigi Rizzo 	}
697ce3ee1e7SLuigi Rizzo 	if (op && msg)
698b6e66be2SVincenzo Maffione 		nm_prinf("%s %s to %d (was %d)", op, msg, *v, oldv);
699ce3ee1e7SLuigi Rizzo 	return *v;
700ce3ee1e7SLuigi Rizzo }
701ce3ee1e7SLuigi Rizzo 
702f9790aebSLuigi Rizzo 
703ce3ee1e7SLuigi Rizzo /*
704ce3ee1e7SLuigi Rizzo  * packet-dump function, user-supplied or static buffer.
705ce3ee1e7SLuigi Rizzo  * The destination buffer must be at least 30+4*len
706ce3ee1e7SLuigi Rizzo  */
707ce3ee1e7SLuigi Rizzo const char *
708ce3ee1e7SLuigi Rizzo nm_dump_buf(char *p, int len, int lim, char *dst)
709ce3ee1e7SLuigi Rizzo {
710ce3ee1e7SLuigi Rizzo 	static char _dst[8192];
711ce3ee1e7SLuigi Rizzo 	int i, j, i0;
712ce3ee1e7SLuigi Rizzo 	static char hex[] ="0123456789abcdef";
713ce3ee1e7SLuigi Rizzo 	char *o;	/* output position */
714ce3ee1e7SLuigi Rizzo 
715ce3ee1e7SLuigi Rizzo #define P_HI(x)	hex[((x) & 0xf0)>>4]
716ce3ee1e7SLuigi Rizzo #define P_LO(x)	hex[((x) & 0xf)]
717ce3ee1e7SLuigi Rizzo #define P_C(x)	((x) >= 0x20 && (x) <= 0x7e ? (x) : '.')
718ce3ee1e7SLuigi Rizzo 	if (!dst)
719ce3ee1e7SLuigi Rizzo 		dst = _dst;
720ce3ee1e7SLuigi Rizzo 	if (lim <= 0 || lim > len)
721ce3ee1e7SLuigi Rizzo 		lim = len;
722ce3ee1e7SLuigi Rizzo 	o = dst;
723ce3ee1e7SLuigi Rizzo 	sprintf(o, "buf 0x%p len %d lim %d\n", p, len, lim);
724ce3ee1e7SLuigi Rizzo 	o += strlen(o);
725ce3ee1e7SLuigi Rizzo 	/* hexdump routine */
726ce3ee1e7SLuigi Rizzo 	for (i = 0; i < lim; ) {
727ce3ee1e7SLuigi Rizzo 		sprintf(o, "%5d: ", i);
728ce3ee1e7SLuigi Rizzo 		o += strlen(o);
729ce3ee1e7SLuigi Rizzo 		memset(o, ' ', 48);
730ce3ee1e7SLuigi Rizzo 		i0 = i;
731ce3ee1e7SLuigi Rizzo 		for (j=0; j < 16 && i < lim; i++, j++) {
732ce3ee1e7SLuigi Rizzo 			o[j*3] = P_HI(p[i]);
733ce3ee1e7SLuigi Rizzo 			o[j*3+1] = P_LO(p[i]);
734ce3ee1e7SLuigi Rizzo 		}
735ce3ee1e7SLuigi Rizzo 		i = i0;
736ce3ee1e7SLuigi Rizzo 		for (j=0; j < 16 && i < lim; i++, j++)
737ce3ee1e7SLuigi Rizzo 			o[j + 48] = P_C(p[i]);
738ce3ee1e7SLuigi Rizzo 		o[j+48] = '\n';
739ce3ee1e7SLuigi Rizzo 		o += j+49;
740ce3ee1e7SLuigi Rizzo 	}
741ce3ee1e7SLuigi Rizzo 	*o = '\0';
742ce3ee1e7SLuigi Rizzo #undef P_HI
743ce3ee1e7SLuigi Rizzo #undef P_LO
744ce3ee1e7SLuigi Rizzo #undef P_C
745ce3ee1e7SLuigi Rizzo 	return dst;
746ce3ee1e7SLuigi Rizzo }
747f196ce38SLuigi Rizzo 
748f18be576SLuigi Rizzo 
749ae10d1afSLuigi Rizzo /*
750ae10d1afSLuigi Rizzo  * Fetch configuration from the device, to cope with dynamic
751ae10d1afSLuigi Rizzo  * reconfigurations after loading the module.
752ae10d1afSLuigi Rizzo  */
75389cc2556SLuigi Rizzo /* call with NMG_LOCK held */
754f9790aebSLuigi Rizzo int
755ae10d1afSLuigi Rizzo netmap_update_config(struct netmap_adapter *na)
756ae10d1afSLuigi Rizzo {
7572ff91c17SVincenzo Maffione 	struct nm_config_info info;
758ae10d1afSLuigi Rizzo 
7592ff91c17SVincenzo Maffione 	bzero(&info, sizeof(info));
7606641c68bSLuigi Rizzo 	if (na->nm_config == NULL ||
7612ff91c17SVincenzo Maffione 	    na->nm_config(na, &info)) {
762ae10d1afSLuigi Rizzo 		/* take whatever we had at init time */
7632ff91c17SVincenzo Maffione 		info.num_tx_rings = na->num_tx_rings;
7642ff91c17SVincenzo Maffione 		info.num_tx_descs = na->num_tx_desc;
7652ff91c17SVincenzo Maffione 		info.num_rx_rings = na->num_rx_rings;
7662ff91c17SVincenzo Maffione 		info.num_rx_descs = na->num_rx_desc;
7672ff91c17SVincenzo Maffione 		info.rx_buf_maxsize = na->rx_buf_maxsize;
768ae10d1afSLuigi Rizzo 	}
769ae10d1afSLuigi Rizzo 
7702ff91c17SVincenzo Maffione 	if (na->num_tx_rings == info.num_tx_rings &&
7712ff91c17SVincenzo Maffione 	    na->num_tx_desc == info.num_tx_descs &&
7722ff91c17SVincenzo Maffione 	    na->num_rx_rings == info.num_rx_rings &&
7732ff91c17SVincenzo Maffione 	    na->num_rx_desc == info.num_rx_descs &&
7742ff91c17SVincenzo Maffione 	    na->rx_buf_maxsize == info.rx_buf_maxsize)
775ae10d1afSLuigi Rizzo 		return 0; /* nothing changed */
776f9790aebSLuigi Rizzo 	if (na->active_fds == 0) {
7772ff91c17SVincenzo Maffione 		na->num_tx_rings = info.num_tx_rings;
7782ff91c17SVincenzo Maffione 		na->num_tx_desc = info.num_tx_descs;
7792ff91c17SVincenzo Maffione 		na->num_rx_rings = info.num_rx_rings;
7802ff91c17SVincenzo Maffione 		na->num_rx_desc = info.num_rx_descs;
7812ff91c17SVincenzo Maffione 		na->rx_buf_maxsize = info.rx_buf_maxsize;
782b6e66be2SVincenzo Maffione 		if (netmap_verbose)
783b6e66be2SVincenzo Maffione 			nm_prinf("configuration changed for %s: txring %d x %d, "
784cfa866f6SMatt Macy 				"rxring %d x %d, rxbufsz %d",
785cfa866f6SMatt Macy 				na->name, na->num_tx_rings, na->num_tx_desc,
786cfa866f6SMatt Macy 				na->num_rx_rings, na->num_rx_desc, na->rx_buf_maxsize);
787ae10d1afSLuigi Rizzo 		return 0;
788ae10d1afSLuigi Rizzo 	}
789b6e66be2SVincenzo Maffione 	nm_prerr("WARNING: configuration changed for %s while active: "
7902ff91c17SVincenzo Maffione 		"txring %d x %d, rxring %d x %d, rxbufsz %d",
7912ff91c17SVincenzo Maffione 		na->name, info.num_tx_rings, info.num_tx_descs,
7922ff91c17SVincenzo Maffione 		info.num_rx_rings, info.num_rx_descs,
7932ff91c17SVincenzo Maffione 		info.rx_buf_maxsize);
794ae10d1afSLuigi Rizzo 	return 1;
795ae10d1afSLuigi Rizzo }
796ae10d1afSLuigi Rizzo 
79737e3a6d3SLuigi Rizzo /* nm_sync callbacks for the host rings */
79837e3a6d3SLuigi Rizzo static int netmap_txsync_to_host(struct netmap_kring *kring, int flags);
79937e3a6d3SLuigi Rizzo static int netmap_rxsync_from_host(struct netmap_kring *kring, int flags);
800f0ea3689SLuigi Rizzo 
801f0ea3689SLuigi Rizzo /* create the krings array and initialize the fields common to all adapters.
802f0ea3689SLuigi Rizzo  * The array layout is this:
803f0ea3689SLuigi Rizzo  *
804f0ea3689SLuigi Rizzo  *                    +----------+
805f0ea3689SLuigi Rizzo  * na->tx_rings ----->|          | \
806f0ea3689SLuigi Rizzo  *                    |          |  } na->num_tx_ring
807f0ea3689SLuigi Rizzo  *                    |          | /
808f0ea3689SLuigi Rizzo  *                    +----------+
809f0ea3689SLuigi Rizzo  *                    |          |    host tx kring
810f0ea3689SLuigi Rizzo  * na->rx_rings ----> +----------+
811f0ea3689SLuigi Rizzo  *                    |          | \
812f0ea3689SLuigi Rizzo  *                    |          |  } na->num_rx_rings
813f0ea3689SLuigi Rizzo  *                    |          | /
814f0ea3689SLuigi Rizzo  *                    +----------+
815f0ea3689SLuigi Rizzo  *                    |          |    host rx kring
816f0ea3689SLuigi Rizzo  *                    +----------+
817f0ea3689SLuigi Rizzo  * na->tailroom ----->|          | \
818f0ea3689SLuigi Rizzo  *                    |          |  } tailroom bytes
819f0ea3689SLuigi Rizzo  *                    |          | /
820f0ea3689SLuigi Rizzo  *                    +----------+
821f0ea3689SLuigi Rizzo  *
822f0ea3689SLuigi Rizzo  * Note: for compatibility, host krings are created even when not needed.
823f0ea3689SLuigi Rizzo  * The tailroom space is currently used by vale ports for allocating leases.
824f0ea3689SLuigi Rizzo  */
82589cc2556SLuigi Rizzo /* call with NMG_LOCK held */
826f9790aebSLuigi Rizzo int
827f0ea3689SLuigi Rizzo netmap_krings_create(struct netmap_adapter *na, u_int tailroom)
828f9790aebSLuigi Rizzo {
829f9790aebSLuigi Rizzo 	u_int i, len, ndesc;
830f9790aebSLuigi Rizzo 	struct netmap_kring *kring;
831847bf383SLuigi Rizzo 	u_int n[NR_TXRX];
832847bf383SLuigi Rizzo 	enum txrx t;
83319c4ec08SVincenzo Maffione 	int err = 0;
834f9790aebSLuigi Rizzo 
835c3e9b4dbSLuiz Otavio O Souza 	if (na->tx_rings != NULL) {
836b6e66be2SVincenzo Maffione 		if (netmap_debug & NM_DEBUG_ON)
837b6e66be2SVincenzo Maffione 			nm_prerr("warning: krings were already created");
838c3e9b4dbSLuiz Otavio O Souza 		return 0;
839c3e9b4dbSLuiz Otavio O Souza 	}
840c3e9b4dbSLuiz Otavio O Souza 
841f0ea3689SLuigi Rizzo 	/* account for the (possibly fake) host rings */
8422a7db7a6SVincenzo Maffione 	n[NR_TX] = netmap_all_rings(na, NR_TX);
8432a7db7a6SVincenzo Maffione 	n[NR_RX] = netmap_all_rings(na, NR_RX);
844f0ea3689SLuigi Rizzo 
8452ff91c17SVincenzo Maffione 	len = (n[NR_TX] + n[NR_RX]) *
8462ff91c17SVincenzo Maffione 		(sizeof(struct netmap_kring) + sizeof(struct netmap_kring *))
8472ff91c17SVincenzo Maffione 		+ tailroom;
848f9790aebSLuigi Rizzo 
849c3e9b4dbSLuiz Otavio O Souza 	na->tx_rings = nm_os_malloc((size_t)len);
850f9790aebSLuigi Rizzo 	if (na->tx_rings == NULL) {
851b6e66be2SVincenzo Maffione 		nm_prerr("Cannot allocate krings");
852f9790aebSLuigi Rizzo 		return ENOMEM;
853f9790aebSLuigi Rizzo 	}
854847bf383SLuigi Rizzo 	na->rx_rings = na->tx_rings + n[NR_TX];
8552ff91c17SVincenzo Maffione 	na->tailroom = na->rx_rings + n[NR_RX];
8562ff91c17SVincenzo Maffione 
8572ff91c17SVincenzo Maffione 	/* link the krings in the krings array */
8582ff91c17SVincenzo Maffione 	kring = (struct netmap_kring *)((char *)na->tailroom + tailroom);
8592ff91c17SVincenzo Maffione 	for (i = 0; i < n[NR_TX] + n[NR_RX]; i++) {
8602ff91c17SVincenzo Maffione 		na->tx_rings[i] = kring;
8612ff91c17SVincenzo Maffione 		kring++;
8622ff91c17SVincenzo Maffione 	}
863f9790aebSLuigi Rizzo 
86417885a7bSLuigi Rizzo 	/*
86517885a7bSLuigi Rizzo 	 * All fields in krings are 0 except the one initialized below.
86617885a7bSLuigi Rizzo 	 * but better be explicit on important kring fields.
86717885a7bSLuigi Rizzo 	 */
868847bf383SLuigi Rizzo 	for_rx_tx(t) {
869847bf383SLuigi Rizzo 		ndesc = nma_get_ndesc(na, t);
870847bf383SLuigi Rizzo 		for (i = 0; i < n[t]; i++) {
8712ff91c17SVincenzo Maffione 			kring = NMR(na, t)[i];
872f9790aebSLuigi Rizzo 			bzero(kring, sizeof(*kring));
8732ff91c17SVincenzo Maffione 			kring->notify_na = na;
87417885a7bSLuigi Rizzo 			kring->ring_id = i;
875847bf383SLuigi Rizzo 			kring->tx = t;
876f9790aebSLuigi Rizzo 			kring->nkr_num_slots = ndesc;
87737e3a6d3SLuigi Rizzo 			kring->nr_mode = NKR_NETMAP_OFF;
87837e3a6d3SLuigi Rizzo 			kring->nr_pending_mode = NKR_NETMAP_OFF;
879847bf383SLuigi Rizzo 			if (i < nma_get_nrings(na, t)) {
880847bf383SLuigi Rizzo 				kring->nm_sync = (t == NR_TX ? na->nm_txsync : na->nm_rxsync);
88137e3a6d3SLuigi Rizzo 			} else {
8822ff91c17SVincenzo Maffione 				if (!(na->na_flags & NAF_HOST_RINGS))
8832ff91c17SVincenzo Maffione 					kring->nr_kflags |= NKR_FAKERING;
884847bf383SLuigi Rizzo 				kring->nm_sync = (t == NR_TX ?
88537e3a6d3SLuigi Rizzo 						netmap_txsync_to_host:
88637e3a6d3SLuigi Rizzo 						netmap_rxsync_from_host);
887f0ea3689SLuigi Rizzo 			}
888847bf383SLuigi Rizzo 			kring->nm_notify = na->nm_notify;
889847bf383SLuigi Rizzo 			kring->rhead = kring->rcur = kring->nr_hwcur = 0;
890f9790aebSLuigi Rizzo 			/*
89117885a7bSLuigi Rizzo 			 * IMPORTANT: Always keep one slot empty.
892f9790aebSLuigi Rizzo 			 */
893847bf383SLuigi Rizzo 			kring->rtail = kring->nr_hwtail = (t == NR_TX ? ndesc - 1 : 0);
894847bf383SLuigi Rizzo 			snprintf(kring->name, sizeof(kring->name) - 1, "%s %s%d", na->name,
895847bf383SLuigi Rizzo 					nm_txrx2str(t), i);
89675f4f3edSVincenzo Maffione 			nm_prdis("ktx %s h %d c %d t %d",
897f0ea3689SLuigi Rizzo 				kring->name, kring->rhead, kring->rcur, kring->rtail);
89819c4ec08SVincenzo Maffione 			err = nm_os_selinfo_init(&kring->si, kring->name);
89919c4ec08SVincenzo Maffione 			if (err) {
90019c4ec08SVincenzo Maffione 				netmap_krings_delete(na);
90119c4ec08SVincenzo Maffione 				return err;
90219c4ec08SVincenzo Maffione 			}
903847bf383SLuigi Rizzo 			mtx_init(&kring->q_lock, (t == NR_TX ? "nm_txq_lock" : "nm_rxq_lock"), NULL, MTX_DEF);
90419c4ec08SVincenzo Maffione 			kring->na = na;	/* setting this field marks the mutex as initialized */
905f9790aebSLuigi Rizzo 		}
90619c4ec08SVincenzo Maffione 		err = nm_os_selinfo_init(&na->si[t], na->name);
90719c4ec08SVincenzo Maffione 		if (err) {
90819c4ec08SVincenzo Maffione 			netmap_krings_delete(na);
90919c4ec08SVincenzo Maffione 			return err;
910f0ea3689SLuigi Rizzo 		}
91119c4ec08SVincenzo Maffione 	}
912f9790aebSLuigi Rizzo 
913f9790aebSLuigi Rizzo 	return 0;
914f9790aebSLuigi Rizzo }
915f9790aebSLuigi Rizzo 
916f9790aebSLuigi Rizzo 
917f0ea3689SLuigi Rizzo /* undo the actions performed by netmap_krings_create */
91889cc2556SLuigi Rizzo /* call with NMG_LOCK held */
919f9790aebSLuigi Rizzo void
920f9790aebSLuigi Rizzo netmap_krings_delete(struct netmap_adapter *na)
921f9790aebSLuigi Rizzo {
9222ff91c17SVincenzo Maffione 	struct netmap_kring **kring = na->tx_rings;
923847bf383SLuigi Rizzo 	enum txrx t;
924847bf383SLuigi Rizzo 
925c3e9b4dbSLuiz Otavio O Souza 	if (na->tx_rings == NULL) {
926b6e66be2SVincenzo Maffione 		if (netmap_debug & NM_DEBUG_ON)
927b6e66be2SVincenzo Maffione 			nm_prerr("warning: krings were already deleted");
928c3e9b4dbSLuiz Otavio O Souza 		return;
929c3e9b4dbSLuiz Otavio O Souza 	}
930c3e9b4dbSLuiz Otavio O Souza 
931847bf383SLuigi Rizzo 	for_rx_tx(t)
93237e3a6d3SLuigi Rizzo 		nm_os_selinfo_uninit(&na->si[t]);
933f9790aebSLuigi Rizzo 
934f0ea3689SLuigi Rizzo 	/* we rely on the krings layout described above */
935f0ea3689SLuigi Rizzo 	for ( ; kring != na->tailroom; kring++) {
93619c4ec08SVincenzo Maffione 		if ((*kring)->na != NULL)
9372ff91c17SVincenzo Maffione 			mtx_destroy(&(*kring)->q_lock);
9382ff91c17SVincenzo Maffione 		nm_os_selinfo_uninit(&(*kring)->si);
939f9790aebSLuigi Rizzo 	}
940c3e9b4dbSLuiz Otavio O Souza 	nm_os_free(na->tx_rings);
941f9790aebSLuigi Rizzo 	na->tx_rings = na->rx_rings = na->tailroom = NULL;
942f9790aebSLuigi Rizzo }
943f9790aebSLuigi Rizzo 
944f9790aebSLuigi Rizzo 
94517885a7bSLuigi Rizzo /*
94617885a7bSLuigi Rizzo  * Destructor for NIC ports. They also have an mbuf queue
94717885a7bSLuigi Rizzo  * on the rings connected to the host so we need to purge
94817885a7bSLuigi Rizzo  * them first.
94917885a7bSLuigi Rizzo  */
95089cc2556SLuigi Rizzo /* call with NMG_LOCK held */
95137e3a6d3SLuigi Rizzo void
95217885a7bSLuigi Rizzo netmap_hw_krings_delete(struct netmap_adapter *na)
95317885a7bSLuigi Rizzo {
9542a7db7a6SVincenzo Maffione 	u_int lim = netmap_real_rings(na, NR_RX), i;
95517885a7bSLuigi Rizzo 
9562a7db7a6SVincenzo Maffione 	for (i = nma_get_nrings(na, NR_RX); i < lim; i++) {
9572a7db7a6SVincenzo Maffione 		struct mbq *q = &NMR(na, NR_RX)[i]->rx_queue;
95875f4f3edSVincenzo Maffione 		nm_prdis("destroy sw mbq with len %d", mbq_len(q));
95917885a7bSLuigi Rizzo 		mbq_purge(q);
96037e3a6d3SLuigi Rizzo 		mbq_safe_fini(q);
9612a7db7a6SVincenzo Maffione 	}
96217885a7bSLuigi Rizzo 	netmap_krings_delete(na);
96317885a7bSLuigi Rizzo }
96417885a7bSLuigi Rizzo 
9654f80b14cSVincenzo Maffione static void
9664f80b14cSVincenzo Maffione netmap_mem_drop(struct netmap_adapter *na)
9674f80b14cSVincenzo Maffione {
9684f80b14cSVincenzo Maffione 	int last = netmap_mem_deref(na->nm_mem, na);
9694f80b14cSVincenzo Maffione 	/* if the native allocator had been overrided on regif,
9704f80b14cSVincenzo Maffione 	 * restore it now and drop the temporary one
9714f80b14cSVincenzo Maffione 	 */
9724f80b14cSVincenzo Maffione 	if (last && na->nm_mem_prev) {
9734f80b14cSVincenzo Maffione 		netmap_mem_put(na->nm_mem);
9744f80b14cSVincenzo Maffione 		na->nm_mem = na->nm_mem_prev;
9754f80b14cSVincenzo Maffione 		na->nm_mem_prev = NULL;
9764f80b14cSVincenzo Maffione 	}
9774f80b14cSVincenzo Maffione }
978f9790aebSLuigi Rizzo 
97968b8534bSLuigi Rizzo /*
980847bf383SLuigi Rizzo  * Undo everything that was done in netmap_do_regif(). In particular,
981847bf383SLuigi Rizzo  * call nm_register(ifp,0) to stop netmap mode on the interface and
9824bf50f18SLuigi Rizzo  * revert to normal operation.
98368b8534bSLuigi Rizzo  */
984ce3ee1e7SLuigi Rizzo /* call with NMG_LOCK held */
985847bf383SLuigi Rizzo static void netmap_unset_ringid(struct netmap_priv_d *);
98637e3a6d3SLuigi Rizzo static void netmap_krings_put(struct netmap_priv_d *);
98737e3a6d3SLuigi Rizzo void
988847bf383SLuigi Rizzo netmap_do_unregif(struct netmap_priv_d *priv)
98968b8534bSLuigi Rizzo {
990f9790aebSLuigi Rizzo 	struct netmap_adapter *na = priv->np_na;
99168b8534bSLuigi Rizzo 
992ce3ee1e7SLuigi Rizzo 	NMG_LOCK_ASSERT();
993f9790aebSLuigi Rizzo 	na->active_fds--;
99437e3a6d3SLuigi Rizzo 	/* unset nr_pending_mode and possibly release exclusive mode */
99537e3a6d3SLuigi Rizzo 	netmap_krings_put(priv);
996847bf383SLuigi Rizzo 
997847bf383SLuigi Rizzo #ifdef	WITH_MONITOR
99837e3a6d3SLuigi Rizzo 	/* XXX check whether we have to do something with monitor
99937e3a6d3SLuigi Rizzo 	 * when rings change nr_mode. */
100037e3a6d3SLuigi Rizzo 	if (na->active_fds <= 0) {
1001847bf383SLuigi Rizzo 		/* walk through all the rings and tell any monitor
1002847bf383SLuigi Rizzo 		 * that the port is going to exit netmap mode
1003847bf383SLuigi Rizzo 		 */
1004847bf383SLuigi Rizzo 		netmap_monitor_stop(na);
100537e3a6d3SLuigi Rizzo 	}
1006847bf383SLuigi Rizzo #endif
100737e3a6d3SLuigi Rizzo 
100837e3a6d3SLuigi Rizzo 	if (na->active_fds <= 0 || nm_kring_pending(priv)) {
100937e3a6d3SLuigi Rizzo 		na->nm_register(na, 0);
101037e3a6d3SLuigi Rizzo 	}
101137e3a6d3SLuigi Rizzo 
101237e3a6d3SLuigi Rizzo 	/* delete rings and buffers that are no longer needed */
101337e3a6d3SLuigi Rizzo 	netmap_mem_rings_delete(na);
101437e3a6d3SLuigi Rizzo 
101537e3a6d3SLuigi Rizzo 	if (na->active_fds <= 0) {	/* last instance */
101668b8534bSLuigi Rizzo 		/*
101737e3a6d3SLuigi Rizzo 		 * (TO CHECK) We enter here
1018f18be576SLuigi Rizzo 		 * when the last reference to this file descriptor goes
1019f18be576SLuigi Rizzo 		 * away. This means we cannot have any pending poll()
1020f18be576SLuigi Rizzo 		 * or interrupt routine operating on the structure.
1021ce3ee1e7SLuigi Rizzo 		 * XXX The file may be closed in a thread while
1022ce3ee1e7SLuigi Rizzo 		 * another thread is using it.
1023ce3ee1e7SLuigi Rizzo 		 * Linux keeps the file opened until the last reference
1024ce3ee1e7SLuigi Rizzo 		 * by any outstanding ioctl/poll or mmap is gone.
1025ce3ee1e7SLuigi Rizzo 		 * FreeBSD does not track mmap()s (but we do) and
1026ce3ee1e7SLuigi Rizzo 		 * wakes up any sleeping poll(). Need to check what
1027ce3ee1e7SLuigi Rizzo 		 * happens if the close() occurs while a concurrent
1028ce3ee1e7SLuigi Rizzo 		 * syscall is running.
102968b8534bSLuigi Rizzo 		 */
1030b6e66be2SVincenzo Maffione 		if (netmap_debug & NM_DEBUG_ON)
1031b6e66be2SVincenzo Maffione 			nm_prinf("deleting last instance for %s", na->name);
103237e3a6d3SLuigi Rizzo 
103337e3a6d3SLuigi Rizzo 		if (nm_netmap_on(na)) {
1034b6e66be2SVincenzo Maffione 			nm_prerr("BUG: netmap on while going to delete the krings");
103537e3a6d3SLuigi Rizzo 		}
103637e3a6d3SLuigi Rizzo 
1037f9790aebSLuigi Rizzo 		na->nm_krings_delete(na);
1038d12354a5SVincenzo Maffione 
1039d12354a5SVincenzo Maffione 		/* restore the default number of host tx and rx rings */
1040253b2ec1SVincenzo Maffione 		if (na->na_flags & NAF_HOST_RINGS) {
1041d12354a5SVincenzo Maffione 			na->num_host_tx_rings = 1;
1042d12354a5SVincenzo Maffione 			na->num_host_rx_rings = 1;
1043253b2ec1SVincenzo Maffione 		} else {
1044253b2ec1SVincenzo Maffione 			na->num_host_tx_rings = 0;
1045253b2ec1SVincenzo Maffione 			na->num_host_rx_rings = 0;
1046253b2ec1SVincenzo Maffione 		}
104768b8534bSLuigi Rizzo 	}
104837e3a6d3SLuigi Rizzo 
1049847bf383SLuigi Rizzo 	/* possibily decrement counter of tx_si/rx_si users */
1050847bf383SLuigi Rizzo 	netmap_unset_ringid(priv);
1051f9790aebSLuigi Rizzo 	/* delete the nifp */
1052847bf383SLuigi Rizzo 	netmap_mem_if_delete(na, priv->np_nifp);
1053847bf383SLuigi Rizzo 	/* drop the allocator */
10544f80b14cSVincenzo Maffione 	netmap_mem_drop(na);
1055847bf383SLuigi Rizzo 	/* mark the priv as unregistered */
1056847bf383SLuigi Rizzo 	priv->np_na = NULL;
1057847bf383SLuigi Rizzo 	priv->np_nifp = NULL;
10585819da83SLuigi Rizzo }
105968b8534bSLuigi Rizzo 
106037e3a6d3SLuigi Rizzo struct netmap_priv_d*
106137e3a6d3SLuigi Rizzo netmap_priv_new(void)
106237e3a6d3SLuigi Rizzo {
106337e3a6d3SLuigi Rizzo 	struct netmap_priv_d *priv;
106437e3a6d3SLuigi Rizzo 
1065c3e9b4dbSLuiz Otavio O Souza 	priv = nm_os_malloc(sizeof(struct netmap_priv_d));
106637e3a6d3SLuigi Rizzo 	if (priv == NULL)
106737e3a6d3SLuigi Rizzo 		return NULL;
106837e3a6d3SLuigi Rizzo 	priv->np_refs = 1;
106937e3a6d3SLuigi Rizzo 	nm_os_get_module();
107037e3a6d3SLuigi Rizzo 	return priv;
107137e3a6d3SLuigi Rizzo }
107237e3a6d3SLuigi Rizzo 
1073ce3ee1e7SLuigi Rizzo /*
10748fd44c93SLuigi Rizzo  * Destructor of the netmap_priv_d, called when the fd is closed
10758fd44c93SLuigi Rizzo  * Action: undo all the things done by NIOCREGIF,
10768fd44c93SLuigi Rizzo  * On FreeBSD we need to track whether there are active mmap()s,
10778fd44c93SLuigi Rizzo  * and we use np_active_mmaps for that. On linux, the field is always 0.
10788fd44c93SLuigi Rizzo  * Return: 1 if we can free priv, 0 otherwise.
107989cc2556SLuigi Rizzo  *
1080ce3ee1e7SLuigi Rizzo  */
108189cc2556SLuigi Rizzo /* call with NMG_LOCK held */
108237e3a6d3SLuigi Rizzo void
108337e3a6d3SLuigi Rizzo netmap_priv_delete(struct netmap_priv_d *priv)
1084ce3ee1e7SLuigi Rizzo {
1085f9790aebSLuigi Rizzo 	struct netmap_adapter *na = priv->np_na;
1086ce3ee1e7SLuigi Rizzo 
1087847adfb7SLuigi Rizzo 	/* number of active references to this fd */
10888fd44c93SLuigi Rizzo 	if (--priv->np_refs > 0) {
108937e3a6d3SLuigi Rizzo 		return;
1090ce3ee1e7SLuigi Rizzo 	}
109137e3a6d3SLuigi Rizzo 	nm_os_put_module();
109237e3a6d3SLuigi Rizzo 	if (na) {
1093847bf383SLuigi Rizzo 		netmap_do_unregif(priv);
109437e3a6d3SLuigi Rizzo 	}
109537e3a6d3SLuigi Rizzo 	netmap_unget_na(na, priv->np_ifp);
109637e3a6d3SLuigi Rizzo 	bzero(priv, sizeof(*priv));	/* for safety */
1097c3e9b4dbSLuiz Otavio O Souza 	nm_os_free(priv);
1098f196ce38SLuigi Rizzo }
10995819da83SLuigi Rizzo 
1100f9790aebSLuigi Rizzo 
110189cc2556SLuigi Rizzo /* call with NMG_LOCK *not* held */
1102f9790aebSLuigi Rizzo void
11035819da83SLuigi Rizzo netmap_dtor(void *data)
11045819da83SLuigi Rizzo {
11055819da83SLuigi Rizzo 	struct netmap_priv_d *priv = data;
11065819da83SLuigi Rizzo 
1107ce3ee1e7SLuigi Rizzo 	NMG_LOCK();
110837e3a6d3SLuigi Rizzo 	netmap_priv_delete(priv);
1109ce3ee1e7SLuigi Rizzo 	NMG_UNLOCK();
1110ce3ee1e7SLuigi Rizzo }
111168b8534bSLuigi Rizzo 
1112f18be576SLuigi Rizzo 
111368b8534bSLuigi Rizzo /*
1114c3e9b4dbSLuiz Otavio O Souza  * Handlers for synchronization of the rings from/to the host stack.
1115c3e9b4dbSLuiz Otavio O Souza  * These are associated to a network interface and are just another
1116c3e9b4dbSLuiz Otavio O Souza  * ring pair managed by userspace.
1117c3e9b4dbSLuiz Otavio O Souza  *
1118c3e9b4dbSLuiz Otavio O Souza  * Netmap also supports transparent forwarding (NS_FORWARD and NR_FORWARD
1119c3e9b4dbSLuiz Otavio O Souza  * flags):
1120c3e9b4dbSLuiz Otavio O Souza  *
1121c3e9b4dbSLuiz Otavio O Souza  * - Before releasing buffers on hw RX rings, the application can mark
1122c3e9b4dbSLuiz Otavio O Souza  *   them with the NS_FORWARD flag. During the next RXSYNC or poll(), they
1123c3e9b4dbSLuiz Otavio O Souza  *   will be forwarded to the host stack, similarly to what happened if
1124c3e9b4dbSLuiz Otavio O Souza  *   the application moved them to the host TX ring.
1125c3e9b4dbSLuiz Otavio O Souza  *
1126c3e9b4dbSLuiz Otavio O Souza  * - Before releasing buffers on the host RX ring, the application can
1127c3e9b4dbSLuiz Otavio O Souza  *   mark them with the NS_FORWARD flag. During the next RXSYNC or poll(),
1128c3e9b4dbSLuiz Otavio O Souza  *   they will be forwarded to the hw TX rings, saving the application
1129c3e9b4dbSLuiz Otavio O Souza  *   from doing the same task in user-space.
1130c3e9b4dbSLuiz Otavio O Souza  *
1131c3e9b4dbSLuiz Otavio O Souza  * Transparent fowarding can be enabled per-ring, by setting the NR_FORWARD
1132c3e9b4dbSLuiz Otavio O Souza  * flag, or globally with the netmap_fwd sysctl.
1133c3e9b4dbSLuiz Otavio O Souza  *
1134091fd0abSLuigi Rizzo  * The transfer NIC --> host is relatively easy, just encapsulate
1135091fd0abSLuigi Rizzo  * into mbufs and we are done. The host --> NIC side is slightly
1136091fd0abSLuigi Rizzo  * harder because there might not be room in the tx ring so it
1137091fd0abSLuigi Rizzo  * might take a while before releasing the buffer.
1138091fd0abSLuigi Rizzo  */
1139091fd0abSLuigi Rizzo 
1140f18be576SLuigi Rizzo 
1141091fd0abSLuigi Rizzo /*
1142c3e9b4dbSLuiz Otavio O Souza  * Pass a whole queue of mbufs to the host stack as coming from 'dst'
114317885a7bSLuigi Rizzo  * We do not need to lock because the queue is private.
1144c3e9b4dbSLuiz Otavio O Souza  * After this call the queue is empty.
1145091fd0abSLuigi Rizzo  */
1146091fd0abSLuigi Rizzo static void
1147f9790aebSLuigi Rizzo netmap_send_up(struct ifnet *dst, struct mbq *q)
1148091fd0abSLuigi Rizzo {
1149091fd0abSLuigi Rizzo 	struct mbuf *m;
115037e3a6d3SLuigi Rizzo 	struct mbuf *head = NULL, *prev = NULL;
1151091fd0abSLuigi Rizzo 
1152c3e9b4dbSLuiz Otavio O Souza 	/* Send packets up, outside the lock; head/prev machinery
1153c3e9b4dbSLuiz Otavio O Souza 	 * is only useful for Windows. */
1154f9790aebSLuigi Rizzo 	while ((m = mbq_dequeue(q)) != NULL) {
1155b6e66be2SVincenzo Maffione 		if (netmap_debug & NM_DEBUG_HOST)
1156b6e66be2SVincenzo Maffione 			nm_prinf("sending up pkt %p size %d", m, MBUF_LEN(m));
115737e3a6d3SLuigi Rizzo 		prev = nm_os_send_up(dst, m, prev);
115837e3a6d3SLuigi Rizzo 		if (head == NULL)
115937e3a6d3SLuigi Rizzo 			head = prev;
1160091fd0abSLuigi Rizzo 	}
116137e3a6d3SLuigi Rizzo 	if (head)
116237e3a6d3SLuigi Rizzo 		nm_os_send_up(dst, NULL, head);
116337e3a6d3SLuigi Rizzo 	mbq_fini(q);
1164091fd0abSLuigi Rizzo }
1165091fd0abSLuigi Rizzo 
1166f18be576SLuigi Rizzo 
1167091fd0abSLuigi Rizzo /*
1168c3e9b4dbSLuiz Otavio O Souza  * Scan the buffers from hwcur to ring->head, and put a copy of those
1169c3e9b4dbSLuiz Otavio O Souza  * marked NS_FORWARD (or all of them if forced) into a queue of mbufs.
1170c3e9b4dbSLuiz Otavio O Souza  * Drop remaining packets in the unlikely event
117117885a7bSLuigi Rizzo  * of an mbuf shortage.
1172091fd0abSLuigi Rizzo  */
1173091fd0abSLuigi Rizzo static void
1174091fd0abSLuigi Rizzo netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force)
1175091fd0abSLuigi Rizzo {
117617885a7bSLuigi Rizzo 	u_int const lim = kring->nkr_num_slots - 1;
1177847bf383SLuigi Rizzo 	u_int const head = kring->rhead;
117817885a7bSLuigi Rizzo 	u_int n;
1179f9790aebSLuigi Rizzo 	struct netmap_adapter *na = kring->na;
1180091fd0abSLuigi Rizzo 
118117885a7bSLuigi Rizzo 	for (n = kring->nr_hwcur; n != head; n = nm_next(n, lim)) {
118217885a7bSLuigi Rizzo 		struct mbuf *m;
1183091fd0abSLuigi Rizzo 		struct netmap_slot *slot = &kring->ring->slot[n];
1184091fd0abSLuigi Rizzo 
1185091fd0abSLuigi Rizzo 		if ((slot->flags & NS_FORWARD) == 0 && !force)
1186091fd0abSLuigi Rizzo 			continue;
11874bf50f18SLuigi Rizzo 		if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE(na)) {
118875f4f3edSVincenzo Maffione 			nm_prlim(5, "bad pkt at %d len %d", n, slot->len);
1189091fd0abSLuigi Rizzo 			continue;
1190091fd0abSLuigi Rizzo 		}
1191091fd0abSLuigi Rizzo 		slot->flags &= ~NS_FORWARD; // XXX needed ?
119217885a7bSLuigi Rizzo 		/* XXX TODO: adapt to the case of a multisegment packet */
11934bf50f18SLuigi Rizzo 		m = m_devget(NMB(na, slot), slot->len, 0, na->ifp, NULL);
1194091fd0abSLuigi Rizzo 
1195091fd0abSLuigi Rizzo 		if (m == NULL)
1196091fd0abSLuigi Rizzo 			break;
1197f9790aebSLuigi Rizzo 		mbq_enqueue(q, m);
1198091fd0abSLuigi Rizzo 	}
1199091fd0abSLuigi Rizzo }
1200091fd0abSLuigi Rizzo 
120137e3a6d3SLuigi Rizzo static inline int
120237e3a6d3SLuigi Rizzo _nm_may_forward(struct netmap_kring *kring)
120337e3a6d3SLuigi Rizzo {
120437e3a6d3SLuigi Rizzo 	return	((netmap_fwd || kring->ring->flags & NR_FORWARD) &&
120537e3a6d3SLuigi Rizzo 		 kring->na->na_flags & NAF_HOST_RINGS &&
120637e3a6d3SLuigi Rizzo 		 kring->tx == NR_RX);
120737e3a6d3SLuigi Rizzo }
120837e3a6d3SLuigi Rizzo 
120937e3a6d3SLuigi Rizzo static inline int
121037e3a6d3SLuigi Rizzo nm_may_forward_up(struct netmap_kring *kring)
121137e3a6d3SLuigi Rizzo {
121237e3a6d3SLuigi Rizzo 	return	_nm_may_forward(kring) &&
121337e3a6d3SLuigi Rizzo 		 kring->ring_id != kring->na->num_rx_rings;
121437e3a6d3SLuigi Rizzo }
121537e3a6d3SLuigi Rizzo 
121637e3a6d3SLuigi Rizzo static inline int
1217c3e9b4dbSLuiz Otavio O Souza nm_may_forward_down(struct netmap_kring *kring, int sync_flags)
121837e3a6d3SLuigi Rizzo {
121937e3a6d3SLuigi Rizzo 	return	_nm_may_forward(kring) &&
1220c3e9b4dbSLuiz Otavio O Souza 		 (sync_flags & NAF_CAN_FORWARD_DOWN) &&
122137e3a6d3SLuigi Rizzo 		 kring->ring_id == kring->na->num_rx_rings;
122237e3a6d3SLuigi Rizzo }
1223f18be576SLuigi Rizzo 
1224091fd0abSLuigi Rizzo /*
122517885a7bSLuigi Rizzo  * Send to the NIC rings packets marked NS_FORWARD between
1226c3e9b4dbSLuiz Otavio O Souza  * kring->nr_hwcur and kring->rhead.
1227c3e9b4dbSLuiz Otavio O Souza  * Called under kring->rx_queue.lock on the sw rx ring.
1228c3e9b4dbSLuiz Otavio O Souza  *
1229c3e9b4dbSLuiz Otavio O Souza  * It can only be called if the user opened all the TX hw rings,
1230c3e9b4dbSLuiz Otavio O Souza  * see NAF_CAN_FORWARD_DOWN flag.
1231c3e9b4dbSLuiz Otavio O Souza  * We can touch the TX netmap rings (slots, head and cur) since
1232c3e9b4dbSLuiz Otavio O Souza  * we are in poll/ioctl system call context, and the application
1233c3e9b4dbSLuiz Otavio O Souza  * is not supposed to touch the ring (using a different thread)
1234c3e9b4dbSLuiz Otavio O Souza  * during the execution of the system call.
1235091fd0abSLuigi Rizzo  */
123617885a7bSLuigi Rizzo static u_int
1237091fd0abSLuigi Rizzo netmap_sw_to_nic(struct netmap_adapter *na)
1238091fd0abSLuigi Rizzo {
12392ff91c17SVincenzo Maffione 	struct netmap_kring *kring = na->rx_rings[na->num_rx_rings];
124017885a7bSLuigi Rizzo 	struct netmap_slot *rxslot = kring->ring->slot;
124117885a7bSLuigi Rizzo 	u_int i, rxcur = kring->nr_hwcur;
124217885a7bSLuigi Rizzo 	u_int const head = kring->rhead;
124317885a7bSLuigi Rizzo 	u_int const src_lim = kring->nkr_num_slots - 1;
124417885a7bSLuigi Rizzo 	u_int sent = 0;
1245ce3ee1e7SLuigi Rizzo 
124617885a7bSLuigi Rizzo 	/* scan rings to find space, then fill as much as possible */
124717885a7bSLuigi Rizzo 	for (i = 0; i < na->num_tx_rings; i++) {
12482ff91c17SVincenzo Maffione 		struct netmap_kring *kdst = na->tx_rings[i];
124917885a7bSLuigi Rizzo 		struct netmap_ring *rdst = kdst->ring;
125017885a7bSLuigi Rizzo 		u_int const dst_lim = kdst->nkr_num_slots - 1;
1251ce3ee1e7SLuigi Rizzo 
125217885a7bSLuigi Rizzo 		/* XXX do we trust ring or kring->rcur,rtail ? */
125317885a7bSLuigi Rizzo 		for (; rxcur != head && !nm_ring_empty(rdst);
125417885a7bSLuigi Rizzo 		     rxcur = nm_next(rxcur, src_lim) ) {
1255091fd0abSLuigi Rizzo 			struct netmap_slot *src, *dst, tmp;
125637e3a6d3SLuigi Rizzo 			u_int dst_head = rdst->head;
125717885a7bSLuigi Rizzo 
125817885a7bSLuigi Rizzo 			src = &rxslot[rxcur];
125917885a7bSLuigi Rizzo 			if ((src->flags & NS_FORWARD) == 0 && !netmap_fwd)
126017885a7bSLuigi Rizzo 				continue;
126117885a7bSLuigi Rizzo 
126217885a7bSLuigi Rizzo 			sent++;
126317885a7bSLuigi Rizzo 
126437e3a6d3SLuigi Rizzo 			dst = &rdst->slot[dst_head];
126517885a7bSLuigi Rizzo 
1266091fd0abSLuigi Rizzo 			tmp = *src;
126717885a7bSLuigi Rizzo 
1268091fd0abSLuigi Rizzo 			src->buf_idx = dst->buf_idx;
1269091fd0abSLuigi Rizzo 			src->flags = NS_BUF_CHANGED;
1270091fd0abSLuigi Rizzo 
1271091fd0abSLuigi Rizzo 			dst->buf_idx = tmp.buf_idx;
1272091fd0abSLuigi Rizzo 			dst->len = tmp.len;
1273091fd0abSLuigi Rizzo 			dst->flags = NS_BUF_CHANGED;
1274091fd0abSLuigi Rizzo 
127537e3a6d3SLuigi Rizzo 			rdst->head = rdst->cur = nm_next(dst_head, dst_lim);
1276091fd0abSLuigi Rizzo 		}
1277c3e9b4dbSLuiz Otavio O Souza 		/* if (sent) XXX txsync ? it would be just an optimization */
1278091fd0abSLuigi Rizzo 	}
127917885a7bSLuigi Rizzo 	return sent;
1280091fd0abSLuigi Rizzo }
1281091fd0abSLuigi Rizzo 
1282f18be576SLuigi Rizzo 
1283091fd0abSLuigi Rizzo /*
1284ce3ee1e7SLuigi Rizzo  * netmap_txsync_to_host() passes packets up. We are called from a
128502ad4083SLuigi Rizzo  * system call in user process context, and the only contention
128602ad4083SLuigi Rizzo  * can be among multiple user threads erroneously calling
1287091fd0abSLuigi Rizzo  * this routine concurrently.
128868b8534bSLuigi Rizzo  */
128937e3a6d3SLuigi Rizzo static int
129037e3a6d3SLuigi Rizzo netmap_txsync_to_host(struct netmap_kring *kring, int flags)
129168b8534bSLuigi Rizzo {
129237e3a6d3SLuigi Rizzo 	struct netmap_adapter *na = kring->na;
129317885a7bSLuigi Rizzo 	u_int const lim = kring->nkr_num_slots - 1;
1294f0ea3689SLuigi Rizzo 	u_int const head = kring->rhead;
1295f9790aebSLuigi Rizzo 	struct mbq q;
129668b8534bSLuigi Rizzo 
129717885a7bSLuigi Rizzo 	/* Take packets from hwcur to head and pass them up.
1298c3e9b4dbSLuiz Otavio O Souza 	 * Force hwcur = head since netmap_grab_packets() stops at head
129968b8534bSLuigi Rizzo 	 */
1300f9790aebSLuigi Rizzo 	mbq_init(&q);
130117885a7bSLuigi Rizzo 	netmap_grab_packets(kring, &q, 1 /* force */);
130275f4f3edSVincenzo Maffione 	nm_prdis("have %d pkts in queue", mbq_len(&q));
130317885a7bSLuigi Rizzo 	kring->nr_hwcur = head;
130417885a7bSLuigi Rizzo 	kring->nr_hwtail = head + lim;
130517885a7bSLuigi Rizzo 	if (kring->nr_hwtail > lim)
130617885a7bSLuigi Rizzo 		kring->nr_hwtail -= lim + 1;
130768b8534bSLuigi Rizzo 
1308f9790aebSLuigi Rizzo 	netmap_send_up(na->ifp, &q);
130937e3a6d3SLuigi Rizzo 	return 0;
1310f18be576SLuigi Rizzo }
1311f18be576SLuigi Rizzo 
1312f18be576SLuigi Rizzo 
131368b8534bSLuigi Rizzo /*
131402ad4083SLuigi Rizzo  * rxsync backend for packets coming from the host stack.
131517885a7bSLuigi Rizzo  * They have been put in kring->rx_queue by netmap_transmit().
131617885a7bSLuigi Rizzo  * We protect access to the kring using kring->rx_queue.lock
131702ad4083SLuigi Rizzo  *
1318c3e9b4dbSLuiz Otavio O Souza  * also moves to the nic hw rings any packet the user has marked
1319c3e9b4dbSLuiz Otavio O Souza  * for transparent-mode forwarding, then sets the NR_FORWARD
1320c3e9b4dbSLuiz Otavio O Souza  * flag in the kring to let the caller push them out
132168b8534bSLuigi Rizzo  */
13228fd44c93SLuigi Rizzo static int
132337e3a6d3SLuigi Rizzo netmap_rxsync_from_host(struct netmap_kring *kring, int flags)
132468b8534bSLuigi Rizzo {
132537e3a6d3SLuigi Rizzo 	struct netmap_adapter *na = kring->na;
132668b8534bSLuigi Rizzo 	struct netmap_ring *ring = kring->ring;
132717885a7bSLuigi Rizzo 	u_int nm_i, n;
132817885a7bSLuigi Rizzo 	u_int const lim = kring->nkr_num_slots - 1;
1329f0ea3689SLuigi Rizzo 	u_int const head = kring->rhead;
133017885a7bSLuigi Rizzo 	int ret = 0;
1331847bf383SLuigi Rizzo 	struct mbq *q = &kring->rx_queue, fq;
133268b8534bSLuigi Rizzo 
1333847bf383SLuigi Rizzo 	mbq_init(&fq); /* fq holds packets to be freed */
1334847bf383SLuigi Rizzo 
1335997b054cSLuigi Rizzo 	mbq_lock(q);
133617885a7bSLuigi Rizzo 
133717885a7bSLuigi Rizzo 	/* First part: import newly received packets */
133817885a7bSLuigi Rizzo 	n = mbq_len(q);
133917885a7bSLuigi Rizzo 	if (n) { /* grab packets from the queue */
134017885a7bSLuigi Rizzo 		struct mbuf *m;
134117885a7bSLuigi Rizzo 		uint32_t stop_i;
134217885a7bSLuigi Rizzo 
134317885a7bSLuigi Rizzo 		nm_i = kring->nr_hwtail;
1344c3e9b4dbSLuiz Otavio O Souza 		stop_i = nm_prev(kring->nr_hwcur, lim);
134517885a7bSLuigi Rizzo 		while ( nm_i != stop_i && (m = mbq_dequeue(q)) != NULL ) {
134617885a7bSLuigi Rizzo 			int len = MBUF_LEN(m);
134717885a7bSLuigi Rizzo 			struct netmap_slot *slot = &ring->slot[nm_i];
134817885a7bSLuigi Rizzo 
13494bf50f18SLuigi Rizzo 			m_copydata(m, 0, len, NMB(na, slot));
135075f4f3edSVincenzo Maffione 			nm_prdis("nm %d len %d", nm_i, len);
1351b6e66be2SVincenzo Maffione 			if (netmap_debug & NM_DEBUG_HOST)
1352b6e66be2SVincenzo Maffione 				nm_prinf("%s", nm_dump_buf(NMB(na, slot),len, 128, NULL));
135317885a7bSLuigi Rizzo 
135417885a7bSLuigi Rizzo 			slot->len = len;
13554f80b14cSVincenzo Maffione 			slot->flags = 0;
135617885a7bSLuigi Rizzo 			nm_i = nm_next(nm_i, lim);
1357847bf383SLuigi Rizzo 			mbq_enqueue(&fq, m);
135864ae02c3SLuigi Rizzo 		}
135917885a7bSLuigi Rizzo 		kring->nr_hwtail = nm_i;
136064ae02c3SLuigi Rizzo 	}
136117885a7bSLuigi Rizzo 
136217885a7bSLuigi Rizzo 	/*
136317885a7bSLuigi Rizzo 	 * Second part: skip past packets that userspace has released.
136417885a7bSLuigi Rizzo 	 */
136517885a7bSLuigi Rizzo 	nm_i = kring->nr_hwcur;
136617885a7bSLuigi Rizzo 	if (nm_i != head) { /* something was released */
1367c3e9b4dbSLuiz Otavio O Souza 		if (nm_may_forward_down(kring, flags)) {
136817885a7bSLuigi Rizzo 			ret = netmap_sw_to_nic(na);
136937e3a6d3SLuigi Rizzo 			if (ret > 0) {
137037e3a6d3SLuigi Rizzo 				kring->nr_kflags |= NR_FORWARD;
137137e3a6d3SLuigi Rizzo 				ret = 0;
137237e3a6d3SLuigi Rizzo 			}
137337e3a6d3SLuigi Rizzo 		}
137417885a7bSLuigi Rizzo 		kring->nr_hwcur = head;
137564ae02c3SLuigi Rizzo 	}
137617885a7bSLuigi Rizzo 
1377997b054cSLuigi Rizzo 	mbq_unlock(q);
1378847bf383SLuigi Rizzo 
1379847bf383SLuigi Rizzo 	mbq_purge(&fq);
138037e3a6d3SLuigi Rizzo 	mbq_fini(&fq);
1381847bf383SLuigi Rizzo 
138217885a7bSLuigi Rizzo 	return ret;
138368b8534bSLuigi Rizzo }
138468b8534bSLuigi Rizzo 
138568b8534bSLuigi Rizzo 
1386f9790aebSLuigi Rizzo /* Get a netmap adapter for the port.
1387f9790aebSLuigi Rizzo  *
1388f9790aebSLuigi Rizzo  * If it is possible to satisfy the request, return 0
1389f9790aebSLuigi Rizzo  * with *na containing the netmap adapter found.
1390f9790aebSLuigi Rizzo  * Otherwise return an error code, with *na containing NULL.
1391f9790aebSLuigi Rizzo  *
1392f9790aebSLuigi Rizzo  * When the port is attached to a bridge, we always return
1393f9790aebSLuigi Rizzo  * EBUSY.
1394f9790aebSLuigi Rizzo  * Otherwise, if the port is already bound to a file descriptor,
1395f9790aebSLuigi Rizzo  * then we unconditionally return the existing adapter into *na.
1396f9790aebSLuigi Rizzo  * In all the other cases, we return (into *na) either native,
1397f9790aebSLuigi Rizzo  * generic or NULL, according to the following table:
1398f9790aebSLuigi Rizzo  *
1399f9790aebSLuigi Rizzo  *					native_support
1400f9790aebSLuigi Rizzo  * active_fds   dev.netmap.admode         YES     NO
1401f9790aebSLuigi Rizzo  * -------------------------------------------------------
1402f9790aebSLuigi Rizzo  *    >0              *                 NA(ifp) NA(ifp)
1403f9790aebSLuigi Rizzo  *
1404f9790aebSLuigi Rizzo  *     0        NETMAP_ADMODE_BEST      NATIVE  GENERIC
1405f9790aebSLuigi Rizzo  *     0        NETMAP_ADMODE_NATIVE    NATIVE   NULL
1406f9790aebSLuigi Rizzo  *     0        NETMAP_ADMODE_GENERIC   GENERIC GENERIC
1407f9790aebSLuigi Rizzo  *
1408f9790aebSLuigi Rizzo  */
140937e3a6d3SLuigi Rizzo static void netmap_hw_dtor(struct netmap_adapter *); /* needed by NM_IS_NATIVE() */
1410f9790aebSLuigi Rizzo int
1411c3e9b4dbSLuiz Otavio O Souza netmap_get_hw_na(struct ifnet *ifp, struct netmap_mem_d *nmd, struct netmap_adapter **na)
1412f9790aebSLuigi Rizzo {
1413f9790aebSLuigi Rizzo 	/* generic support */
1414f9790aebSLuigi Rizzo 	int i = netmap_admode;	/* Take a snapshot. */
1415f9790aebSLuigi Rizzo 	struct netmap_adapter *prev_na;
1416847bf383SLuigi Rizzo 	int error = 0;
1417f9790aebSLuigi Rizzo 
1418f9790aebSLuigi Rizzo 	*na = NULL; /* default */
1419f9790aebSLuigi Rizzo 
1420f9790aebSLuigi Rizzo 	/* reset in case of invalid value */
1421f9790aebSLuigi Rizzo 	if (i < NETMAP_ADMODE_BEST || i >= NETMAP_ADMODE_LAST)
1422f9790aebSLuigi Rizzo 		i = netmap_admode = NETMAP_ADMODE_BEST;
1423f9790aebSLuigi Rizzo 
142437e3a6d3SLuigi Rizzo 	if (NM_NA_VALID(ifp)) {
14254bf50f18SLuigi Rizzo 		prev_na = NA(ifp);
1426f9790aebSLuigi Rizzo 		/* If an adapter already exists, return it if
1427f9790aebSLuigi Rizzo 		 * there are active file descriptors or if
1428f9790aebSLuigi Rizzo 		 * netmap is not forced to use generic
1429f9790aebSLuigi Rizzo 		 * adapters.
1430f9790aebSLuigi Rizzo 		 */
14314bf50f18SLuigi Rizzo 		if (NETMAP_OWNED_BY_ANY(prev_na)
14324bf50f18SLuigi Rizzo 			|| i != NETMAP_ADMODE_GENERIC
14334bf50f18SLuigi Rizzo 			|| prev_na->na_flags & NAF_FORCE_NATIVE
14344bf50f18SLuigi Rizzo #ifdef WITH_PIPES
14354bf50f18SLuigi Rizzo 			/* ugly, but we cannot allow an adapter switch
14364bf50f18SLuigi Rizzo 			 * if some pipe is referring to this one
14374bf50f18SLuigi Rizzo 			 */
14384bf50f18SLuigi Rizzo 			|| prev_na->na_next_pipe > 0
14394bf50f18SLuigi Rizzo #endif
14404bf50f18SLuigi Rizzo 		) {
14414bf50f18SLuigi Rizzo 			*na = prev_na;
1442c3e9b4dbSLuiz Otavio O Souza 			goto assign_mem;
1443f9790aebSLuigi Rizzo 		}
1444f9790aebSLuigi Rizzo 	}
1445f9790aebSLuigi Rizzo 
1446f9790aebSLuigi Rizzo 	/* If there isn't native support and netmap is not allowed
1447f9790aebSLuigi Rizzo 	 * to use generic adapters, we cannot satisfy the request.
1448f9790aebSLuigi Rizzo 	 */
144937e3a6d3SLuigi Rizzo 	if (!NM_IS_NATIVE(ifp) && i == NETMAP_ADMODE_NATIVE)
1450f2637526SLuigi Rizzo 		return EOPNOTSUPP;
1451f9790aebSLuigi Rizzo 
1452f9790aebSLuigi Rizzo 	/* Otherwise, create a generic adapter and return it,
1453f9790aebSLuigi Rizzo 	 * saving the previously used netmap adapter, if any.
1454f9790aebSLuigi Rizzo 	 *
1455f9790aebSLuigi Rizzo 	 * Note that here 'prev_na', if not NULL, MUST be a
1456f9790aebSLuigi Rizzo 	 * native adapter, and CANNOT be a generic one. This is
1457f9790aebSLuigi Rizzo 	 * true because generic adapters are created on demand, and
1458f9790aebSLuigi Rizzo 	 * destroyed when not used anymore. Therefore, if the adapter
1459f9790aebSLuigi Rizzo 	 * currently attached to an interface 'ifp' is generic, it
1460f9790aebSLuigi Rizzo 	 * must be that
1461f9790aebSLuigi Rizzo 	 * (NA(ifp)->active_fds > 0 || NETMAP_OWNED_BY_KERN(NA(ifp))).
1462f9790aebSLuigi Rizzo 	 * Consequently, if NA(ifp) is generic, we will enter one of
1463f9790aebSLuigi Rizzo 	 * the branches above. This ensures that we never override
1464f9790aebSLuigi Rizzo 	 * a generic adapter with another generic adapter.
1465f9790aebSLuigi Rizzo 	 */
1466f9790aebSLuigi Rizzo 	error = generic_netmap_attach(ifp);
1467f9790aebSLuigi Rizzo 	if (error)
1468f9790aebSLuigi Rizzo 		return error;
1469f9790aebSLuigi Rizzo 
1470f9790aebSLuigi Rizzo 	*na = NA(ifp);
1471c3e9b4dbSLuiz Otavio O Souza 
1472c3e9b4dbSLuiz Otavio O Souza assign_mem:
1473c3e9b4dbSLuiz Otavio O Souza 	if (nmd != NULL && !((*na)->na_flags & NAF_MEM_OWNER) &&
1474c3e9b4dbSLuiz Otavio O Souza 	    (*na)->active_fds == 0 && ((*na)->nm_mem != nmd)) {
14754f80b14cSVincenzo Maffione 		(*na)->nm_mem_prev = (*na)->nm_mem;
1476c3e9b4dbSLuiz Otavio O Souza 		(*na)->nm_mem = netmap_mem_get(nmd);
1477f9790aebSLuigi Rizzo 	}
1478f9790aebSLuigi Rizzo 
1479c3e9b4dbSLuiz Otavio O Souza 	return 0;
1480c3e9b4dbSLuiz Otavio O Souza }
1481f9790aebSLuigi Rizzo 
148268b8534bSLuigi Rizzo /*
1483ce3ee1e7SLuigi Rizzo  * MUST BE CALLED UNDER NMG_LOCK()
1484ce3ee1e7SLuigi Rizzo  *
1485f2637526SLuigi Rizzo  * Get a refcounted reference to a netmap adapter attached
14862ff91c17SVincenzo Maffione  * to the interface specified by req.
1487ce3ee1e7SLuigi Rizzo  * This is always called in the execution of an ioctl().
1488ce3ee1e7SLuigi Rizzo  *
1489f2637526SLuigi Rizzo  * Return ENXIO if the interface specified by the request does
1490f2637526SLuigi Rizzo  * not exist, ENOTSUP if netmap is not supported by the interface,
1491f2637526SLuigi Rizzo  * EBUSY if the interface is already attached to a bridge,
1492f2637526SLuigi Rizzo  * EINVAL if parameters are invalid, ENOMEM if needed resources
1493f2637526SLuigi Rizzo  * could not be allocated.
1494f2637526SLuigi Rizzo  * If successful, hold a reference to the netmap adapter.
1495f18be576SLuigi Rizzo  *
14962ff91c17SVincenzo Maffione  * If the interface specified by req is a system one, also keep
149737e3a6d3SLuigi Rizzo  * a reference to it and return a valid *ifp.
149868b8534bSLuigi Rizzo  */
1499f9790aebSLuigi Rizzo int
15002ff91c17SVincenzo Maffione netmap_get_na(struct nmreq_header *hdr,
15012ff91c17SVincenzo Maffione 	      struct netmap_adapter **na, struct ifnet **ifp,
15022ff91c17SVincenzo Maffione 	      struct netmap_mem_d *nmd, int create)
150368b8534bSLuigi Rizzo {
1504cfa866f6SMatt Macy 	struct nmreq_register *req = (struct nmreq_register *)(uintptr_t)hdr->nr_body;
1505f9790aebSLuigi Rizzo 	int error = 0;
1506f0ea3689SLuigi Rizzo 	struct netmap_adapter *ret = NULL;
1507c3e9b4dbSLuiz Otavio O Souza 	int nmd_ref = 0;
1508f9790aebSLuigi Rizzo 
1509f9790aebSLuigi Rizzo 	*na = NULL;     /* default return value */
151037e3a6d3SLuigi Rizzo 	*ifp = NULL;
1511f196ce38SLuigi Rizzo 
15122ff91c17SVincenzo Maffione 	if (hdr->nr_reqtype != NETMAP_REQ_REGISTER) {
15132ff91c17SVincenzo Maffione 		return EINVAL;
15142ff91c17SVincenzo Maffione 	}
15152ff91c17SVincenzo Maffione 
15162ff91c17SVincenzo Maffione 	if (req->nr_mode == NR_REG_PIPE_MASTER ||
15172ff91c17SVincenzo Maffione 			req->nr_mode == NR_REG_PIPE_SLAVE) {
15182ff91c17SVincenzo Maffione 		/* Do not accept deprecated pipe modes. */
1519b6e66be2SVincenzo Maffione 		nm_prerr("Deprecated pipe nr_mode, use xx{yy or xx}yy syntax");
15202ff91c17SVincenzo Maffione 		return EINVAL;
15212ff91c17SVincenzo Maffione 	}
15222ff91c17SVincenzo Maffione 
1523ce3ee1e7SLuigi Rizzo 	NMG_LOCK_ASSERT();
1524ce3ee1e7SLuigi Rizzo 
1525c3e9b4dbSLuiz Otavio O Souza 	/* if the request contain a memid, try to find the
1526c3e9b4dbSLuiz Otavio O Souza 	 * corresponding memory region
1527c3e9b4dbSLuiz Otavio O Souza 	 */
15282ff91c17SVincenzo Maffione 	if (nmd == NULL && req->nr_mem_id) {
15292ff91c17SVincenzo Maffione 		nmd = netmap_mem_find(req->nr_mem_id);
1530c3e9b4dbSLuiz Otavio O Souza 		if (nmd == NULL)
1531c3e9b4dbSLuiz Otavio O Souza 			return EINVAL;
1532c3e9b4dbSLuiz Otavio O Souza 		/* keep the rereference */
1533c3e9b4dbSLuiz Otavio O Souza 		nmd_ref = 1;
1534c3e9b4dbSLuiz Otavio O Souza 	}
1535c3e9b4dbSLuiz Otavio O Souza 
153637e3a6d3SLuigi Rizzo 	/* We cascade through all possible types of netmap adapter.
15374bf50f18SLuigi Rizzo 	 * All netmap_get_*_na() functions return an error and an na,
15384bf50f18SLuigi Rizzo 	 * with the following combinations:
15394bf50f18SLuigi Rizzo 	 *
15404bf50f18SLuigi Rizzo 	 * error    na
15414bf50f18SLuigi Rizzo 	 *   0	   NULL		type doesn't match
15424bf50f18SLuigi Rizzo 	 *  !0	   NULL		type matches, but na creation/lookup failed
15434bf50f18SLuigi Rizzo 	 *   0	  !NULL		type matches and na created/found
15444bf50f18SLuigi Rizzo 	 *  !0    !NULL		impossible
15454bf50f18SLuigi Rizzo 	 */
1546b6e66be2SVincenzo Maffione 	error = netmap_get_null_na(hdr, na, nmd, create);
154737e3a6d3SLuigi Rizzo 	if (error || *na != NULL)
1548c3e9b4dbSLuiz Otavio O Souza 		goto out;
154937e3a6d3SLuigi Rizzo 
15504bf50f18SLuigi Rizzo 	/* try to see if this is a monitor port */
15512ff91c17SVincenzo Maffione 	error = netmap_get_monitor_na(hdr, na, nmd, create);
15524bf50f18SLuigi Rizzo 	if (error || *na != NULL)
1553c3e9b4dbSLuiz Otavio O Souza 		goto out;
15544bf50f18SLuigi Rizzo 
15554bf50f18SLuigi Rizzo 	/* try to see if this is a pipe port */
15562ff91c17SVincenzo Maffione 	error = netmap_get_pipe_na(hdr, na, nmd, create);
1557f0ea3689SLuigi Rizzo 	if (error || *na != NULL)
1558c3e9b4dbSLuiz Otavio O Souza 		goto out;
1559ce3ee1e7SLuigi Rizzo 
15604bf50f18SLuigi Rizzo 	/* try to see if this is a bridge port */
15612a7db7a6SVincenzo Maffione 	error = netmap_get_vale_na(hdr, na, nmd, create);
1562f0ea3689SLuigi Rizzo 	if (error)
1563c3e9b4dbSLuiz Otavio O Souza 		goto out;
1564f0ea3689SLuigi Rizzo 
1565f0ea3689SLuigi Rizzo 	if (*na != NULL) /* valid match in netmap_get_bdg_na() */
1566847bf383SLuigi Rizzo 		goto out;
1567f0ea3689SLuigi Rizzo 
156889cc2556SLuigi Rizzo 	/*
156989cc2556SLuigi Rizzo 	 * This must be a hardware na, lookup the name in the system.
157089cc2556SLuigi Rizzo 	 * Note that by hardware we actually mean "it shows up in ifconfig".
157189cc2556SLuigi Rizzo 	 * This may still be a tap, a veth/epair, or even a
157289cc2556SLuigi Rizzo 	 * persistent VALE port.
157389cc2556SLuigi Rizzo 	 */
15742ff91c17SVincenzo Maffione 	*ifp = ifunit_ref(hdr->nr_name);
157537e3a6d3SLuigi Rizzo 	if (*ifp == NULL) {
1576c3e9b4dbSLuiz Otavio O Souza 		error = ENXIO;
1577c3e9b4dbSLuiz Otavio O Souza 		goto out;
1578f196ce38SLuigi Rizzo 	}
1579ce3ee1e7SLuigi Rizzo 
1580c3e9b4dbSLuiz Otavio O Souza 	error = netmap_get_hw_na(*ifp, nmd, &ret);
1581f9790aebSLuigi Rizzo 	if (error)
1582f9790aebSLuigi Rizzo 		goto out;
1583f18be576SLuigi Rizzo 
1584f9790aebSLuigi Rizzo 	*na = ret;
1585f9790aebSLuigi Rizzo 	netmap_adapter_get(ret);
1586f0ea3689SLuigi Rizzo 
1587d12354a5SVincenzo Maffione 	/*
1588d12354a5SVincenzo Maffione 	 * if the adapter supports the host rings and it is not alread open,
1589d12354a5SVincenzo Maffione 	 * try to set the number of host rings as requested by the user
1590d12354a5SVincenzo Maffione 	 */
1591d12354a5SVincenzo Maffione 	if (((*na)->na_flags & NAF_HOST_RINGS) && (*na)->active_fds == 0) {
1592d12354a5SVincenzo Maffione 		if (req->nr_host_tx_rings)
1593d12354a5SVincenzo Maffione 			(*na)->num_host_tx_rings = req->nr_host_tx_rings;
1594d12354a5SVincenzo Maffione 		if (req->nr_host_rx_rings)
1595d12354a5SVincenzo Maffione 			(*na)->num_host_rx_rings = req->nr_host_rx_rings;
1596d12354a5SVincenzo Maffione 	}
1597d12354a5SVincenzo Maffione 	nm_prdis("%s: host tx %d rx %u", (*na)->name, (*na)->num_host_tx_rings,
1598d12354a5SVincenzo Maffione 			(*na)->num_host_rx_rings);
1599d12354a5SVincenzo Maffione 
1600f9790aebSLuigi Rizzo out:
160137e3a6d3SLuigi Rizzo 	if (error) {
160237e3a6d3SLuigi Rizzo 		if (ret)
1603f0ea3689SLuigi Rizzo 			netmap_adapter_put(ret);
160437e3a6d3SLuigi Rizzo 		if (*ifp) {
160537e3a6d3SLuigi Rizzo 			if_rele(*ifp);
160637e3a6d3SLuigi Rizzo 			*ifp = NULL;
160737e3a6d3SLuigi Rizzo 		}
160837e3a6d3SLuigi Rizzo 	}
1609c3e9b4dbSLuiz Otavio O Souza 	if (nmd_ref)
1610c3e9b4dbSLuiz Otavio O Souza 		netmap_mem_put(nmd);
1611f18be576SLuigi Rizzo 
16125ab0d24dSLuigi Rizzo 	return error;
16135ab0d24dSLuigi Rizzo }
1614ce3ee1e7SLuigi Rizzo 
161537e3a6d3SLuigi Rizzo /* undo netmap_get_na() */
161637e3a6d3SLuigi Rizzo void
161737e3a6d3SLuigi Rizzo netmap_unget_na(struct netmap_adapter *na, struct ifnet *ifp)
161837e3a6d3SLuigi Rizzo {
161937e3a6d3SLuigi Rizzo 	if (ifp)
162037e3a6d3SLuigi Rizzo 		if_rele(ifp);
162137e3a6d3SLuigi Rizzo 	if (na)
162237e3a6d3SLuigi Rizzo 		netmap_adapter_put(na);
162337e3a6d3SLuigi Rizzo }
162437e3a6d3SLuigi Rizzo 
162537e3a6d3SLuigi Rizzo 
162637e3a6d3SLuigi Rizzo #define NM_FAIL_ON(t) do {						\
162737e3a6d3SLuigi Rizzo 	if (unlikely(t)) {						\
162875f4f3edSVincenzo Maffione 		nm_prlim(5, "%s: fail '" #t "' "				\
162937e3a6d3SLuigi Rizzo 			"h %d c %d t %d "				\
163037e3a6d3SLuigi Rizzo 			"rh %d rc %d rt %d "				\
163137e3a6d3SLuigi Rizzo 			"hc %d ht %d",					\
163237e3a6d3SLuigi Rizzo 			kring->name,					\
163337e3a6d3SLuigi Rizzo 			head, cur, ring->tail,				\
163437e3a6d3SLuigi Rizzo 			kring->rhead, kring->rcur, kring->rtail,	\
163537e3a6d3SLuigi Rizzo 			kring->nr_hwcur, kring->nr_hwtail);		\
163637e3a6d3SLuigi Rizzo 		return kring->nkr_num_slots;				\
163737e3a6d3SLuigi Rizzo 	}								\
163837e3a6d3SLuigi Rizzo } while (0)
1639ce3ee1e7SLuigi Rizzo 
1640f9790aebSLuigi Rizzo /*
1641f9790aebSLuigi Rizzo  * validate parameters on entry for *_txsync()
1642f9790aebSLuigi Rizzo  * Returns ring->cur if ok, or something >= kring->nkr_num_slots
164317885a7bSLuigi Rizzo  * in case of error.
1644f9790aebSLuigi Rizzo  *
164517885a7bSLuigi Rizzo  * rhead, rcur and rtail=hwtail are stored from previous round.
164617885a7bSLuigi Rizzo  * hwcur is the next packet to send to the ring.
1647f9790aebSLuigi Rizzo  *
164817885a7bSLuigi Rizzo  * We want
164917885a7bSLuigi Rizzo  *    hwcur <= *rhead <= head <= cur <= tail = *rtail <= hwtail
1650f9790aebSLuigi Rizzo  *
165117885a7bSLuigi Rizzo  * hwcur, rhead, rtail and hwtail are reliable
1652f9790aebSLuigi Rizzo  */
165337e3a6d3SLuigi Rizzo u_int
165437e3a6d3SLuigi Rizzo nm_txsync_prologue(struct netmap_kring *kring, struct netmap_ring *ring)
1655f9790aebSLuigi Rizzo {
165617885a7bSLuigi Rizzo 	u_int head = ring->head; /* read only once */
1657f9790aebSLuigi Rizzo 	u_int cur = ring->cur; /* read only once */
1658f9790aebSLuigi Rizzo 	u_int n = kring->nkr_num_slots;
1659ce3ee1e7SLuigi Rizzo 
166075f4f3edSVincenzo Maffione 	nm_prdis(5, "%s kcur %d ktail %d head %d cur %d tail %d",
166117885a7bSLuigi Rizzo 		kring->name,
166217885a7bSLuigi Rizzo 		kring->nr_hwcur, kring->nr_hwtail,
166317885a7bSLuigi Rizzo 		ring->head, ring->cur, ring->tail);
166417885a7bSLuigi Rizzo #if 1 /* kernel sanity checks; but we can trust the kring. */
166537e3a6d3SLuigi Rizzo 	NM_FAIL_ON(kring->nr_hwcur >= n || kring->rhead >= n ||
166637e3a6d3SLuigi Rizzo 	    kring->rtail >= n ||  kring->nr_hwtail >= n);
1667f9790aebSLuigi Rizzo #endif /* kernel sanity checks */
166817885a7bSLuigi Rizzo 	/*
166937e3a6d3SLuigi Rizzo 	 * user sanity checks. We only use head,
167037e3a6d3SLuigi Rizzo 	 * A, B, ... are possible positions for head:
167117885a7bSLuigi Rizzo 	 *
167237e3a6d3SLuigi Rizzo 	 *  0    A  rhead   B  rtail   C  n-1
167337e3a6d3SLuigi Rizzo 	 *  0    D  rtail   E  rhead   F  n-1
167417885a7bSLuigi Rizzo 	 *
167517885a7bSLuigi Rizzo 	 * B, F, D are valid. A, C, E are wrong
167617885a7bSLuigi Rizzo 	 */
167717885a7bSLuigi Rizzo 	if (kring->rtail >= kring->rhead) {
167817885a7bSLuigi Rizzo 		/* want rhead <= head <= rtail */
167937e3a6d3SLuigi Rizzo 		NM_FAIL_ON(head < kring->rhead || head > kring->rtail);
168017885a7bSLuigi Rizzo 		/* and also head <= cur <= rtail */
168137e3a6d3SLuigi Rizzo 		NM_FAIL_ON(cur < head || cur > kring->rtail);
168217885a7bSLuigi Rizzo 	} else { /* here rtail < rhead */
168317885a7bSLuigi Rizzo 		/* we need head outside rtail .. rhead */
168437e3a6d3SLuigi Rizzo 		NM_FAIL_ON(head > kring->rtail && head < kring->rhead);
168517885a7bSLuigi Rizzo 
168617885a7bSLuigi Rizzo 		/* two cases now: head <= rtail or head >= rhead  */
168717885a7bSLuigi Rizzo 		if (head <= kring->rtail) {
168817885a7bSLuigi Rizzo 			/* want head <= cur <= rtail */
168937e3a6d3SLuigi Rizzo 			NM_FAIL_ON(cur < head || cur > kring->rtail);
169017885a7bSLuigi Rizzo 		} else { /* head >= rhead */
169117885a7bSLuigi Rizzo 			/* cur must be outside rtail..head */
169237e3a6d3SLuigi Rizzo 			NM_FAIL_ON(cur > kring->rtail && cur < head);
1693f18be576SLuigi Rizzo 		}
1694f9790aebSLuigi Rizzo 	}
169517885a7bSLuigi Rizzo 	if (ring->tail != kring->rtail) {
169675f4f3edSVincenzo Maffione 		nm_prlim(5, "%s tail overwritten was %d need %d", kring->name,
169717885a7bSLuigi Rizzo 			ring->tail, kring->rtail);
169817885a7bSLuigi Rizzo 		ring->tail = kring->rtail;
169917885a7bSLuigi Rizzo 	}
170017885a7bSLuigi Rizzo 	kring->rhead = head;
170117885a7bSLuigi Rizzo 	kring->rcur = cur;
170217885a7bSLuigi Rizzo 	return head;
170368b8534bSLuigi Rizzo }
170468b8534bSLuigi Rizzo 
170568b8534bSLuigi Rizzo 
170668b8534bSLuigi Rizzo /*
1707f9790aebSLuigi Rizzo  * validate parameters on entry for *_rxsync()
170817885a7bSLuigi Rizzo  * Returns ring->head if ok, kring->nkr_num_slots on error.
1709f9790aebSLuigi Rizzo  *
171017885a7bSLuigi Rizzo  * For a valid configuration,
171117885a7bSLuigi Rizzo  * hwcur <= head <= cur <= tail <= hwtail
1712f9790aebSLuigi Rizzo  *
171317885a7bSLuigi Rizzo  * We only consider head and cur.
171417885a7bSLuigi Rizzo  * hwcur and hwtail are reliable.
1715f9790aebSLuigi Rizzo  *
1716f9790aebSLuigi Rizzo  */
171737e3a6d3SLuigi Rizzo u_int
171837e3a6d3SLuigi Rizzo nm_rxsync_prologue(struct netmap_kring *kring, struct netmap_ring *ring)
1719f9790aebSLuigi Rizzo {
172017885a7bSLuigi Rizzo 	uint32_t const n = kring->nkr_num_slots;
172117885a7bSLuigi Rizzo 	uint32_t head, cur;
1722f9790aebSLuigi Rizzo 
172375f4f3edSVincenzo Maffione 	nm_prdis(5,"%s kc %d kt %d h %d c %d t %d",
172417885a7bSLuigi Rizzo 		kring->name,
172517885a7bSLuigi Rizzo 		kring->nr_hwcur, kring->nr_hwtail,
172617885a7bSLuigi Rizzo 		ring->head, ring->cur, ring->tail);
172717885a7bSLuigi Rizzo 	/*
172817885a7bSLuigi Rizzo 	 * Before storing the new values, we should check they do not
172917885a7bSLuigi Rizzo 	 * move backwards. However:
173017885a7bSLuigi Rizzo 	 * - head is not an issue because the previous value is hwcur;
173117885a7bSLuigi Rizzo 	 * - cur could in principle go back, however it does not matter
173217885a7bSLuigi Rizzo 	 *   because we are processing a brand new rxsync()
173317885a7bSLuigi Rizzo 	 */
173417885a7bSLuigi Rizzo 	cur = kring->rcur = ring->cur;	/* read only once */
173517885a7bSLuigi Rizzo 	head = kring->rhead = ring->head;	/* read only once */
1736f9790aebSLuigi Rizzo #if 1 /* kernel sanity checks */
173737e3a6d3SLuigi Rizzo 	NM_FAIL_ON(kring->nr_hwcur >= n || kring->nr_hwtail >= n);
1738f9790aebSLuigi Rizzo #endif /* kernel sanity checks */
1739f9790aebSLuigi Rizzo 	/* user sanity checks */
174017885a7bSLuigi Rizzo 	if (kring->nr_hwtail >= kring->nr_hwcur) {
174117885a7bSLuigi Rizzo 		/* want hwcur <= rhead <= hwtail */
174237e3a6d3SLuigi Rizzo 		NM_FAIL_ON(head < kring->nr_hwcur || head > kring->nr_hwtail);
174317885a7bSLuigi Rizzo 		/* and also rhead <= rcur <= hwtail */
174437e3a6d3SLuigi Rizzo 		NM_FAIL_ON(cur < head || cur > kring->nr_hwtail);
1745f9790aebSLuigi Rizzo 	} else {
174617885a7bSLuigi Rizzo 		/* we need rhead outside hwtail..hwcur */
174737e3a6d3SLuigi Rizzo 		NM_FAIL_ON(head < kring->nr_hwcur && head > kring->nr_hwtail);
174817885a7bSLuigi Rizzo 		/* two cases now: head <= hwtail or head >= hwcur  */
174917885a7bSLuigi Rizzo 		if (head <= kring->nr_hwtail) {
175017885a7bSLuigi Rizzo 			/* want head <= cur <= hwtail */
175137e3a6d3SLuigi Rizzo 			NM_FAIL_ON(cur < head || cur > kring->nr_hwtail);
175217885a7bSLuigi Rizzo 		} else {
175317885a7bSLuigi Rizzo 			/* cur must be outside hwtail..head */
175437e3a6d3SLuigi Rizzo 			NM_FAIL_ON(cur < head && cur > kring->nr_hwtail);
1755f9790aebSLuigi Rizzo 		}
1756f9790aebSLuigi Rizzo 	}
175717885a7bSLuigi Rizzo 	if (ring->tail != kring->rtail) {
175875f4f3edSVincenzo Maffione 		nm_prlim(5, "%s tail overwritten was %d need %d",
175917885a7bSLuigi Rizzo 			kring->name,
176017885a7bSLuigi Rizzo 			ring->tail, kring->rtail);
176117885a7bSLuigi Rizzo 		ring->tail = kring->rtail;
176217885a7bSLuigi Rizzo 	}
176317885a7bSLuigi Rizzo 	return head;
1764f9790aebSLuigi Rizzo }
1765f9790aebSLuigi Rizzo 
176617885a7bSLuigi Rizzo 
1767f9790aebSLuigi Rizzo /*
176868b8534bSLuigi Rizzo  * Error routine called when txsync/rxsync detects an error.
176917885a7bSLuigi Rizzo  * Can't do much more than resetting head = cur = hwcur, tail = hwtail
177068b8534bSLuigi Rizzo  * Return 1 on reinit.
1771506cc70cSLuigi Rizzo  *
1772506cc70cSLuigi Rizzo  * This routine is only called by the upper half of the kernel.
1773506cc70cSLuigi Rizzo  * It only reads hwcur (which is changed only by the upper half, too)
177417885a7bSLuigi Rizzo  * and hwtail (which may be changed by the lower half, but only on
1775506cc70cSLuigi Rizzo  * a tx ring and only to increase it, so any error will be recovered
1776506cc70cSLuigi Rizzo  * on the next call). For the above, we don't strictly need to call
1777506cc70cSLuigi Rizzo  * it under lock.
177868b8534bSLuigi Rizzo  */
177968b8534bSLuigi Rizzo int
178068b8534bSLuigi Rizzo netmap_ring_reinit(struct netmap_kring *kring)
178168b8534bSLuigi Rizzo {
178268b8534bSLuigi Rizzo 	struct netmap_ring *ring = kring->ring;
178368b8534bSLuigi Rizzo 	u_int i, lim = kring->nkr_num_slots - 1;
178468b8534bSLuigi Rizzo 	int errors = 0;
178568b8534bSLuigi Rizzo 
1786ce3ee1e7SLuigi Rizzo 	// XXX KASSERT nm_kr_tryget
178775f4f3edSVincenzo Maffione 	nm_prlim(10, "called for %s", kring->name);
178817885a7bSLuigi Rizzo 	// XXX probably wrong to trust userspace
178917885a7bSLuigi Rizzo 	kring->rhead = ring->head;
179017885a7bSLuigi Rizzo 	kring->rcur  = ring->cur;
179117885a7bSLuigi Rizzo 	kring->rtail = ring->tail;
179217885a7bSLuigi Rizzo 
179368b8534bSLuigi Rizzo 	if (ring->cur > lim)
179468b8534bSLuigi Rizzo 		errors++;
179517885a7bSLuigi Rizzo 	if (ring->head > lim)
179617885a7bSLuigi Rizzo 		errors++;
179717885a7bSLuigi Rizzo 	if (ring->tail > lim)
179817885a7bSLuigi Rizzo 		errors++;
179968b8534bSLuigi Rizzo 	for (i = 0; i <= lim; i++) {
180068b8534bSLuigi Rizzo 		u_int idx = ring->slot[i].buf_idx;
180168b8534bSLuigi Rizzo 		u_int len = ring->slot[i].len;
1802847bf383SLuigi Rizzo 		if (idx < 2 || idx >= kring->na->na_lut.objtotal) {
180375f4f3edSVincenzo Maffione 			nm_prlim(5, "bad index at slot %d idx %d len %d ", i, idx, len);
180468b8534bSLuigi Rizzo 			ring->slot[i].buf_idx = 0;
180568b8534bSLuigi Rizzo 			ring->slot[i].len = 0;
18064bf50f18SLuigi Rizzo 		} else if (len > NETMAP_BUF_SIZE(kring->na)) {
180768b8534bSLuigi Rizzo 			ring->slot[i].len = 0;
180875f4f3edSVincenzo Maffione 			nm_prlim(5, "bad len at slot %d idx %d len %d", i, idx, len);
180968b8534bSLuigi Rizzo 		}
181068b8534bSLuigi Rizzo 	}
181168b8534bSLuigi Rizzo 	if (errors) {
181275f4f3edSVincenzo Maffione 		nm_prlim(10, "total %d errors", errors);
181375f4f3edSVincenzo Maffione 		nm_prlim(10, "%s reinit, cur %d -> %d tail %d -> %d",
181417885a7bSLuigi Rizzo 			kring->name,
181568b8534bSLuigi Rizzo 			ring->cur, kring->nr_hwcur,
181617885a7bSLuigi Rizzo 			ring->tail, kring->nr_hwtail);
181717885a7bSLuigi Rizzo 		ring->head = kring->rhead = kring->nr_hwcur;
181817885a7bSLuigi Rizzo 		ring->cur  = kring->rcur  = kring->nr_hwcur;
181917885a7bSLuigi Rizzo 		ring->tail = kring->rtail = kring->nr_hwtail;
182068b8534bSLuigi Rizzo 	}
182168b8534bSLuigi Rizzo 	return (errors ? 1 : 0);
182268b8534bSLuigi Rizzo }
182368b8534bSLuigi Rizzo 
18244bf50f18SLuigi Rizzo /* interpret the ringid and flags fields of an nmreq, by translating them
18254bf50f18SLuigi Rizzo  * into a pair of intervals of ring indices:
18264bf50f18SLuigi Rizzo  *
18274bf50f18SLuigi Rizzo  * [priv->np_txqfirst, priv->np_txqlast) and
18284bf50f18SLuigi Rizzo  * [priv->np_rxqfirst, priv->np_rxqlast)
18294bf50f18SLuigi Rizzo  *
183068b8534bSLuigi Rizzo  */
18314bf50f18SLuigi Rizzo int
18322ff91c17SVincenzo Maffione netmap_interp_ringid(struct netmap_priv_d *priv, uint32_t nr_mode,
18332ff91c17SVincenzo Maffione 			uint16_t nr_ringid, uint64_t nr_flags)
183468b8534bSLuigi Rizzo {
1835f9790aebSLuigi Rizzo 	struct netmap_adapter *na = priv->np_na;
183637e3a6d3SLuigi Rizzo 	int excluded_direction[] = { NR_TX_RINGS_ONLY, NR_RX_RINGS_ONLY };
1837847bf383SLuigi Rizzo 	enum txrx t;
18382ff91c17SVincenzo Maffione 	u_int j;
183968b8534bSLuigi Rizzo 
184037e3a6d3SLuigi Rizzo 	for_rx_tx(t) {
18412ff91c17SVincenzo Maffione 		if (nr_flags & excluded_direction[t]) {
184237e3a6d3SLuigi Rizzo 			priv->np_qfirst[t] = priv->np_qlast[t] = 0;
184337e3a6d3SLuigi Rizzo 			continue;
184437e3a6d3SLuigi Rizzo 		}
18452ff91c17SVincenzo Maffione 		switch (nr_mode) {
1846f0ea3689SLuigi Rizzo 		case NR_REG_ALL_NIC:
1847b6e66be2SVincenzo Maffione 		case NR_REG_NULL:
1848847bf383SLuigi Rizzo 			priv->np_qfirst[t] = 0;
1849847bf383SLuigi Rizzo 			priv->np_qlast[t] = nma_get_nrings(na, t);
185075f4f3edSVincenzo Maffione 			nm_prdis("ALL/PIPE: %s %d %d", nm_txrx2str(t),
185137e3a6d3SLuigi Rizzo 				priv->np_qfirst[t], priv->np_qlast[t]);
1852f0ea3689SLuigi Rizzo 			break;
1853f0ea3689SLuigi Rizzo 		case NR_REG_SW:
1854f0ea3689SLuigi Rizzo 		case NR_REG_NIC_SW:
1855f0ea3689SLuigi Rizzo 			if (!(na->na_flags & NAF_HOST_RINGS)) {
1856b6e66be2SVincenzo Maffione 				nm_prerr("host rings not supported");
1857f0ea3689SLuigi Rizzo 				return EINVAL;
1858f0ea3689SLuigi Rizzo 			}
18592ff91c17SVincenzo Maffione 			priv->np_qfirst[t] = (nr_mode == NR_REG_SW ?
1860847bf383SLuigi Rizzo 				nma_get_nrings(na, t) : 0);
18612a7db7a6SVincenzo Maffione 			priv->np_qlast[t] = netmap_all_rings(na, t);
186275f4f3edSVincenzo Maffione 			nm_prdis("%s: %s %d %d", nr_mode == NR_REG_SW ? "SW" : "NIC+SW",
186337e3a6d3SLuigi Rizzo 				nm_txrx2str(t),
186437e3a6d3SLuigi Rizzo 				priv->np_qfirst[t], priv->np_qlast[t]);
1865f0ea3689SLuigi Rizzo 			break;
1866f0ea3689SLuigi Rizzo 		case NR_REG_ONE_NIC:
18672ff91c17SVincenzo Maffione 			if (nr_ringid >= na->num_tx_rings &&
18682ff91c17SVincenzo Maffione 					nr_ringid >= na->num_rx_rings) {
1869b6e66be2SVincenzo Maffione 				nm_prerr("invalid ring id %d", nr_ringid);
1870f0ea3689SLuigi Rizzo 				return EINVAL;
1871f0ea3689SLuigi Rizzo 			}
1872f0ea3689SLuigi Rizzo 			/* if not enough rings, use the first one */
18732ff91c17SVincenzo Maffione 			j = nr_ringid;
1874847bf383SLuigi Rizzo 			if (j >= nma_get_nrings(na, t))
1875f0ea3689SLuigi Rizzo 				j = 0;
1876847bf383SLuigi Rizzo 			priv->np_qfirst[t] = j;
1877847bf383SLuigi Rizzo 			priv->np_qlast[t] = j + 1;
187875f4f3edSVincenzo Maffione 			nm_prdis("ONE_NIC: %s %d %d", nm_txrx2str(t),
187937e3a6d3SLuigi Rizzo 				priv->np_qfirst[t], priv->np_qlast[t]);
1880f0ea3689SLuigi Rizzo 			break;
1881d12354a5SVincenzo Maffione 		case NR_REG_ONE_SW:
1882d12354a5SVincenzo Maffione 			if (!(na->na_flags & NAF_HOST_RINGS)) {
1883d12354a5SVincenzo Maffione 				nm_prerr("host rings not supported");
1884d12354a5SVincenzo Maffione 				return EINVAL;
1885d12354a5SVincenzo Maffione 			}
1886d12354a5SVincenzo Maffione 			if (nr_ringid >= na->num_host_tx_rings &&
1887d12354a5SVincenzo Maffione 					nr_ringid >= na->num_host_rx_rings) {
1888d12354a5SVincenzo Maffione 				nm_prerr("invalid ring id %d", nr_ringid);
1889d12354a5SVincenzo Maffione 				return EINVAL;
1890d12354a5SVincenzo Maffione 			}
1891d12354a5SVincenzo Maffione 			/* if not enough rings, use the first one */
1892d12354a5SVincenzo Maffione 			j = nr_ringid;
1893d12354a5SVincenzo Maffione 			if (j >= nma_get_host_nrings(na, t))
1894d12354a5SVincenzo Maffione 				j = 0;
1895d12354a5SVincenzo Maffione 			priv->np_qfirst[t] = nma_get_nrings(na, t) + j;
1896d12354a5SVincenzo Maffione 			priv->np_qlast[t] = nma_get_nrings(na, t) + j + 1;
1897d12354a5SVincenzo Maffione 			nm_prdis("ONE_SW: %s %d %d", nm_txrx2str(t),
1898d12354a5SVincenzo Maffione 				priv->np_qfirst[t], priv->np_qlast[t]);
1899d12354a5SVincenzo Maffione 			break;
1900f0ea3689SLuigi Rizzo 		default:
1901b6e66be2SVincenzo Maffione 			nm_prerr("invalid regif type %d", nr_mode);
1902f0ea3689SLuigi Rizzo 			return EINVAL;
190368b8534bSLuigi Rizzo 		}
190437e3a6d3SLuigi Rizzo 	}
1905b6e66be2SVincenzo Maffione 	priv->np_flags = nr_flags;
19064bf50f18SLuigi Rizzo 
1907c3e9b4dbSLuiz Otavio O Souza 	/* Allow transparent forwarding mode in the host --> nic
1908c3e9b4dbSLuiz Otavio O Souza 	 * direction only if all the TX hw rings have been opened. */
1909c3e9b4dbSLuiz Otavio O Souza 	if (priv->np_qfirst[NR_TX] == 0 &&
1910c3e9b4dbSLuiz Otavio O Souza 			priv->np_qlast[NR_TX] >= na->num_tx_rings) {
1911c3e9b4dbSLuiz Otavio O Souza 		priv->np_sync_flags |= NAF_CAN_FORWARD_DOWN;
1912c3e9b4dbSLuiz Otavio O Souza 	}
1913c3e9b4dbSLuiz Otavio O Souza 
1914ae10d1afSLuigi Rizzo 	if (netmap_verbose) {
1915b6e66be2SVincenzo Maffione 		nm_prinf("%s: tx [%d,%d) rx [%d,%d) id %d",
19164bf50f18SLuigi Rizzo 			na->name,
1917847bf383SLuigi Rizzo 			priv->np_qfirst[NR_TX],
1918847bf383SLuigi Rizzo 			priv->np_qlast[NR_TX],
1919847bf383SLuigi Rizzo 			priv->np_qfirst[NR_RX],
1920847bf383SLuigi Rizzo 			priv->np_qlast[NR_RX],
19212ff91c17SVincenzo Maffione 			nr_ringid);
1922ae10d1afSLuigi Rizzo 	}
192368b8534bSLuigi Rizzo 	return 0;
192468b8534bSLuigi Rizzo }
192568b8534bSLuigi Rizzo 
19264bf50f18SLuigi Rizzo 
19274bf50f18SLuigi Rizzo /*
19284bf50f18SLuigi Rizzo  * Set the ring ID. For devices with a single queue, a request
19294bf50f18SLuigi Rizzo  * for all rings is the same as a single ring.
19304bf50f18SLuigi Rizzo  */
19314bf50f18SLuigi Rizzo static int
19322ff91c17SVincenzo Maffione netmap_set_ringid(struct netmap_priv_d *priv, uint32_t nr_mode,
19332ff91c17SVincenzo Maffione 		uint16_t nr_ringid, uint64_t nr_flags)
19344bf50f18SLuigi Rizzo {
19354bf50f18SLuigi Rizzo 	struct netmap_adapter *na = priv->np_na;
19364bf50f18SLuigi Rizzo 	int error;
1937847bf383SLuigi Rizzo 	enum txrx t;
19384bf50f18SLuigi Rizzo 
19392ff91c17SVincenzo Maffione 	error = netmap_interp_ringid(priv, nr_mode, nr_ringid, nr_flags);
19404bf50f18SLuigi Rizzo 	if (error) {
19414bf50f18SLuigi Rizzo 		return error;
19424bf50f18SLuigi Rizzo 	}
19434bf50f18SLuigi Rizzo 
19442ff91c17SVincenzo Maffione 	priv->np_txpoll = (nr_flags & NR_NO_TX_POLL) ? 0 : 1;
19454bf50f18SLuigi Rizzo 
19464bf50f18SLuigi Rizzo 	/* optimization: count the users registered for more than
19474bf50f18SLuigi Rizzo 	 * one ring, which are the ones sleeping on the global queue.
19484bf50f18SLuigi Rizzo 	 * The default netmap_notify() callback will then
19494bf50f18SLuigi Rizzo 	 * avoid signaling the global queue if nobody is using it
19504bf50f18SLuigi Rizzo 	 */
1951847bf383SLuigi Rizzo 	for_rx_tx(t) {
1952847bf383SLuigi Rizzo 		if (nm_si_user(priv, t))
1953847bf383SLuigi Rizzo 			na->si_users[t]++;
1954847bf383SLuigi Rizzo 	}
19554bf50f18SLuigi Rizzo 	return 0;
19564bf50f18SLuigi Rizzo }
19574bf50f18SLuigi Rizzo 
1958847bf383SLuigi Rizzo static void
1959847bf383SLuigi Rizzo netmap_unset_ringid(struct netmap_priv_d *priv)
1960847bf383SLuigi Rizzo {
1961847bf383SLuigi Rizzo 	struct netmap_adapter *na = priv->np_na;
1962847bf383SLuigi Rizzo 	enum txrx t;
1963847bf383SLuigi Rizzo 
1964847bf383SLuigi Rizzo 	for_rx_tx(t) {
1965847bf383SLuigi Rizzo 		if (nm_si_user(priv, t))
1966847bf383SLuigi Rizzo 			na->si_users[t]--;
1967847bf383SLuigi Rizzo 		priv->np_qfirst[t] = priv->np_qlast[t] = 0;
1968847bf383SLuigi Rizzo 	}
1969847bf383SLuigi Rizzo 	priv->np_flags = 0;
1970847bf383SLuigi Rizzo 	priv->np_txpoll = 0;
1971b6e66be2SVincenzo Maffione 	priv->np_kloop_state = 0;
1972847bf383SLuigi Rizzo }
1973847bf383SLuigi Rizzo 
1974847bf383SLuigi Rizzo 
197537e3a6d3SLuigi Rizzo /* Set the nr_pending_mode for the requested rings.
197637e3a6d3SLuigi Rizzo  * If requested, also try to get exclusive access to the rings, provided
197737e3a6d3SLuigi Rizzo  * the rings we want to bind are not exclusively owned by a previous bind.
1978847bf383SLuigi Rizzo  */
1979847bf383SLuigi Rizzo static int
198037e3a6d3SLuigi Rizzo netmap_krings_get(struct netmap_priv_d *priv)
1981847bf383SLuigi Rizzo {
1982847bf383SLuigi Rizzo 	struct netmap_adapter *na = priv->np_na;
1983847bf383SLuigi Rizzo 	u_int i;
1984847bf383SLuigi Rizzo 	struct netmap_kring *kring;
1985847bf383SLuigi Rizzo 	int excl = (priv->np_flags & NR_EXCLUSIVE);
1986847bf383SLuigi Rizzo 	enum txrx t;
1987847bf383SLuigi Rizzo 
1988b6e66be2SVincenzo Maffione 	if (netmap_debug & NM_DEBUG_ON)
1989b6e66be2SVincenzo Maffione 		nm_prinf("%s: grabbing tx [%d, %d) rx [%d, %d)",
1990847bf383SLuigi Rizzo 			na->name,
1991847bf383SLuigi Rizzo 			priv->np_qfirst[NR_TX],
1992847bf383SLuigi Rizzo 			priv->np_qlast[NR_TX],
1993847bf383SLuigi Rizzo 			priv->np_qfirst[NR_RX],
1994847bf383SLuigi Rizzo 			priv->np_qlast[NR_RX]);
1995847bf383SLuigi Rizzo 
1996847bf383SLuigi Rizzo 	/* first round: check that all the requested rings
1997847bf383SLuigi Rizzo 	 * are neither alread exclusively owned, nor we
1998847bf383SLuigi Rizzo 	 * want exclusive ownership when they are already in use
1999847bf383SLuigi Rizzo 	 */
2000847bf383SLuigi Rizzo 	for_rx_tx(t) {
2001847bf383SLuigi Rizzo 		for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
20022ff91c17SVincenzo Maffione 			kring = NMR(na, t)[i];
2003847bf383SLuigi Rizzo 			if ((kring->nr_kflags & NKR_EXCLUSIVE) ||
2004847bf383SLuigi Rizzo 			    (kring->users && excl))
2005847bf383SLuigi Rizzo 			{
200675f4f3edSVincenzo Maffione 				nm_prdis("ring %s busy", kring->name);
2007847bf383SLuigi Rizzo 				return EBUSY;
2008847bf383SLuigi Rizzo 			}
2009847bf383SLuigi Rizzo 		}
2010847bf383SLuigi Rizzo 	}
2011847bf383SLuigi Rizzo 
201237e3a6d3SLuigi Rizzo 	/* second round: increment usage count (possibly marking them
201337e3a6d3SLuigi Rizzo 	 * as exclusive) and set the nr_pending_mode
2014847bf383SLuigi Rizzo 	 */
2015847bf383SLuigi Rizzo 	for_rx_tx(t) {
2016847bf383SLuigi Rizzo 		for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
20172ff91c17SVincenzo Maffione 			kring = NMR(na, t)[i];
2018847bf383SLuigi Rizzo 			kring->users++;
2019847bf383SLuigi Rizzo 			if (excl)
2020847bf383SLuigi Rizzo 				kring->nr_kflags |= NKR_EXCLUSIVE;
202137e3a6d3SLuigi Rizzo 	                kring->nr_pending_mode = NKR_NETMAP_ON;
2022847bf383SLuigi Rizzo 		}
2023847bf383SLuigi Rizzo 	}
2024847bf383SLuigi Rizzo 
2025847bf383SLuigi Rizzo 	return 0;
2026847bf383SLuigi Rizzo 
2027847bf383SLuigi Rizzo }
2028847bf383SLuigi Rizzo 
202937e3a6d3SLuigi Rizzo /* Undo netmap_krings_get(). This is done by clearing the exclusive mode
203037e3a6d3SLuigi Rizzo  * if was asked on regif, and unset the nr_pending_mode if we are the
203137e3a6d3SLuigi Rizzo  * last users of the involved rings. */
2032847bf383SLuigi Rizzo static void
203337e3a6d3SLuigi Rizzo netmap_krings_put(struct netmap_priv_d *priv)
2034847bf383SLuigi Rizzo {
2035847bf383SLuigi Rizzo 	struct netmap_adapter *na = priv->np_na;
2036847bf383SLuigi Rizzo 	u_int i;
2037847bf383SLuigi Rizzo 	struct netmap_kring *kring;
2038847bf383SLuigi Rizzo 	int excl = (priv->np_flags & NR_EXCLUSIVE);
2039847bf383SLuigi Rizzo 	enum txrx t;
2040847bf383SLuigi Rizzo 
204175f4f3edSVincenzo Maffione 	nm_prdis("%s: releasing tx [%d, %d) rx [%d, %d)",
2042847bf383SLuigi Rizzo 			na->name,
2043847bf383SLuigi Rizzo 			priv->np_qfirst[NR_TX],
2044847bf383SLuigi Rizzo 			priv->np_qlast[NR_TX],
2045847bf383SLuigi Rizzo 			priv->np_qfirst[NR_RX],
2046847bf383SLuigi Rizzo 			priv->np_qlast[MR_RX]);
2047847bf383SLuigi Rizzo 
2048847bf383SLuigi Rizzo 	for_rx_tx(t) {
2049847bf383SLuigi Rizzo 		for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
20502ff91c17SVincenzo Maffione 			kring = NMR(na, t)[i];
2051847bf383SLuigi Rizzo 			if (excl)
2052847bf383SLuigi Rizzo 				kring->nr_kflags &= ~NKR_EXCLUSIVE;
2053847bf383SLuigi Rizzo 			kring->users--;
205437e3a6d3SLuigi Rizzo 			if (kring->users == 0)
205537e3a6d3SLuigi Rizzo 				kring->nr_pending_mode = NKR_NETMAP_OFF;
2056847bf383SLuigi Rizzo 		}
2057847bf383SLuigi Rizzo 	}
2058847bf383SLuigi Rizzo }
2059847bf383SLuigi Rizzo 
20602ff91c17SVincenzo Maffione static int
20612ff91c17SVincenzo Maffione nm_priv_rx_enabled(struct netmap_priv_d *priv)
20622ff91c17SVincenzo Maffione {
20632ff91c17SVincenzo Maffione 	return (priv->np_qfirst[NR_RX] != priv->np_qlast[NR_RX]);
20642ff91c17SVincenzo Maffione }
20652ff91c17SVincenzo Maffione 
2066b6e66be2SVincenzo Maffione /* Validate the CSB entries for both directions (atok and ktoa).
2067b6e66be2SVincenzo Maffione  * To be called under NMG_LOCK(). */
2068b6e66be2SVincenzo Maffione static int
2069b6e66be2SVincenzo Maffione netmap_csb_validate(struct netmap_priv_d *priv, struct nmreq_opt_csb *csbo)
2070b6e66be2SVincenzo Maffione {
2071b6e66be2SVincenzo Maffione 	struct nm_csb_atok *csb_atok_base =
2072b6e66be2SVincenzo Maffione 		(struct nm_csb_atok *)(uintptr_t)csbo->csb_atok;
2073b6e66be2SVincenzo Maffione 	struct nm_csb_ktoa *csb_ktoa_base =
2074b6e66be2SVincenzo Maffione 		(struct nm_csb_ktoa *)(uintptr_t)csbo->csb_ktoa;
2075b6e66be2SVincenzo Maffione 	enum txrx t;
2076b6e66be2SVincenzo Maffione 	int num_rings[NR_TXRX], tot_rings;
2077b6e66be2SVincenzo Maffione 	size_t entry_size[2];
2078b6e66be2SVincenzo Maffione 	void *csb_start[2];
2079b6e66be2SVincenzo Maffione 	int i;
2080b6e66be2SVincenzo Maffione 
2081b6e66be2SVincenzo Maffione 	if (priv->np_kloop_state & NM_SYNC_KLOOP_RUNNING) {
2082b6e66be2SVincenzo Maffione 		nm_prerr("Cannot update CSB while kloop is running");
2083b6e66be2SVincenzo Maffione 		return EBUSY;
2084b6e66be2SVincenzo Maffione 	}
2085b6e66be2SVincenzo Maffione 
2086b6e66be2SVincenzo Maffione 	tot_rings = 0;
2087b6e66be2SVincenzo Maffione 	for_rx_tx(t) {
2088b6e66be2SVincenzo Maffione 		num_rings[t] = priv->np_qlast[t] - priv->np_qfirst[t];
2089b6e66be2SVincenzo Maffione 		tot_rings += num_rings[t];
2090b6e66be2SVincenzo Maffione 	}
2091b6e66be2SVincenzo Maffione 	if (tot_rings <= 0)
2092b6e66be2SVincenzo Maffione 		return 0;
2093b6e66be2SVincenzo Maffione 
2094b6e66be2SVincenzo Maffione 	if (!(priv->np_flags & NR_EXCLUSIVE)) {
2095b6e66be2SVincenzo Maffione 		nm_prerr("CSB mode requires NR_EXCLUSIVE");
2096b6e66be2SVincenzo Maffione 		return EINVAL;
2097b6e66be2SVincenzo Maffione 	}
2098b6e66be2SVincenzo Maffione 
2099b6e66be2SVincenzo Maffione 	entry_size[0] = sizeof(*csb_atok_base);
2100b6e66be2SVincenzo Maffione 	entry_size[1] = sizeof(*csb_ktoa_base);
2101b6e66be2SVincenzo Maffione 	csb_start[0] = (void *)csb_atok_base;
2102b6e66be2SVincenzo Maffione 	csb_start[1] = (void *)csb_ktoa_base;
2103b6e66be2SVincenzo Maffione 
2104b6e66be2SVincenzo Maffione 	for (i = 0; i < 2; i++) {
2105b6e66be2SVincenzo Maffione 		/* On Linux we could use access_ok() to simplify
2106b6e66be2SVincenzo Maffione 		 * the validation. However, the advantage of
2107b6e66be2SVincenzo Maffione 		 * this approach is that it works also on
2108b6e66be2SVincenzo Maffione 		 * FreeBSD. */
2109b6e66be2SVincenzo Maffione 		size_t csb_size = tot_rings * entry_size[i];
2110b6e66be2SVincenzo Maffione 		void *tmp;
2111b6e66be2SVincenzo Maffione 		int err;
2112b6e66be2SVincenzo Maffione 
2113b6e66be2SVincenzo Maffione 		if ((uintptr_t)csb_start[i] & (entry_size[i]-1)) {
2114b6e66be2SVincenzo Maffione 			nm_prerr("Unaligned CSB address");
2115b6e66be2SVincenzo Maffione 			return EINVAL;
2116b6e66be2SVincenzo Maffione 		}
2117b6e66be2SVincenzo Maffione 
2118b6e66be2SVincenzo Maffione 		tmp = nm_os_malloc(csb_size);
2119b6e66be2SVincenzo Maffione 		if (!tmp)
2120b6e66be2SVincenzo Maffione 			return ENOMEM;
2121b6e66be2SVincenzo Maffione 		if (i == 0) {
2122b6e66be2SVincenzo Maffione 			/* Application --> kernel direction. */
2123b6e66be2SVincenzo Maffione 			err = copyin(csb_start[i], tmp, csb_size);
2124b6e66be2SVincenzo Maffione 		} else {
2125b6e66be2SVincenzo Maffione 			/* Kernel --> application direction. */
2126b6e66be2SVincenzo Maffione 			memset(tmp, 0, csb_size);
2127b6e66be2SVincenzo Maffione 			err = copyout(tmp, csb_start[i], csb_size);
2128b6e66be2SVincenzo Maffione 		}
2129b6e66be2SVincenzo Maffione 		nm_os_free(tmp);
2130b6e66be2SVincenzo Maffione 		if (err) {
2131b6e66be2SVincenzo Maffione 			nm_prerr("Invalid CSB address");
2132b6e66be2SVincenzo Maffione 			return err;
2133b6e66be2SVincenzo Maffione 		}
2134b6e66be2SVincenzo Maffione 	}
2135b6e66be2SVincenzo Maffione 
2136b6e66be2SVincenzo Maffione 	priv->np_csb_atok_base = csb_atok_base;
2137b6e66be2SVincenzo Maffione 	priv->np_csb_ktoa_base = csb_ktoa_base;
2138b6e66be2SVincenzo Maffione 
2139b6e66be2SVincenzo Maffione 	/* Initialize the CSB. */
2140b6e66be2SVincenzo Maffione 	for_rx_tx(t) {
2141b6e66be2SVincenzo Maffione 		for (i = 0; i < num_rings[t]; i++) {
2142b6e66be2SVincenzo Maffione 			struct netmap_kring *kring =
2143b6e66be2SVincenzo Maffione 				NMR(priv->np_na, t)[i + priv->np_qfirst[t]];
2144b6e66be2SVincenzo Maffione 			struct nm_csb_atok *csb_atok = csb_atok_base + i;
2145b6e66be2SVincenzo Maffione 			struct nm_csb_ktoa *csb_ktoa = csb_ktoa_base + i;
2146b6e66be2SVincenzo Maffione 
2147b6e66be2SVincenzo Maffione 			if (t == NR_RX) {
2148b6e66be2SVincenzo Maffione 				csb_atok += num_rings[NR_TX];
2149b6e66be2SVincenzo Maffione 				csb_ktoa += num_rings[NR_TX];
2150b6e66be2SVincenzo Maffione 			}
2151b6e66be2SVincenzo Maffione 
2152b6e66be2SVincenzo Maffione 			CSB_WRITE(csb_atok, head, kring->rhead);
2153b6e66be2SVincenzo Maffione 			CSB_WRITE(csb_atok, cur, kring->rcur);
2154b6e66be2SVincenzo Maffione 			CSB_WRITE(csb_atok, appl_need_kick, 1);
2155b6e66be2SVincenzo Maffione 			CSB_WRITE(csb_atok, sync_flags, 1);
2156b6e66be2SVincenzo Maffione 			CSB_WRITE(csb_ktoa, hwcur, kring->nr_hwcur);
2157b6e66be2SVincenzo Maffione 			CSB_WRITE(csb_ktoa, hwtail, kring->nr_hwtail);
2158b6e66be2SVincenzo Maffione 			CSB_WRITE(csb_ktoa, kern_need_kick, 1);
2159b6e66be2SVincenzo Maffione 
2160b6e66be2SVincenzo Maffione 			nm_prinf("csb_init for kring %s: head %u, cur %u, "
2161b6e66be2SVincenzo Maffione 				"hwcur %u, hwtail %u", kring->name,
2162b6e66be2SVincenzo Maffione 				kring->rhead, kring->rcur, kring->nr_hwcur,
2163b6e66be2SVincenzo Maffione 				kring->nr_hwtail);
2164b6e66be2SVincenzo Maffione 		}
2165b6e66be2SVincenzo Maffione 	}
2166b6e66be2SVincenzo Maffione 
2167b6e66be2SVincenzo Maffione 	return 0;
2168b6e66be2SVincenzo Maffione }
2169b6e66be2SVincenzo Maffione 
217077a2baf5SVincenzo Maffione /* Ensure that the netmap adapter can support the given MTU.
217177a2baf5SVincenzo Maffione  * @return EINVAL if the na cannot be set to mtu, 0 otherwise.
217277a2baf5SVincenzo Maffione  */
217377a2baf5SVincenzo Maffione int
217477a2baf5SVincenzo Maffione netmap_buf_size_validate(const struct netmap_adapter *na, unsigned mtu) {
217577a2baf5SVincenzo Maffione 	unsigned nbs = NETMAP_BUF_SIZE(na);
217677a2baf5SVincenzo Maffione 
217777a2baf5SVincenzo Maffione 	if (mtu <= na->rx_buf_maxsize) {
217877a2baf5SVincenzo Maffione 		/* The MTU fits a single NIC slot. We only
217977a2baf5SVincenzo Maffione 		 * Need to check that netmap buffers are
218077a2baf5SVincenzo Maffione 		 * large enough to hold an MTU. NS_MOREFRAG
218177a2baf5SVincenzo Maffione 		 * cannot be used in this case. */
218277a2baf5SVincenzo Maffione 		if (nbs < mtu) {
218377a2baf5SVincenzo Maffione 			nm_prerr("error: netmap buf size (%u) "
218477a2baf5SVincenzo Maffione 				 "< device MTU (%u)", nbs, mtu);
218577a2baf5SVincenzo Maffione 			return EINVAL;
218677a2baf5SVincenzo Maffione 		}
218777a2baf5SVincenzo Maffione 	} else {
218877a2baf5SVincenzo Maffione 		/* More NIC slots may be needed to receive
218977a2baf5SVincenzo Maffione 		 * or transmit a single packet. Check that
219077a2baf5SVincenzo Maffione 		 * the adapter supports NS_MOREFRAG and that
219177a2baf5SVincenzo Maffione 		 * netmap buffers are large enough to hold
219277a2baf5SVincenzo Maffione 		 * the maximum per-slot size. */
219377a2baf5SVincenzo Maffione 		if (!(na->na_flags & NAF_MOREFRAG)) {
219477a2baf5SVincenzo Maffione 			nm_prerr("error: large MTU (%d) needed "
219577a2baf5SVincenzo Maffione 				 "but %s does not support "
219677a2baf5SVincenzo Maffione 				 "NS_MOREFRAG", mtu,
219777a2baf5SVincenzo Maffione 				 na->ifp->if_xname);
219877a2baf5SVincenzo Maffione 			return EINVAL;
219977a2baf5SVincenzo Maffione 		} else if (nbs < na->rx_buf_maxsize) {
220077a2baf5SVincenzo Maffione 			nm_prerr("error: using NS_MOREFRAG on "
220177a2baf5SVincenzo Maffione 				 "%s requires netmap buf size "
220277a2baf5SVincenzo Maffione 				 ">= %u", na->ifp->if_xname,
220377a2baf5SVincenzo Maffione 				 na->rx_buf_maxsize);
220477a2baf5SVincenzo Maffione 			return EINVAL;
220577a2baf5SVincenzo Maffione 		} else {
220677a2baf5SVincenzo Maffione 			nm_prinf("info: netmap application on "
220777a2baf5SVincenzo Maffione 				 "%s needs to support "
220877a2baf5SVincenzo Maffione 				 "NS_MOREFRAG "
220977a2baf5SVincenzo Maffione 				 "(MTU=%u,netmap_buf_size=%u)",
221077a2baf5SVincenzo Maffione 				 na->ifp->if_xname, mtu, nbs);
221177a2baf5SVincenzo Maffione 		}
221277a2baf5SVincenzo Maffione 	}
221377a2baf5SVincenzo Maffione 	return 0;
221477a2baf5SVincenzo Maffione }
221577a2baf5SVincenzo Maffione 
221677a2baf5SVincenzo Maffione 
2217f18be576SLuigi Rizzo /*
2218f18be576SLuigi Rizzo  * possibly move the interface to netmap-mode.
2219f18be576SLuigi Rizzo  * If success it returns a pointer to netmap_if, otherwise NULL.
2220ce3ee1e7SLuigi Rizzo  * This must be called with NMG_LOCK held.
22214bf50f18SLuigi Rizzo  *
22224bf50f18SLuigi Rizzo  * The following na callbacks are called in the process:
22234bf50f18SLuigi Rizzo  *
22244bf50f18SLuigi Rizzo  * na->nm_config()			[by netmap_update_config]
22254bf50f18SLuigi Rizzo  * (get current number and size of rings)
22264bf50f18SLuigi Rizzo  *
22274bf50f18SLuigi Rizzo  *  	We have a generic one for linux (netmap_linux_config).
22284bf50f18SLuigi Rizzo  *  	The bwrap has to override this, since it has to forward
22294bf50f18SLuigi Rizzo  *  	the request to the wrapped adapter (netmap_bwrap_config).
22304bf50f18SLuigi Rizzo  *
22314bf50f18SLuigi Rizzo  *
2232847bf383SLuigi Rizzo  * na->nm_krings_create()
22334bf50f18SLuigi Rizzo  * (create and init the krings array)
22344bf50f18SLuigi Rizzo  *
22354bf50f18SLuigi Rizzo  * 	One of the following:
22364bf50f18SLuigi Rizzo  *
22374bf50f18SLuigi Rizzo  *	* netmap_hw_krings_create, 			(hw ports)
22384bf50f18SLuigi Rizzo  *		creates the standard layout for the krings
22394bf50f18SLuigi Rizzo  * 		and adds the mbq (used for the host rings).
22404bf50f18SLuigi Rizzo  *
22414bf50f18SLuigi Rizzo  * 	* netmap_vp_krings_create			(VALE ports)
22424bf50f18SLuigi Rizzo  * 		add leases and scratchpads
22434bf50f18SLuigi Rizzo  *
22444bf50f18SLuigi Rizzo  * 	* netmap_pipe_krings_create			(pipes)
22454bf50f18SLuigi Rizzo  * 		create the krings and rings of both ends and
22464bf50f18SLuigi Rizzo  * 		cross-link them
22474bf50f18SLuigi Rizzo  *
22484bf50f18SLuigi Rizzo  *      * netmap_monitor_krings_create 			(monitors)
22494bf50f18SLuigi Rizzo  *      	avoid allocating the mbq
22504bf50f18SLuigi Rizzo  *
22514bf50f18SLuigi Rizzo  *      * netmap_bwrap_krings_create			(bwraps)
22524bf50f18SLuigi Rizzo  *      	create both the brap krings array,
22534bf50f18SLuigi Rizzo  *      	the krings array of the wrapped adapter, and
22544bf50f18SLuigi Rizzo  *      	(if needed) the fake array for the host adapter
22554bf50f18SLuigi Rizzo  *
22564bf50f18SLuigi Rizzo  * na->nm_register(, 1)
22574bf50f18SLuigi Rizzo  * (put the adapter in netmap mode)
22584bf50f18SLuigi Rizzo  *
22594bf50f18SLuigi Rizzo  * 	This may be one of the following:
22604bf50f18SLuigi Rizzo  *
226137e3a6d3SLuigi Rizzo  * 	* netmap_hw_reg				        (hw ports)
22624bf50f18SLuigi Rizzo  * 		checks that the ifp is still there, then calls
22634bf50f18SLuigi Rizzo  * 		the hardware specific callback;
22644bf50f18SLuigi Rizzo  *
22654bf50f18SLuigi Rizzo  * 	* netmap_vp_reg					(VALE ports)
22664bf50f18SLuigi Rizzo  *		If the port is connected to a bridge,
22674bf50f18SLuigi Rizzo  *		set the NAF_NETMAP_ON flag under the
22684bf50f18SLuigi Rizzo  *		bridge write lock.
22694bf50f18SLuigi Rizzo  *
22704bf50f18SLuigi Rizzo  *	* netmap_pipe_reg				(pipes)
22714bf50f18SLuigi Rizzo  *		inform the other pipe end that it is no
2272453130d9SPedro F. Giffuni  *		longer responsible for the lifetime of this
22734bf50f18SLuigi Rizzo  *		pipe end
22744bf50f18SLuigi Rizzo  *
22754bf50f18SLuigi Rizzo  *	* netmap_monitor_reg				(monitors)
22764bf50f18SLuigi Rizzo  *		intercept the sync callbacks of the monitored
22774bf50f18SLuigi Rizzo  *		rings
22784bf50f18SLuigi Rizzo  *
227937e3a6d3SLuigi Rizzo  *	* netmap_bwrap_reg				(bwraps)
22804bf50f18SLuigi Rizzo  *		cross-link the bwrap and hwna rings,
22814bf50f18SLuigi Rizzo  *		forward the request to the hwna, override
22824bf50f18SLuigi Rizzo  *		the hwna notify callback (to get the frames
22834bf50f18SLuigi Rizzo  *		coming from outside go through the bridge).
22844bf50f18SLuigi Rizzo  *
22854bf50f18SLuigi Rizzo  *
2286f18be576SLuigi Rizzo  */
2287847bf383SLuigi Rizzo int
2288f9790aebSLuigi Rizzo netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
22892ff91c17SVincenzo Maffione 	uint32_t nr_mode, uint16_t nr_ringid, uint64_t nr_flags)
2290f18be576SLuigi Rizzo {
2291f18be576SLuigi Rizzo 	struct netmap_if *nifp = NULL;
2292847bf383SLuigi Rizzo 	int error;
2293f18be576SLuigi Rizzo 
2294ce3ee1e7SLuigi Rizzo 	NMG_LOCK_ASSERT();
2295f9790aebSLuigi Rizzo 	priv->np_na = na;     /* store the reference */
2296847bf383SLuigi Rizzo 	error = netmap_mem_finalize(na->nm_mem, na);
2297ce3ee1e7SLuigi Rizzo 	if (error)
2298847bf383SLuigi Rizzo 		goto err;
2299847bf383SLuigi Rizzo 
2300847bf383SLuigi Rizzo 	if (na->active_fds == 0) {
23012ff91c17SVincenzo Maffione 
23022ff91c17SVincenzo Maffione 		/* cache the allocator info in the na */
23032ff91c17SVincenzo Maffione 		error = netmap_mem_get_lut(na->nm_mem, &na->na_lut);
23042ff91c17SVincenzo Maffione 		if (error)
23052ff91c17SVincenzo Maffione 			goto err_drop_mem;
230675f4f3edSVincenzo Maffione 		nm_prdis("lut %p bufs %u size %u", na->na_lut.lut, na->na_lut.objtotal,
23072ff91c17SVincenzo Maffione 					    na->na_lut.objsize);
23082ff91c17SVincenzo Maffione 
23092ff91c17SVincenzo Maffione 		/* ring configuration may have changed, fetch from the card */
23102ff91c17SVincenzo Maffione 		netmap_update_config(na);
2311cfa866f6SMatt Macy 	}
23122ff91c17SVincenzo Maffione 
2313cfa866f6SMatt Macy 	/* compute the range of tx and rx rings to monitor */
2314cfa866f6SMatt Macy 	error = netmap_set_ringid(priv, nr_mode, nr_ringid, nr_flags);
2315cfa866f6SMatt Macy 	if (error)
2316cfa866f6SMatt Macy 		goto err_put_lut;
2317cfa866f6SMatt Macy 
2318cfa866f6SMatt Macy 	if (na->active_fds == 0) {
2319847bf383SLuigi Rizzo 		/*
2320847bf383SLuigi Rizzo 		 * If this is the first registration of the adapter,
23214f80b14cSVincenzo Maffione 		 * perform sanity checks and create the in-kernel view
23224f80b14cSVincenzo Maffione 		 * of the netmap rings (the netmap krings).
2323847bf383SLuigi Rizzo 		 */
23242ff91c17SVincenzo Maffione 		if (na->ifp && nm_priv_rx_enabled(priv)) {
23254f80b14cSVincenzo Maffione 			/* This netmap adapter is attached to an ifnet. */
23264f80b14cSVincenzo Maffione 			unsigned mtu = nm_os_ifnet_mtu(na->ifp);
23274f80b14cSVincenzo Maffione 
232875f4f3edSVincenzo Maffione 			nm_prdis("%s: mtu %d rx_buf_maxsize %d netmap_buf_size %d",
232977a2baf5SVincenzo Maffione 				na->name, mtu, na->rx_buf_maxsize, NETMAP_BUF_SIZE(na));
2330cfa866f6SMatt Macy 
2331cfa866f6SMatt Macy 			if (na->rx_buf_maxsize == 0) {
2332b6e66be2SVincenzo Maffione 				nm_prerr("%s: error: rx_buf_maxsize == 0", na->name);
2333cfa866f6SMatt Macy 				error = EIO;
2334cfa866f6SMatt Macy 				goto err_drop_mem;
2335cfa866f6SMatt Macy 			}
23362ff91c17SVincenzo Maffione 
233777a2baf5SVincenzo Maffione 			error = netmap_buf_size_validate(na, mtu);
233877a2baf5SVincenzo Maffione 			if (error)
23394f80b14cSVincenzo Maffione 				goto err_drop_mem;
23404f80b14cSVincenzo Maffione 		}
2341847bf383SLuigi Rizzo 
2342847bf383SLuigi Rizzo 		/*
2343847bf383SLuigi Rizzo 		 * Depending on the adapter, this may also create
2344847bf383SLuigi Rizzo 		 * the netmap rings themselves
2345847bf383SLuigi Rizzo 		 */
2346847bf383SLuigi Rizzo 		error = na->nm_krings_create(na);
2347847bf383SLuigi Rizzo 		if (error)
23482ff91c17SVincenzo Maffione 			goto err_put_lut;
2349847bf383SLuigi Rizzo 
2350ce3ee1e7SLuigi Rizzo 	}
2351847bf383SLuigi Rizzo 
235237e3a6d3SLuigi Rizzo 	/* now the krings must exist and we can check whether some
235337e3a6d3SLuigi Rizzo 	 * previous bind has exclusive ownership on them, and set
235437e3a6d3SLuigi Rizzo 	 * nr_pending_mode
2355847bf383SLuigi Rizzo 	 */
235637e3a6d3SLuigi Rizzo 	error = netmap_krings_get(priv);
2357847bf383SLuigi Rizzo 	if (error)
235837e3a6d3SLuigi Rizzo 		goto err_del_krings;
235937e3a6d3SLuigi Rizzo 
236037e3a6d3SLuigi Rizzo 	/* create all needed missing netmap rings */
236137e3a6d3SLuigi Rizzo 	error = netmap_mem_rings_create(na);
236237e3a6d3SLuigi Rizzo 	if (error)
236337e3a6d3SLuigi Rizzo 		goto err_rel_excl;
2364847bf383SLuigi Rizzo 
2365847bf383SLuigi Rizzo 	/* in all cases, create a new netmap if */
2366c3e9b4dbSLuiz Otavio O Souza 	nifp = netmap_mem_if_new(na, priv);
2367847bf383SLuigi Rizzo 	if (nifp == NULL) {
2368f18be576SLuigi Rizzo 		error = ENOMEM;
2369cfa866f6SMatt Macy 		goto err_rel_excl;
2370ce3ee1e7SLuigi Rizzo 	}
2371847bf383SLuigi Rizzo 
237237e3a6d3SLuigi Rizzo 	if (nm_kring_pending(priv)) {
237337e3a6d3SLuigi Rizzo 		/* Some kring is switching mode, tell the adapter to
237437e3a6d3SLuigi Rizzo 		 * react on this. */
237537e3a6d3SLuigi Rizzo 		error = na->nm_register(na, 1);
237637e3a6d3SLuigi Rizzo 		if (error)
23772ff91c17SVincenzo Maffione 			goto err_del_if;
237837e3a6d3SLuigi Rizzo 	}
237937e3a6d3SLuigi Rizzo 
238037e3a6d3SLuigi Rizzo 	/* Commit the reference. */
238137e3a6d3SLuigi Rizzo 	na->active_fds++;
238237e3a6d3SLuigi Rizzo 
2383ce3ee1e7SLuigi Rizzo 	/*
2384847bf383SLuigi Rizzo 	 * advertise that the interface is ready by setting np_nifp.
2385847bf383SLuigi Rizzo 	 * The barrier is needed because readers (poll, *SYNC and mmap)
2386ce3ee1e7SLuigi Rizzo 	 * check for priv->np_nifp != NULL without locking
2387ce3ee1e7SLuigi Rizzo 	 */
2388847bf383SLuigi Rizzo 	mb(); /* make sure previous writes are visible to all CPUs */
2389ce3ee1e7SLuigi Rizzo 	priv->np_nifp = nifp;
2390847bf383SLuigi Rizzo 
2391847bf383SLuigi Rizzo 	return 0;
2392847bf383SLuigi Rizzo 
239337e3a6d3SLuigi Rizzo err_del_if:
2394847bf383SLuigi Rizzo 	netmap_mem_if_delete(na, nifp);
23954f80b14cSVincenzo Maffione err_rel_excl:
23964f80b14cSVincenzo Maffione 	netmap_krings_put(priv);
2397cfa866f6SMatt Macy 	netmap_mem_rings_delete(na);
2398847bf383SLuigi Rizzo err_del_krings:
2399847bf383SLuigi Rizzo 	if (na->active_fds == 0)
2400847bf383SLuigi Rizzo 		na->nm_krings_delete(na);
24012ff91c17SVincenzo Maffione err_put_lut:
24022ff91c17SVincenzo Maffione 	if (na->active_fds == 0)
24032ff91c17SVincenzo Maffione 		memset(&na->na_lut, 0, sizeof(na->na_lut));
2404847bf383SLuigi Rizzo err_drop_mem:
24054f80b14cSVincenzo Maffione 	netmap_mem_drop(na);
2406847bf383SLuigi Rizzo err:
2407847bf383SLuigi Rizzo 	priv->np_na = NULL;
2408847bf383SLuigi Rizzo 	return error;
2409ce3ee1e7SLuigi Rizzo }
2410847bf383SLuigi Rizzo 
2411847bf383SLuigi Rizzo 
2412847bf383SLuigi Rizzo /*
241337e3a6d3SLuigi Rizzo  * update kring and ring at the end of rxsync/txsync.
2414847bf383SLuigi Rizzo  */
2415847bf383SLuigi Rizzo static inline void
241637e3a6d3SLuigi Rizzo nm_sync_finalize(struct netmap_kring *kring)
2417847bf383SLuigi Rizzo {
241837e3a6d3SLuigi Rizzo 	/*
241937e3a6d3SLuigi Rizzo 	 * Update ring tail to what the kernel knows
242037e3a6d3SLuigi Rizzo 	 * After txsync: head/rhead/hwcur might be behind cur/rcur
242137e3a6d3SLuigi Rizzo 	 * if no carrier.
242237e3a6d3SLuigi Rizzo 	 */
2423847bf383SLuigi Rizzo 	kring->ring->tail = kring->rtail = kring->nr_hwtail;
2424847bf383SLuigi Rizzo 
242575f4f3edSVincenzo Maffione 	nm_prdis(5, "%s now hwcur %d hwtail %d head %d cur %d tail %d",
2426847bf383SLuigi Rizzo 		kring->name, kring->nr_hwcur, kring->nr_hwtail,
2427847bf383SLuigi Rizzo 		kring->rhead, kring->rcur, kring->rtail);
2428847bf383SLuigi Rizzo }
2429847bf383SLuigi Rizzo 
2430c3e9b4dbSLuiz Otavio O Souza /* set ring timestamp */
2431c3e9b4dbSLuiz Otavio O Souza static inline void
2432c3e9b4dbSLuiz Otavio O Souza ring_timestamp_set(struct netmap_ring *ring)
2433c3e9b4dbSLuiz Otavio O Souza {
2434c3e9b4dbSLuiz Otavio O Souza 	if (netmap_no_timestamp == 0 || ring->flags & NR_TIMESTAMP) {
2435c3e9b4dbSLuiz Otavio O Souza 		microtime(&ring->ts);
2436c3e9b4dbSLuiz Otavio O Souza 	}
2437c3e9b4dbSLuiz Otavio O Souza }
2438c3e9b4dbSLuiz Otavio O Souza 
24392ff91c17SVincenzo Maffione static int nmreq_copyin(struct nmreq_header *, int);
24402ff91c17SVincenzo Maffione static int nmreq_copyout(struct nmreq_header *, int);
24412ff91c17SVincenzo Maffione static int nmreq_checkoptions(struct nmreq_header *);
2442c3e9b4dbSLuiz Otavio O Souza 
244368b8534bSLuigi Rizzo /*
244468b8534bSLuigi Rizzo  * ioctl(2) support for the "netmap" device.
244568b8534bSLuigi Rizzo  *
244668b8534bSLuigi Rizzo  * Following a list of accepted commands:
24472ff91c17SVincenzo Maffione  * - NIOCCTRL		device control API
24482ff91c17SVincenzo Maffione  * - NIOCTXSYNC		sync TX rings
24492ff91c17SVincenzo Maffione  * - NIOCRXSYNC		sync RX rings
245068b8534bSLuigi Rizzo  * - SIOCGIFADDR	just for convenience
24512ff91c17SVincenzo Maffione  * - NIOCGINFO		deprecated (legacy API)
24522ff91c17SVincenzo Maffione  * - NIOCREGIF		deprecated (legacy API)
245368b8534bSLuigi Rizzo  *
245468b8534bSLuigi Rizzo  * Return 0 on success, errno otherwise.
245568b8534bSLuigi Rizzo  */
2456f9790aebSLuigi Rizzo int
24572ff91c17SVincenzo Maffione netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data,
24582ff91c17SVincenzo Maffione 		struct thread *td, int nr_body_is_user)
245968b8534bSLuigi Rizzo {
2460c3e9b4dbSLuiz Otavio O Souza 	struct mbq q;	/* packets from RX hw queues to host stack */
2461ce3ee1e7SLuigi Rizzo 	struct netmap_adapter *na = NULL;
2462c3e9b4dbSLuiz Otavio O Souza 	struct netmap_mem_d *nmd = NULL;
246337e3a6d3SLuigi Rizzo 	struct ifnet *ifp = NULL;
246437e3a6d3SLuigi Rizzo 	int error = 0;
2465f0ea3689SLuigi Rizzo 	u_int i, qfirst, qlast;
24662ff91c17SVincenzo Maffione 	struct netmap_kring **krings;
2467c3e9b4dbSLuiz Otavio O Souza 	int sync_flags;
2468847bf383SLuigi Rizzo 	enum txrx t;
246968b8534bSLuigi Rizzo 
24702ff91c17SVincenzo Maffione 	switch (cmd) {
24712ff91c17SVincenzo Maffione 	case NIOCCTRL: {
24722ff91c17SVincenzo Maffione 		struct nmreq_header *hdr = (struct nmreq_header *)data;
24732ff91c17SVincenzo Maffione 
24742ff91c17SVincenzo Maffione 		if (hdr->nr_version < NETMAP_MIN_API ||
24752ff91c17SVincenzo Maffione 		    hdr->nr_version > NETMAP_MAX_API) {
2476b6e66be2SVincenzo Maffione 			nm_prerr("API mismatch: got %d need %d",
2477b6e66be2SVincenzo Maffione 				hdr->nr_version, NETMAP_API);
247817885a7bSLuigi Rizzo 			return EINVAL;
247917885a7bSLuigi Rizzo 		}
248068b8534bSLuigi Rizzo 
24812ff91c17SVincenzo Maffione 		/* Make a kernel-space copy of the user-space nr_body.
24822ff91c17SVincenzo Maffione 		 * For convenince, the nr_body pointer and the pointers
24832ff91c17SVincenzo Maffione 		 * in the options list will be replaced with their
24842ff91c17SVincenzo Maffione 		 * kernel-space counterparts. The original pointers are
24852ff91c17SVincenzo Maffione 		 * saved internally and later restored by nmreq_copyout
24862ff91c17SVincenzo Maffione 		 */
24872ff91c17SVincenzo Maffione 		error = nmreq_copyin(hdr, nr_body_is_user);
248837e3a6d3SLuigi Rizzo 		if (error) {
24892ff91c17SVincenzo Maffione 			return error;
2490ce3ee1e7SLuigi Rizzo 		}
2491ce3ee1e7SLuigi Rizzo 
24922ff91c17SVincenzo Maffione 		/* Sanitize hdr->nr_name. */
24932ff91c17SVincenzo Maffione 		hdr->nr_name[sizeof(hdr->nr_name) - 1] = '\0';
249468b8534bSLuigi Rizzo 
24952ff91c17SVincenzo Maffione 		switch (hdr->nr_reqtype) {
24962ff91c17SVincenzo Maffione 		case NETMAP_REQ_REGISTER: {
24972ff91c17SVincenzo Maffione 			struct nmreq_register *req =
2498cfa866f6SMatt Macy 				(struct nmreq_register *)(uintptr_t)hdr->nr_body;
2499b6e66be2SVincenzo Maffione 			struct netmap_if *nifp;
2500b6e66be2SVincenzo Maffione 
25012ff91c17SVincenzo Maffione 			/* Protect access to priv from concurrent requests. */
2502ce3ee1e7SLuigi Rizzo 			NMG_LOCK();
2503ce3ee1e7SLuigi Rizzo 			do {
25042ff91c17SVincenzo Maffione 				struct nmreq_option *opt;
2505b6e66be2SVincenzo Maffione 				u_int memflags;
2506ce3ee1e7SLuigi Rizzo 
2507847bf383SLuigi Rizzo 				if (priv->np_nifp != NULL) {	/* thread already registered */
2508f0ea3689SLuigi Rizzo 					error = EBUSY;
2509506cc70cSLuigi Rizzo 					break;
2510506cc70cSLuigi Rizzo 				}
2511c3e9b4dbSLuiz Otavio O Souza 
25122ff91c17SVincenzo Maffione #ifdef WITH_EXTMEM
2513253b2ec1SVincenzo Maffione 				opt = nmreq_getoption(hdr, NETMAP_REQ_OPT_EXTMEM);
25142ff91c17SVincenzo Maffione 				if (opt != NULL) {
25152ff91c17SVincenzo Maffione 					struct nmreq_opt_extmem *e =
25162ff91c17SVincenzo Maffione 						(struct nmreq_opt_extmem *)opt;
25172ff91c17SVincenzo Maffione 
25182ff91c17SVincenzo Maffione 					nmd = netmap_mem_ext_create(e->nro_usrptr,
25192ff91c17SVincenzo Maffione 							&e->nro_info, &error);
25202ff91c17SVincenzo Maffione 					opt->nro_status = error;
25212ff91c17SVincenzo Maffione 					if (nmd == NULL)
25222ff91c17SVincenzo Maffione 						break;
25232ff91c17SVincenzo Maffione 				}
25242ff91c17SVincenzo Maffione #endif /* WITH_EXTMEM */
25252ff91c17SVincenzo Maffione 
25262ff91c17SVincenzo Maffione 				if (nmd == NULL && req->nr_mem_id) {
2527c3e9b4dbSLuiz Otavio O Souza 					/* find the allocator and get a reference */
25282ff91c17SVincenzo Maffione 					nmd = netmap_mem_find(req->nr_mem_id);
2529c3e9b4dbSLuiz Otavio O Souza 					if (nmd == NULL) {
2530b6e66be2SVincenzo Maffione 						if (netmap_verbose) {
2531b6e66be2SVincenzo Maffione 							nm_prerr("%s: failed to find mem_id %u",
2532b6e66be2SVincenzo Maffione 									hdr->nr_name, req->nr_mem_id);
2533b6e66be2SVincenzo Maffione 						}
2534c3e9b4dbSLuiz Otavio O Souza 						error = EINVAL;
2535c3e9b4dbSLuiz Otavio O Souza 						break;
2536c3e9b4dbSLuiz Otavio O Souza 					}
2537c3e9b4dbSLuiz Otavio O Souza 				}
253868b8534bSLuigi Rizzo 				/* find the interface and a reference */
25392ff91c17SVincenzo Maffione 				error = netmap_get_na(hdr, &na, &ifp, nmd,
254037e3a6d3SLuigi Rizzo 						      1 /* create */); /* keep reference */
254168b8534bSLuigi Rizzo 				if (error)
2542ce3ee1e7SLuigi Rizzo 					break;
2543f9790aebSLuigi Rizzo 				if (NETMAP_OWNED_BY_KERN(na)) {
2544ce3ee1e7SLuigi Rizzo 					error = EBUSY;
2545ce3ee1e7SLuigi Rizzo 					break;
2546f196ce38SLuigi Rizzo 				}
254737e3a6d3SLuigi Rizzo 
25482ff91c17SVincenzo Maffione 				if (na->virt_hdr_len && !(req->nr_flags & NR_ACCEPT_VNET_HDR)) {
2549b6e66be2SVincenzo Maffione 					nm_prerr("virt_hdr_len=%d, but application does "
2550b6e66be2SVincenzo Maffione 						"not accept it", na->virt_hdr_len);
255137e3a6d3SLuigi Rizzo 					error = EIO;
255237e3a6d3SLuigi Rizzo 					break;
255337e3a6d3SLuigi Rizzo 				}
255437e3a6d3SLuigi Rizzo 
25552ff91c17SVincenzo Maffione 				error = netmap_do_regif(priv, na, req->nr_mode,
25562ff91c17SVincenzo Maffione 							req->nr_ringid, req->nr_flags);
2557847bf383SLuigi Rizzo 				if (error) {    /* reg. failed, release priv and ref */
2558ce3ee1e7SLuigi Rizzo 					break;
255968b8534bSLuigi Rizzo 				}
2560b6e66be2SVincenzo Maffione 
2561253b2ec1SVincenzo Maffione 				opt = nmreq_getoption(hdr, NETMAP_REQ_OPT_CSB);
2562b6e66be2SVincenzo Maffione 				if (opt != NULL) {
2563b6e66be2SVincenzo Maffione 					struct nmreq_opt_csb *csbo =
2564b6e66be2SVincenzo Maffione 						(struct nmreq_opt_csb *)opt;
2565b6e66be2SVincenzo Maffione 					error = netmap_csb_validate(priv, csbo);
2566b6e66be2SVincenzo Maffione 					opt->nro_status = error;
2567b6e66be2SVincenzo Maffione 					if (error) {
2568b6e66be2SVincenzo Maffione 						netmap_do_unregif(priv);
2569b6e66be2SVincenzo Maffione 						break;
2570b6e66be2SVincenzo Maffione 					}
2571b6e66be2SVincenzo Maffione 				}
2572b6e66be2SVincenzo Maffione 
2573847bf383SLuigi Rizzo 				nifp = priv->np_nifp;
257468b8534bSLuigi Rizzo 
257568b8534bSLuigi Rizzo 				/* return the offset of the netmap_if object */
25762ff91c17SVincenzo Maffione 				req->nr_rx_rings = na->num_rx_rings;
25772ff91c17SVincenzo Maffione 				req->nr_tx_rings = na->num_tx_rings;
25782ff91c17SVincenzo Maffione 				req->nr_rx_slots = na->num_rx_desc;
25792ff91c17SVincenzo Maffione 				req->nr_tx_slots = na->num_tx_desc;
2580d12354a5SVincenzo Maffione 				req->nr_host_tx_rings = na->num_host_tx_rings;
2581d12354a5SVincenzo Maffione 				req->nr_host_rx_rings = na->num_host_rx_rings;
25822ff91c17SVincenzo Maffione 				error = netmap_mem_get_info(na->nm_mem, &req->nr_memsize, &memflags,
25832ff91c17SVincenzo Maffione 					&req->nr_mem_id);
2584ce3ee1e7SLuigi Rizzo 				if (error) {
2585847bf383SLuigi Rizzo 					netmap_do_unregif(priv);
2586ce3ee1e7SLuigi Rizzo 					break;
2587ce3ee1e7SLuigi Rizzo 				}
2588ce3ee1e7SLuigi Rizzo 				if (memflags & NETMAP_MEM_PRIVATE) {
25893d819cb6SLuigi Rizzo 					*(uint32_t *)(uintptr_t)&nifp->ni_flags |= NI_PRIV_MEM;
2590ce3ee1e7SLuigi Rizzo 				}
2591847bf383SLuigi Rizzo 				for_rx_tx(t) {
2592847bf383SLuigi Rizzo 					priv->np_si[t] = nm_si_user(priv, t) ?
25932ff91c17SVincenzo Maffione 						&na->si[t] : &NMR(na, t)[priv->np_qfirst[t]]->si;
2594847bf383SLuigi Rizzo 				}
2595f0ea3689SLuigi Rizzo 
25962ff91c17SVincenzo Maffione 				if (req->nr_extra_bufs) {
259737e3a6d3SLuigi Rizzo 					if (netmap_verbose)
2598b6e66be2SVincenzo Maffione 						nm_prinf("requested %d extra buffers",
25992ff91c17SVincenzo Maffione 							req->nr_extra_bufs);
26002ff91c17SVincenzo Maffione 					req->nr_extra_bufs = netmap_extra_alloc(na,
26012ff91c17SVincenzo Maffione 						&nifp->ni_bufs_head, req->nr_extra_bufs);
260237e3a6d3SLuigi Rizzo 					if (netmap_verbose)
2603b6e66be2SVincenzo Maffione 						nm_prinf("got %d extra buffers", req->nr_extra_bufs);
2604f0ea3689SLuigi Rizzo 				}
26052ff91c17SVincenzo Maffione 				req->nr_offset = netmap_mem_if_offset(na->nm_mem, nifp);
26062ff91c17SVincenzo Maffione 
26072ff91c17SVincenzo Maffione 				error = nmreq_checkoptions(hdr);
26082ff91c17SVincenzo Maffione 				if (error) {
26092ff91c17SVincenzo Maffione 					netmap_do_unregif(priv);
26102ff91c17SVincenzo Maffione 					break;
26112ff91c17SVincenzo Maffione 				}
261237e3a6d3SLuigi Rizzo 
261337e3a6d3SLuigi Rizzo 				/* store ifp reference so that priv destructor may release it */
261437e3a6d3SLuigi Rizzo 				priv->np_ifp = ifp;
2615ce3ee1e7SLuigi Rizzo 			} while (0);
2616c3e9b4dbSLuiz Otavio O Souza 			if (error) {
2617c3e9b4dbSLuiz Otavio O Souza 				netmap_unget_na(na, ifp);
2618c3e9b4dbSLuiz Otavio O Souza 			}
2619c3e9b4dbSLuiz Otavio O Souza 			/* release the reference from netmap_mem_find() or
2620c3e9b4dbSLuiz Otavio O Souza 			 * netmap_mem_ext_create()
2621c3e9b4dbSLuiz Otavio O Souza 			 */
2622c3e9b4dbSLuiz Otavio O Souza 			if (nmd)
2623c3e9b4dbSLuiz Otavio O Souza 				netmap_mem_put(nmd);
2624ce3ee1e7SLuigi Rizzo 			NMG_UNLOCK();
262568b8534bSLuigi Rizzo 			break;
26262ff91c17SVincenzo Maffione 		}
26272ff91c17SVincenzo Maffione 
26282ff91c17SVincenzo Maffione 		case NETMAP_REQ_PORT_INFO_GET: {
26292ff91c17SVincenzo Maffione 			struct nmreq_port_info_get *req =
2630cfa866f6SMatt Macy 				(struct nmreq_port_info_get *)(uintptr_t)hdr->nr_body;
26312ff91c17SVincenzo Maffione 
26322ff91c17SVincenzo Maffione 			NMG_LOCK();
26332ff91c17SVincenzo Maffione 			do {
26342ff91c17SVincenzo Maffione 				u_int memflags;
26352ff91c17SVincenzo Maffione 
26362ff91c17SVincenzo Maffione 				if (hdr->nr_name[0] != '\0') {
26372ff91c17SVincenzo Maffione 					/* Build a nmreq_register out of the nmreq_port_info_get,
26382ff91c17SVincenzo Maffione 					 * so that we can call netmap_get_na(). */
26392ff91c17SVincenzo Maffione 					struct nmreq_register regreq;
26402ff91c17SVincenzo Maffione 					bzero(&regreq, sizeof(regreq));
2641b6e66be2SVincenzo Maffione 					regreq.nr_mode = NR_REG_ALL_NIC;
26422ff91c17SVincenzo Maffione 					regreq.nr_tx_slots = req->nr_tx_slots;
26432ff91c17SVincenzo Maffione 					regreq.nr_rx_slots = req->nr_rx_slots;
26442ff91c17SVincenzo Maffione 					regreq.nr_tx_rings = req->nr_tx_rings;
26452ff91c17SVincenzo Maffione 					regreq.nr_rx_rings = req->nr_rx_rings;
2646d12354a5SVincenzo Maffione 					regreq.nr_host_tx_rings = req->nr_host_tx_rings;
2647d12354a5SVincenzo Maffione 					regreq.nr_host_rx_rings = req->nr_host_rx_rings;
26482ff91c17SVincenzo Maffione 					regreq.nr_mem_id = req->nr_mem_id;
26492ff91c17SVincenzo Maffione 
26502ff91c17SVincenzo Maffione 					/* get a refcount */
26512ff91c17SVincenzo Maffione 					hdr->nr_reqtype = NETMAP_REQ_REGISTER;
2652cfa866f6SMatt Macy 					hdr->nr_body = (uintptr_t)&regreq;
26532ff91c17SVincenzo Maffione 					error = netmap_get_na(hdr, &na, &ifp, NULL, 1 /* create */);
26542ff91c17SVincenzo Maffione 					hdr->nr_reqtype = NETMAP_REQ_PORT_INFO_GET; /* reset type */
2655cfa866f6SMatt Macy 					hdr->nr_body = (uintptr_t)req; /* reset nr_body */
26562ff91c17SVincenzo Maffione 					if (error) {
26572ff91c17SVincenzo Maffione 						na = NULL;
26582ff91c17SVincenzo Maffione 						ifp = NULL;
26592ff91c17SVincenzo Maffione 						break;
26602ff91c17SVincenzo Maffione 					}
26612ff91c17SVincenzo Maffione 					nmd = na->nm_mem; /* get memory allocator */
26622ff91c17SVincenzo Maffione 				} else {
26632ff91c17SVincenzo Maffione 					nmd = netmap_mem_find(req->nr_mem_id ? req->nr_mem_id : 1);
26642ff91c17SVincenzo Maffione 					if (nmd == NULL) {
2665b6e66be2SVincenzo Maffione 						if (netmap_verbose)
2666b6e66be2SVincenzo Maffione 							nm_prerr("%s: failed to find mem_id %u",
2667b6e66be2SVincenzo Maffione 									hdr->nr_name,
2668b6e66be2SVincenzo Maffione 									req->nr_mem_id ? req->nr_mem_id : 1);
26692ff91c17SVincenzo Maffione 						error = EINVAL;
26702ff91c17SVincenzo Maffione 						break;
26712ff91c17SVincenzo Maffione 					}
26722ff91c17SVincenzo Maffione 				}
26732ff91c17SVincenzo Maffione 
26742ff91c17SVincenzo Maffione 				error = netmap_mem_get_info(nmd, &req->nr_memsize, &memflags,
26752ff91c17SVincenzo Maffione 					&req->nr_mem_id);
26762ff91c17SVincenzo Maffione 				if (error)
26772ff91c17SVincenzo Maffione 					break;
26782ff91c17SVincenzo Maffione 				if (na == NULL) /* only memory info */
26792ff91c17SVincenzo Maffione 					break;
26802ff91c17SVincenzo Maffione 				netmap_update_config(na);
26812ff91c17SVincenzo Maffione 				req->nr_rx_rings = na->num_rx_rings;
26822ff91c17SVincenzo Maffione 				req->nr_tx_rings = na->num_tx_rings;
26832ff91c17SVincenzo Maffione 				req->nr_rx_slots = na->num_rx_desc;
26842ff91c17SVincenzo Maffione 				req->nr_tx_slots = na->num_tx_desc;
2685d12354a5SVincenzo Maffione 				req->nr_host_tx_rings = na->num_host_tx_rings;
2686d12354a5SVincenzo Maffione 				req->nr_host_rx_rings = na->num_host_rx_rings;
26872ff91c17SVincenzo Maffione 			} while (0);
26882ff91c17SVincenzo Maffione 			netmap_unget_na(na, ifp);
26892ff91c17SVincenzo Maffione 			NMG_UNLOCK();
26902ff91c17SVincenzo Maffione 			break;
26912ff91c17SVincenzo Maffione 		}
26922ff91c17SVincenzo Maffione #ifdef WITH_VALE
26932ff91c17SVincenzo Maffione 		case NETMAP_REQ_VALE_ATTACH: {
2694b6e66be2SVincenzo Maffione 			error = netmap_vale_attach(hdr, NULL /* userspace request */);
26952ff91c17SVincenzo Maffione 			break;
26962ff91c17SVincenzo Maffione 		}
26972ff91c17SVincenzo Maffione 
26982ff91c17SVincenzo Maffione 		case NETMAP_REQ_VALE_DETACH: {
2699b6e66be2SVincenzo Maffione 			error = netmap_vale_detach(hdr, NULL /* userspace request */);
27002ff91c17SVincenzo Maffione 			break;
27012ff91c17SVincenzo Maffione 		}
27022ff91c17SVincenzo Maffione 
27032ff91c17SVincenzo Maffione 		case NETMAP_REQ_VALE_LIST: {
2704b6e66be2SVincenzo Maffione 			error = netmap_vale_list(hdr);
27052ff91c17SVincenzo Maffione 			break;
27062ff91c17SVincenzo Maffione 		}
27072ff91c17SVincenzo Maffione 
27082ff91c17SVincenzo Maffione 		case NETMAP_REQ_PORT_HDR_SET: {
27092ff91c17SVincenzo Maffione 			struct nmreq_port_hdr *req =
2710cfa866f6SMatt Macy 				(struct nmreq_port_hdr *)(uintptr_t)hdr->nr_body;
27112ff91c17SVincenzo Maffione 			/* Build a nmreq_register out of the nmreq_port_hdr,
27122ff91c17SVincenzo Maffione 			 * so that we can call netmap_get_bdg_na(). */
27132ff91c17SVincenzo Maffione 			struct nmreq_register regreq;
27142ff91c17SVincenzo Maffione 			bzero(&regreq, sizeof(regreq));
2715b6e66be2SVincenzo Maffione 			regreq.nr_mode = NR_REG_ALL_NIC;
2716b6e66be2SVincenzo Maffione 
27172ff91c17SVincenzo Maffione 			/* For now we only support virtio-net headers, and only for
27182ff91c17SVincenzo Maffione 			 * VALE ports, but this may change in future. Valid lengths
27192ff91c17SVincenzo Maffione 			 * for the virtio-net header are 0 (no header), 10 and 12. */
27202ff91c17SVincenzo Maffione 			if (req->nr_hdr_len != 0 &&
27212ff91c17SVincenzo Maffione 				req->nr_hdr_len != sizeof(struct nm_vnet_hdr) &&
27222ff91c17SVincenzo Maffione 					req->nr_hdr_len != 12) {
2723b6e66be2SVincenzo Maffione 				if (netmap_verbose)
2724b6e66be2SVincenzo Maffione 					nm_prerr("invalid hdr_len %u", req->nr_hdr_len);
27252ff91c17SVincenzo Maffione 				error = EINVAL;
27262ff91c17SVincenzo Maffione 				break;
27272ff91c17SVincenzo Maffione 			}
27282ff91c17SVincenzo Maffione 			NMG_LOCK();
27292ff91c17SVincenzo Maffione 			hdr->nr_reqtype = NETMAP_REQ_REGISTER;
2730cfa866f6SMatt Macy 			hdr->nr_body = (uintptr_t)&regreq;
27312a7db7a6SVincenzo Maffione 			error = netmap_get_vale_na(hdr, &na, NULL, 0);
27322ff91c17SVincenzo Maffione 			hdr->nr_reqtype = NETMAP_REQ_PORT_HDR_SET;
2733cfa866f6SMatt Macy 			hdr->nr_body = (uintptr_t)req;
27342ff91c17SVincenzo Maffione 			if (na && !error) {
27352ff91c17SVincenzo Maffione 				struct netmap_vp_adapter *vpna =
27362ff91c17SVincenzo Maffione 					(struct netmap_vp_adapter *)na;
27372ff91c17SVincenzo Maffione 				na->virt_hdr_len = req->nr_hdr_len;
27382ff91c17SVincenzo Maffione 				if (na->virt_hdr_len) {
27392ff91c17SVincenzo Maffione 					vpna->mfs = NETMAP_BUF_SIZE(na);
27402ff91c17SVincenzo Maffione 				}
2741b6e66be2SVincenzo Maffione 				if (netmap_verbose)
2742b6e66be2SVincenzo Maffione 					nm_prinf("Using vnet_hdr_len %d for %p", na->virt_hdr_len, na);
27432ff91c17SVincenzo Maffione 				netmap_adapter_put(na);
27442ff91c17SVincenzo Maffione 			} else if (!na) {
27452ff91c17SVincenzo Maffione 				error = ENXIO;
27462ff91c17SVincenzo Maffione 			}
27472ff91c17SVincenzo Maffione 			NMG_UNLOCK();
27482ff91c17SVincenzo Maffione 			break;
27492ff91c17SVincenzo Maffione 		}
27502ff91c17SVincenzo Maffione 
27512ff91c17SVincenzo Maffione 		case NETMAP_REQ_PORT_HDR_GET: {
27522ff91c17SVincenzo Maffione 			/* Get vnet-header length for this netmap port */
27532ff91c17SVincenzo Maffione 			struct nmreq_port_hdr *req =
2754cfa866f6SMatt Macy 				(struct nmreq_port_hdr *)(uintptr_t)hdr->nr_body;
27552ff91c17SVincenzo Maffione 			/* Build a nmreq_register out of the nmreq_port_hdr,
27562ff91c17SVincenzo Maffione 			 * so that we can call netmap_get_bdg_na(). */
27572ff91c17SVincenzo Maffione 			struct nmreq_register regreq;
27582ff91c17SVincenzo Maffione 			struct ifnet *ifp;
27592ff91c17SVincenzo Maffione 
27602ff91c17SVincenzo Maffione 			bzero(&regreq, sizeof(regreq));
2761b6e66be2SVincenzo Maffione 			regreq.nr_mode = NR_REG_ALL_NIC;
27622ff91c17SVincenzo Maffione 			NMG_LOCK();
27632ff91c17SVincenzo Maffione 			hdr->nr_reqtype = NETMAP_REQ_REGISTER;
2764cfa866f6SMatt Macy 			hdr->nr_body = (uintptr_t)&regreq;
27652ff91c17SVincenzo Maffione 			error = netmap_get_na(hdr, &na, &ifp, NULL, 0);
27662ff91c17SVincenzo Maffione 			hdr->nr_reqtype = NETMAP_REQ_PORT_HDR_GET;
2767cfa866f6SMatt Macy 			hdr->nr_body = (uintptr_t)req;
27682ff91c17SVincenzo Maffione 			if (na && !error) {
27692ff91c17SVincenzo Maffione 				req->nr_hdr_len = na->virt_hdr_len;
27702ff91c17SVincenzo Maffione 			}
27712ff91c17SVincenzo Maffione 			netmap_unget_na(na, ifp);
27722ff91c17SVincenzo Maffione 			NMG_UNLOCK();
27732ff91c17SVincenzo Maffione 			break;
27742ff91c17SVincenzo Maffione 		}
27752ff91c17SVincenzo Maffione 
27762ff91c17SVincenzo Maffione 		case NETMAP_REQ_VALE_NEWIF: {
27772ff91c17SVincenzo Maffione 			error = nm_vi_create(hdr);
27782ff91c17SVincenzo Maffione 			break;
27792ff91c17SVincenzo Maffione 		}
27802ff91c17SVincenzo Maffione 
27812ff91c17SVincenzo Maffione 		case NETMAP_REQ_VALE_DELIF: {
27822ff91c17SVincenzo Maffione 			error = nm_vi_destroy(hdr->nr_name);
27832ff91c17SVincenzo Maffione 			break;
27842ff91c17SVincenzo Maffione 		}
27852ff91c17SVincenzo Maffione 
27862ff91c17SVincenzo Maffione 		case NETMAP_REQ_VALE_POLLING_ENABLE:
27872ff91c17SVincenzo Maffione 		case NETMAP_REQ_VALE_POLLING_DISABLE: {
27882ff91c17SVincenzo Maffione 			error = nm_bdg_polling(hdr);
27892ff91c17SVincenzo Maffione 			break;
27902ff91c17SVincenzo Maffione 		}
27912ff91c17SVincenzo Maffione #endif  /* WITH_VALE */
27922ff91c17SVincenzo Maffione 		case NETMAP_REQ_POOLS_INFO_GET: {
2793b6e66be2SVincenzo Maffione 			/* Get information from the memory allocator used for
2794b6e66be2SVincenzo Maffione 			 * hdr->nr_name. */
27952ff91c17SVincenzo Maffione 			struct nmreq_pools_info *req =
2796cfa866f6SMatt Macy 				(struct nmreq_pools_info *)(uintptr_t)hdr->nr_body;
27972ff91c17SVincenzo Maffione 			NMG_LOCK();
2798b6e66be2SVincenzo Maffione 			do {
2799b6e66be2SVincenzo Maffione 				/* Build a nmreq_register out of the nmreq_pools_info,
2800b6e66be2SVincenzo Maffione 				 * so that we can call netmap_get_na(). */
2801b6e66be2SVincenzo Maffione 				struct nmreq_register regreq;
2802b6e66be2SVincenzo Maffione 				bzero(&regreq, sizeof(regreq));
2803b6e66be2SVincenzo Maffione 				regreq.nr_mem_id = req->nr_mem_id;
2804b6e66be2SVincenzo Maffione 				regreq.nr_mode = NR_REG_ALL_NIC;
2805b6e66be2SVincenzo Maffione 
2806b6e66be2SVincenzo Maffione 				hdr->nr_reqtype = NETMAP_REQ_REGISTER;
2807b6e66be2SVincenzo Maffione 				hdr->nr_body = (uintptr_t)&regreq;
2808b6e66be2SVincenzo Maffione 				error = netmap_get_na(hdr, &na, &ifp, NULL, 1 /* create */);
2809b6e66be2SVincenzo Maffione 				hdr->nr_reqtype = NETMAP_REQ_POOLS_INFO_GET; /* reset type */
2810b6e66be2SVincenzo Maffione 				hdr->nr_body = (uintptr_t)req; /* reset nr_body */
2811b6e66be2SVincenzo Maffione 				if (error) {
2812b6e66be2SVincenzo Maffione 					na = NULL;
2813b6e66be2SVincenzo Maffione 					ifp = NULL;
2814b6e66be2SVincenzo Maffione 					break;
28152ff91c17SVincenzo Maffione 				}
2816b6e66be2SVincenzo Maffione 				nmd = na->nm_mem; /* grab the memory allocator */
2817b6e66be2SVincenzo Maffione 				if (nmd == NULL) {
2818b6e66be2SVincenzo Maffione 					error = EINVAL;
2819b6e66be2SVincenzo Maffione 					break;
2820b6e66be2SVincenzo Maffione 				}
2821b6e66be2SVincenzo Maffione 
2822b6e66be2SVincenzo Maffione 				/* Finalize the memory allocator, get the pools
2823b6e66be2SVincenzo Maffione 				 * information and release the allocator. */
2824b6e66be2SVincenzo Maffione 				error = netmap_mem_finalize(nmd, na);
2825b6e66be2SVincenzo Maffione 				if (error) {
2826b6e66be2SVincenzo Maffione 					break;
2827b6e66be2SVincenzo Maffione 				}
2828b6e66be2SVincenzo Maffione 				error = netmap_mem_pools_info_get(req, nmd);
2829b6e66be2SVincenzo Maffione 				netmap_mem_drop(na);
2830b6e66be2SVincenzo Maffione 			} while (0);
2831b6e66be2SVincenzo Maffione 			netmap_unget_na(na, ifp);
28322ff91c17SVincenzo Maffione 			NMG_UNLOCK();
28332ff91c17SVincenzo Maffione 			break;
28342ff91c17SVincenzo Maffione 		}
28352ff91c17SVincenzo Maffione 
2836b6e66be2SVincenzo Maffione 		case NETMAP_REQ_CSB_ENABLE: {
2837b6e66be2SVincenzo Maffione 			struct nmreq_option *opt;
2838b6e66be2SVincenzo Maffione 
2839253b2ec1SVincenzo Maffione 			opt = nmreq_getoption(hdr, NETMAP_REQ_OPT_CSB);
2840b6e66be2SVincenzo Maffione 			if (opt == NULL) {
2841b6e66be2SVincenzo Maffione 				error = EINVAL;
2842b6e66be2SVincenzo Maffione 			} else {
2843b6e66be2SVincenzo Maffione 				struct nmreq_opt_csb *csbo =
2844b6e66be2SVincenzo Maffione 					(struct nmreq_opt_csb *)opt;
2845b6e66be2SVincenzo Maffione 				NMG_LOCK();
2846b6e66be2SVincenzo Maffione 				error = netmap_csb_validate(priv, csbo);
2847b6e66be2SVincenzo Maffione 				NMG_UNLOCK();
2848b6e66be2SVincenzo Maffione 				opt->nro_status = error;
2849b6e66be2SVincenzo Maffione 			}
2850b6e66be2SVincenzo Maffione 			break;
2851b6e66be2SVincenzo Maffione 		}
2852b6e66be2SVincenzo Maffione 
2853b6e66be2SVincenzo Maffione 		case NETMAP_REQ_SYNC_KLOOP_START: {
2854b6e66be2SVincenzo Maffione 			error = netmap_sync_kloop(priv, hdr);
2855b6e66be2SVincenzo Maffione 			break;
2856b6e66be2SVincenzo Maffione 		}
2857b6e66be2SVincenzo Maffione 
2858b6e66be2SVincenzo Maffione 		case NETMAP_REQ_SYNC_KLOOP_STOP: {
2859b6e66be2SVincenzo Maffione 			error = netmap_sync_kloop_stop(priv);
2860b6e66be2SVincenzo Maffione 			break;
2861b6e66be2SVincenzo Maffione 		}
2862b6e66be2SVincenzo Maffione 
28632ff91c17SVincenzo Maffione 		default: {
28642ff91c17SVincenzo Maffione 			error = EINVAL;
28652ff91c17SVincenzo Maffione 			break;
28662ff91c17SVincenzo Maffione 		}
28672ff91c17SVincenzo Maffione 		}
28682ff91c17SVincenzo Maffione 		/* Write back request body to userspace and reset the
28692ff91c17SVincenzo Maffione 		 * user-space pointer. */
28702ff91c17SVincenzo Maffione 		error = nmreq_copyout(hdr, error);
28712ff91c17SVincenzo Maffione 		break;
28722ff91c17SVincenzo Maffione 	}
287368b8534bSLuigi Rizzo 
287468b8534bSLuigi Rizzo 	case NIOCTXSYNC:
28752ff91c17SVincenzo Maffione 	case NIOCRXSYNC: {
2876b6e66be2SVincenzo Maffione 		if (unlikely(priv->np_nifp == NULL)) {
2877506cc70cSLuigi Rizzo 			error = ENXIO;
2878506cc70cSLuigi Rizzo 			break;
2879506cc70cSLuigi Rizzo 		}
28806641c68bSLuigi Rizzo 		mb(); /* make sure following reads are not from cache */
28818241616dSLuigi Rizzo 
2882b6e66be2SVincenzo Maffione 		if (unlikely(priv->np_csb_atok_base)) {
2883b6e66be2SVincenzo Maffione 			nm_prerr("Invalid sync in CSB mode");
2884b6e66be2SVincenzo Maffione 			error = EBUSY;
28858241616dSLuigi Rizzo 			break;
28868241616dSLuigi Rizzo 		}
28878241616dSLuigi Rizzo 
2888b6e66be2SVincenzo Maffione 		na = priv->np_na;      /* we have a reference */
2889b6e66be2SVincenzo Maffione 
2890c3e9b4dbSLuiz Otavio O Souza 		mbq_init(&q);
2891847bf383SLuigi Rizzo 		t = (cmd == NIOCTXSYNC ? NR_TX : NR_RX);
2892847bf383SLuigi Rizzo 		krings = NMR(na, t);
2893847bf383SLuigi Rizzo 		qfirst = priv->np_qfirst[t];
2894847bf383SLuigi Rizzo 		qlast = priv->np_qlast[t];
2895c3e9b4dbSLuiz Otavio O Souza 		sync_flags = priv->np_sync_flags;
289668b8534bSLuigi Rizzo 
2897f0ea3689SLuigi Rizzo 		for (i = qfirst; i < qlast; i++) {
28982ff91c17SVincenzo Maffione 			struct netmap_kring *kring = krings[i];
289937e3a6d3SLuigi Rizzo 			struct netmap_ring *ring = kring->ring;
290037e3a6d3SLuigi Rizzo 
290137e3a6d3SLuigi Rizzo 			if (unlikely(nm_kr_tryget(kring, 1, &error))) {
290237e3a6d3SLuigi Rizzo 				error = (error ? EIO : 0);
290337e3a6d3SLuigi Rizzo 				continue;
2904ce3ee1e7SLuigi Rizzo 			}
290537e3a6d3SLuigi Rizzo 
290668b8534bSLuigi Rizzo 			if (cmd == NIOCTXSYNC) {
2907b6e66be2SVincenzo Maffione 				if (netmap_debug & NM_DEBUG_TXSYNC)
2908b6e66be2SVincenzo Maffione 					nm_prinf("pre txsync ring %d cur %d hwcur %d",
290937e3a6d3SLuigi Rizzo 					    i, ring->cur,
291068b8534bSLuigi Rizzo 					    kring->nr_hwcur);
291137e3a6d3SLuigi Rizzo 				if (nm_txsync_prologue(kring, ring) >= kring->nkr_num_slots) {
291217885a7bSLuigi Rizzo 					netmap_ring_reinit(kring);
2913c3e9b4dbSLuiz Otavio O Souza 				} else if (kring->nm_sync(kring, sync_flags | NAF_FORCE_RECLAIM) == 0) {
291437e3a6d3SLuigi Rizzo 					nm_sync_finalize(kring);
291517885a7bSLuigi Rizzo 				}
2916b6e66be2SVincenzo Maffione 				if (netmap_debug & NM_DEBUG_TXSYNC)
2917b6e66be2SVincenzo Maffione 					nm_prinf("post txsync ring %d cur %d hwcur %d",
291837e3a6d3SLuigi Rizzo 					    i, ring->cur,
291968b8534bSLuigi Rizzo 					    kring->nr_hwcur);
292068b8534bSLuigi Rizzo 			} else {
292137e3a6d3SLuigi Rizzo 				if (nm_rxsync_prologue(kring, ring) >= kring->nkr_num_slots) {
2922847bf383SLuigi Rizzo 					netmap_ring_reinit(kring);
2923c3e9b4dbSLuiz Otavio O Souza 				}
2924c3e9b4dbSLuiz Otavio O Souza 				if (nm_may_forward_up(kring)) {
2925c3e9b4dbSLuiz Otavio O Souza 					/* transparent forwarding, see netmap_poll() */
2926c3e9b4dbSLuiz Otavio O Souza 					netmap_grab_packets(kring, &q, netmap_fwd);
2927c3e9b4dbSLuiz Otavio O Souza 				}
2928c3e9b4dbSLuiz Otavio O Souza 				if (kring->nm_sync(kring, sync_flags | NAF_FORCE_READ) == 0) {
292937e3a6d3SLuigi Rizzo 					nm_sync_finalize(kring);
2930847bf383SLuigi Rizzo 				}
2931c3e9b4dbSLuiz Otavio O Souza 				ring_timestamp_set(ring);
293268b8534bSLuigi Rizzo 			}
2933ce3ee1e7SLuigi Rizzo 			nm_kr_put(kring);
293468b8534bSLuigi Rizzo 		}
293568b8534bSLuigi Rizzo 
2936c3e9b4dbSLuiz Otavio O Souza 		if (mbq_peek(&q)) {
2937c3e9b4dbSLuiz Otavio O Souza 			netmap_send_up(na->ifp, &q);
2938c3e9b4dbSLuiz Otavio O Souza 		}
2939c3e9b4dbSLuiz Otavio O Souza 
294068b8534bSLuigi Rizzo 		break;
294168b8534bSLuigi Rizzo 	}
2942f196ce38SLuigi Rizzo 
29432ff91c17SVincenzo Maffione 	default: {
29442ff91c17SVincenzo Maffione 		return netmap_ioctl_legacy(priv, cmd, data, td);
29452ff91c17SVincenzo Maffione 		break;
29462ff91c17SVincenzo Maffione 	}
294768b8534bSLuigi Rizzo 	}
294868b8534bSLuigi Rizzo 
294968b8534bSLuigi Rizzo 	return (error);
295068b8534bSLuigi Rizzo }
295168b8534bSLuigi Rizzo 
29522ff91c17SVincenzo Maffione size_t
29532ff91c17SVincenzo Maffione nmreq_size_by_type(uint16_t nr_reqtype)
29542ff91c17SVincenzo Maffione {
29552ff91c17SVincenzo Maffione 	switch (nr_reqtype) {
29562ff91c17SVincenzo Maffione 	case NETMAP_REQ_REGISTER:
29572ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_register);
29582ff91c17SVincenzo Maffione 	case NETMAP_REQ_PORT_INFO_GET:
29592ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_port_info_get);
29602ff91c17SVincenzo Maffione 	case NETMAP_REQ_VALE_ATTACH:
29612ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_vale_attach);
29622ff91c17SVincenzo Maffione 	case NETMAP_REQ_VALE_DETACH:
29632ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_vale_detach);
29642ff91c17SVincenzo Maffione 	case NETMAP_REQ_VALE_LIST:
29652ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_vale_list);
29662ff91c17SVincenzo Maffione 	case NETMAP_REQ_PORT_HDR_SET:
29672ff91c17SVincenzo Maffione 	case NETMAP_REQ_PORT_HDR_GET:
29682ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_port_hdr);
29692ff91c17SVincenzo Maffione 	case NETMAP_REQ_VALE_NEWIF:
29702ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_vale_newif);
29712ff91c17SVincenzo Maffione 	case NETMAP_REQ_VALE_DELIF:
2972b6e66be2SVincenzo Maffione 	case NETMAP_REQ_SYNC_KLOOP_STOP:
2973b6e66be2SVincenzo Maffione 	case NETMAP_REQ_CSB_ENABLE:
29742ff91c17SVincenzo Maffione 		return 0;
29752ff91c17SVincenzo Maffione 	case NETMAP_REQ_VALE_POLLING_ENABLE:
29762ff91c17SVincenzo Maffione 	case NETMAP_REQ_VALE_POLLING_DISABLE:
29772ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_vale_polling);
29782ff91c17SVincenzo Maffione 	case NETMAP_REQ_POOLS_INFO_GET:
29792ff91c17SVincenzo Maffione 		return sizeof(struct nmreq_pools_info);
2980b6e66be2SVincenzo Maffione 	case NETMAP_REQ_SYNC_KLOOP_START:
2981b6e66be2SVincenzo Maffione 		return sizeof(struct nmreq_sync_kloop_start);
29822ff91c17SVincenzo Maffione 	}
29832ff91c17SVincenzo Maffione 	return 0;
29842ff91c17SVincenzo Maffione }
29852ff91c17SVincenzo Maffione 
29862ff91c17SVincenzo Maffione static size_t
2987b6e66be2SVincenzo Maffione nmreq_opt_size_by_type(uint32_t nro_reqtype, uint64_t nro_size)
29882ff91c17SVincenzo Maffione {
29892ff91c17SVincenzo Maffione 	size_t rv = sizeof(struct nmreq_option);
29902ff91c17SVincenzo Maffione #ifdef NETMAP_REQ_OPT_DEBUG
29912ff91c17SVincenzo Maffione 	if (nro_reqtype & NETMAP_REQ_OPT_DEBUG)
29922ff91c17SVincenzo Maffione 		return (nro_reqtype & ~NETMAP_REQ_OPT_DEBUG);
29932ff91c17SVincenzo Maffione #endif /* NETMAP_REQ_OPT_DEBUG */
29942ff91c17SVincenzo Maffione 	switch (nro_reqtype) {
29952ff91c17SVincenzo Maffione #ifdef WITH_EXTMEM
29962ff91c17SVincenzo Maffione 	case NETMAP_REQ_OPT_EXTMEM:
29972ff91c17SVincenzo Maffione 		rv = sizeof(struct nmreq_opt_extmem);
29982ff91c17SVincenzo Maffione 		break;
29992ff91c17SVincenzo Maffione #endif /* WITH_EXTMEM */
3000b6e66be2SVincenzo Maffione 	case NETMAP_REQ_OPT_SYNC_KLOOP_EVENTFDS:
3001b6e66be2SVincenzo Maffione 		if (nro_size >= rv)
3002b6e66be2SVincenzo Maffione 			rv = nro_size;
3003b6e66be2SVincenzo Maffione 		break;
3004b6e66be2SVincenzo Maffione 	case NETMAP_REQ_OPT_CSB:
3005b6e66be2SVincenzo Maffione 		rv = sizeof(struct nmreq_opt_csb);
3006b6e66be2SVincenzo Maffione 		break;
30075faab778SVincenzo Maffione 	case NETMAP_REQ_OPT_SYNC_KLOOP_MODE:
30085faab778SVincenzo Maffione 		rv = sizeof(struct nmreq_opt_sync_kloop_mode);
30095faab778SVincenzo Maffione 		break;
30102ff91c17SVincenzo Maffione 	}
30112ff91c17SVincenzo Maffione 	/* subtract the common header */
30122ff91c17SVincenzo Maffione 	return rv - sizeof(struct nmreq_option);
30132ff91c17SVincenzo Maffione }
30142ff91c17SVincenzo Maffione 
3015253b2ec1SVincenzo Maffione /*
3016253b2ec1SVincenzo Maffione  * nmreq_copyin: create an in-kernel version of the request.
3017253b2ec1SVincenzo Maffione  *
3018253b2ec1SVincenzo Maffione  * We build the following data structure:
3019253b2ec1SVincenzo Maffione  *
3020253b2ec1SVincenzo Maffione  * hdr -> +-------+                buf
3021253b2ec1SVincenzo Maffione  *        |       |          +---------------+
3022253b2ec1SVincenzo Maffione  *        +-------+          |usr body ptr   |
3023253b2ec1SVincenzo Maffione  *        |options|-.        +---------------+
3024253b2ec1SVincenzo Maffione  *        +-------+ |        |usr options ptr|
3025253b2ec1SVincenzo Maffione  *        |body   |--------->+---------------+
3026253b2ec1SVincenzo Maffione  *        +-------+ |        |               |
3027253b2ec1SVincenzo Maffione  *                  |        |  copy of body |
3028253b2ec1SVincenzo Maffione  *                  |        |               |
3029253b2ec1SVincenzo Maffione  *                  |        +---------------+
3030253b2ec1SVincenzo Maffione  *                  |        |    NULL       |
3031253b2ec1SVincenzo Maffione  *                  |        +---------------+
3032253b2ec1SVincenzo Maffione  *                  |    .---|               |\
3033253b2ec1SVincenzo Maffione  *                  |    |   +---------------+ |
3034253b2ec1SVincenzo Maffione  *                  | .------|               | |
3035253b2ec1SVincenzo Maffione  *                  | |  |   +---------------+  \ option table
3036253b2ec1SVincenzo Maffione  *                  | |  |   |      ...      |  / indexed by option
3037253b2ec1SVincenzo Maffione  *                  | |  |   +---------------+ |  type
3038253b2ec1SVincenzo Maffione  *                  | |  |   |               | |
3039253b2ec1SVincenzo Maffione  *                  | |  |   +---------------+/
3040253b2ec1SVincenzo Maffione  *                  | |  |   |usr next ptr 1 |
3041253b2ec1SVincenzo Maffione  *                  `-|----->+---------------+
3042253b2ec1SVincenzo Maffione  *                    |  |   | copy of opt 1 |
3043253b2ec1SVincenzo Maffione  *                    |  |   |               |
3044253b2ec1SVincenzo Maffione  *                    |  | .-| nro_next      |
3045253b2ec1SVincenzo Maffione  *                    |  | | +---------------+
3046253b2ec1SVincenzo Maffione  *                    |  | | |usr next ptr 2 |
3047253b2ec1SVincenzo Maffione  *                    |  `-`>+---------------+
3048253b2ec1SVincenzo Maffione  *                    |      | copy of opt 2 |
3049253b2ec1SVincenzo Maffione  *                    |      |               |
3050253b2ec1SVincenzo Maffione  *                    |    .-| nro_next      |
3051253b2ec1SVincenzo Maffione  *                    |    | +---------------+
3052253b2ec1SVincenzo Maffione  *                    |    | |               |
3053253b2ec1SVincenzo Maffione  *                    ~    ~ ~      ...      ~
3054253b2ec1SVincenzo Maffione  *                    |    .-|               |
3055253b2ec1SVincenzo Maffione  *                    `----->+---------------+
3056253b2ec1SVincenzo Maffione  *                         | |usr next ptr n |
3057253b2ec1SVincenzo Maffione  *                         `>+---------------+
3058253b2ec1SVincenzo Maffione  *                           | copy of opt n |
3059253b2ec1SVincenzo Maffione  *                           |               |
3060253b2ec1SVincenzo Maffione  *                           | nro_next(NULL)|
3061253b2ec1SVincenzo Maffione  *                           +---------------+
3062253b2ec1SVincenzo Maffione  *
3063253b2ec1SVincenzo Maffione  * The options and body fields of the hdr structure are overwritten
3064253b2ec1SVincenzo Maffione  * with in-kernel valid pointers inside the buf. The original user
3065253b2ec1SVincenzo Maffione  * pointers are saved in the buf and restored on copyout.
3066253b2ec1SVincenzo Maffione  * The list of options is copied and the pointers adjusted. The
3067253b2ec1SVincenzo Maffione  * original pointers are saved before the option they belonged.
3068253b2ec1SVincenzo Maffione  *
3069253b2ec1SVincenzo Maffione  * The option table has an entry for every availabe option.  Entries
3070253b2ec1SVincenzo Maffione  * for options that have not been passed contain NULL.
3071253b2ec1SVincenzo Maffione  *
3072253b2ec1SVincenzo Maffione  */
3073253b2ec1SVincenzo Maffione 
30742ff91c17SVincenzo Maffione int
30752ff91c17SVincenzo Maffione nmreq_copyin(struct nmreq_header *hdr, int nr_body_is_user)
30762ff91c17SVincenzo Maffione {
30772ff91c17SVincenzo Maffione 	size_t rqsz, optsz, bufsz;
3078253b2ec1SVincenzo Maffione 	int error = 0;
30792ff91c17SVincenzo Maffione 	char *ker = NULL, *p;
3080253b2ec1SVincenzo Maffione 	struct nmreq_option **next, *src, **opt_tab;
30812ff91c17SVincenzo Maffione 	struct nmreq_option buf;
30822ff91c17SVincenzo Maffione 	uint64_t *ptrs;
30832ff91c17SVincenzo Maffione 
3084b6e66be2SVincenzo Maffione 	if (hdr->nr_reserved) {
3085b6e66be2SVincenzo Maffione 		if (netmap_verbose)
3086b6e66be2SVincenzo Maffione 			nm_prerr("nr_reserved must be zero");
30872ff91c17SVincenzo Maffione 		return EINVAL;
3088b6e66be2SVincenzo Maffione 	}
30892ff91c17SVincenzo Maffione 
30902ff91c17SVincenzo Maffione 	if (!nr_body_is_user)
30912ff91c17SVincenzo Maffione 		return 0;
30922ff91c17SVincenzo Maffione 
30932ff91c17SVincenzo Maffione 	hdr->nr_reserved = nr_body_is_user;
30942ff91c17SVincenzo Maffione 
30952ff91c17SVincenzo Maffione 	/* compute the total size of the buffer */
30962ff91c17SVincenzo Maffione 	rqsz = nmreq_size_by_type(hdr->nr_reqtype);
30972ff91c17SVincenzo Maffione 	if (rqsz > NETMAP_REQ_MAXSIZE) {
30982ff91c17SVincenzo Maffione 		error = EMSGSIZE;
30992ff91c17SVincenzo Maffione 		goto out_err;
31002ff91c17SVincenzo Maffione 	}
3101cfa866f6SMatt Macy 	if ((rqsz && hdr->nr_body == (uintptr_t)NULL) ||
3102cfa866f6SMatt Macy 		(!rqsz && hdr->nr_body != (uintptr_t)NULL)) {
31032ff91c17SVincenzo Maffione 		/* Request body expected, but not found; or
31042ff91c17SVincenzo Maffione 		 * request body found but unexpected. */
3105b6e66be2SVincenzo Maffione 		if (netmap_verbose)
3106b6e66be2SVincenzo Maffione 			nm_prerr("nr_body expected but not found, or vice versa");
31072ff91c17SVincenzo Maffione 		error = EINVAL;
31082ff91c17SVincenzo Maffione 		goto out_err;
31092ff91c17SVincenzo Maffione 	}
31102ff91c17SVincenzo Maffione 
3111253b2ec1SVincenzo Maffione 	bufsz = 2 * sizeof(void *) + rqsz +
3112253b2ec1SVincenzo Maffione 		NETMAP_REQ_OPT_MAX * sizeof(opt_tab);
3113253b2ec1SVincenzo Maffione 	/* compute the size of the buf below the option table.
3114253b2ec1SVincenzo Maffione 	 * It must contain a copy of every received option structure.
3115253b2ec1SVincenzo Maffione 	 * For every option we also need to store a copy of the user
3116253b2ec1SVincenzo Maffione 	 * list pointer.
3117253b2ec1SVincenzo Maffione 	 */
31182ff91c17SVincenzo Maffione 	optsz = 0;
3119cfa866f6SMatt Macy 	for (src = (struct nmreq_option *)(uintptr_t)hdr->nr_options; src;
3120cfa866f6SMatt Macy 	     src = (struct nmreq_option *)(uintptr_t)buf.nro_next)
31212ff91c17SVincenzo Maffione 	{
31222ff91c17SVincenzo Maffione 		error = copyin(src, &buf, sizeof(*src));
31232ff91c17SVincenzo Maffione 		if (error)
31242ff91c17SVincenzo Maffione 			goto out_err;
31252ff91c17SVincenzo Maffione 		optsz += sizeof(*src);
3126b6e66be2SVincenzo Maffione 		optsz += nmreq_opt_size_by_type(buf.nro_reqtype, buf.nro_size);
31272ff91c17SVincenzo Maffione 		if (rqsz + optsz > NETMAP_REQ_MAXSIZE) {
31282ff91c17SVincenzo Maffione 			error = EMSGSIZE;
31292ff91c17SVincenzo Maffione 			goto out_err;
31302ff91c17SVincenzo Maffione 		}
3131253b2ec1SVincenzo Maffione 		bufsz += sizeof(void *);
31322ff91c17SVincenzo Maffione 	}
3133253b2ec1SVincenzo Maffione 	bufsz += optsz;
31342ff91c17SVincenzo Maffione 
31352ff91c17SVincenzo Maffione 	ker = nm_os_malloc(bufsz);
31362ff91c17SVincenzo Maffione 	if (ker == NULL) {
31372ff91c17SVincenzo Maffione 		error = ENOMEM;
31382ff91c17SVincenzo Maffione 		goto out_err;
31392ff91c17SVincenzo Maffione 	}
3140253b2ec1SVincenzo Maffione 	p = ker;	/* write pointer into the buffer */
31412ff91c17SVincenzo Maffione 
31422ff91c17SVincenzo Maffione 	/* make a copy of the user pointers */
31432ff91c17SVincenzo Maffione 	ptrs = (uint64_t*)p;
31442ff91c17SVincenzo Maffione 	*ptrs++ = hdr->nr_body;
31452ff91c17SVincenzo Maffione 	*ptrs++ = hdr->nr_options;
31462ff91c17SVincenzo Maffione 	p = (char *)ptrs;
31472ff91c17SVincenzo Maffione 
31482ff91c17SVincenzo Maffione 	/* copy the body */
3149cfa866f6SMatt Macy 	error = copyin((void *)(uintptr_t)hdr->nr_body, p, rqsz);
31502ff91c17SVincenzo Maffione 	if (error)
31512ff91c17SVincenzo Maffione 		goto out_restore;
31522ff91c17SVincenzo Maffione 	/* overwrite the user pointer with the in-kernel one */
3153cfa866f6SMatt Macy 	hdr->nr_body = (uintptr_t)p;
31542ff91c17SVincenzo Maffione 	p += rqsz;
3155253b2ec1SVincenzo Maffione 	/* start of the options table */
3156253b2ec1SVincenzo Maffione 	opt_tab = (struct nmreq_option **)p;
3157253b2ec1SVincenzo Maffione 	p += sizeof(opt_tab) * NETMAP_REQ_OPT_MAX;
31582ff91c17SVincenzo Maffione 
31592ff91c17SVincenzo Maffione 	/* copy the options */
31602ff91c17SVincenzo Maffione 	next = (struct nmreq_option **)&hdr->nr_options;
31612ff91c17SVincenzo Maffione 	src = *next;
31622ff91c17SVincenzo Maffione 	while (src) {
31632ff91c17SVincenzo Maffione 		struct nmreq_option *opt;
31642ff91c17SVincenzo Maffione 
31652ff91c17SVincenzo Maffione 		/* copy the option header */
31662ff91c17SVincenzo Maffione 		ptrs = (uint64_t *)p;
31672ff91c17SVincenzo Maffione 		opt = (struct nmreq_option *)(ptrs + 1);
31682ff91c17SVincenzo Maffione 		error = copyin(src, opt, sizeof(*src));
31692ff91c17SVincenzo Maffione 		if (error)
31702ff91c17SVincenzo Maffione 			goto out_restore;
31712ff91c17SVincenzo Maffione 		/* make a copy of the user next pointer */
31722ff91c17SVincenzo Maffione 		*ptrs = opt->nro_next;
31732ff91c17SVincenzo Maffione 		/* overwrite the user pointer with the in-kernel one */
31742ff91c17SVincenzo Maffione 		*next = opt;
31752ff91c17SVincenzo Maffione 
31762ff91c17SVincenzo Maffione 		/* initialize the option as not supported.
31772ff91c17SVincenzo Maffione 		 * Recognized options will update this field.
31782ff91c17SVincenzo Maffione 		 */
31792ff91c17SVincenzo Maffione 		opt->nro_status = EOPNOTSUPP;
31802ff91c17SVincenzo Maffione 
3181253b2ec1SVincenzo Maffione 		/* check for invalid types */
3182253b2ec1SVincenzo Maffione 		if (opt->nro_reqtype < 1) {
3183253b2ec1SVincenzo Maffione 			if (netmap_verbose)
3184253b2ec1SVincenzo Maffione 				nm_prinf("invalid option type: %u", opt->nro_reqtype);
3185253b2ec1SVincenzo Maffione 			opt->nro_status = EINVAL;
3186253b2ec1SVincenzo Maffione 			error = EINVAL;
3187253b2ec1SVincenzo Maffione 			goto next;
3188253b2ec1SVincenzo Maffione 		}
3189253b2ec1SVincenzo Maffione 
3190253b2ec1SVincenzo Maffione 		if (opt->nro_reqtype >= NETMAP_REQ_OPT_MAX) {
3191253b2ec1SVincenzo Maffione 			/* opt->nro_status is already EOPNOTSUPP */
3192253b2ec1SVincenzo Maffione 			error = EOPNOTSUPP;
3193253b2ec1SVincenzo Maffione 			goto next;
3194253b2ec1SVincenzo Maffione 		}
3195253b2ec1SVincenzo Maffione 
3196253b2ec1SVincenzo Maffione 		/* if the type is valid, index the option in the table
3197253b2ec1SVincenzo Maffione 		 * unless it is a duplicate.
3198253b2ec1SVincenzo Maffione 		 */
3199253b2ec1SVincenzo Maffione 		if (opt_tab[opt->nro_reqtype] != NULL) {
3200253b2ec1SVincenzo Maffione 			if (netmap_verbose)
3201253b2ec1SVincenzo Maffione 				nm_prinf("duplicate option: %u", opt->nro_reqtype);
3202253b2ec1SVincenzo Maffione 			opt->nro_status = EINVAL;
3203253b2ec1SVincenzo Maffione 			opt_tab[opt->nro_reqtype]->nro_status = EINVAL;
3204253b2ec1SVincenzo Maffione 			error = EINVAL;
3205253b2ec1SVincenzo Maffione 			goto next;
3206253b2ec1SVincenzo Maffione 		}
3207253b2ec1SVincenzo Maffione 		opt_tab[opt->nro_reqtype] = opt;
3208253b2ec1SVincenzo Maffione 
32092ff91c17SVincenzo Maffione 		p = (char *)(opt + 1);
32102ff91c17SVincenzo Maffione 
32112ff91c17SVincenzo Maffione 		/* copy the option body */
3212b6e66be2SVincenzo Maffione 		optsz = nmreq_opt_size_by_type(opt->nro_reqtype,
3213b6e66be2SVincenzo Maffione 						opt->nro_size);
32142ff91c17SVincenzo Maffione 		if (optsz) {
32152ff91c17SVincenzo Maffione 			/* the option body follows the option header */
32162ff91c17SVincenzo Maffione 			error = copyin(src + 1, p, optsz);
32172ff91c17SVincenzo Maffione 			if (error)
32182ff91c17SVincenzo Maffione 				goto out_restore;
32192ff91c17SVincenzo Maffione 			p += optsz;
32202ff91c17SVincenzo Maffione 		}
32212ff91c17SVincenzo Maffione 
3222253b2ec1SVincenzo Maffione 	next:
32232ff91c17SVincenzo Maffione 		/* move to next option */
32242ff91c17SVincenzo Maffione 		next = (struct nmreq_option **)&opt->nro_next;
32252ff91c17SVincenzo Maffione 		src = *next;
32262ff91c17SVincenzo Maffione 	}
3227253b2ec1SVincenzo Maffione 	if (error)
3228253b2ec1SVincenzo Maffione 		nmreq_copyout(hdr, error);
3229253b2ec1SVincenzo Maffione 	return error;
32302ff91c17SVincenzo Maffione 
32312ff91c17SVincenzo Maffione out_restore:
32322ff91c17SVincenzo Maffione 	ptrs = (uint64_t *)ker;
32332ff91c17SVincenzo Maffione 	hdr->nr_body = *ptrs++;
32342ff91c17SVincenzo Maffione 	hdr->nr_options = *ptrs++;
32352ff91c17SVincenzo Maffione 	hdr->nr_reserved = 0;
32362ff91c17SVincenzo Maffione 	nm_os_free(ker);
32372ff91c17SVincenzo Maffione out_err:
32382ff91c17SVincenzo Maffione 	return error;
32392ff91c17SVincenzo Maffione }
32402ff91c17SVincenzo Maffione 
32412ff91c17SVincenzo Maffione static int
32422ff91c17SVincenzo Maffione nmreq_copyout(struct nmreq_header *hdr, int rerror)
32432ff91c17SVincenzo Maffione {
32442ff91c17SVincenzo Maffione 	struct nmreq_option *src, *dst;
3245cfa866f6SMatt Macy 	void *ker = (void *)(uintptr_t)hdr->nr_body, *bufstart;
32462ff91c17SVincenzo Maffione 	uint64_t *ptrs;
32472ff91c17SVincenzo Maffione 	size_t bodysz;
32482ff91c17SVincenzo Maffione 	int error;
32492ff91c17SVincenzo Maffione 
32502ff91c17SVincenzo Maffione 	if (!hdr->nr_reserved)
32512ff91c17SVincenzo Maffione 		return rerror;
32522ff91c17SVincenzo Maffione 
32532ff91c17SVincenzo Maffione 	/* restore the user pointers in the header */
32542ff91c17SVincenzo Maffione 	ptrs = (uint64_t *)ker - 2;
32552ff91c17SVincenzo Maffione 	bufstart = ptrs;
32562ff91c17SVincenzo Maffione 	hdr->nr_body = *ptrs++;
3257cfa866f6SMatt Macy 	src = (struct nmreq_option *)(uintptr_t)hdr->nr_options;
32582ff91c17SVincenzo Maffione 	hdr->nr_options = *ptrs;
32592ff91c17SVincenzo Maffione 
32602ff91c17SVincenzo Maffione 	if (!rerror) {
32612ff91c17SVincenzo Maffione 		/* copy the body */
32622ff91c17SVincenzo Maffione 		bodysz = nmreq_size_by_type(hdr->nr_reqtype);
3263cfa866f6SMatt Macy 		error = copyout(ker, (void *)(uintptr_t)hdr->nr_body, bodysz);
32642ff91c17SVincenzo Maffione 		if (error) {
32652ff91c17SVincenzo Maffione 			rerror = error;
32662ff91c17SVincenzo Maffione 			goto out;
32672ff91c17SVincenzo Maffione 		}
32682ff91c17SVincenzo Maffione 	}
32692ff91c17SVincenzo Maffione 
32702ff91c17SVincenzo Maffione 	/* copy the options */
3271cfa866f6SMatt Macy 	dst = (struct nmreq_option *)(uintptr_t)hdr->nr_options;
32722ff91c17SVincenzo Maffione 	while (src) {
32732ff91c17SVincenzo Maffione 		size_t optsz;
32742ff91c17SVincenzo Maffione 		uint64_t next;
32752ff91c17SVincenzo Maffione 
32762ff91c17SVincenzo Maffione 		/* restore the user pointer */
32772ff91c17SVincenzo Maffione 		next = src->nro_next;
32782ff91c17SVincenzo Maffione 		ptrs = (uint64_t *)src - 1;
32792ff91c17SVincenzo Maffione 		src->nro_next = *ptrs;
32802ff91c17SVincenzo Maffione 
32812ff91c17SVincenzo Maffione 		/* always copy the option header */
32822ff91c17SVincenzo Maffione 		error = copyout(src, dst, sizeof(*src));
32832ff91c17SVincenzo Maffione 		if (error) {
32842ff91c17SVincenzo Maffione 			rerror = error;
32852ff91c17SVincenzo Maffione 			goto out;
32862ff91c17SVincenzo Maffione 		}
32872ff91c17SVincenzo Maffione 
32882ff91c17SVincenzo Maffione 		/* copy the option body only if there was no error */
32892ff91c17SVincenzo Maffione 		if (!rerror && !src->nro_status) {
3290b6e66be2SVincenzo Maffione 			optsz = nmreq_opt_size_by_type(src->nro_reqtype,
3291b6e66be2SVincenzo Maffione 							src->nro_size);
32922ff91c17SVincenzo Maffione 			if (optsz) {
32932ff91c17SVincenzo Maffione 				error = copyout(src + 1, dst + 1, optsz);
32942ff91c17SVincenzo Maffione 				if (error) {
32952ff91c17SVincenzo Maffione 					rerror = error;
32962ff91c17SVincenzo Maffione 					goto out;
32972ff91c17SVincenzo Maffione 				}
32982ff91c17SVincenzo Maffione 			}
32992ff91c17SVincenzo Maffione 		}
3300cfa866f6SMatt Macy 		src = (struct nmreq_option *)(uintptr_t)next;
3301cfa866f6SMatt Macy 		dst = (struct nmreq_option *)(uintptr_t)*ptrs;
33022ff91c17SVincenzo Maffione 	}
33032ff91c17SVincenzo Maffione 
33042ff91c17SVincenzo Maffione 
33052ff91c17SVincenzo Maffione out:
33062ff91c17SVincenzo Maffione 	hdr->nr_reserved = 0;
33072ff91c17SVincenzo Maffione 	nm_os_free(bufstart);
33082ff91c17SVincenzo Maffione 	return rerror;
33092ff91c17SVincenzo Maffione }
33102ff91c17SVincenzo Maffione 
33112ff91c17SVincenzo Maffione struct nmreq_option *
3312253b2ec1SVincenzo Maffione nmreq_getoption(struct nmreq_header *hdr, uint16_t reqtype)
33132ff91c17SVincenzo Maffione {
3314253b2ec1SVincenzo Maffione 	struct nmreq_option **opt_tab;
3315253b2ec1SVincenzo Maffione 
3316253b2ec1SVincenzo Maffione 	if (!hdr->nr_options)
33172ff91c17SVincenzo Maffione 		return NULL;
33182ff91c17SVincenzo Maffione 
3319*760fa2abSVincenzo Maffione 	opt_tab = (struct nmreq_option **)((uintptr_t)hdr->nr_options) -
3320*760fa2abSVincenzo Maffione 	    (NETMAP_REQ_OPT_MAX + 1);
3321253b2ec1SVincenzo Maffione 	return opt_tab[reqtype];
33222ff91c17SVincenzo Maffione }
33232ff91c17SVincenzo Maffione 
33242ff91c17SVincenzo Maffione static int
33252ff91c17SVincenzo Maffione nmreq_checkoptions(struct nmreq_header *hdr)
33262ff91c17SVincenzo Maffione {
33272ff91c17SVincenzo Maffione 	struct nmreq_option *opt;
33282ff91c17SVincenzo Maffione 	/* return error if there is still any option
33292ff91c17SVincenzo Maffione 	 * marked as not supported
33302ff91c17SVincenzo Maffione 	 */
33312ff91c17SVincenzo Maffione 
3332cfa866f6SMatt Macy 	for (opt = (struct nmreq_option *)(uintptr_t)hdr->nr_options; opt;
3333cfa866f6SMatt Macy 	     opt = (struct nmreq_option *)(uintptr_t)opt->nro_next)
33342ff91c17SVincenzo Maffione 		if (opt->nro_status == EOPNOTSUPP)
33352ff91c17SVincenzo Maffione 			return EOPNOTSUPP;
33362ff91c17SVincenzo Maffione 
33372ff91c17SVincenzo Maffione 	return 0;
33382ff91c17SVincenzo Maffione }
333968b8534bSLuigi Rizzo 
334068b8534bSLuigi Rizzo /*
334168b8534bSLuigi Rizzo  * select(2) and poll(2) handlers for the "netmap" device.
334268b8534bSLuigi Rizzo  *
334368b8534bSLuigi Rizzo  * Can be called for one or more queues.
334468b8534bSLuigi Rizzo  * Return true the event mask corresponding to ready events.
33458c9874f5SVincenzo Maffione  * If there are no ready events (and 'sr' is not NULL), do a
33468c9874f5SVincenzo Maffione  * selrecord on either individual selinfo or on the global one.
334768b8534bSLuigi Rizzo  * Device-dependent parts (locking and sync of tx/rx rings)
334868b8534bSLuigi Rizzo  * are done through callbacks.
3349f196ce38SLuigi Rizzo  *
335001c7d25fSLuigi Rizzo  * On linux, arguments are really pwait, the poll table, and 'td' is struct file *
335101c7d25fSLuigi Rizzo  * The first one is remapped to pwait as selrecord() uses the name as an
335201c7d25fSLuigi Rizzo  * hidden argument.
335368b8534bSLuigi Rizzo  */
3354f9790aebSLuigi Rizzo int
335537e3a6d3SLuigi Rizzo netmap_poll(struct netmap_priv_d *priv, int events, NM_SELRECORD_T *sr)
335668b8534bSLuigi Rizzo {
335768b8534bSLuigi Rizzo 	struct netmap_adapter *na;
335868b8534bSLuigi Rizzo 	struct netmap_kring *kring;
335937e3a6d3SLuigi Rizzo 	struct netmap_ring *ring;
3360b6e66be2SVincenzo Maffione 	u_int i, want[NR_TXRX], revents = 0;
3361b6e66be2SVincenzo Maffione 	NM_SELINFO_T *si[NR_TXRX];
3362847bf383SLuigi Rizzo #define want_tx want[NR_TX]
3363847bf383SLuigi Rizzo #define want_rx want[NR_RX]
3364c3e9b4dbSLuiz Otavio O Souza 	struct mbq q;	/* packets from RX hw queues to host stack */
336501c7d25fSLuigi Rizzo 
3366f9790aebSLuigi Rizzo 	/*
3367f9790aebSLuigi Rizzo 	 * In order to avoid nested locks, we need to "double check"
3368f9790aebSLuigi Rizzo 	 * txsync and rxsync if we decide to do a selrecord().
3369f9790aebSLuigi Rizzo 	 * retry_tx (and retry_rx, later) prevent looping forever.
3370f9790aebSLuigi Rizzo 	 */
337117885a7bSLuigi Rizzo 	int retry_tx = 1, retry_rx = 1;
3372ce3ee1e7SLuigi Rizzo 
3373c3e9b4dbSLuiz Otavio O Souza 	/* Transparent mode: send_down is 1 if we have found some
3374c3e9b4dbSLuiz Otavio O Souza 	 * packets to forward (host RX ring --> NIC) during the rx
3375c3e9b4dbSLuiz Otavio O Souza 	 * scan and we have not sent them down to the NIC yet.
3376c3e9b4dbSLuiz Otavio O Souza 	 * Transparent mode requires to bind all rings to a single
3377c3e9b4dbSLuiz Otavio O Souza 	 * file descriptor.
3378f0ea3689SLuigi Rizzo 	 */
337937e3a6d3SLuigi Rizzo 	int send_down = 0;
3380c3e9b4dbSLuiz Otavio O Souza 	int sync_flags = priv->np_sync_flags;
338137e3a6d3SLuigi Rizzo 
338237e3a6d3SLuigi Rizzo 	mbq_init(&q);
338368b8534bSLuigi Rizzo 
3384b6e66be2SVincenzo Maffione 	if (unlikely(priv->np_nifp == NULL)) {
33858241616dSLuigi Rizzo 		return POLLERR;
33868241616dSLuigi Rizzo 	}
3387847bf383SLuigi Rizzo 	mb(); /* make sure following reads are not from cache */
33888241616dSLuigi Rizzo 
3389f9790aebSLuigi Rizzo 	na = priv->np_na;
3390f9790aebSLuigi Rizzo 
3391b6e66be2SVincenzo Maffione 	if (unlikely(!nm_netmap_on(na)))
339268b8534bSLuigi Rizzo 		return POLLERR;
339368b8534bSLuigi Rizzo 
3394b6e66be2SVincenzo Maffione 	if (unlikely(priv->np_csb_atok_base)) {
3395b6e66be2SVincenzo Maffione 		nm_prerr("Invalid poll in CSB mode");
3396b6e66be2SVincenzo Maffione 		return POLLERR;
3397b6e66be2SVincenzo Maffione 	}
3398b6e66be2SVincenzo Maffione 
3399b6e66be2SVincenzo Maffione 	if (netmap_debug & NM_DEBUG_ON)
3400b6e66be2SVincenzo Maffione 		nm_prinf("device %s events 0x%x", na->name, events);
340168b8534bSLuigi Rizzo 	want_tx = events & (POLLOUT | POLLWRNORM);
340268b8534bSLuigi Rizzo 	want_rx = events & (POLLIN | POLLRDNORM);
340368b8534bSLuigi Rizzo 
340468b8534bSLuigi Rizzo 	/*
3405b6e66be2SVincenzo Maffione 	 * If the card has more than one queue AND the file descriptor is
3406b6e66be2SVincenzo Maffione 	 * bound to all of them, we sleep on the "global" selinfo, otherwise
3407b6e66be2SVincenzo Maffione 	 * we sleep on individual selinfo (FreeBSD only allows two selinfo's
3408b6e66be2SVincenzo Maffione 	 * per file descriptor).
3409ce3ee1e7SLuigi Rizzo 	 * The interrupt routine in the driver wake one or the other
3410ce3ee1e7SLuigi Rizzo 	 * (or both) depending on which clients are active.
341168b8534bSLuigi Rizzo 	 *
341268b8534bSLuigi Rizzo 	 * rxsync() is only called if we run out of buffers on a POLLIN.
341368b8534bSLuigi Rizzo 	 * txsync() is called if we run out of buffers on POLLOUT, or
341468b8534bSLuigi Rizzo 	 * there are pending packets to send. The latter can be disabled
341568b8534bSLuigi Rizzo 	 * passing NETMAP_NO_TX_POLL in the NIOCREG call.
341668b8534bSLuigi Rizzo 	 */
34175faab778SVincenzo Maffione 	si[NR_RX] = priv->np_si[NR_RX];
34185faab778SVincenzo Maffione 	si[NR_TX] = priv->np_si[NR_TX];
341964ae02c3SLuigi Rizzo 
34204f80b14cSVincenzo Maffione #ifdef __FreeBSD__
342168b8534bSLuigi Rizzo 	/*
3422f9790aebSLuigi Rizzo 	 * We start with a lock free round which is cheap if we have
3423f9790aebSLuigi Rizzo 	 * slots available. If this fails, then lock and call the sync
34244f80b14cSVincenzo Maffione 	 * routines. We can't do this on Linux, as the contract says
34254f80b14cSVincenzo Maffione 	 * that we must call nm_os_selrecord() unconditionally.
342668b8534bSLuigi Rizzo 	 */
342737e3a6d3SLuigi Rizzo 	if (want_tx) {
342858e18542SVincenzo Maffione 		const enum txrx t = NR_TX;
342958e18542SVincenzo Maffione 		for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
34302ff91c17SVincenzo Maffione 			kring = NMR(na, t)[i];
343158e18542SVincenzo Maffione 			if (kring->ring->cur != kring->ring->tail) {
343258e18542SVincenzo Maffione 				/* Some unseen TX space is available, so what
343358e18542SVincenzo Maffione 				 * we don't need to run txsync. */
343437e3a6d3SLuigi Rizzo 				revents |= want[t];
343558e18542SVincenzo Maffione 				want[t] = 0;
343658e18542SVincenzo Maffione 				break;
343737e3a6d3SLuigi Rizzo 			}
343837e3a6d3SLuigi Rizzo 		}
343937e3a6d3SLuigi Rizzo 	}
344037e3a6d3SLuigi Rizzo 	if (want_rx) {
344158e18542SVincenzo Maffione 		const enum txrx t = NR_RX;
3442e1ed1fbdSVincenzo Maffione 		int rxsync_needed = 0;
3443e1ed1fbdSVincenzo Maffione 
344437e3a6d3SLuigi Rizzo 		for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
34452ff91c17SVincenzo Maffione 			kring = NMR(na, t)[i];
344658e18542SVincenzo Maffione 			if (kring->ring->cur == kring->ring->tail
344758e18542SVincenzo Maffione 				|| kring->rhead != kring->ring->head) {
344858e18542SVincenzo Maffione 				/* There are no unseen packets on this ring,
344958e18542SVincenzo Maffione 				 * or there are some buffers to be returned
345058e18542SVincenzo Maffione 				 * to the netmap port. We therefore go ahead
345158e18542SVincenzo Maffione 				 * and run rxsync. */
3452e1ed1fbdSVincenzo Maffione 				rxsync_needed = 1;
3453e1ed1fbdSVincenzo Maffione 				break;
345437e3a6d3SLuigi Rizzo 			}
345537e3a6d3SLuigi Rizzo 		}
345658e18542SVincenzo Maffione 		if (!rxsync_needed) {
345758e18542SVincenzo Maffione 			revents |= want_rx;
345858e18542SVincenzo Maffione 			want_rx = 0;
345958e18542SVincenzo Maffione 		}
346037e3a6d3SLuigi Rizzo 	}
34614f80b14cSVincenzo Maffione #endif
34624f80b14cSVincenzo Maffione 
34634f80b14cSVincenzo Maffione #ifdef linux
34644f80b14cSVincenzo Maffione 	/* The selrecord must be unconditional on linux. */
3465b6e66be2SVincenzo Maffione 	nm_os_selrecord(sr, si[NR_RX]);
3466b6e66be2SVincenzo Maffione 	nm_os_selrecord(sr, si[NR_TX]);
34674f80b14cSVincenzo Maffione #endif /* linux */
346868b8534bSLuigi Rizzo 
346968b8534bSLuigi Rizzo 	/*
347017885a7bSLuigi Rizzo 	 * If we want to push packets out (priv->np_txpoll) or
347117885a7bSLuigi Rizzo 	 * want_tx is still set, we must issue txsync calls
347217885a7bSLuigi Rizzo 	 * (on all rings, to avoid that the tx rings stall).
3473f9790aebSLuigi Rizzo 	 * Fortunately, normal tx mode has np_txpoll set.
347468b8534bSLuigi Rizzo 	 */
347568b8534bSLuigi Rizzo 	if (priv->np_txpoll || want_tx) {
347617885a7bSLuigi Rizzo 		/*
347717885a7bSLuigi Rizzo 		 * The first round checks if anyone is ready, if not
347817885a7bSLuigi Rizzo 		 * do a selrecord and another round to handle races.
347917885a7bSLuigi Rizzo 		 * want_tx goes to 0 if any space is found, and is
348017885a7bSLuigi Rizzo 		 * used to skip rings with no pending transmissions.
3481ce3ee1e7SLuigi Rizzo 		 */
3482091fd0abSLuigi Rizzo flush_tx:
348337e3a6d3SLuigi Rizzo 		for (i = priv->np_qfirst[NR_TX]; i < priv->np_qlast[NR_TX]; i++) {
348417885a7bSLuigi Rizzo 			int found = 0;
348517885a7bSLuigi Rizzo 
34862ff91c17SVincenzo Maffione 			kring = na->tx_rings[i];
348737e3a6d3SLuigi Rizzo 			ring = kring->ring;
348837e3a6d3SLuigi Rizzo 
34894f80b14cSVincenzo Maffione 			/*
34904f80b14cSVincenzo Maffione 			 * Don't try to txsync this TX ring if we already found some
34914f80b14cSVincenzo Maffione 			 * space in some of the TX rings (want_tx == 0) and there are no
34924f80b14cSVincenzo Maffione 			 * TX slots in this ring that need to be flushed to the NIC
34932ff91c17SVincenzo Maffione 			 * (head == hwcur).
34944f80b14cSVincenzo Maffione 			 */
34952ff91c17SVincenzo Maffione 			if (!send_down && !want_tx && ring->head == kring->nr_hwcur)
349668b8534bSLuigi Rizzo 				continue;
349737e3a6d3SLuigi Rizzo 
349837e3a6d3SLuigi Rizzo 			if (nm_kr_tryget(kring, 1, &revents))
349917885a7bSLuigi Rizzo 				continue;
350037e3a6d3SLuigi Rizzo 
350137e3a6d3SLuigi Rizzo 			if (nm_txsync_prologue(kring, ring) >= kring->nkr_num_slots) {
350217885a7bSLuigi Rizzo 				netmap_ring_reinit(kring);
350317885a7bSLuigi Rizzo 				revents |= POLLERR;
350417885a7bSLuigi Rizzo 			} else {
3505c3e9b4dbSLuiz Otavio O Souza 				if (kring->nm_sync(kring, sync_flags))
350668b8534bSLuigi Rizzo 					revents |= POLLERR;
3507847bf383SLuigi Rizzo 				else
350837e3a6d3SLuigi Rizzo 					nm_sync_finalize(kring);
350917885a7bSLuigi Rizzo 			}
351068b8534bSLuigi Rizzo 
351117885a7bSLuigi Rizzo 			/*
351217885a7bSLuigi Rizzo 			 * If we found new slots, notify potential
351317885a7bSLuigi Rizzo 			 * listeners on the same ring.
351417885a7bSLuigi Rizzo 			 * Since we just did a txsync, look at the copies
351517885a7bSLuigi Rizzo 			 * of cur,tail in the kring.
3516f9790aebSLuigi Rizzo 			 */
351717885a7bSLuigi Rizzo 			found = kring->rcur != kring->rtail;
351817885a7bSLuigi Rizzo 			nm_kr_put(kring);
351917885a7bSLuigi Rizzo 			if (found) { /* notify other listeners */
352068b8534bSLuigi Rizzo 				revents |= want_tx;
352168b8534bSLuigi Rizzo 				want_tx = 0;
35224f80b14cSVincenzo Maffione #ifndef linux
3523847bf383SLuigi Rizzo 				kring->nm_notify(kring, 0);
35244f80b14cSVincenzo Maffione #endif /* linux */
352568b8534bSLuigi Rizzo 			}
3526ce3ee1e7SLuigi Rizzo 		}
352737e3a6d3SLuigi Rizzo 		/* if there were any packet to forward we must have handled them by now */
352837e3a6d3SLuigi Rizzo 		send_down = 0;
352937e3a6d3SLuigi Rizzo 		if (want_tx && retry_tx && sr) {
35304f80b14cSVincenzo Maffione #ifndef linux
3531b6e66be2SVincenzo Maffione 			nm_os_selrecord(sr, si[NR_TX]);
35324f80b14cSVincenzo Maffione #endif /* !linux */
3533ce3ee1e7SLuigi Rizzo 			retry_tx = 0;
3534ce3ee1e7SLuigi Rizzo 			goto flush_tx;
353568b8534bSLuigi Rizzo 		}
353668b8534bSLuigi Rizzo 	}
353768b8534bSLuigi Rizzo 
353868b8534bSLuigi Rizzo 	/*
353917885a7bSLuigi Rizzo 	 * If want_rx is still set scan receive rings.
354068b8534bSLuigi Rizzo 	 * Do it on all rings because otherwise we starve.
354168b8534bSLuigi Rizzo 	 */
354268b8534bSLuigi Rizzo 	if (want_rx) {
354389cc2556SLuigi Rizzo 		/* two rounds here for race avoidance */
3544ce3ee1e7SLuigi Rizzo do_retry_rx:
3545847bf383SLuigi Rizzo 		for (i = priv->np_qfirst[NR_RX]; i < priv->np_qlast[NR_RX]; i++) {
354617885a7bSLuigi Rizzo 			int found = 0;
354717885a7bSLuigi Rizzo 
35482ff91c17SVincenzo Maffione 			kring = na->rx_rings[i];
354937e3a6d3SLuigi Rizzo 			ring = kring->ring;
3550ce3ee1e7SLuigi Rizzo 
355137e3a6d3SLuigi Rizzo 			if (unlikely(nm_kr_tryget(kring, 1, &revents)))
355217885a7bSLuigi Rizzo 				continue;
3553ce3ee1e7SLuigi Rizzo 
355437e3a6d3SLuigi Rizzo 			if (nm_rxsync_prologue(kring, ring) >= kring->nkr_num_slots) {
3555847bf383SLuigi Rizzo 				netmap_ring_reinit(kring);
3556847bf383SLuigi Rizzo 				revents |= POLLERR;
3557847bf383SLuigi Rizzo 			}
3558847bf383SLuigi Rizzo 			/* now we can use kring->rcur, rtail */
3559847bf383SLuigi Rizzo 
356017885a7bSLuigi Rizzo 			/*
3561c3e9b4dbSLuiz Otavio O Souza 			 * transparent mode support: collect packets from
3562c3e9b4dbSLuiz Otavio O Souza 			 * hw rxring(s) that have been released by the user
3563ce3ee1e7SLuigi Rizzo 			 */
356437e3a6d3SLuigi Rizzo 			if (nm_may_forward_up(kring)) {
3565091fd0abSLuigi Rizzo 				netmap_grab_packets(kring, &q, netmap_fwd);
3566091fd0abSLuigi Rizzo 			}
356768b8534bSLuigi Rizzo 
3568c3e9b4dbSLuiz Otavio O Souza 			/* Clear the NR_FORWARD flag anyway, it may be set by
3569c3e9b4dbSLuiz Otavio O Souza 			 * the nm_sync() below only on for the host RX ring (see
3570c3e9b4dbSLuiz Otavio O Souza 			 * netmap_rxsync_from_host()). */
357137e3a6d3SLuigi Rizzo 			kring->nr_kflags &= ~NR_FORWARD;
3572c3e9b4dbSLuiz Otavio O Souza 			if (kring->nm_sync(kring, sync_flags))
357368b8534bSLuigi Rizzo 				revents |= POLLERR;
3574847bf383SLuigi Rizzo 			else
357537e3a6d3SLuigi Rizzo 				nm_sync_finalize(kring);
3576c3e9b4dbSLuiz Otavio O Souza 			send_down |= (kring->nr_kflags & NR_FORWARD);
3577c3e9b4dbSLuiz Otavio O Souza 			ring_timestamp_set(ring);
357817885a7bSLuigi Rizzo 			found = kring->rcur != kring->rtail;
357917885a7bSLuigi Rizzo 			nm_kr_put(kring);
358017885a7bSLuigi Rizzo 			if (found) {
358168b8534bSLuigi Rizzo 				revents |= want_rx;
3582ce3ee1e7SLuigi Rizzo 				retry_rx = 0;
35834f80b14cSVincenzo Maffione #ifndef linux
3584847bf383SLuigi Rizzo 				kring->nm_notify(kring, 0);
35854f80b14cSVincenzo Maffione #endif /* linux */
358668b8534bSLuigi Rizzo 			}
358768b8534bSLuigi Rizzo 		}
358817885a7bSLuigi Rizzo 
35894f80b14cSVincenzo Maffione #ifndef linux
359037e3a6d3SLuigi Rizzo 		if (retry_rx && sr) {
3591b6e66be2SVincenzo Maffione 			nm_os_selrecord(sr, si[NR_RX]);
359237e3a6d3SLuigi Rizzo 		}
35934f80b14cSVincenzo Maffione #endif /* !linux */
3594c3e9b4dbSLuiz Otavio O Souza 		if (send_down || retry_rx) {
359517885a7bSLuigi Rizzo 			retry_rx = 0;
359617885a7bSLuigi Rizzo 			if (send_down)
359717885a7bSLuigi Rizzo 				goto flush_tx; /* and retry_rx */
359817885a7bSLuigi Rizzo 			else
3599ce3ee1e7SLuigi Rizzo 				goto do_retry_rx;
3600ce3ee1e7SLuigi Rizzo 		}
360168b8534bSLuigi Rizzo 	}
3602091fd0abSLuigi Rizzo 
360317885a7bSLuigi Rizzo 	/*
3604c3e9b4dbSLuiz Otavio O Souza 	 * Transparent mode: released bufs (i.e. between kring->nr_hwcur and
3605c3e9b4dbSLuiz Otavio O Souza 	 * ring->head) marked with NS_FORWARD on hw rx rings are passed up
3606c3e9b4dbSLuiz Otavio O Souza 	 * to the host stack.
3607ce3ee1e7SLuigi Rizzo 	 */
3608091fd0abSLuigi Rizzo 
3609c3e9b4dbSLuiz Otavio O Souza 	if (mbq_peek(&q)) {
3610f9790aebSLuigi Rizzo 		netmap_send_up(na->ifp, &q);
361137e3a6d3SLuigi Rizzo 	}
361268b8534bSLuigi Rizzo 
361368b8534bSLuigi Rizzo 	return (revents);
3614847bf383SLuigi Rizzo #undef want_tx
3615847bf383SLuigi Rizzo #undef want_rx
361668b8534bSLuigi Rizzo }
361768b8534bSLuigi Rizzo 
36184f80b14cSVincenzo Maffione int
36194f80b14cSVincenzo Maffione nma_intr_enable(struct netmap_adapter *na, int onoff)
36204f80b14cSVincenzo Maffione {
36214f80b14cSVincenzo Maffione 	bool changed = false;
36224f80b14cSVincenzo Maffione 	enum txrx t;
36234f80b14cSVincenzo Maffione 	int i;
36244f80b14cSVincenzo Maffione 
36254f80b14cSVincenzo Maffione 	for_rx_tx(t) {
36264f80b14cSVincenzo Maffione 		for (i = 0; i < nma_get_nrings(na, t); i++) {
36272ff91c17SVincenzo Maffione 			struct netmap_kring *kring = NMR(na, t)[i];
36284f80b14cSVincenzo Maffione 			int on = !(kring->nr_kflags & NKR_NOINTR);
36294f80b14cSVincenzo Maffione 
36304f80b14cSVincenzo Maffione 			if (!!onoff != !!on) {
36314f80b14cSVincenzo Maffione 				changed = true;
36324f80b14cSVincenzo Maffione 			}
36334f80b14cSVincenzo Maffione 			if (onoff) {
36344f80b14cSVincenzo Maffione 				kring->nr_kflags &= ~NKR_NOINTR;
36354f80b14cSVincenzo Maffione 			} else {
36364f80b14cSVincenzo Maffione 				kring->nr_kflags |= NKR_NOINTR;
36374f80b14cSVincenzo Maffione 			}
36384f80b14cSVincenzo Maffione 		}
36394f80b14cSVincenzo Maffione 	}
36404f80b14cSVincenzo Maffione 
36414f80b14cSVincenzo Maffione 	if (!changed) {
36424f80b14cSVincenzo Maffione 		return 0; /* nothing to do */
36434f80b14cSVincenzo Maffione 	}
36444f80b14cSVincenzo Maffione 
36454f80b14cSVincenzo Maffione 	if (!na->nm_intr) {
3646b6e66be2SVincenzo Maffione 		nm_prerr("Cannot %s interrupts for %s", onoff ? "enable" : "disable",
36474f80b14cSVincenzo Maffione 		  na->name);
36484f80b14cSVincenzo Maffione 		return -1;
36494f80b14cSVincenzo Maffione 	}
36504f80b14cSVincenzo Maffione 
36514f80b14cSVincenzo Maffione 	na->nm_intr(na, onoff);
36524f80b14cSVincenzo Maffione 
36534f80b14cSVincenzo Maffione 	return 0;
36544f80b14cSVincenzo Maffione }
36554f80b14cSVincenzo Maffione 
365617885a7bSLuigi Rizzo 
365717885a7bSLuigi Rizzo /*-------------------- driver support routines -------------------*/
365868b8534bSLuigi Rizzo 
365989cc2556SLuigi Rizzo /* default notify callback */
3660f9790aebSLuigi Rizzo static int
3661847bf383SLuigi Rizzo netmap_notify(struct netmap_kring *kring, int flags)
3662f9790aebSLuigi Rizzo {
36632ff91c17SVincenzo Maffione 	struct netmap_adapter *na = kring->notify_na;
3664847bf383SLuigi Rizzo 	enum txrx t = kring->tx;
3665f9790aebSLuigi Rizzo 
366637e3a6d3SLuigi Rizzo 	nm_os_selwakeup(&kring->si);
366789cc2556SLuigi Rizzo 	/* optimization: avoid a wake up on the global
366889cc2556SLuigi Rizzo 	 * queue if nobody has registered for more
366989cc2556SLuigi Rizzo 	 * than one ring
367089cc2556SLuigi Rizzo 	 */
3671847bf383SLuigi Rizzo 	if (na->si_users[t] > 0)
367237e3a6d3SLuigi Rizzo 		nm_os_selwakeup(&na->si[t]);
3673847bf383SLuigi Rizzo 
367437e3a6d3SLuigi Rizzo 	return NM_IRQ_COMPLETED;
3675f9790aebSLuigi Rizzo }
3676f9790aebSLuigi Rizzo 
367789cc2556SLuigi Rizzo /* called by all routines that create netmap_adapters.
367837e3a6d3SLuigi Rizzo  * provide some defaults and get a reference to the
367937e3a6d3SLuigi Rizzo  * memory allocator
368089cc2556SLuigi Rizzo  */
3681f9790aebSLuigi Rizzo int
3682f9790aebSLuigi Rizzo netmap_attach_common(struct netmap_adapter *na)
3683f9790aebSLuigi Rizzo {
36842ff91c17SVincenzo Maffione 	if (!na->rx_buf_maxsize) {
36852ff91c17SVincenzo Maffione 		/* Set a conservative default (larger is safer). */
36862ff91c17SVincenzo Maffione 		na->rx_buf_maxsize = PAGE_SIZE;
36872ff91c17SVincenzo Maffione 	}
36882ff91c17SVincenzo Maffione 
368917885a7bSLuigi Rizzo #ifdef __FreeBSD__
369037e3a6d3SLuigi Rizzo 	if (na->na_flags & NAF_HOST_RINGS && na->ifp) {
369137e3a6d3SLuigi Rizzo 		na->if_input = na->ifp->if_input; /* for netmap_send_up */
36924bf50f18SLuigi Rizzo 	}
36934f80b14cSVincenzo Maffione 	na->pdev = na; /* make sure netmap_mem_map() is called */
369437e3a6d3SLuigi Rizzo #endif /* __FreeBSD__ */
36952a7db7a6SVincenzo Maffione 	if (na->na_flags & NAF_HOST_RINGS) {
36962a7db7a6SVincenzo Maffione 		if (na->num_host_rx_rings == 0)
36972a7db7a6SVincenzo Maffione 			na->num_host_rx_rings = 1;
36982a7db7a6SVincenzo Maffione 		if (na->num_host_tx_rings == 0)
36992a7db7a6SVincenzo Maffione 			na->num_host_tx_rings = 1;
37002a7db7a6SVincenzo Maffione 	}
3701f9790aebSLuigi Rizzo 	if (na->nm_krings_create == NULL) {
370289cc2556SLuigi Rizzo 		/* we assume that we have been called by a driver,
370389cc2556SLuigi Rizzo 		 * since other port types all provide their own
370489cc2556SLuigi Rizzo 		 * nm_krings_create
370589cc2556SLuigi Rizzo 		 */
3706f9790aebSLuigi Rizzo 		na->nm_krings_create = netmap_hw_krings_create;
370717885a7bSLuigi Rizzo 		na->nm_krings_delete = netmap_hw_krings_delete;
3708f9790aebSLuigi Rizzo 	}
3709f9790aebSLuigi Rizzo 	if (na->nm_notify == NULL)
3710f9790aebSLuigi Rizzo 		na->nm_notify = netmap_notify;
3711f9790aebSLuigi Rizzo 	na->active_fds = 0;
3712f9790aebSLuigi Rizzo 
3713c3e9b4dbSLuiz Otavio O Souza 	if (na->nm_mem == NULL) {
37144bf50f18SLuigi Rizzo 		/* use the global allocator */
3715c3e9b4dbSLuiz Otavio O Souza 		na->nm_mem = netmap_mem_get(&nm_mem);
3716c3e9b4dbSLuiz Otavio O Souza 	}
3717847bf383SLuigi Rizzo #ifdef WITH_VALE
37184bf50f18SLuigi Rizzo 	if (na->nm_bdg_attach == NULL)
37194bf50f18SLuigi Rizzo 		/* no special nm_bdg_attach callback. On VALE
37204bf50f18SLuigi Rizzo 		 * attach, we need to interpose a bwrap
37214bf50f18SLuigi Rizzo 		 */
37222a7db7a6SVincenzo Maffione 		na->nm_bdg_attach = netmap_default_bdg_attach;
3723847bf383SLuigi Rizzo #endif
372437e3a6d3SLuigi Rizzo 
3725f9790aebSLuigi Rizzo 	return 0;
3726f9790aebSLuigi Rizzo }
3727f9790aebSLuigi Rizzo 
372837e3a6d3SLuigi Rizzo /* Wrapper for the register callback provided netmap-enabled
372937e3a6d3SLuigi Rizzo  * hardware drivers.
373037e3a6d3SLuigi Rizzo  * nm_iszombie(na) means that the driver module has been
37314bf50f18SLuigi Rizzo  * unloaded, so we cannot call into it.
373237e3a6d3SLuigi Rizzo  * nm_os_ifnet_lock() must guarantee mutual exclusion with
373337e3a6d3SLuigi Rizzo  * module unloading.
37344bf50f18SLuigi Rizzo  */
37354bf50f18SLuigi Rizzo static int
373637e3a6d3SLuigi Rizzo netmap_hw_reg(struct netmap_adapter *na, int onoff)
37374bf50f18SLuigi Rizzo {
37384bf50f18SLuigi Rizzo 	struct netmap_hw_adapter *hwna =
37394bf50f18SLuigi Rizzo 		(struct netmap_hw_adapter*)na;
374037e3a6d3SLuigi Rizzo 	int error = 0;
37414bf50f18SLuigi Rizzo 
374237e3a6d3SLuigi Rizzo 	nm_os_ifnet_lock();
37434bf50f18SLuigi Rizzo 
374437e3a6d3SLuigi Rizzo 	if (nm_iszombie(na)) {
374537e3a6d3SLuigi Rizzo 		if (onoff) {
374637e3a6d3SLuigi Rizzo 			error = ENXIO;
374737e3a6d3SLuigi Rizzo 		} else if (na != NULL) {
374837e3a6d3SLuigi Rizzo 			na->na_flags &= ~NAF_NETMAP_ON;
374937e3a6d3SLuigi Rizzo 		}
375037e3a6d3SLuigi Rizzo 		goto out;
375137e3a6d3SLuigi Rizzo 	}
375237e3a6d3SLuigi Rizzo 
375337e3a6d3SLuigi Rizzo 	error = hwna->nm_hw_register(na, onoff);
375437e3a6d3SLuigi Rizzo 
375537e3a6d3SLuigi Rizzo out:
375637e3a6d3SLuigi Rizzo 	nm_os_ifnet_unlock();
375737e3a6d3SLuigi Rizzo 
375837e3a6d3SLuigi Rizzo 	return error;
375937e3a6d3SLuigi Rizzo }
376037e3a6d3SLuigi Rizzo 
376137e3a6d3SLuigi Rizzo static void
376237e3a6d3SLuigi Rizzo netmap_hw_dtor(struct netmap_adapter *na)
376337e3a6d3SLuigi Rizzo {
37642a7db7a6SVincenzo Maffione 	if (na->ifp == NULL)
376537e3a6d3SLuigi Rizzo 		return;
376637e3a6d3SLuigi Rizzo 
37672a7db7a6SVincenzo Maffione 	NM_DETACH_NA(na->ifp);
37684bf50f18SLuigi Rizzo }
37694bf50f18SLuigi Rizzo 
3770f18be576SLuigi Rizzo 
377168b8534bSLuigi Rizzo /*
3772c3e9b4dbSLuiz Otavio O Souza  * Allocate a netmap_adapter object, and initialize it from the
377337e3a6d3SLuigi Rizzo  * 'arg' passed by the driver on attach.
3774c3e9b4dbSLuiz Otavio O Souza  * We allocate a block of memory of 'size' bytes, which has room
3775c3e9b4dbSLuiz Otavio O Souza  * for struct netmap_adapter plus additional room private to
3776c3e9b4dbSLuiz Otavio O Souza  * the caller.
377768b8534bSLuigi Rizzo  * Return 0 on success, ENOMEM otherwise.
377868b8534bSLuigi Rizzo  */
3779c3e9b4dbSLuiz Otavio O Souza int
37804f80b14cSVincenzo Maffione netmap_attach_ext(struct netmap_adapter *arg, size_t size, int override_reg)
378168b8534bSLuigi Rizzo {
3782f9790aebSLuigi Rizzo 	struct netmap_hw_adapter *hwna = NULL;
378337e3a6d3SLuigi Rizzo 	struct ifnet *ifp = NULL;
378468b8534bSLuigi Rizzo 
3785c3e9b4dbSLuiz Otavio O Souza 	if (size < sizeof(struct netmap_hw_adapter)) {
3786b6e66be2SVincenzo Maffione 		if (netmap_debug & NM_DEBUG_ON)
3787b6e66be2SVincenzo Maffione 			nm_prerr("Invalid netmap adapter size %d", (int)size);
3788c3e9b4dbSLuiz Otavio O Souza 		return EINVAL;
3789c3e9b4dbSLuiz Otavio O Souza 	}
3790c3e9b4dbSLuiz Otavio O Souza 
3791b6e66be2SVincenzo Maffione 	if (arg == NULL || arg->ifp == NULL) {
3792b6e66be2SVincenzo Maffione 		if (netmap_debug & NM_DEBUG_ON)
3793b6e66be2SVincenzo Maffione 			nm_prerr("either arg or arg->ifp is NULL");
37942a7db7a6SVincenzo Maffione 		return EINVAL;
3795b6e66be2SVincenzo Maffione 	}
3796b6e66be2SVincenzo Maffione 
3797b6e66be2SVincenzo Maffione 	if (arg->num_tx_rings == 0 || arg->num_rx_rings == 0) {
3798b6e66be2SVincenzo Maffione 		if (netmap_debug & NM_DEBUG_ON)
3799b6e66be2SVincenzo Maffione 			nm_prerr("%s: invalid rings tx %d rx %d",
3800b6e66be2SVincenzo Maffione 				arg->name, arg->num_tx_rings, arg->num_rx_rings);
3801b6e66be2SVincenzo Maffione 		return EINVAL;
3802b6e66be2SVincenzo Maffione 	}
38034f80b14cSVincenzo Maffione 
380437e3a6d3SLuigi Rizzo 	ifp = arg->ifp;
38052a7db7a6SVincenzo Maffione 	if (NM_NA_CLASH(ifp)) {
38064f80b14cSVincenzo Maffione 		/* If NA(ifp) is not null but there is no valid netmap
38074f80b14cSVincenzo Maffione 		 * adapter it means that someone else is using the same
38084f80b14cSVincenzo Maffione 		 * pointer (e.g. ax25_ptr on linux). This happens for
38094f80b14cSVincenzo Maffione 		 * instance when also PF_RING is in use. */
3810b6e66be2SVincenzo Maffione 		nm_prerr("Error: netmap adapter hook is busy");
38114f80b14cSVincenzo Maffione 		return EBUSY;
38124f80b14cSVincenzo Maffione 	}
38134f80b14cSVincenzo Maffione 
3814c3e9b4dbSLuiz Otavio O Souza 	hwna = nm_os_malloc(size);
3815f9790aebSLuigi Rizzo 	if (hwna == NULL)
3816ae10d1afSLuigi Rizzo 		goto fail;
3817f9790aebSLuigi Rizzo 	hwna->up = *arg;
3818847bf383SLuigi Rizzo 	hwna->up.na_flags |= NAF_HOST_RINGS | NAF_NATIVE;
3819b6e66be2SVincenzo Maffione 	strlcpy(hwna->up.name, ifp->if_xname, sizeof(hwna->up.name));
38204f80b14cSVincenzo Maffione 	if (override_reg) {
38214bf50f18SLuigi Rizzo 		hwna->nm_hw_register = hwna->up.nm_register;
382237e3a6d3SLuigi Rizzo 		hwna->up.nm_register = netmap_hw_reg;
38234f80b14cSVincenzo Maffione 	}
3824f9790aebSLuigi Rizzo 	if (netmap_attach_common(&hwna->up)) {
3825c3e9b4dbSLuiz Otavio O Souza 		nm_os_free(hwna);
3826f9790aebSLuigi Rizzo 		goto fail;
3827f9790aebSLuigi Rizzo 	}
3828f9790aebSLuigi Rizzo 	netmap_adapter_get(&hwna->up);
3829f9790aebSLuigi Rizzo 
383037e3a6d3SLuigi Rizzo 	NM_ATTACH_NA(ifp, &hwna->up);
383137e3a6d3SLuigi Rizzo 
38322a7db7a6SVincenzo Maffione 	nm_os_onattach(ifp);
38332a7db7a6SVincenzo Maffione 
383437e3a6d3SLuigi Rizzo 	if (arg->nm_dtor == NULL) {
383537e3a6d3SLuigi Rizzo 		hwna->up.nm_dtor = netmap_hw_dtor;
383637e3a6d3SLuigi Rizzo 	}
3837f9790aebSLuigi Rizzo 
38381ef2a881SVincenzo Maffione 	if_printf(ifp, "netmap queues/slots: TX %d/%d, RX %d/%d\n",
3839d82f9014SRui Paulo 	    hwna->up.num_tx_rings, hwna->up.num_tx_desc,
3840d82f9014SRui Paulo 	    hwna->up.num_rx_rings, hwna->up.num_rx_desc);
3841ae10d1afSLuigi Rizzo 	return 0;
384268b8534bSLuigi Rizzo 
3843ae10d1afSLuigi Rizzo fail:
3844b6e66be2SVincenzo Maffione 	nm_prerr("fail, arg %p ifp %p na %p", arg, ifp, hwna);
3845f9790aebSLuigi Rizzo 	return (hwna ? EINVAL : ENOMEM);
384668b8534bSLuigi Rizzo }
384768b8534bSLuigi Rizzo 
384868b8534bSLuigi Rizzo 
384937e3a6d3SLuigi Rizzo int
385037e3a6d3SLuigi Rizzo netmap_attach(struct netmap_adapter *arg)
385137e3a6d3SLuigi Rizzo {
38524f80b14cSVincenzo Maffione 	return netmap_attach_ext(arg, sizeof(struct netmap_hw_adapter),
38534f80b14cSVincenzo Maffione 			1 /* override nm_reg */);
385437e3a6d3SLuigi Rizzo }
385537e3a6d3SLuigi Rizzo 
385637e3a6d3SLuigi Rizzo 
3857f9790aebSLuigi Rizzo void
3858f9790aebSLuigi Rizzo NM_DBG(netmap_adapter_get)(struct netmap_adapter *na)
3859f9790aebSLuigi Rizzo {
3860f9790aebSLuigi Rizzo 	if (!na) {
3861f9790aebSLuigi Rizzo 		return;
3862f9790aebSLuigi Rizzo 	}
3863f9790aebSLuigi Rizzo 
3864f9790aebSLuigi Rizzo 	refcount_acquire(&na->na_refcount);
3865f9790aebSLuigi Rizzo }
3866f9790aebSLuigi Rizzo 
3867f9790aebSLuigi Rizzo 
3868f9790aebSLuigi Rizzo /* returns 1 iff the netmap_adapter is destroyed */
3869f9790aebSLuigi Rizzo int
3870f9790aebSLuigi Rizzo NM_DBG(netmap_adapter_put)(struct netmap_adapter *na)
3871f9790aebSLuigi Rizzo {
3872f9790aebSLuigi Rizzo 	if (!na)
3873f9790aebSLuigi Rizzo 		return 1;
3874f9790aebSLuigi Rizzo 
3875f9790aebSLuigi Rizzo 	if (!refcount_release(&na->na_refcount))
3876f9790aebSLuigi Rizzo 		return 0;
3877f9790aebSLuigi Rizzo 
3878f9790aebSLuigi Rizzo 	if (na->nm_dtor)
3879f9790aebSLuigi Rizzo 		na->nm_dtor(na);
3880f9790aebSLuigi Rizzo 
38814f80b14cSVincenzo Maffione 	if (na->tx_rings) { /* XXX should not happen */
3882b6e66be2SVincenzo Maffione 		if (netmap_debug & NM_DEBUG_ON)
3883b6e66be2SVincenzo Maffione 			nm_prerr("freeing leftover tx_rings");
38844f80b14cSVincenzo Maffione 		na->nm_krings_delete(na);
38854f80b14cSVincenzo Maffione 	}
38864f80b14cSVincenzo Maffione 	netmap_pipe_dealloc(na);
38874f80b14cSVincenzo Maffione 	if (na->nm_mem)
38884f80b14cSVincenzo Maffione 		netmap_mem_put(na->nm_mem);
38894f80b14cSVincenzo Maffione 	bzero(na, sizeof(*na));
38904f80b14cSVincenzo Maffione 	nm_os_free(na);
3891f9790aebSLuigi Rizzo 
3892f9790aebSLuigi Rizzo 	return 1;
3893f9790aebSLuigi Rizzo }
3894f9790aebSLuigi Rizzo 
389589cc2556SLuigi Rizzo /* nm_krings_create callback for all hardware native adapters */
3896f9790aebSLuigi Rizzo int
3897f9790aebSLuigi Rizzo netmap_hw_krings_create(struct netmap_adapter *na)
3898f9790aebSLuigi Rizzo {
3899f0ea3689SLuigi Rizzo 	int ret = netmap_krings_create(na, 0);
390017885a7bSLuigi Rizzo 	if (ret == 0) {
390117885a7bSLuigi Rizzo 		/* initialize the mbq for the sw rx ring */
39022a7db7a6SVincenzo Maffione 		u_int lim = netmap_real_rings(na, NR_RX), i;
39032a7db7a6SVincenzo Maffione 		for (i = na->num_rx_rings; i < lim; i++) {
39042a7db7a6SVincenzo Maffione 			mbq_safe_init(&NMR(na, NR_RX)[i]->rx_queue);
39052a7db7a6SVincenzo Maffione 		}
390675f4f3edSVincenzo Maffione 		nm_prdis("initialized sw rx queue %d", na->num_rx_rings);
390717885a7bSLuigi Rizzo 	}
390817885a7bSLuigi Rizzo 	return ret;
3909f9790aebSLuigi Rizzo }
3910f9790aebSLuigi Rizzo 
3911f9790aebSLuigi Rizzo 
3912f9790aebSLuigi Rizzo 
391368b8534bSLuigi Rizzo /*
391489cc2556SLuigi Rizzo  * Called on module unload by the netmap-enabled drivers
391568b8534bSLuigi Rizzo  */
391668b8534bSLuigi Rizzo void
391768b8534bSLuigi Rizzo netmap_detach(struct ifnet *ifp)
391868b8534bSLuigi Rizzo {
391968b8534bSLuigi Rizzo 	struct netmap_adapter *na = NA(ifp);
392068b8534bSLuigi Rizzo 
392168b8534bSLuigi Rizzo 	if (!na)
392268b8534bSLuigi Rizzo 		return;
392368b8534bSLuigi Rizzo 
3924f9790aebSLuigi Rizzo 	NMG_LOCK();
392537e3a6d3SLuigi Rizzo 	netmap_set_all_rings(na, NM_KR_LOCKED);
3926847bf383SLuigi Rizzo 	/*
3927847bf383SLuigi Rizzo 	 * if the netmap adapter is not native, somebody
3928847bf383SLuigi Rizzo 	 * changed it, so we can not release it here.
392937e3a6d3SLuigi Rizzo 	 * The NAF_ZOMBIE flag will notify the new owner that
3930847bf383SLuigi Rizzo 	 * the driver is gone.
3931847bf383SLuigi Rizzo 	 */
39324f80b14cSVincenzo Maffione 	if (!(na->na_flags & NAF_NATIVE) || !netmap_adapter_put(na)) {
39334f80b14cSVincenzo Maffione 		na->na_flags |= NAF_ZOMBIE;
3934847bf383SLuigi Rizzo 	}
393537e3a6d3SLuigi Rizzo 	/* give active users a chance to notice that NAF_ZOMBIE has been
393637e3a6d3SLuigi Rizzo 	 * turned on, so that they can stop and return an error to userspace.
393737e3a6d3SLuigi Rizzo 	 * Note that this becomes a NOP if there are no active users and,
393837e3a6d3SLuigi Rizzo 	 * therefore, the put() above has deleted the na, since now NA(ifp) is
393937e3a6d3SLuigi Rizzo 	 * NULL.
394037e3a6d3SLuigi Rizzo 	 */
3941f9790aebSLuigi Rizzo 	netmap_enable_all_rings(ifp);
3942f9790aebSLuigi Rizzo 	NMG_UNLOCK();
3943ae10d1afSLuigi Rizzo }
3944f18be576SLuigi Rizzo 
3945f18be576SLuigi Rizzo 
394668b8534bSLuigi Rizzo /*
394702ad4083SLuigi Rizzo  * Intercept packets from the network stack and pass them
394802ad4083SLuigi Rizzo  * to netmap as incoming packets on the 'software' ring.
394917885a7bSLuigi Rizzo  *
395017885a7bSLuigi Rizzo  * We only store packets in a bounded mbq and then copy them
395117885a7bSLuigi Rizzo  * in the relevant rxsync routine.
395217885a7bSLuigi Rizzo  *
3953ce3ee1e7SLuigi Rizzo  * We rely on the OS to make sure that the ifp and na do not go
3954ce3ee1e7SLuigi Rizzo  * away (typically the caller checks for IFF_DRV_RUNNING or the like).
3955ce3ee1e7SLuigi Rizzo  * In nm_register() or whenever there is a reinitialization,
3956f9790aebSLuigi Rizzo  * we make sure to make the mode change visible here.
395768b8534bSLuigi Rizzo  */
395868b8534bSLuigi Rizzo int
3959ce3ee1e7SLuigi Rizzo netmap_transmit(struct ifnet *ifp, struct mbuf *m)
396068b8534bSLuigi Rizzo {
396168b8534bSLuigi Rizzo 	struct netmap_adapter *na = NA(ifp);
396237e3a6d3SLuigi Rizzo 	struct netmap_kring *kring, *tx_kring;
396317885a7bSLuigi Rizzo 	u_int len = MBUF_LEN(m);
396417885a7bSLuigi Rizzo 	u_int error = ENOBUFS;
396537e3a6d3SLuigi Rizzo 	unsigned int txr;
396617885a7bSLuigi Rizzo 	struct mbq *q;
3967c3e9b4dbSLuiz Otavio O Souza 	int busy;
39682a7db7a6SVincenzo Maffione 	u_int i;
396968b8534bSLuigi Rizzo 
39702a7db7a6SVincenzo Maffione 	i = MBUF_TXQ(m);
39712a7db7a6SVincenzo Maffione 	if (i >= na->num_host_rx_rings) {
39722a7db7a6SVincenzo Maffione 		i = i % na->num_host_rx_rings;
39732a7db7a6SVincenzo Maffione 	}
39742a7db7a6SVincenzo Maffione 	kring = NMR(na, NR_RX)[nma_get_nrings(na, NR_RX) + i];
39752a7db7a6SVincenzo Maffione 
3976ce3ee1e7SLuigi Rizzo 	// XXX [Linux] we do not need this lock
3977ce3ee1e7SLuigi Rizzo 	// if we follow the down/configure/up protocol -gl
3978ce3ee1e7SLuigi Rizzo 	// mtx_lock(&na->core_lock);
397917885a7bSLuigi Rizzo 
39804bf50f18SLuigi Rizzo 	if (!nm_netmap_on(na)) {
3981b6e66be2SVincenzo Maffione 		nm_prerr("%s not in netmap mode anymore", na->name);
3982ce3ee1e7SLuigi Rizzo 		error = ENXIO;
3983ce3ee1e7SLuigi Rizzo 		goto done;
3984ce3ee1e7SLuigi Rizzo 	}
3985ce3ee1e7SLuigi Rizzo 
398637e3a6d3SLuigi Rizzo 	txr = MBUF_TXQ(m);
398737e3a6d3SLuigi Rizzo 	if (txr >= na->num_tx_rings) {
398837e3a6d3SLuigi Rizzo 		txr %= na->num_tx_rings;
398937e3a6d3SLuigi Rizzo 	}
39902ff91c17SVincenzo Maffione 	tx_kring = NMR(na, NR_TX)[txr];
399137e3a6d3SLuigi Rizzo 
399237e3a6d3SLuigi Rizzo 	if (tx_kring->nr_mode == NKR_NETMAP_OFF) {
399337e3a6d3SLuigi Rizzo 		return MBUF_TRANSMIT(na, ifp, m);
399437e3a6d3SLuigi Rizzo 	}
399537e3a6d3SLuigi Rizzo 
399617885a7bSLuigi Rizzo 	q = &kring->rx_queue;
399717885a7bSLuigi Rizzo 
3998ce3ee1e7SLuigi Rizzo 	// XXX reconsider long packets if we handle fragments
39994bf50f18SLuigi Rizzo 	if (len > NETMAP_BUF_SIZE(na)) { /* too long for us */
4000b6e66be2SVincenzo Maffione 		nm_prerr("%s from_host, drop packet size %d > %d", na->name,
40014bf50f18SLuigi Rizzo 			len, NETMAP_BUF_SIZE(na));
4002ce3ee1e7SLuigi Rizzo 		goto done;
4003849bec0eSLuigi Rizzo 	}
400417885a7bSLuigi Rizzo 
40052a7db7a6SVincenzo Maffione 	if (!netmap_generic_hwcsum) {
40062a7db7a6SVincenzo Maffione 		if (nm_os_mbuf_has_csum_offld(m)) {
400775f4f3edSVincenzo Maffione 			nm_prlim(1, "%s drop mbuf that needs checksum offload", na->name);
40082a7db7a6SVincenzo Maffione 			goto done;
40092a7db7a6SVincenzo Maffione 		}
40102a7db7a6SVincenzo Maffione 	}
40112a7db7a6SVincenzo Maffione 
40122a7db7a6SVincenzo Maffione 	if (nm_os_mbuf_has_seg_offld(m)) {
401375f4f3edSVincenzo Maffione 		nm_prlim(1, "%s drop mbuf that needs generic segmentation offload", na->name);
401437e3a6d3SLuigi Rizzo 		goto done;
401537e3a6d3SLuigi Rizzo 	}
401637e3a6d3SLuigi Rizzo 
401789a9a5b5SVincenzo Maffione #ifdef __FreeBSD__
401889a9a5b5SVincenzo Maffione 	ETHER_BPF_MTAP(ifp, m);
401989a9a5b5SVincenzo Maffione #endif /* __FreeBSD__ */
402089a9a5b5SVincenzo Maffione 
4021c3e9b4dbSLuiz Otavio O Souza 	/* protect against netmap_rxsync_from_host(), netmap_sw_to_nic()
402217885a7bSLuigi Rizzo 	 * and maybe other instances of netmap_transmit (the latter
402317885a7bSLuigi Rizzo 	 * not possible on Linux).
4024c3e9b4dbSLuiz Otavio O Souza 	 * We enqueue the mbuf only if we are sure there is going to be
4025c3e9b4dbSLuiz Otavio O Souza 	 * enough room in the host RX ring, otherwise we drop it.
4026ce3ee1e7SLuigi Rizzo 	 */
4027997b054cSLuigi Rizzo 	mbq_lock(q);
402817885a7bSLuigi Rizzo 
4029c3e9b4dbSLuiz Otavio O Souza 	busy = kring->nr_hwtail - kring->nr_hwcur;
4030c3e9b4dbSLuiz Otavio O Souza 	if (busy < 0)
4031c3e9b4dbSLuiz Otavio O Souza 		busy += kring->nkr_num_slots;
4032c3e9b4dbSLuiz Otavio O Souza 	if (busy + mbq_len(q) >= kring->nkr_num_slots - 1) {
403375f4f3edSVincenzo Maffione 		nm_prlim(2, "%s full hwcur %d hwtail %d qlen %d", na->name,
4034c3e9b4dbSLuiz Otavio O Souza 			kring->nr_hwcur, kring->nr_hwtail, mbq_len(q));
4035ce3ee1e7SLuigi Rizzo 	} else {
403617885a7bSLuigi Rizzo 		mbq_enqueue(q, m);
403775f4f3edSVincenzo Maffione 		nm_prdis(2, "%s %d bufs in queue", na->name, mbq_len(q));
403817885a7bSLuigi Rizzo 		/* notify outside the lock */
403917885a7bSLuigi Rizzo 		m = NULL;
404068b8534bSLuigi Rizzo 		error = 0;
4041ce3ee1e7SLuigi Rizzo 	}
4042997b054cSLuigi Rizzo 	mbq_unlock(q);
4043ce3ee1e7SLuigi Rizzo 
404468b8534bSLuigi Rizzo done:
404517885a7bSLuigi Rizzo 	if (m)
404668b8534bSLuigi Rizzo 		m_freem(m);
404717885a7bSLuigi Rizzo 	/* unconditionally wake up listeners */
4048847bf383SLuigi Rizzo 	kring->nm_notify(kring, 0);
404989cc2556SLuigi Rizzo 	/* this is normally netmap_notify(), but for nics
405089cc2556SLuigi Rizzo 	 * connected to a bridge it is netmap_bwrap_intr_notify(),
405189cc2556SLuigi Rizzo 	 * that possibly forwards the frames through the switch
405289cc2556SLuigi Rizzo 	 */
405368b8534bSLuigi Rizzo 
405468b8534bSLuigi Rizzo 	return (error);
405568b8534bSLuigi Rizzo }
405668b8534bSLuigi Rizzo 
405768b8534bSLuigi Rizzo 
405868b8534bSLuigi Rizzo /*
405968b8534bSLuigi Rizzo  * netmap_reset() is called by the driver routines when reinitializing
406068b8534bSLuigi Rizzo  * a ring. The driver is in charge of locking to protect the kring.
4061f9790aebSLuigi Rizzo  * If native netmap mode is not set just return NULL.
406237e3a6d3SLuigi Rizzo  * If native netmap mode is set, in particular, we have to set nr_mode to
406337e3a6d3SLuigi Rizzo  * NKR_NETMAP_ON.
406468b8534bSLuigi Rizzo  */
406568b8534bSLuigi Rizzo struct netmap_slot *
4066ce3ee1e7SLuigi Rizzo netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n,
406768b8534bSLuigi Rizzo 	u_int new_cur)
406868b8534bSLuigi Rizzo {
406968b8534bSLuigi Rizzo 	struct netmap_kring *kring;
4070506cc70cSLuigi Rizzo 	int new_hwofs, lim;
407168b8534bSLuigi Rizzo 
40724bf50f18SLuigi Rizzo 	if (!nm_native_on(na)) {
407375f4f3edSVincenzo Maffione 		nm_prdis("interface not in native netmap mode");
407468b8534bSLuigi Rizzo 		return NULL;	/* nothing to reinitialize */
4075ce3ee1e7SLuigi Rizzo 	}
407668b8534bSLuigi Rizzo 
4077ce3ee1e7SLuigi Rizzo 	/* XXX note- in the new scheme, we are not guaranteed to be
4078ce3ee1e7SLuigi Rizzo 	 * under lock (e.g. when called on a device reset).
4079ce3ee1e7SLuigi Rizzo 	 * In this case, we should set a flag and do not trust too
4080ce3ee1e7SLuigi Rizzo 	 * much the values. In practice: TODO
4081ce3ee1e7SLuigi Rizzo 	 * - set a RESET flag somewhere in the kring
4082ce3ee1e7SLuigi Rizzo 	 * - do the processing in a conservative way
4083ce3ee1e7SLuigi Rizzo 	 * - let the *sync() fixup at the end.
4084ce3ee1e7SLuigi Rizzo 	 */
408564ae02c3SLuigi Rizzo 	if (tx == NR_TX) {
40868241616dSLuigi Rizzo 		if (n >= na->num_tx_rings)
40878241616dSLuigi Rizzo 			return NULL;
408837e3a6d3SLuigi Rizzo 
40892ff91c17SVincenzo Maffione 		kring = na->tx_rings[n];
409037e3a6d3SLuigi Rizzo 
409137e3a6d3SLuigi Rizzo 		if (kring->nr_pending_mode == NKR_NETMAP_OFF) {
409237e3a6d3SLuigi Rizzo 			kring->nr_mode = NKR_NETMAP_OFF;
409337e3a6d3SLuigi Rizzo 			return NULL;
409437e3a6d3SLuigi Rizzo 		}
409537e3a6d3SLuigi Rizzo 
409617885a7bSLuigi Rizzo 		// XXX check whether we should use hwcur or rcur
4097506cc70cSLuigi Rizzo 		new_hwofs = kring->nr_hwcur - new_cur;
409864ae02c3SLuigi Rizzo 	} else {
40998241616dSLuigi Rizzo 		if (n >= na->num_rx_rings)
41008241616dSLuigi Rizzo 			return NULL;
41012ff91c17SVincenzo Maffione 		kring = na->rx_rings[n];
410237e3a6d3SLuigi Rizzo 
410337e3a6d3SLuigi Rizzo 		if (kring->nr_pending_mode == NKR_NETMAP_OFF) {
410437e3a6d3SLuigi Rizzo 			kring->nr_mode = NKR_NETMAP_OFF;
410537e3a6d3SLuigi Rizzo 			return NULL;
410637e3a6d3SLuigi Rizzo 		}
410737e3a6d3SLuigi Rizzo 
410817885a7bSLuigi Rizzo 		new_hwofs = kring->nr_hwtail - new_cur;
410964ae02c3SLuigi Rizzo 	}
411064ae02c3SLuigi Rizzo 	lim = kring->nkr_num_slots - 1;
4111506cc70cSLuigi Rizzo 	if (new_hwofs > lim)
4112506cc70cSLuigi Rizzo 		new_hwofs -= lim + 1;
4113506cc70cSLuigi Rizzo 
4114ce3ee1e7SLuigi Rizzo 	/* Always set the new offset value and realign the ring. */
4115b6e66be2SVincenzo Maffione 	if (netmap_debug & NM_DEBUG_ON)
4116b6e66be2SVincenzo Maffione 	    nm_prinf("%s %s%d hwofs %d -> %d, hwtail %d -> %d",
41174bf50f18SLuigi Rizzo 		na->name,
411817885a7bSLuigi Rizzo 		tx == NR_TX ? "TX" : "RX", n,
4119ce3ee1e7SLuigi Rizzo 		kring->nkr_hwofs, new_hwofs,
412017885a7bSLuigi Rizzo 		kring->nr_hwtail,
412117885a7bSLuigi Rizzo 		tx == NR_TX ? lim : kring->nr_hwtail);
4122506cc70cSLuigi Rizzo 	kring->nkr_hwofs = new_hwofs;
412317885a7bSLuigi Rizzo 	if (tx == NR_TX) {
412417885a7bSLuigi Rizzo 		kring->nr_hwtail = kring->nr_hwcur + lim;
412517885a7bSLuigi Rizzo 		if (kring->nr_hwtail > lim)
412617885a7bSLuigi Rizzo 			kring->nr_hwtail -= lim + 1;
412717885a7bSLuigi Rizzo 	}
4128506cc70cSLuigi Rizzo 
412968b8534bSLuigi Rizzo 	/*
4130ce3ee1e7SLuigi Rizzo 	 * Wakeup on the individual and global selwait
4131506cc70cSLuigi Rizzo 	 * We do the wakeup here, but the ring is not yet reconfigured.
4132506cc70cSLuigi Rizzo 	 * However, we are under lock so there are no races.
413368b8534bSLuigi Rizzo 	 */
413437e3a6d3SLuigi Rizzo 	kring->nr_mode = NKR_NETMAP_ON;
4135847bf383SLuigi Rizzo 	kring->nm_notify(kring, 0);
413668b8534bSLuigi Rizzo 	return kring->ring->slot;
413768b8534bSLuigi Rizzo }
413868b8534bSLuigi Rizzo 
413968b8534bSLuigi Rizzo 
4140ce3ee1e7SLuigi Rizzo /*
4141f9790aebSLuigi Rizzo  * Dispatch rx/tx interrupts to the netmap rings.
4142ce3ee1e7SLuigi Rizzo  *
4143ce3ee1e7SLuigi Rizzo  * "work_done" is non-null on the RX path, NULL for the TX path.
4144ce3ee1e7SLuigi Rizzo  * We rely on the OS to make sure that there is only one active
4145ce3ee1e7SLuigi Rizzo  * instance per queue, and that there is appropriate locking.
4146849bec0eSLuigi Rizzo  *
4147f9790aebSLuigi Rizzo  * The 'notify' routine depends on what the ring is attached to.
4148f9790aebSLuigi Rizzo  * - for a netmap file descriptor, do a selwakeup on the individual
4149f9790aebSLuigi Rizzo  *   waitqueue, plus one on the global one if needed
41504bf50f18SLuigi Rizzo  *   (see netmap_notify)
41514bf50f18SLuigi Rizzo  * - for a nic connected to a switch, call the proper forwarding routine
41524bf50f18SLuigi Rizzo  *   (see netmap_bwrap_intr_notify)
4153f9790aebSLuigi Rizzo  */
415437e3a6d3SLuigi Rizzo int
415537e3a6d3SLuigi Rizzo netmap_common_irq(struct netmap_adapter *na, u_int q, u_int *work_done)
4156f9790aebSLuigi Rizzo {
4157f9790aebSLuigi Rizzo 	struct netmap_kring *kring;
4158847bf383SLuigi Rizzo 	enum txrx t = (work_done ? NR_RX : NR_TX);
4159f9790aebSLuigi Rizzo 
4160f9790aebSLuigi Rizzo 	q &= NETMAP_RING_MASK;
4161f9790aebSLuigi Rizzo 
4162b6e66be2SVincenzo Maffione 	if (netmap_debug & (NM_DEBUG_RXINTR|NM_DEBUG_TXINTR)) {
4163b6e66be2SVincenzo Maffione 	        nm_prlim(5, "received %s queue %d", work_done ? "RX" : "TX" , q);
4164f9790aebSLuigi Rizzo 	}
4165f9790aebSLuigi Rizzo 
4166847bf383SLuigi Rizzo 	if (q >= nma_get_nrings(na, t))
416737e3a6d3SLuigi Rizzo 		return NM_IRQ_PASS; // not a physical queue
4168847bf383SLuigi Rizzo 
41692ff91c17SVincenzo Maffione 	kring = NMR(na, t)[q];
4170847bf383SLuigi Rizzo 
417137e3a6d3SLuigi Rizzo 	if (kring->nr_mode == NKR_NETMAP_OFF) {
417237e3a6d3SLuigi Rizzo 		return NM_IRQ_PASS;
417337e3a6d3SLuigi Rizzo 	}
417437e3a6d3SLuigi Rizzo 
4175847bf383SLuigi Rizzo 	if (t == NR_RX) {
4176f9790aebSLuigi Rizzo 		kring->nr_kflags |= NKR_PENDINTR;	// XXX atomic ?
4177f9790aebSLuigi Rizzo 		*work_done = 1; /* do not fire napi again */
4178f9790aebSLuigi Rizzo 	}
417937e3a6d3SLuigi Rizzo 
418037e3a6d3SLuigi Rizzo 	return kring->nm_notify(kring, 0);
4181f9790aebSLuigi Rizzo }
4182f9790aebSLuigi Rizzo 
418317885a7bSLuigi Rizzo 
4184f9790aebSLuigi Rizzo /*
4185f9790aebSLuigi Rizzo  * Default functions to handle rx/tx interrupts from a physical device.
4186f9790aebSLuigi Rizzo  * "work_done" is non-null on the RX path, NULL for the TX path.
4187f9790aebSLuigi Rizzo  *
418837e3a6d3SLuigi Rizzo  * If the card is not in netmap mode, simply return NM_IRQ_PASS,
4189ce3ee1e7SLuigi Rizzo  * so that the caller proceeds with regular processing.
419037e3a6d3SLuigi Rizzo  * Otherwise call netmap_common_irq().
4191ce3ee1e7SLuigi Rizzo  *
4192ce3ee1e7SLuigi Rizzo  * If the card is connected to a netmap file descriptor,
4193ce3ee1e7SLuigi Rizzo  * do a selwakeup on the individual queue, plus one on the global one
4194ce3ee1e7SLuigi Rizzo  * if needed (multiqueue card _and_ there are multiqueue listeners),
419537e3a6d3SLuigi Rizzo  * and return NR_IRQ_COMPLETED.
4196ce3ee1e7SLuigi Rizzo  *
4197ce3ee1e7SLuigi Rizzo  * Finally, if called on rx from an interface connected to a switch,
419837e3a6d3SLuigi Rizzo  * calls the proper forwarding routine.
41991a26580eSLuigi Rizzo  */
4200babc7c12SLuigi Rizzo int
4201ce3ee1e7SLuigi Rizzo netmap_rx_irq(struct ifnet *ifp, u_int q, u_int *work_done)
42021a26580eSLuigi Rizzo {
42034bf50f18SLuigi Rizzo 	struct netmap_adapter *na = NA(ifp);
42044bf50f18SLuigi Rizzo 
42054bf50f18SLuigi Rizzo 	/*
42064bf50f18SLuigi Rizzo 	 * XXX emulated netmap mode sets NAF_SKIP_INTR so
42074bf50f18SLuigi Rizzo 	 * we still use the regular driver even though the previous
42084bf50f18SLuigi Rizzo 	 * check fails. It is unclear whether we should use
42094bf50f18SLuigi Rizzo 	 * nm_native_on() here.
42104bf50f18SLuigi Rizzo 	 */
42114bf50f18SLuigi Rizzo 	if (!nm_netmap_on(na))
421237e3a6d3SLuigi Rizzo 		return NM_IRQ_PASS;
4213849bec0eSLuigi Rizzo 
42144bf50f18SLuigi Rizzo 	if (na->na_flags & NAF_SKIP_INTR) {
421575f4f3edSVincenzo Maffione 		nm_prdis("use regular interrupt");
421637e3a6d3SLuigi Rizzo 		return NM_IRQ_PASS;
42178241616dSLuigi Rizzo 	}
42188241616dSLuigi Rizzo 
421937e3a6d3SLuigi Rizzo 	return netmap_common_irq(na, q, work_done);
42201a26580eSLuigi Rizzo }
42211a26580eSLuigi Rizzo 
42222a7db7a6SVincenzo Maffione /* set/clear native flags and if_transmit/netdev_ops */
42232a7db7a6SVincenzo Maffione void
42242a7db7a6SVincenzo Maffione nm_set_native_flags(struct netmap_adapter *na)
42252a7db7a6SVincenzo Maffione {
42262a7db7a6SVincenzo Maffione 	struct ifnet *ifp = na->ifp;
42272a7db7a6SVincenzo Maffione 
42282a7db7a6SVincenzo Maffione 	/* We do the setup for intercepting packets only if we are the
42292a7db7a6SVincenzo Maffione 	 * first user of this adapapter. */
42302a7db7a6SVincenzo Maffione 	if (na->active_fds > 0) {
42312a7db7a6SVincenzo Maffione 		return;
42322a7db7a6SVincenzo Maffione 	}
42332a7db7a6SVincenzo Maffione 
42342a7db7a6SVincenzo Maffione 	na->na_flags |= NAF_NETMAP_ON;
42352a7db7a6SVincenzo Maffione 	nm_os_onenter(ifp);
42362a7db7a6SVincenzo Maffione 	nm_update_hostrings_mode(na);
42372a7db7a6SVincenzo Maffione }
42382a7db7a6SVincenzo Maffione 
42392a7db7a6SVincenzo Maffione void
42402a7db7a6SVincenzo Maffione nm_clear_native_flags(struct netmap_adapter *na)
42412a7db7a6SVincenzo Maffione {
42422a7db7a6SVincenzo Maffione 	struct ifnet *ifp = na->ifp;
42432a7db7a6SVincenzo Maffione 
42442a7db7a6SVincenzo Maffione 	/* We undo the setup for intercepting packets only if we are the
4245b6e66be2SVincenzo Maffione 	 * last user of this adapter. */
42462a7db7a6SVincenzo Maffione 	if (na->active_fds > 0) {
42472a7db7a6SVincenzo Maffione 		return;
42482a7db7a6SVincenzo Maffione 	}
42492a7db7a6SVincenzo Maffione 
42502a7db7a6SVincenzo Maffione 	nm_update_hostrings_mode(na);
42512a7db7a6SVincenzo Maffione 	nm_os_onexit(ifp);
42522a7db7a6SVincenzo Maffione 
42532a7db7a6SVincenzo Maffione 	na->na_flags &= ~NAF_NETMAP_ON;
42542a7db7a6SVincenzo Maffione }
42552a7db7a6SVincenzo Maffione 
425675f4f3edSVincenzo Maffione void
425775f4f3edSVincenzo Maffione netmap_krings_mode_commit(struct netmap_adapter *na, int onoff)
425875f4f3edSVincenzo Maffione {
425975f4f3edSVincenzo Maffione 	enum txrx t;
426075f4f3edSVincenzo Maffione 
426175f4f3edSVincenzo Maffione 	for_rx_tx(t) {
426275f4f3edSVincenzo Maffione 		int i;
426375f4f3edSVincenzo Maffione 
426475f4f3edSVincenzo Maffione 		for (i = 0; i < netmap_real_rings(na, t); i++) {
426575f4f3edSVincenzo Maffione 			struct netmap_kring *kring = NMR(na, t)[i];
426675f4f3edSVincenzo Maffione 
426775f4f3edSVincenzo Maffione 			if (onoff && nm_kring_pending_on(kring))
426875f4f3edSVincenzo Maffione 				kring->nr_mode = NKR_NETMAP_ON;
426975f4f3edSVincenzo Maffione 			else if (!onoff && nm_kring_pending_off(kring))
427075f4f3edSVincenzo Maffione 				kring->nr_mode = NKR_NETMAP_OFF;
427175f4f3edSVincenzo Maffione 		}
427275f4f3edSVincenzo Maffione 	}
427375f4f3edSVincenzo Maffione }
427475f4f3edSVincenzo Maffione 
427501c7d25fSLuigi Rizzo /*
4276f9790aebSLuigi Rizzo  * Module loader and unloader
4277f196ce38SLuigi Rizzo  *
4278f9790aebSLuigi Rizzo  * netmap_init() creates the /dev/netmap device and initializes
4279f9790aebSLuigi Rizzo  * all global variables. Returns 0 on success, errno on failure
4280f9790aebSLuigi Rizzo  * (but there is no chance)
4281f9790aebSLuigi Rizzo  *
4282f9790aebSLuigi Rizzo  * netmap_fini() destroys everything.
4283f196ce38SLuigi Rizzo  */
4284babc7c12SLuigi Rizzo 
4285babc7c12SLuigi Rizzo static struct cdev *netmap_dev; /* /dev/netmap character device. */
4286f9790aebSLuigi Rizzo extern struct cdevsw netmap_cdevsw;
4287babc7c12SLuigi Rizzo 
428817885a7bSLuigi Rizzo 
4289f9790aebSLuigi Rizzo void
429068b8534bSLuigi Rizzo netmap_fini(void)
429168b8534bSLuigi Rizzo {
4292f9790aebSLuigi Rizzo 	if (netmap_dev)
429368b8534bSLuigi Rizzo 		destroy_dev(netmap_dev);
429437e3a6d3SLuigi Rizzo 	/* we assume that there are no longer netmap users */
429537e3a6d3SLuigi Rizzo 	nm_os_ifnet_fini();
429637e3a6d3SLuigi Rizzo 	netmap_uninit_bridges();
4297ce3ee1e7SLuigi Rizzo 	netmap_mem_fini();
4298ce3ee1e7SLuigi Rizzo 	NMG_LOCK_DESTROY();
4299b6e66be2SVincenzo Maffione 	nm_prinf("netmap: unloaded module.");
430068b8534bSLuigi Rizzo }
430168b8534bSLuigi Rizzo 
430217885a7bSLuigi Rizzo 
4303f9790aebSLuigi Rizzo int
4304f9790aebSLuigi Rizzo netmap_init(void)
430568b8534bSLuigi Rizzo {
4306f9790aebSLuigi Rizzo 	int error;
430768b8534bSLuigi Rizzo 
4308f9790aebSLuigi Rizzo 	NMG_LOCK_INIT();
430968b8534bSLuigi Rizzo 
4310f9790aebSLuigi Rizzo 	error = netmap_mem_init();
4311f9790aebSLuigi Rizzo 	if (error != 0)
4312f9790aebSLuigi Rizzo 		goto fail;
4313c929ca72SLuigi Rizzo 	/*
4314c929ca72SLuigi Rizzo 	 * MAKEDEV_ETERNAL_KLD avoids an expensive check on syscalls
4315c929ca72SLuigi Rizzo 	 * when the module is compiled in.
4316c929ca72SLuigi Rizzo 	 * XXX could use make_dev_credv() to get error number
4317c929ca72SLuigi Rizzo 	 */
43180e73f29aSLuigi Rizzo 	netmap_dev = make_dev_credf(MAKEDEV_ETERNAL_KLD,
431911c0b69cSAdrian Chadd 		&netmap_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0600,
43200e73f29aSLuigi Rizzo 			      "netmap");
4321f9790aebSLuigi Rizzo 	if (!netmap_dev)
4322f9790aebSLuigi Rizzo 		goto fail;
4323f9790aebSLuigi Rizzo 
4324847bf383SLuigi Rizzo 	error = netmap_init_bridges();
4325847bf383SLuigi Rizzo 	if (error)
4326847bf383SLuigi Rizzo 		goto fail;
4327847bf383SLuigi Rizzo 
43284bf50f18SLuigi Rizzo #ifdef __FreeBSD__
432937e3a6d3SLuigi Rizzo 	nm_os_vi_init_index();
43304bf50f18SLuigi Rizzo #endif
4331847bf383SLuigi Rizzo 
433237e3a6d3SLuigi Rizzo 	error = nm_os_ifnet_init();
433337e3a6d3SLuigi Rizzo 	if (error)
433437e3a6d3SLuigi Rizzo 		goto fail;
433537e3a6d3SLuigi Rizzo 
4336b6e66be2SVincenzo Maffione 	nm_prinf("netmap: loaded module");
4337f9790aebSLuigi Rizzo 	return (0);
4338f9790aebSLuigi Rizzo fail:
433968b8534bSLuigi Rizzo 	netmap_fini();
4340f9790aebSLuigi Rizzo 	return (EINVAL); /* may be incorrect */
434168b8534bSLuigi Rizzo }
4342