xref: /freebsd/sys/dev/netmap/if_ptnet.c (revision 2ec213aba4bfecb47d7439a6d4b46b88a382eaff)
1a9e644cdSLuigi Rizzo /*-
2a9e644cdSLuigi Rizzo  * Copyright (c) 2016, Vincenzo Maffione
3a9e644cdSLuigi Rizzo  * All rights reserved.
4a9e644cdSLuigi Rizzo  *
5a9e644cdSLuigi Rizzo  * Redistribution and use in source and binary forms, with or without
6a9e644cdSLuigi Rizzo  * modification, are permitted provided that the following conditions
7a9e644cdSLuigi Rizzo  * are met:
8a9e644cdSLuigi Rizzo  * 1. Redistributions of source code must retain the above copyright
9a9e644cdSLuigi Rizzo  *    notice unmodified, this list of conditions, and the following
10a9e644cdSLuigi Rizzo  *    disclaimer.
11a9e644cdSLuigi Rizzo  * 2. Redistributions in binary form must reproduce the above copyright
12a9e644cdSLuigi Rizzo  *    notice, this list of conditions and the following disclaimer in the
13a9e644cdSLuigi Rizzo  *    documentation and/or other materials provided with the distribution.
14a9e644cdSLuigi Rizzo  *
15a9e644cdSLuigi Rizzo  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16a9e644cdSLuigi Rizzo  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17a9e644cdSLuigi Rizzo  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18a9e644cdSLuigi Rizzo  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19a9e644cdSLuigi Rizzo  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20a9e644cdSLuigi Rizzo  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21a9e644cdSLuigi Rizzo  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22a9e644cdSLuigi Rizzo  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23a9e644cdSLuigi Rizzo  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24a9e644cdSLuigi Rizzo  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25a9e644cdSLuigi Rizzo  *
26a9e644cdSLuigi Rizzo  * $FreeBSD$
27a9e644cdSLuigi Rizzo  */
28a9e644cdSLuigi Rizzo 
29a9e644cdSLuigi Rizzo /* Driver for ptnet paravirtualized network device. */
30a9e644cdSLuigi Rizzo 
31a9e644cdSLuigi Rizzo #include <sys/cdefs.h>
32a9e644cdSLuigi Rizzo 
33a9e644cdSLuigi Rizzo #include <sys/types.h>
34a9e644cdSLuigi Rizzo #include <sys/param.h>
35a9e644cdSLuigi Rizzo #include <sys/systm.h>
36a9e644cdSLuigi Rizzo #include <sys/kernel.h>
37a9e644cdSLuigi Rizzo #include <sys/sockio.h>
38a9e644cdSLuigi Rizzo #include <sys/mbuf.h>
39a9e644cdSLuigi Rizzo #include <sys/malloc.h>
40a9e644cdSLuigi Rizzo #include <sys/module.h>
41a9e644cdSLuigi Rizzo #include <sys/socket.h>
42a9e644cdSLuigi Rizzo #include <sys/sysctl.h>
43a9e644cdSLuigi Rizzo #include <sys/lock.h>
44a9e644cdSLuigi Rizzo #include <sys/mutex.h>
45a9e644cdSLuigi Rizzo #include <sys/taskqueue.h>
46a9e644cdSLuigi Rizzo #include <sys/smp.h>
47a9e644cdSLuigi Rizzo #include <sys/time.h>
48a9e644cdSLuigi Rizzo #include <machine/smp.h>
49a9e644cdSLuigi Rizzo 
50a9e644cdSLuigi Rizzo #include <vm/uma.h>
51a9e644cdSLuigi Rizzo #include <vm/vm.h>
52a9e644cdSLuigi Rizzo #include <vm/pmap.h>
53a9e644cdSLuigi Rizzo 
54a9e644cdSLuigi Rizzo #include <net/ethernet.h>
55a9e644cdSLuigi Rizzo #include <net/if.h>
56a9e644cdSLuigi Rizzo #include <net/if_var.h>
57a9e644cdSLuigi Rizzo #include <net/if_arp.h>
58a9e644cdSLuigi Rizzo #include <net/if_dl.h>
59a9e644cdSLuigi Rizzo #include <net/if_types.h>
60a9e644cdSLuigi Rizzo #include <net/if_media.h>
61a9e644cdSLuigi Rizzo #include <net/if_vlan_var.h>
62a9e644cdSLuigi Rizzo #include <net/bpf.h>
63a9e644cdSLuigi Rizzo 
64a9e644cdSLuigi Rizzo #include <netinet/in_systm.h>
65a9e644cdSLuigi Rizzo #include <netinet/in.h>
66a9e644cdSLuigi Rizzo #include <netinet/ip.h>
67a9e644cdSLuigi Rizzo #include <netinet/ip6.h>
68a9e644cdSLuigi Rizzo #include <netinet6/ip6_var.h>
69a9e644cdSLuigi Rizzo #include <netinet/udp.h>
70a9e644cdSLuigi Rizzo #include <netinet/tcp.h>
71a9e644cdSLuigi Rizzo 
72a9e644cdSLuigi Rizzo #include <machine/bus.h>
73a9e644cdSLuigi Rizzo #include <machine/resource.h>
74a9e644cdSLuigi Rizzo #include <sys/bus.h>
75a9e644cdSLuigi Rizzo #include <sys/rman.h>
76a9e644cdSLuigi Rizzo 
77a9e644cdSLuigi Rizzo #include <dev/pci/pcivar.h>
78a9e644cdSLuigi Rizzo #include <dev/pci/pcireg.h>
79a9e644cdSLuigi Rizzo 
80a9e644cdSLuigi Rizzo #include "opt_inet.h"
81a9e644cdSLuigi Rizzo #include "opt_inet6.h"
82a9e644cdSLuigi Rizzo 
83a9e644cdSLuigi Rizzo #include <sys/selinfo.h>
84a9e644cdSLuigi Rizzo #include <net/netmap.h>
85a9e644cdSLuigi Rizzo #include <dev/netmap/netmap_kern.h>
86a9e644cdSLuigi Rizzo #include <net/netmap_virt.h>
87a9e644cdSLuigi Rizzo #include <dev/netmap/netmap_mem2.h>
88a9e644cdSLuigi Rizzo #include <dev/virtio/network/virtio_net.h>
89a9e644cdSLuigi Rizzo 
90*2ec213abSVincenzo Maffione #ifdef WITH_PTNETMAP
91*2ec213abSVincenzo Maffione 
92a9e644cdSLuigi Rizzo #ifndef INET
93a9e644cdSLuigi Rizzo #error "INET not defined, cannot support offloadings"
94a9e644cdSLuigi Rizzo #endif
95a9e644cdSLuigi Rizzo 
96a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000
97a9e644cdSLuigi Rizzo static uint64_t	ptnet_get_counter(if_t, ift_counter);
98a9e644cdSLuigi Rizzo #else
99a9e644cdSLuigi Rizzo typedef struct ifnet *if_t;
100a9e644cdSLuigi Rizzo #define if_getsoftc(_ifp)   (_ifp)->if_softc
101a9e644cdSLuigi Rizzo #endif
102a9e644cdSLuigi Rizzo 
103a9e644cdSLuigi Rizzo //#define PTNETMAP_STATS
104a9e644cdSLuigi Rizzo //#define DEBUG
105a9e644cdSLuigi Rizzo #ifdef DEBUG
106a9e644cdSLuigi Rizzo #define DBG(x) x
107a9e644cdSLuigi Rizzo #else   /* !DEBUG */
108a9e644cdSLuigi Rizzo #define DBG(x)
109a9e644cdSLuigi Rizzo #endif  /* !DEBUG */
110a9e644cdSLuigi Rizzo 
111a9e644cdSLuigi Rizzo extern int ptnet_vnet_hdr; /* Tunable parameter */
112a9e644cdSLuigi Rizzo 
113a9e644cdSLuigi Rizzo struct ptnet_softc;
114a9e644cdSLuigi Rizzo 
115a9e644cdSLuigi Rizzo struct ptnet_queue_stats {
116a9e644cdSLuigi Rizzo 	uint64_t	packets; /* if_[io]packets */
117a9e644cdSLuigi Rizzo 	uint64_t	bytes;	 /* if_[io]bytes */
118a9e644cdSLuigi Rizzo 	uint64_t	errors;	 /* if_[io]errors */
119a9e644cdSLuigi Rizzo 	uint64_t	iqdrops; /* if_iqdrops */
120a9e644cdSLuigi Rizzo 	uint64_t	mcasts;  /* if_[io]mcasts */
121a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
122a9e644cdSLuigi Rizzo 	uint64_t	intrs;
123a9e644cdSLuigi Rizzo 	uint64_t	kicks;
124a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */
125a9e644cdSLuigi Rizzo };
126a9e644cdSLuigi Rizzo 
127a9e644cdSLuigi Rizzo struct ptnet_queue {
128a9e644cdSLuigi Rizzo 	struct ptnet_softc		*sc;
129a9e644cdSLuigi Rizzo 	struct				resource *irq;
130a9e644cdSLuigi Rizzo 	void				*cookie;
131a9e644cdSLuigi Rizzo 	int				kring_id;
132b6e66be2SVincenzo Maffione 	struct nm_csb_atok		*atok;
133b6e66be2SVincenzo Maffione 	struct nm_csb_ktoa		*ktoa;
134a9e644cdSLuigi Rizzo 	unsigned int			kick;
135a9e644cdSLuigi Rizzo 	struct mtx			lock;
136a9e644cdSLuigi Rizzo 	struct buf_ring			*bufring; /* for TX queues */
137a9e644cdSLuigi Rizzo 	struct ptnet_queue_stats	stats;
138a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
139a9e644cdSLuigi Rizzo 	struct ptnet_queue_stats	last_stats;
140a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */
141a9e644cdSLuigi Rizzo 	struct taskqueue		*taskq;
142a9e644cdSLuigi Rizzo 	struct task			task;
143a9e644cdSLuigi Rizzo 	char				lock_name[16];
144a9e644cdSLuigi Rizzo };
145a9e644cdSLuigi Rizzo 
146a9e644cdSLuigi Rizzo #define PTNET_Q_LOCK(_pq)	mtx_lock(&(_pq)->lock)
147a9e644cdSLuigi Rizzo #define PTNET_Q_TRYLOCK(_pq)	mtx_trylock(&(_pq)->lock)
148a9e644cdSLuigi Rizzo #define PTNET_Q_UNLOCK(_pq)	mtx_unlock(&(_pq)->lock)
149a9e644cdSLuigi Rizzo 
150a9e644cdSLuigi Rizzo struct ptnet_softc {
151a9e644cdSLuigi Rizzo 	device_t		dev;
152a9e644cdSLuigi Rizzo 	if_t			ifp;
153a9e644cdSLuigi Rizzo 	struct ifmedia		media;
154a9e644cdSLuigi Rizzo 	struct mtx		lock;
155a9e644cdSLuigi Rizzo 	char			lock_name[16];
156a9e644cdSLuigi Rizzo 	char			hwaddr[ETHER_ADDR_LEN];
157a9e644cdSLuigi Rizzo 
158a9e644cdSLuigi Rizzo 	/* Mirror of PTFEAT register. */
159a9e644cdSLuigi Rizzo 	uint32_t		ptfeatures;
160a9e644cdSLuigi Rizzo 	unsigned int		vnet_hdr_len;
161a9e644cdSLuigi Rizzo 
162a9e644cdSLuigi Rizzo 	/* PCI BARs support. */
163a9e644cdSLuigi Rizzo 	struct resource		*iomem;
164a9e644cdSLuigi Rizzo 	struct resource		*msix_mem;
165a9e644cdSLuigi Rizzo 
166a9e644cdSLuigi Rizzo 	unsigned int		num_rings;
167a9e644cdSLuigi Rizzo 	unsigned int		num_tx_rings;
168a9e644cdSLuigi Rizzo 	struct ptnet_queue	*queues;
169a9e644cdSLuigi Rizzo 	struct ptnet_queue	*rxqueues;
170b6e66be2SVincenzo Maffione 	struct nm_csb_atok	*csb_gh;
171b6e66be2SVincenzo Maffione 	struct nm_csb_ktoa	*csb_hg;
172a9e644cdSLuigi Rizzo 
173a9e644cdSLuigi Rizzo 	unsigned int		min_tx_space;
174a9e644cdSLuigi Rizzo 
175a9e644cdSLuigi Rizzo 	struct netmap_pt_guest_adapter *ptna;
176a9e644cdSLuigi Rizzo 
177a9e644cdSLuigi Rizzo 	struct callout		tick;
178a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
179a9e644cdSLuigi Rizzo 	struct timeval		last_ts;
180a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */
181a9e644cdSLuigi Rizzo };
182a9e644cdSLuigi Rizzo 
183a9e644cdSLuigi Rizzo #define PTNET_CORE_LOCK(_sc)	mtx_lock(&(_sc)->lock)
184a9e644cdSLuigi Rizzo #define PTNET_CORE_UNLOCK(_sc)	mtx_unlock(&(_sc)->lock)
185a9e644cdSLuigi Rizzo 
186a9e644cdSLuigi Rizzo static int	ptnet_probe(device_t);
187a9e644cdSLuigi Rizzo static int	ptnet_attach(device_t);
188a9e644cdSLuigi Rizzo static int	ptnet_detach(device_t);
189a9e644cdSLuigi Rizzo static int	ptnet_suspend(device_t);
190a9e644cdSLuigi Rizzo static int	ptnet_resume(device_t);
191a9e644cdSLuigi Rizzo static int	ptnet_shutdown(device_t);
192a9e644cdSLuigi Rizzo 
193a9e644cdSLuigi Rizzo static void	ptnet_init(void *opaque);
194a9e644cdSLuigi Rizzo static int	ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data);
195a9e644cdSLuigi Rizzo static int	ptnet_init_locked(struct ptnet_softc *sc);
196a9e644cdSLuigi Rizzo static int	ptnet_stop(struct ptnet_softc *sc);
197a9e644cdSLuigi Rizzo static int	ptnet_transmit(if_t ifp, struct mbuf *m);
198a9e644cdSLuigi Rizzo static int	ptnet_drain_transmit_queue(struct ptnet_queue *pq,
199a9e644cdSLuigi Rizzo 					   unsigned int budget,
200a9e644cdSLuigi Rizzo 					   bool may_resched);
201a9e644cdSLuigi Rizzo static void	ptnet_qflush(if_t ifp);
202a9e644cdSLuigi Rizzo static void	ptnet_tx_task(void *context, int pending);
203a9e644cdSLuigi Rizzo 
204a9e644cdSLuigi Rizzo static int	ptnet_media_change(if_t ifp);
205a9e644cdSLuigi Rizzo static void	ptnet_media_status(if_t ifp, struct ifmediareq *ifmr);
206a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
207a9e644cdSLuigi Rizzo static void	ptnet_tick(void *opaque);
208a9e644cdSLuigi Rizzo #endif
209a9e644cdSLuigi Rizzo 
210a9e644cdSLuigi Rizzo static int	ptnet_irqs_init(struct ptnet_softc *sc);
211a9e644cdSLuigi Rizzo static void	ptnet_irqs_fini(struct ptnet_softc *sc);
212a9e644cdSLuigi Rizzo 
213b6e66be2SVincenzo Maffione static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd);
2142ff91c17SVincenzo Maffione static int      ptnet_nm_config(struct netmap_adapter *na,
2152ff91c17SVincenzo Maffione 				struct nm_config_info *info);
216a9e644cdSLuigi Rizzo static void	ptnet_update_vnet_hdr(struct ptnet_softc *sc);
217a9e644cdSLuigi Rizzo static int	ptnet_nm_register(struct netmap_adapter *na, int onoff);
218a9e644cdSLuigi Rizzo static int	ptnet_nm_txsync(struct netmap_kring *kring, int flags);
219a9e644cdSLuigi Rizzo static int	ptnet_nm_rxsync(struct netmap_kring *kring, int flags);
2204f80b14cSVincenzo Maffione static void	ptnet_nm_intr(struct netmap_adapter *na, int onoff);
221a9e644cdSLuigi Rizzo 
222a9e644cdSLuigi Rizzo static void	ptnet_tx_intr(void *opaque);
223a9e644cdSLuigi Rizzo static void	ptnet_rx_intr(void *opaque);
224a9e644cdSLuigi Rizzo 
225a9e644cdSLuigi Rizzo static unsigned	ptnet_rx_discard(struct netmap_kring *kring,
226a9e644cdSLuigi Rizzo 				 unsigned int head);
227a9e644cdSLuigi Rizzo static int	ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget,
228a9e644cdSLuigi Rizzo 			     bool may_resched);
229a9e644cdSLuigi Rizzo static void	ptnet_rx_task(void *context, int pending);
230a9e644cdSLuigi Rizzo 
231a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING
232a9e644cdSLuigi Rizzo static poll_handler_t ptnet_poll;
233a9e644cdSLuigi Rizzo #endif
234a9e644cdSLuigi Rizzo 
235a9e644cdSLuigi Rizzo static device_method_t ptnet_methods[] = {
236a9e644cdSLuigi Rizzo 	DEVMETHOD(device_probe,			ptnet_probe),
237a9e644cdSLuigi Rizzo 	DEVMETHOD(device_attach,		ptnet_attach),
238a9e644cdSLuigi Rizzo 	DEVMETHOD(device_detach,		ptnet_detach),
239a9e644cdSLuigi Rizzo 	DEVMETHOD(device_suspend,		ptnet_suspend),
240a9e644cdSLuigi Rizzo 	DEVMETHOD(device_resume,		ptnet_resume),
241a9e644cdSLuigi Rizzo 	DEVMETHOD(device_shutdown,		ptnet_shutdown),
242a9e644cdSLuigi Rizzo 	DEVMETHOD_END
243a9e644cdSLuigi Rizzo };
244a9e644cdSLuigi Rizzo 
245a9e644cdSLuigi Rizzo static driver_t ptnet_driver = {
246a9e644cdSLuigi Rizzo 	"ptnet",
247a9e644cdSLuigi Rizzo 	ptnet_methods,
248a9e644cdSLuigi Rizzo 	sizeof(struct ptnet_softc)
249a9e644cdSLuigi Rizzo };
250a9e644cdSLuigi Rizzo 
251a9e644cdSLuigi Rizzo /* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */
252a9e644cdSLuigi Rizzo static devclass_t ptnet_devclass;
253a9e644cdSLuigi Rizzo DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, ptnet_devclass,
254a9e644cdSLuigi Rizzo 		      NULL, NULL, SI_ORDER_MIDDLE + 2);
255a9e644cdSLuigi Rizzo 
256a9e644cdSLuigi Rizzo static int
257a9e644cdSLuigi Rizzo ptnet_probe(device_t dev)
258a9e644cdSLuigi Rizzo {
259a9e644cdSLuigi Rizzo 	if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID ||
260a9e644cdSLuigi Rizzo 		pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) {
261a9e644cdSLuigi Rizzo 		return (ENXIO);
262a9e644cdSLuigi Rizzo 	}
263a9e644cdSLuigi Rizzo 
264a9e644cdSLuigi Rizzo 	device_set_desc(dev, "ptnet network adapter");
265a9e644cdSLuigi Rizzo 
266a9e644cdSLuigi Rizzo 	return (BUS_PROBE_DEFAULT);
267a9e644cdSLuigi Rizzo }
268a9e644cdSLuigi Rizzo 
269a9e644cdSLuigi Rizzo static inline void ptnet_kick(struct ptnet_queue *pq)
270a9e644cdSLuigi Rizzo {
271a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
272a9e644cdSLuigi Rizzo 	pq->stats.kicks ++;
273a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */
274a9e644cdSLuigi Rizzo 	bus_write_4(pq->sc->iomem, pq->kick, 0);
275a9e644cdSLuigi Rizzo }
276a9e644cdSLuigi Rizzo 
277a9e644cdSLuigi Rizzo #define PTNET_BUF_RING_SIZE	4096
278a9e644cdSLuigi Rizzo #define PTNET_RX_BUDGET		512
279a9e644cdSLuigi Rizzo #define PTNET_RX_BATCH		1
280a9e644cdSLuigi Rizzo #define PTNET_TX_BUDGET		512
281a9e644cdSLuigi Rizzo #define PTNET_TX_BATCH		64
282a9e644cdSLuigi Rizzo #define PTNET_HDR_SIZE		sizeof(struct virtio_net_hdr_mrg_rxbuf)
283a9e644cdSLuigi Rizzo #define PTNET_MAX_PKT_SIZE	65536
284a9e644cdSLuigi Rizzo 
285d337c8c7SVincenzo Maffione #define PTNET_CSUM_OFFLOAD	(CSUM_TCP | CSUM_UDP)
286d337c8c7SVincenzo Maffione #define PTNET_CSUM_OFFLOAD_IPV6	(CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
287a9e644cdSLuigi Rizzo #define PTNET_ALL_OFFLOAD	(CSUM_TSO | PTNET_CSUM_OFFLOAD |\
288a9e644cdSLuigi Rizzo 				 PTNET_CSUM_OFFLOAD_IPV6)
289a9e644cdSLuigi Rizzo 
290a9e644cdSLuigi Rizzo static int
291a9e644cdSLuigi Rizzo ptnet_attach(device_t dev)
292a9e644cdSLuigi Rizzo {
293844a6f0cSLuigi Rizzo 	uint32_t ptfeatures = 0;
294a9e644cdSLuigi Rizzo 	unsigned int num_rx_rings, num_tx_rings;
295a9e644cdSLuigi Rizzo 	struct netmap_adapter na_arg;
296a9e644cdSLuigi Rizzo 	unsigned int nifp_offset;
297a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc;
298a9e644cdSLuigi Rizzo 	if_t ifp;
299a9e644cdSLuigi Rizzo 	uint32_t macreg;
300a9e644cdSLuigi Rizzo 	int err, rid;
301a9e644cdSLuigi Rizzo 	int i;
302a9e644cdSLuigi Rizzo 
303a9e644cdSLuigi Rizzo 	sc = device_get_softc(dev);
304a9e644cdSLuigi Rizzo 	sc->dev = dev;
305a9e644cdSLuigi Rizzo 
306a9e644cdSLuigi Rizzo 	/* Setup PCI resources. */
307a9e644cdSLuigi Rizzo 	pci_enable_busmaster(dev);
308a9e644cdSLuigi Rizzo 
309a9e644cdSLuigi Rizzo 	rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR);
310a9e644cdSLuigi Rizzo 	sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
311a9e644cdSLuigi Rizzo 					   RF_ACTIVE);
312a9e644cdSLuigi Rizzo 	if (sc->iomem == NULL) {
313a9e644cdSLuigi Rizzo 		device_printf(dev, "Failed to map I/O BAR\n");
314a9e644cdSLuigi Rizzo 		return (ENXIO);
315a9e644cdSLuigi Rizzo 	}
316a9e644cdSLuigi Rizzo 
317844a6f0cSLuigi Rizzo 	/* Negotiate features with the hypervisor. */
318a9e644cdSLuigi Rizzo 	if (ptnet_vnet_hdr) {
319a9e644cdSLuigi Rizzo 		ptfeatures |= PTNETMAP_F_VNET_HDR;
320a9e644cdSLuigi Rizzo 	}
321a9e644cdSLuigi Rizzo 	bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */
322a9e644cdSLuigi Rizzo 	ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */
323a9e644cdSLuigi Rizzo 	sc->ptfeatures = ptfeatures;
324a9e644cdSLuigi Rizzo 
32546023447SVincenzo Maffione 	num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
32646023447SVincenzo Maffione 	num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
32746023447SVincenzo Maffione 	sc->num_rings = num_tx_rings + num_rx_rings;
32846023447SVincenzo Maffione 	sc->num_tx_rings = num_tx_rings;
32946023447SVincenzo Maffione 
330b6e66be2SVincenzo Maffione 	if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) {
33146023447SVincenzo Maffione 		device_printf(dev, "CSB cannot handle that many rings (%u)\n",
33246023447SVincenzo Maffione 				sc->num_rings);
33346023447SVincenzo Maffione 		err = ENOMEM;
33446023447SVincenzo Maffione 		goto err_path;
33546023447SVincenzo Maffione 	}
33646023447SVincenzo Maffione 
33746023447SVincenzo Maffione 	/* Allocate CSB and carry out CSB allocation protocol. */
33846023447SVincenzo Maffione 	sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO,
33946023447SVincenzo Maffione 				  (size_t)0, -1UL, PAGE_SIZE, 0);
34046023447SVincenzo Maffione 	if (sc->csb_gh == NULL) {
341a9e644cdSLuigi Rizzo 		device_printf(dev, "Failed to allocate CSB\n");
342a9e644cdSLuigi Rizzo 		err = ENOMEM;
343a9e644cdSLuigi Rizzo 		goto err_path;
344a9e644cdSLuigi Rizzo 	}
345b6e66be2SVincenzo Maffione 	sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE);
346a9e644cdSLuigi Rizzo 
347a9e644cdSLuigi Rizzo 	{
348a2a74091SLuigi Rizzo 		/*
349a2a74091SLuigi Rizzo 		 * We use uint64_t rather than vm_paddr_t since we
350a2a74091SLuigi Rizzo 		 * need 64 bit addresses even on 32 bit platforms.
351a2a74091SLuigi Rizzo 		 */
35246023447SVincenzo Maffione 		uint64_t paddr = vtophys(sc->csb_gh);
353a9e644cdSLuigi Rizzo 
35446023447SVincenzo Maffione 		/* CSB allocation protocol: write to BAH first, then
35546023447SVincenzo Maffione 		 * to BAL (for both GH and HG sections). */
35646023447SVincenzo Maffione 		bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH,
357a9e644cdSLuigi Rizzo 				(paddr >> 32) & 0xffffffff);
35846023447SVincenzo Maffione 		bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL,
35946023447SVincenzo Maffione 				paddr & 0xffffffff);
36046023447SVincenzo Maffione 		paddr = vtophys(sc->csb_hg);
36146023447SVincenzo Maffione 		bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH,
36246023447SVincenzo Maffione 				(paddr >> 32) & 0xffffffff);
36346023447SVincenzo Maffione 		bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL,
36446023447SVincenzo Maffione 				paddr & 0xffffffff);
365a9e644cdSLuigi Rizzo 	}
366a9e644cdSLuigi Rizzo 
367a9e644cdSLuigi Rizzo 	/* Allocate and initialize per-queue data structures. */
368ac2fffa4SPedro F. Giffuni 	sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings,
369a9e644cdSLuigi Rizzo 			    M_DEVBUF, M_NOWAIT | M_ZERO);
370a9e644cdSLuigi Rizzo 	if (sc->queues == NULL) {
371a9e644cdSLuigi Rizzo 		err = ENOMEM;
372a9e644cdSLuigi Rizzo 		goto err_path;
373a9e644cdSLuigi Rizzo 	}
374a9e644cdSLuigi Rizzo 	sc->rxqueues = sc->queues + num_tx_rings;
375a9e644cdSLuigi Rizzo 
376a9e644cdSLuigi Rizzo 	for (i = 0; i < sc->num_rings; i++) {
377a9e644cdSLuigi Rizzo 		struct ptnet_queue *pq = sc->queues + i;
378a9e644cdSLuigi Rizzo 
379a9e644cdSLuigi Rizzo 		pq->sc = sc;
380a9e644cdSLuigi Rizzo 		pq->kring_id = i;
381a9e644cdSLuigi Rizzo 		pq->kick = PTNET_IO_KICK_BASE + 4 * i;
382b6e66be2SVincenzo Maffione 		pq->atok = sc->csb_gh + i;
383b6e66be2SVincenzo Maffione 		pq->ktoa = sc->csb_hg + i;
384a9e644cdSLuigi Rizzo 		snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d",
385a9e644cdSLuigi Rizzo 			 device_get_nameunit(dev), i);
386a9e644cdSLuigi Rizzo 		mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF);
387a9e644cdSLuigi Rizzo 		if (i >= num_tx_rings) {
388a9e644cdSLuigi Rizzo 			/* RX queue: fix kring_id. */
389a9e644cdSLuigi Rizzo 			pq->kring_id -= num_tx_rings;
390a9e644cdSLuigi Rizzo 		} else {
391a9e644cdSLuigi Rizzo 			/* TX queue: allocate buf_ring. */
392a9e644cdSLuigi Rizzo 			pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE,
393a9e644cdSLuigi Rizzo 						M_DEVBUF, M_NOWAIT, &pq->lock);
394a9e644cdSLuigi Rizzo 			if (pq->bufring == NULL) {
395a9e644cdSLuigi Rizzo 				err = ENOMEM;
396a9e644cdSLuigi Rizzo 				goto err_path;
397a9e644cdSLuigi Rizzo 			}
398a9e644cdSLuigi Rizzo 		}
399a9e644cdSLuigi Rizzo 	}
400a9e644cdSLuigi Rizzo 
401a9e644cdSLuigi Rizzo 	sc->min_tx_space = 64; /* Safe initial value. */
402a9e644cdSLuigi Rizzo 
403a9e644cdSLuigi Rizzo 	err = ptnet_irqs_init(sc);
404a9e644cdSLuigi Rizzo 	if (err) {
405a9e644cdSLuigi Rizzo 		goto err_path;
406a9e644cdSLuigi Rizzo 	}
407a9e644cdSLuigi Rizzo 
408a9e644cdSLuigi Rizzo 	/* Setup Ethernet interface. */
409a9e644cdSLuigi Rizzo 	sc->ifp = ifp = if_alloc(IFT_ETHER);
410a9e644cdSLuigi Rizzo 	if (ifp == NULL) {
411a9e644cdSLuigi Rizzo 		device_printf(dev, "Failed to allocate ifnet\n");
412a9e644cdSLuigi Rizzo 		err = ENOMEM;
413a9e644cdSLuigi Rizzo 		goto err_path;
414a9e644cdSLuigi Rizzo 	}
415a9e644cdSLuigi Rizzo 
416a9e644cdSLuigi Rizzo 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
417a9e644cdSLuigi Rizzo 	ifp->if_baudrate = IF_Gbps(10);
418a9e644cdSLuigi Rizzo 	ifp->if_softc = sc;
419a9e644cdSLuigi Rizzo 	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
420a9e644cdSLuigi Rizzo 	ifp->if_init = ptnet_init;
421a9e644cdSLuigi Rizzo 	ifp->if_ioctl = ptnet_ioctl;
422a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000
423a9e644cdSLuigi Rizzo 	ifp->if_get_counter = ptnet_get_counter;
424a9e644cdSLuigi Rizzo #endif
425a9e644cdSLuigi Rizzo 	ifp->if_transmit = ptnet_transmit;
426a9e644cdSLuigi Rizzo 	ifp->if_qflush = ptnet_qflush;
427a9e644cdSLuigi Rizzo 
428a9e644cdSLuigi Rizzo 	ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change,
429a9e644cdSLuigi Rizzo 		     ptnet_media_status);
430a9e644cdSLuigi Rizzo 	ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
431a9e644cdSLuigi Rizzo 	ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX);
432a9e644cdSLuigi Rizzo 
433a9e644cdSLuigi Rizzo 	macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI);
434a9e644cdSLuigi Rizzo 	sc->hwaddr[0] = (macreg >> 8) & 0xff;
435a9e644cdSLuigi Rizzo 	sc->hwaddr[1] = macreg & 0xff;
436a9e644cdSLuigi Rizzo 	macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO);
437a9e644cdSLuigi Rizzo 	sc->hwaddr[2] = (macreg >> 24) & 0xff;
438a9e644cdSLuigi Rizzo 	sc->hwaddr[3] = (macreg >> 16) & 0xff;
439a9e644cdSLuigi Rizzo 	sc->hwaddr[4] = (macreg >> 8) & 0xff;
440a9e644cdSLuigi Rizzo 	sc->hwaddr[5] = macreg & 0xff;
441a9e644cdSLuigi Rizzo 
442a9e644cdSLuigi Rizzo 	ether_ifattach(ifp, sc->hwaddr);
443a9e644cdSLuigi Rizzo 
444a9e644cdSLuigi Rizzo 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
445a9e644cdSLuigi Rizzo 	ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
446a9e644cdSLuigi Rizzo 
447a9e644cdSLuigi Rizzo 	if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) {
448a9e644cdSLuigi Rizzo 		/* Similarly to what the vtnet driver does, we can emulate
449a9e644cdSLuigi Rizzo 		 * VLAN offloadings by inserting and removing the 802.1Q
450a9e644cdSLuigi Rizzo 		 * header during transmit and receive. We are then able
451a9e644cdSLuigi Rizzo 		 * to do checksum offloading of VLAN frames. */
452a9e644cdSLuigi Rizzo 		ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6
453a9e644cdSLuigi Rizzo 					| IFCAP_VLAN_HWCSUM
454a9e644cdSLuigi Rizzo 					| IFCAP_TSO | IFCAP_LRO
455a9e644cdSLuigi Rizzo 					| IFCAP_VLAN_HWTSO
456a9e644cdSLuigi Rizzo 					| IFCAP_VLAN_HWTAGGING;
457a9e644cdSLuigi Rizzo 	}
458a9e644cdSLuigi Rizzo 
459a9e644cdSLuigi Rizzo 	ifp->if_capenable = ifp->if_capabilities;
460a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING
461a9e644cdSLuigi Rizzo 	/* Don't enable polling by default. */
462a9e644cdSLuigi Rizzo 	ifp->if_capabilities |= IFCAP_POLLING;
463a9e644cdSLuigi Rizzo #endif
464a9e644cdSLuigi Rizzo 	snprintf(sc->lock_name, sizeof(sc->lock_name),
465a9e644cdSLuigi Rizzo 		 "%s", device_get_nameunit(dev));
466a9e644cdSLuigi Rizzo 	mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF);
467a9e644cdSLuigi Rizzo 	callout_init_mtx(&sc->tick, &sc->lock, 0);
468a9e644cdSLuigi Rizzo 
469a9e644cdSLuigi Rizzo 	/* Prepare a netmap_adapter struct instance to do netmap_attach(). */
470a9e644cdSLuigi Rizzo 	nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS);
471a9e644cdSLuigi Rizzo 	memset(&na_arg, 0, sizeof(na_arg));
472a9e644cdSLuigi Rizzo 	na_arg.ifp = ifp;
473a9e644cdSLuigi Rizzo 	na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
474a9e644cdSLuigi Rizzo 	na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
475a9e644cdSLuigi Rizzo 	na_arg.num_tx_rings = num_tx_rings;
476a9e644cdSLuigi Rizzo 	na_arg.num_rx_rings = num_rx_rings;
477a9e644cdSLuigi Rizzo 	na_arg.nm_config = ptnet_nm_config;
478a9e644cdSLuigi Rizzo 	na_arg.nm_krings_create = ptnet_nm_krings_create;
479a9e644cdSLuigi Rizzo 	na_arg.nm_krings_delete = ptnet_nm_krings_delete;
480a9e644cdSLuigi Rizzo 	na_arg.nm_dtor = ptnet_nm_dtor;
4814f80b14cSVincenzo Maffione 	na_arg.nm_intr = ptnet_nm_intr;
482a9e644cdSLuigi Rizzo 	na_arg.nm_register = ptnet_nm_register;
483a9e644cdSLuigi Rizzo 	na_arg.nm_txsync = ptnet_nm_txsync;
484a9e644cdSLuigi Rizzo 	na_arg.nm_rxsync = ptnet_nm_rxsync;
485a9e644cdSLuigi Rizzo 
48646023447SVincenzo Maffione 	netmap_pt_guest_attach(&na_arg, nifp_offset,
487844a6f0cSLuigi Rizzo                                 bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID));
488a9e644cdSLuigi Rizzo 
489a9e644cdSLuigi Rizzo 	/* Now a netmap adapter for this ifp has been allocated, and it
490a9e644cdSLuigi Rizzo 	 * can be accessed through NA(ifp). We also have to initialize the CSB
491a9e644cdSLuigi Rizzo 	 * pointer. */
492a9e644cdSLuigi Rizzo 	sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp);
493a9e644cdSLuigi Rizzo 
494a9e644cdSLuigi Rizzo 	/* If virtio-net header was negotiated, set the virt_hdr_len field in
495a9e644cdSLuigi Rizzo 	 * the netmap adapter, to inform users that this netmap adapter requires
496a9e644cdSLuigi Rizzo 	 * the application to deal with the headers. */
497a9e644cdSLuigi Rizzo 	ptnet_update_vnet_hdr(sc);
498a9e644cdSLuigi Rizzo 
499a9e644cdSLuigi Rizzo 	device_printf(dev, "%s() completed\n", __func__);
500a9e644cdSLuigi Rizzo 
501a9e644cdSLuigi Rizzo 	return (0);
502a9e644cdSLuigi Rizzo 
503a9e644cdSLuigi Rizzo err_path:
504a9e644cdSLuigi Rizzo 	ptnet_detach(dev);
505a9e644cdSLuigi Rizzo 	return err;
506a9e644cdSLuigi Rizzo }
507a9e644cdSLuigi Rizzo 
508b6e66be2SVincenzo Maffione /* Stop host sync-kloop if it was running. */
509b6e66be2SVincenzo Maffione static void
510b6e66be2SVincenzo Maffione ptnet_device_shutdown(struct ptnet_softc *sc)
511b6e66be2SVincenzo Maffione {
512b6e66be2SVincenzo Maffione 	ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
513b6e66be2SVincenzo Maffione 	bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0);
514b6e66be2SVincenzo Maffione 	bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0);
515b6e66be2SVincenzo Maffione 	bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0);
516b6e66be2SVincenzo Maffione 	bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0);
517b6e66be2SVincenzo Maffione }
518b6e66be2SVincenzo Maffione 
519a9e644cdSLuigi Rizzo static int
520a9e644cdSLuigi Rizzo ptnet_detach(device_t dev)
521a9e644cdSLuigi Rizzo {
522a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = device_get_softc(dev);
523a9e644cdSLuigi Rizzo 	int i;
524a9e644cdSLuigi Rizzo 
525b6e66be2SVincenzo Maffione 	ptnet_device_shutdown(sc);
526b6e66be2SVincenzo Maffione 
527a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING
528a9e644cdSLuigi Rizzo 	if (sc->ifp->if_capenable & IFCAP_POLLING) {
529a9e644cdSLuigi Rizzo 		ether_poll_deregister(sc->ifp);
530a9e644cdSLuigi Rizzo 	}
531a9e644cdSLuigi Rizzo #endif
532a9e644cdSLuigi Rizzo 	callout_drain(&sc->tick);
533a9e644cdSLuigi Rizzo 
534a9e644cdSLuigi Rizzo 	if (sc->queues) {
535a9e644cdSLuigi Rizzo 		/* Drain taskqueues before calling if_detach. */
536a9e644cdSLuigi Rizzo 		for (i = 0; i < sc->num_rings; i++) {
537a9e644cdSLuigi Rizzo 			struct ptnet_queue *pq = sc->queues + i;
538a9e644cdSLuigi Rizzo 
539a9e644cdSLuigi Rizzo 			if (pq->taskq) {
540a9e644cdSLuigi Rizzo 				taskqueue_drain(pq->taskq, &pq->task);
541a9e644cdSLuigi Rizzo 			}
542a9e644cdSLuigi Rizzo 		}
543a9e644cdSLuigi Rizzo 	}
544a9e644cdSLuigi Rizzo 
545a9e644cdSLuigi Rizzo 	if (sc->ifp) {
546a9e644cdSLuigi Rizzo 		ether_ifdetach(sc->ifp);
547a9e644cdSLuigi Rizzo 
548a9e644cdSLuigi Rizzo 		/* Uninitialize netmap adapters for this device. */
549a9e644cdSLuigi Rizzo 		netmap_detach(sc->ifp);
550a9e644cdSLuigi Rizzo 
551a9e644cdSLuigi Rizzo 		ifmedia_removeall(&sc->media);
552a9e644cdSLuigi Rizzo 		if_free(sc->ifp);
553a9e644cdSLuigi Rizzo 		sc->ifp = NULL;
554a9e644cdSLuigi Rizzo 	}
555a9e644cdSLuigi Rizzo 
556a9e644cdSLuigi Rizzo 	ptnet_irqs_fini(sc);
557a9e644cdSLuigi Rizzo 
55846023447SVincenzo Maffione 	if (sc->csb_gh) {
55946023447SVincenzo Maffione 		contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF);
56046023447SVincenzo Maffione 		sc->csb_gh = NULL;
56146023447SVincenzo Maffione 		sc->csb_hg = NULL;
562a9e644cdSLuigi Rizzo 	}
563a9e644cdSLuigi Rizzo 
564a9e644cdSLuigi Rizzo 	if (sc->queues) {
565a9e644cdSLuigi Rizzo 		for (i = 0; i < sc->num_rings; i++) {
566a9e644cdSLuigi Rizzo 			struct ptnet_queue *pq = sc->queues + i;
567a9e644cdSLuigi Rizzo 
568a9e644cdSLuigi Rizzo 			if (mtx_initialized(&pq->lock)) {
569a9e644cdSLuigi Rizzo 				mtx_destroy(&pq->lock);
570a9e644cdSLuigi Rizzo 			}
571a9e644cdSLuigi Rizzo 			if (pq->bufring != NULL) {
572a9e644cdSLuigi Rizzo 				buf_ring_free(pq->bufring, M_DEVBUF);
573a9e644cdSLuigi Rizzo 			}
574a9e644cdSLuigi Rizzo 		}
575a9e644cdSLuigi Rizzo 		free(sc->queues, M_DEVBUF);
576a9e644cdSLuigi Rizzo 		sc->queues = NULL;
577a9e644cdSLuigi Rizzo 	}
578a9e644cdSLuigi Rizzo 
579a9e644cdSLuigi Rizzo 	if (sc->iomem) {
580a9e644cdSLuigi Rizzo 		bus_release_resource(dev, SYS_RES_IOPORT,
581a9e644cdSLuigi Rizzo 				     PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem);
582a9e644cdSLuigi Rizzo 		sc->iomem = NULL;
583a9e644cdSLuigi Rizzo 	}
584a9e644cdSLuigi Rizzo 
585a9e644cdSLuigi Rizzo 	mtx_destroy(&sc->lock);
586a9e644cdSLuigi Rizzo 
587a9e644cdSLuigi Rizzo 	device_printf(dev, "%s() completed\n", __func__);
588a9e644cdSLuigi Rizzo 
589a9e644cdSLuigi Rizzo 	return (0);
590a9e644cdSLuigi Rizzo }
591a9e644cdSLuigi Rizzo 
592a9e644cdSLuigi Rizzo static int
593a9e644cdSLuigi Rizzo ptnet_suspend(device_t dev)
594a9e644cdSLuigi Rizzo {
595b6e66be2SVincenzo Maffione 	struct ptnet_softc *sc = device_get_softc(dev);
596a9e644cdSLuigi Rizzo 
597a9e644cdSLuigi Rizzo 	(void)sc;
598a9e644cdSLuigi Rizzo 
599a9e644cdSLuigi Rizzo 	return (0);
600a9e644cdSLuigi Rizzo }
601a9e644cdSLuigi Rizzo 
602a9e644cdSLuigi Rizzo static int
603a9e644cdSLuigi Rizzo ptnet_resume(device_t dev)
604a9e644cdSLuigi Rizzo {
605b6e66be2SVincenzo Maffione 	struct ptnet_softc *sc = device_get_softc(dev);
606a9e644cdSLuigi Rizzo 
607a9e644cdSLuigi Rizzo 	(void)sc;
608a9e644cdSLuigi Rizzo 
609a9e644cdSLuigi Rizzo 	return (0);
610a9e644cdSLuigi Rizzo }
611a9e644cdSLuigi Rizzo 
612a9e644cdSLuigi Rizzo static int
613a9e644cdSLuigi Rizzo ptnet_shutdown(device_t dev)
614a9e644cdSLuigi Rizzo {
615b6e66be2SVincenzo Maffione 	struct ptnet_softc *sc = device_get_softc(dev);
616b6e66be2SVincenzo Maffione 
617b6e66be2SVincenzo Maffione 	ptnet_device_shutdown(sc);
618b6e66be2SVincenzo Maffione 
619b6e66be2SVincenzo Maffione 	return (0);
620a9e644cdSLuigi Rizzo }
621a9e644cdSLuigi Rizzo 
622a9e644cdSLuigi Rizzo static int
623a9e644cdSLuigi Rizzo ptnet_irqs_init(struct ptnet_softc *sc)
624a9e644cdSLuigi Rizzo {
625a9e644cdSLuigi Rizzo 	int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR);
626a9e644cdSLuigi Rizzo 	int nvecs = sc->num_rings;
627a9e644cdSLuigi Rizzo 	device_t dev = sc->dev;
628a9e644cdSLuigi Rizzo 	int err = ENOSPC;
629a9e644cdSLuigi Rizzo 	int cpu_cur;
630a9e644cdSLuigi Rizzo 	int i;
631a9e644cdSLuigi Rizzo 
632a9e644cdSLuigi Rizzo 	if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0)  {
633a9e644cdSLuigi Rizzo 		device_printf(dev, "Could not find MSI-X capability\n");
634a9e644cdSLuigi Rizzo 		return (ENXIO);
635a9e644cdSLuigi Rizzo 	}
636a9e644cdSLuigi Rizzo 
637a9e644cdSLuigi Rizzo 	sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
638a9e644cdSLuigi Rizzo 					      &rid, RF_ACTIVE);
639a9e644cdSLuigi Rizzo 	if (sc->msix_mem == NULL) {
640a9e644cdSLuigi Rizzo 		device_printf(dev, "Failed to allocate MSIX PCI BAR\n");
641a9e644cdSLuigi Rizzo 		return (ENXIO);
642a9e644cdSLuigi Rizzo 	}
643a9e644cdSLuigi Rizzo 
644a9e644cdSLuigi Rizzo 	if (pci_msix_count(dev) < nvecs) {
645a9e644cdSLuigi Rizzo 		device_printf(dev, "Not enough MSI-X vectors\n");
646a9e644cdSLuigi Rizzo 		goto err_path;
647a9e644cdSLuigi Rizzo 	}
648a9e644cdSLuigi Rizzo 
649a9e644cdSLuigi Rizzo 	err = pci_alloc_msix(dev, &nvecs);
650a9e644cdSLuigi Rizzo 	if (err) {
651a9e644cdSLuigi Rizzo 		device_printf(dev, "Failed to allocate MSI-X vectors\n");
652a9e644cdSLuigi Rizzo 		goto err_path;
653a9e644cdSLuigi Rizzo 	}
654a9e644cdSLuigi Rizzo 
655a9e644cdSLuigi Rizzo 	for (i = 0; i < nvecs; i++) {
656a9e644cdSLuigi Rizzo 		struct ptnet_queue *pq = sc->queues + i;
657a9e644cdSLuigi Rizzo 
658a9e644cdSLuigi Rizzo 		rid = i + 1;
659a9e644cdSLuigi Rizzo 		pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
660a9e644cdSLuigi Rizzo 						 RF_ACTIVE);
661a9e644cdSLuigi Rizzo 		if (pq->irq == NULL) {
662a9e644cdSLuigi Rizzo 			device_printf(dev, "Failed to allocate interrupt "
663a9e644cdSLuigi Rizzo 					   "for queue #%d\n", i);
664a9e644cdSLuigi Rizzo 			err = ENOSPC;
665a9e644cdSLuigi Rizzo 			goto err_path;
666a9e644cdSLuigi Rizzo 		}
667a9e644cdSLuigi Rizzo 	}
668a9e644cdSLuigi Rizzo 
669a9e644cdSLuigi Rizzo 	cpu_cur = CPU_FIRST();
670a9e644cdSLuigi Rizzo 	for (i = 0; i < nvecs; i++) {
671a9e644cdSLuigi Rizzo 		struct ptnet_queue *pq = sc->queues + i;
672a9e644cdSLuigi Rizzo 		void (*handler)(void *) = ptnet_tx_intr;
673a9e644cdSLuigi Rizzo 
674a9e644cdSLuigi Rizzo 		if (i >= sc->num_tx_rings) {
675a9e644cdSLuigi Rizzo 			handler = ptnet_rx_intr;
676a9e644cdSLuigi Rizzo 		}
677a9e644cdSLuigi Rizzo 		err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE,
678a9e644cdSLuigi Rizzo 				     NULL /* intr_filter */, handler,
679a9e644cdSLuigi Rizzo 				     pq, &pq->cookie);
680a9e644cdSLuigi Rizzo 		if (err) {
681a9e644cdSLuigi Rizzo 			device_printf(dev, "Failed to register intr handler "
682a9e644cdSLuigi Rizzo 					   "for queue #%d\n", i);
683a9e644cdSLuigi Rizzo 			goto err_path;
684a9e644cdSLuigi Rizzo 		}
685a9e644cdSLuigi Rizzo 
686a9e644cdSLuigi Rizzo 		bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i);
687a9e644cdSLuigi Rizzo #if 0
688a9e644cdSLuigi Rizzo 		bus_bind_intr(sc->dev, pq->irq, cpu_cur);
689a9e644cdSLuigi Rizzo #endif
690a9e644cdSLuigi Rizzo 		cpu_cur = CPU_NEXT(cpu_cur);
691a9e644cdSLuigi Rizzo 	}
692a9e644cdSLuigi Rizzo 
693a9e644cdSLuigi Rizzo 	device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs);
694a9e644cdSLuigi Rizzo 
695a9e644cdSLuigi Rizzo 	cpu_cur = CPU_FIRST();
696a9e644cdSLuigi Rizzo 	for (i = 0; i < nvecs; i++) {
697a9e644cdSLuigi Rizzo 		struct ptnet_queue *pq = sc->queues + i;
698a9e644cdSLuigi Rizzo 		static void (*handler)(void *context, int pending);
699a9e644cdSLuigi Rizzo 
700a9e644cdSLuigi Rizzo 		handler = (i < sc->num_tx_rings) ? ptnet_tx_task : ptnet_rx_task;
701a9e644cdSLuigi Rizzo 
702a9e644cdSLuigi Rizzo 		TASK_INIT(&pq->task, 0, handler, pq);
703a9e644cdSLuigi Rizzo 		pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT,
704a9e644cdSLuigi Rizzo 					taskqueue_thread_enqueue, &pq->taskq);
705a9e644cdSLuigi Rizzo 		taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d",
706a9e644cdSLuigi Rizzo 					device_get_nameunit(sc->dev), cpu_cur);
707a9e644cdSLuigi Rizzo 		cpu_cur = CPU_NEXT(cpu_cur);
708a9e644cdSLuigi Rizzo 	}
709a9e644cdSLuigi Rizzo 
710a9e644cdSLuigi Rizzo 	return 0;
711a9e644cdSLuigi Rizzo err_path:
712a9e644cdSLuigi Rizzo 	ptnet_irqs_fini(sc);
713a9e644cdSLuigi Rizzo 	return err;
714a9e644cdSLuigi Rizzo }
715a9e644cdSLuigi Rizzo 
716a9e644cdSLuigi Rizzo static void
717a9e644cdSLuigi Rizzo ptnet_irqs_fini(struct ptnet_softc *sc)
718a9e644cdSLuigi Rizzo {
719a9e644cdSLuigi Rizzo 	device_t dev = sc->dev;
720a9e644cdSLuigi Rizzo 	int i;
721a9e644cdSLuigi Rizzo 
722a9e644cdSLuigi Rizzo 	for (i = 0; i < sc->num_rings; i++) {
723a9e644cdSLuigi Rizzo 		struct ptnet_queue *pq = sc->queues + i;
724a9e644cdSLuigi Rizzo 
725a9e644cdSLuigi Rizzo 		if (pq->taskq) {
726a9e644cdSLuigi Rizzo 			taskqueue_free(pq->taskq);
727a9e644cdSLuigi Rizzo 			pq->taskq = NULL;
728a9e644cdSLuigi Rizzo 		}
729a9e644cdSLuigi Rizzo 
730a9e644cdSLuigi Rizzo 		if (pq->cookie) {
731a9e644cdSLuigi Rizzo 			bus_teardown_intr(dev, pq->irq, pq->cookie);
732a9e644cdSLuigi Rizzo 			pq->cookie = NULL;
733a9e644cdSLuigi Rizzo 		}
734a9e644cdSLuigi Rizzo 
735a9e644cdSLuigi Rizzo 		if (pq->irq) {
736a9e644cdSLuigi Rizzo 			bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq);
737a9e644cdSLuigi Rizzo 			pq->irq = NULL;
738a9e644cdSLuigi Rizzo 		}
739a9e644cdSLuigi Rizzo 	}
740a9e644cdSLuigi Rizzo 
741a9e644cdSLuigi Rizzo 	if (sc->msix_mem) {
742a9e644cdSLuigi Rizzo 		pci_release_msi(dev);
743a9e644cdSLuigi Rizzo 
744a9e644cdSLuigi Rizzo 		bus_release_resource(dev, SYS_RES_MEMORY,
745a9e644cdSLuigi Rizzo 				     PCIR_BAR(PTNETMAP_MSIX_PCI_BAR),
746a9e644cdSLuigi Rizzo 				     sc->msix_mem);
747a9e644cdSLuigi Rizzo 		sc->msix_mem = NULL;
748a9e644cdSLuigi Rizzo 	}
749a9e644cdSLuigi Rizzo }
750a9e644cdSLuigi Rizzo 
751a9e644cdSLuigi Rizzo static void
752a9e644cdSLuigi Rizzo ptnet_init(void *opaque)
753a9e644cdSLuigi Rizzo {
754a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = opaque;
755a9e644cdSLuigi Rizzo 
756a9e644cdSLuigi Rizzo 	PTNET_CORE_LOCK(sc);
757a9e644cdSLuigi Rizzo 	ptnet_init_locked(sc);
758a9e644cdSLuigi Rizzo 	PTNET_CORE_UNLOCK(sc);
759a9e644cdSLuigi Rizzo }
760a9e644cdSLuigi Rizzo 
761a9e644cdSLuigi Rizzo static int
762a9e644cdSLuigi Rizzo ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data)
763a9e644cdSLuigi Rizzo {
764a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(ifp);
765a9e644cdSLuigi Rizzo 	device_t dev = sc->dev;
766a9e644cdSLuigi Rizzo 	struct ifreq *ifr = (struct ifreq *)data;
76724a7d6d3SMatt Macy 	int mask __unused, err = 0;
768a9e644cdSLuigi Rizzo 
769a9e644cdSLuigi Rizzo 	switch (cmd) {
770a9e644cdSLuigi Rizzo 	case SIOCSIFFLAGS:
771a9e644cdSLuigi Rizzo 		device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags);
772a9e644cdSLuigi Rizzo 		PTNET_CORE_LOCK(sc);
773a9e644cdSLuigi Rizzo 		if (ifp->if_flags & IFF_UP) {
774a9e644cdSLuigi Rizzo 			/* Network stack wants the iff to be up. */
775a9e644cdSLuigi Rizzo 			err = ptnet_init_locked(sc);
776a9e644cdSLuigi Rizzo 		} else {
777a9e644cdSLuigi Rizzo 			/* Network stack wants the iff to be down. */
778a9e644cdSLuigi Rizzo 			err = ptnet_stop(sc);
779a9e644cdSLuigi Rizzo 		}
780a9e644cdSLuigi Rizzo 		/* We don't need to do nothing to support IFF_PROMISC,
781a9e644cdSLuigi Rizzo 		 * since that is managed by the backend port. */
782a9e644cdSLuigi Rizzo 		PTNET_CORE_UNLOCK(sc);
783a9e644cdSLuigi Rizzo 		break;
784a9e644cdSLuigi Rizzo 
785a9e644cdSLuigi Rizzo 	case SIOCSIFCAP:
786a9e644cdSLuigi Rizzo 		device_printf(dev, "SIOCSIFCAP %x %x\n",
787a9e644cdSLuigi Rizzo 			      ifr->ifr_reqcap, ifp->if_capenable);
788a9e644cdSLuigi Rizzo 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
789a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING
790a9e644cdSLuigi Rizzo 		if (mask & IFCAP_POLLING) {
791a9e644cdSLuigi Rizzo 			struct ptnet_queue *pq;
792a9e644cdSLuigi Rizzo 			int i;
793a9e644cdSLuigi Rizzo 
794a9e644cdSLuigi Rizzo 			if (ifr->ifr_reqcap & IFCAP_POLLING) {
795a9e644cdSLuigi Rizzo 				err = ether_poll_register(ptnet_poll, ifp);
796a9e644cdSLuigi Rizzo 				if (err) {
797a9e644cdSLuigi Rizzo 					break;
798a9e644cdSLuigi Rizzo 				}
799a9e644cdSLuigi Rizzo 				/* Stop queues and sync with taskqueues. */
800a9e644cdSLuigi Rizzo 				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
801a9e644cdSLuigi Rizzo 				for (i = 0; i < sc->num_rings; i++) {
802a9e644cdSLuigi Rizzo 					pq = sc-> queues + i;
803a9e644cdSLuigi Rizzo 					/* Make sure the worker sees the
804a9e644cdSLuigi Rizzo 					 * IFF_DRV_RUNNING down. */
805a9e644cdSLuigi Rizzo 					PTNET_Q_LOCK(pq);
806b6e66be2SVincenzo Maffione 					pq->atok->appl_need_kick = 0;
807a9e644cdSLuigi Rizzo 					PTNET_Q_UNLOCK(pq);
808a9e644cdSLuigi Rizzo 					/* Wait for rescheduling to finish. */
809a9e644cdSLuigi Rizzo 					if (pq->taskq) {
810a9e644cdSLuigi Rizzo 						taskqueue_drain(pq->taskq,
811a9e644cdSLuigi Rizzo 								&pq->task);
812a9e644cdSLuigi Rizzo 					}
813a9e644cdSLuigi Rizzo 				}
814a9e644cdSLuigi Rizzo 				ifp->if_drv_flags |= IFF_DRV_RUNNING;
815a9e644cdSLuigi Rizzo 			} else {
816a9e644cdSLuigi Rizzo 				err = ether_poll_deregister(ifp);
817a9e644cdSLuigi Rizzo 				for (i = 0; i < sc->num_rings; i++) {
818a9e644cdSLuigi Rizzo 					pq = sc-> queues + i;
819a9e644cdSLuigi Rizzo 					PTNET_Q_LOCK(pq);
820b6e66be2SVincenzo Maffione 					pq->atok->appl_need_kick = 1;
821a9e644cdSLuigi Rizzo 					PTNET_Q_UNLOCK(pq);
822a9e644cdSLuigi Rizzo 				}
823a9e644cdSLuigi Rizzo 			}
824a9e644cdSLuigi Rizzo 		}
825a9e644cdSLuigi Rizzo #endif  /* DEVICE_POLLING */
826a9e644cdSLuigi Rizzo 		ifp->if_capenable = ifr->ifr_reqcap;
827a9e644cdSLuigi Rizzo 		break;
828a9e644cdSLuigi Rizzo 
829a9e644cdSLuigi Rizzo 	case SIOCSIFMTU:
830a9e644cdSLuigi Rizzo 		/* We support any reasonable MTU. */
831a9e644cdSLuigi Rizzo 		if (ifr->ifr_mtu < ETHERMIN ||
832a9e644cdSLuigi Rizzo 				ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) {
833a9e644cdSLuigi Rizzo 			err = EINVAL;
834a9e644cdSLuigi Rizzo 		} else {
835a9e644cdSLuigi Rizzo 			PTNET_CORE_LOCK(sc);
836a9e644cdSLuigi Rizzo 			ifp->if_mtu = ifr->ifr_mtu;
837a9e644cdSLuigi Rizzo 			PTNET_CORE_UNLOCK(sc);
838a9e644cdSLuigi Rizzo 		}
839a9e644cdSLuigi Rizzo 		break;
840a9e644cdSLuigi Rizzo 
841a9e644cdSLuigi Rizzo 	case SIOCSIFMEDIA:
842a9e644cdSLuigi Rizzo 	case SIOCGIFMEDIA:
843a9e644cdSLuigi Rizzo 		err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
844a9e644cdSLuigi Rizzo 		break;
845a9e644cdSLuigi Rizzo 
846a9e644cdSLuigi Rizzo 	default:
847a9e644cdSLuigi Rizzo 		err = ether_ioctl(ifp, cmd, data);
848a9e644cdSLuigi Rizzo 		break;
849a9e644cdSLuigi Rizzo 	}
850a9e644cdSLuigi Rizzo 
851a9e644cdSLuigi Rizzo 	return err;
852a9e644cdSLuigi Rizzo }
853a9e644cdSLuigi Rizzo 
854a9e644cdSLuigi Rizzo static int
855a9e644cdSLuigi Rizzo ptnet_init_locked(struct ptnet_softc *sc)
856a9e644cdSLuigi Rizzo {
857a9e644cdSLuigi Rizzo 	if_t ifp = sc->ifp;
858a9e644cdSLuigi Rizzo 	struct netmap_adapter *na_dr = &sc->ptna->dr.up;
859a9e644cdSLuigi Rizzo 	struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
860a9e644cdSLuigi Rizzo 	unsigned int nm_buf_size;
861a9e644cdSLuigi Rizzo 	int ret;
862a9e644cdSLuigi Rizzo 
863a9e644cdSLuigi Rizzo 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
864a9e644cdSLuigi Rizzo 		return 0; /* nothing to do */
865a9e644cdSLuigi Rizzo 	}
866a9e644cdSLuigi Rizzo 
867a9e644cdSLuigi Rizzo 	device_printf(sc->dev, "%s\n", __func__);
868a9e644cdSLuigi Rizzo 
869a9e644cdSLuigi Rizzo 	/* Translate offload capabilities according to if_capenable. */
870a9e644cdSLuigi Rizzo 	ifp->if_hwassist = 0;
871a9e644cdSLuigi Rizzo 	if (ifp->if_capenable & IFCAP_TXCSUM)
872a9e644cdSLuigi Rizzo 		ifp->if_hwassist |= PTNET_CSUM_OFFLOAD;
873a9e644cdSLuigi Rizzo 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
874a9e644cdSLuigi Rizzo 		ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6;
875a9e644cdSLuigi Rizzo 	if (ifp->if_capenable & IFCAP_TSO4)
876a9e644cdSLuigi Rizzo 		ifp->if_hwassist |= CSUM_IP_TSO;
877a9e644cdSLuigi Rizzo 	if (ifp->if_capenable & IFCAP_TSO6)
878a9e644cdSLuigi Rizzo 		ifp->if_hwassist |= CSUM_IP6_TSO;
879a9e644cdSLuigi Rizzo 
880a9e644cdSLuigi Rizzo 	/*
881a9e644cdSLuigi Rizzo 	 * Prepare the interface for netmap mode access.
882a9e644cdSLuigi Rizzo 	 */
883a9e644cdSLuigi Rizzo 	netmap_update_config(na_dr);
884a9e644cdSLuigi Rizzo 
885a9e644cdSLuigi Rizzo 	ret = netmap_mem_finalize(na_dr->nm_mem, na_dr);
886a9e644cdSLuigi Rizzo 	if (ret) {
887a9e644cdSLuigi Rizzo 		device_printf(sc->dev, "netmap_mem_finalize() failed\n");
888a9e644cdSLuigi Rizzo 		return ret;
889a9e644cdSLuigi Rizzo 	}
890a9e644cdSLuigi Rizzo 
891b6e66be2SVincenzo Maffione 	if (sc->ptna->backend_users == 0) {
892a9e644cdSLuigi Rizzo 		ret = ptnet_nm_krings_create(na_nm);
893a9e644cdSLuigi Rizzo 		if (ret) {
894a9e644cdSLuigi Rizzo 			device_printf(sc->dev, "ptnet_nm_krings_create() "
895a9e644cdSLuigi Rizzo 					       "failed\n");
896a9e644cdSLuigi Rizzo 			goto err_mem_finalize;
897a9e644cdSLuigi Rizzo 		}
898a9e644cdSLuigi Rizzo 
899a9e644cdSLuigi Rizzo 		ret = netmap_mem_rings_create(na_dr);
900a9e644cdSLuigi Rizzo 		if (ret) {
901a9e644cdSLuigi Rizzo 			device_printf(sc->dev, "netmap_mem_rings_create() "
902a9e644cdSLuigi Rizzo 					       "failed\n");
903a9e644cdSLuigi Rizzo 			goto err_rings_create;
904a9e644cdSLuigi Rizzo 		}
905a9e644cdSLuigi Rizzo 
906a9e644cdSLuigi Rizzo 		ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut);
907a9e644cdSLuigi Rizzo 		if (ret) {
908a9e644cdSLuigi Rizzo 			device_printf(sc->dev, "netmap_mem_get_lut() "
909a9e644cdSLuigi Rizzo 					       "failed\n");
910a9e644cdSLuigi Rizzo 			goto err_get_lut;
911a9e644cdSLuigi Rizzo 		}
912a9e644cdSLuigi Rizzo 	}
913a9e644cdSLuigi Rizzo 
914a9e644cdSLuigi Rizzo 	ret = ptnet_nm_register(na_dr, 1 /* on */);
915a9e644cdSLuigi Rizzo 	if (ret) {
916a9e644cdSLuigi Rizzo 		goto err_register;
917a9e644cdSLuigi Rizzo 	}
918a9e644cdSLuigi Rizzo 
919a9e644cdSLuigi Rizzo 	nm_buf_size = NETMAP_BUF_SIZE(na_dr);
920a9e644cdSLuigi Rizzo 
921a9e644cdSLuigi Rizzo 	KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size"));
922a9e644cdSLuigi Rizzo 	sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2;
923a9e644cdSLuigi Rizzo 	device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__,
924a9e644cdSLuigi Rizzo 		      sc->min_tx_space);
925a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
926a9e644cdSLuigi Rizzo 	callout_reset(&sc->tick, hz, ptnet_tick, sc);
927a9e644cdSLuigi Rizzo #endif
928a9e644cdSLuigi Rizzo 
929a9e644cdSLuigi Rizzo 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
930a9e644cdSLuigi Rizzo 
931a9e644cdSLuigi Rizzo 	return 0;
932a9e644cdSLuigi Rizzo 
933a9e644cdSLuigi Rizzo err_register:
934a9e644cdSLuigi Rizzo 	memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut));
935a9e644cdSLuigi Rizzo err_get_lut:
936a9e644cdSLuigi Rizzo 	netmap_mem_rings_delete(na_dr);
937a9e644cdSLuigi Rizzo err_rings_create:
938a9e644cdSLuigi Rizzo 	ptnet_nm_krings_delete(na_nm);
939a9e644cdSLuigi Rizzo err_mem_finalize:
940a9e644cdSLuigi Rizzo 	netmap_mem_deref(na_dr->nm_mem, na_dr);
941a9e644cdSLuigi Rizzo 
942a9e644cdSLuigi Rizzo 	return ret;
943a9e644cdSLuigi Rizzo }
944a9e644cdSLuigi Rizzo 
945a9e644cdSLuigi Rizzo /* To be called under core lock. */
946a9e644cdSLuigi Rizzo static int
947a9e644cdSLuigi Rizzo ptnet_stop(struct ptnet_softc *sc)
948a9e644cdSLuigi Rizzo {
949a9e644cdSLuigi Rizzo 	if_t ifp = sc->ifp;
950a9e644cdSLuigi Rizzo 	struct netmap_adapter *na_dr = &sc->ptna->dr.up;
951a9e644cdSLuigi Rizzo 	struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
952a9e644cdSLuigi Rizzo 	int i;
953a9e644cdSLuigi Rizzo 
954a9e644cdSLuigi Rizzo 	device_printf(sc->dev, "%s\n", __func__);
955a9e644cdSLuigi Rizzo 
956a9e644cdSLuigi Rizzo 	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
957a9e644cdSLuigi Rizzo 		return 0; /* nothing to do */
958a9e644cdSLuigi Rizzo 	}
959a9e644cdSLuigi Rizzo 
960a9e644cdSLuigi Rizzo 	/* Clear the driver-ready flag, and synchronize with all the queues,
961a9e644cdSLuigi Rizzo 	 * so that after this loop we are sure nobody is working anymore with
962a9e644cdSLuigi Rizzo 	 * the device. This scheme is taken from the vtnet driver. */
963a9e644cdSLuigi Rizzo 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
964a9e644cdSLuigi Rizzo 	callout_stop(&sc->tick);
965a9e644cdSLuigi Rizzo 	for (i = 0; i < sc->num_rings; i++) {
966a9e644cdSLuigi Rizzo 		PTNET_Q_LOCK(sc->queues + i);
967a9e644cdSLuigi Rizzo 		PTNET_Q_UNLOCK(sc->queues + i);
968a9e644cdSLuigi Rizzo 	}
969a9e644cdSLuigi Rizzo 
970a9e644cdSLuigi Rizzo 	ptnet_nm_register(na_dr, 0 /* off */);
971a9e644cdSLuigi Rizzo 
972b6e66be2SVincenzo Maffione 	if (sc->ptna->backend_users == 0) {
973a9e644cdSLuigi Rizzo 		netmap_mem_rings_delete(na_dr);
974a9e644cdSLuigi Rizzo 		ptnet_nm_krings_delete(na_nm);
975a9e644cdSLuigi Rizzo 	}
976a9e644cdSLuigi Rizzo 	netmap_mem_deref(na_dr->nm_mem, na_dr);
977a9e644cdSLuigi Rizzo 
978a9e644cdSLuigi Rizzo 	return 0;
979a9e644cdSLuigi Rizzo }
980a9e644cdSLuigi Rizzo 
981a9e644cdSLuigi Rizzo static void
982a9e644cdSLuigi Rizzo ptnet_qflush(if_t ifp)
983a9e644cdSLuigi Rizzo {
984a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(ifp);
985a9e644cdSLuigi Rizzo 	int i;
986a9e644cdSLuigi Rizzo 
987a9e644cdSLuigi Rizzo 	/* Flush all the bufrings and do the interface flush. */
988a9e644cdSLuigi Rizzo 	for (i = 0; i < sc->num_rings; i++) {
989a9e644cdSLuigi Rizzo 		struct ptnet_queue *pq = sc->queues + i;
990a9e644cdSLuigi Rizzo 		struct mbuf *m;
991a9e644cdSLuigi Rizzo 
992a9e644cdSLuigi Rizzo 		PTNET_Q_LOCK(pq);
993a9e644cdSLuigi Rizzo 		if (pq->bufring) {
994a9e644cdSLuigi Rizzo 			while ((m = buf_ring_dequeue_sc(pq->bufring))) {
995a9e644cdSLuigi Rizzo 				m_freem(m);
996a9e644cdSLuigi Rizzo 			}
997a9e644cdSLuigi Rizzo 		}
998a9e644cdSLuigi Rizzo 		PTNET_Q_UNLOCK(pq);
999a9e644cdSLuigi Rizzo 	}
1000a9e644cdSLuigi Rizzo 
1001a9e644cdSLuigi Rizzo 	if_qflush(ifp);
1002a9e644cdSLuigi Rizzo }
1003a9e644cdSLuigi Rizzo 
1004a9e644cdSLuigi Rizzo static int
1005a9e644cdSLuigi Rizzo ptnet_media_change(if_t ifp)
1006a9e644cdSLuigi Rizzo {
1007a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(ifp);
1008a9e644cdSLuigi Rizzo 	struct ifmedia *ifm = &sc->media;
1009a9e644cdSLuigi Rizzo 
1010a9e644cdSLuigi Rizzo 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) {
1011a9e644cdSLuigi Rizzo 		return EINVAL;
1012a9e644cdSLuigi Rizzo 	}
1013a9e644cdSLuigi Rizzo 
1014a9e644cdSLuigi Rizzo 	return 0;
1015a9e644cdSLuigi Rizzo }
1016a9e644cdSLuigi Rizzo 
1017a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000
1018a9e644cdSLuigi Rizzo static uint64_t
1019a9e644cdSLuigi Rizzo ptnet_get_counter(if_t ifp, ift_counter cnt)
1020a9e644cdSLuigi Rizzo {
1021a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(ifp);
1022a9e644cdSLuigi Rizzo 	struct ptnet_queue_stats stats[2];
1023a9e644cdSLuigi Rizzo 	int i;
1024a9e644cdSLuigi Rizzo 
1025a9e644cdSLuigi Rizzo 	/* Accumulate statistics over the queues. */
1026a9e644cdSLuigi Rizzo 	memset(stats, 0, sizeof(stats));
1027a9e644cdSLuigi Rizzo 	for (i = 0; i < sc->num_rings; i++) {
1028a9e644cdSLuigi Rizzo 		struct ptnet_queue *pq = sc->queues + i;
1029a9e644cdSLuigi Rizzo 		int idx = (i < sc->num_tx_rings) ? 0 : 1;
1030a9e644cdSLuigi Rizzo 
1031a9e644cdSLuigi Rizzo 		stats[idx].packets	+= pq->stats.packets;
1032a9e644cdSLuigi Rizzo 		stats[idx].bytes	+= pq->stats.bytes;
1033a9e644cdSLuigi Rizzo 		stats[idx].errors	+= pq->stats.errors;
1034a9e644cdSLuigi Rizzo 		stats[idx].iqdrops	+= pq->stats.iqdrops;
1035a9e644cdSLuigi Rizzo 		stats[idx].mcasts	+= pq->stats.mcasts;
1036a9e644cdSLuigi Rizzo 	}
1037a9e644cdSLuigi Rizzo 
1038a9e644cdSLuigi Rizzo 	switch (cnt) {
1039a9e644cdSLuigi Rizzo 	case IFCOUNTER_IPACKETS:
1040a9e644cdSLuigi Rizzo 		return (stats[1].packets);
1041a9e644cdSLuigi Rizzo 	case IFCOUNTER_IQDROPS:
1042a9e644cdSLuigi Rizzo 		return (stats[1].iqdrops);
1043a9e644cdSLuigi Rizzo 	case IFCOUNTER_IERRORS:
1044a9e644cdSLuigi Rizzo 		return (stats[1].errors);
1045a9e644cdSLuigi Rizzo 	case IFCOUNTER_OPACKETS:
1046a9e644cdSLuigi Rizzo 		return (stats[0].packets);
1047a9e644cdSLuigi Rizzo 	case IFCOUNTER_OBYTES:
1048a9e644cdSLuigi Rizzo 		return (stats[0].bytes);
1049a9e644cdSLuigi Rizzo 	case IFCOUNTER_OMCASTS:
1050a9e644cdSLuigi Rizzo 		return (stats[0].mcasts);
1051a9e644cdSLuigi Rizzo 	default:
1052a9e644cdSLuigi Rizzo 		return (if_get_counter_default(ifp, cnt));
1053a9e644cdSLuigi Rizzo 	}
1054a9e644cdSLuigi Rizzo }
1055a9e644cdSLuigi Rizzo #endif
1056a9e644cdSLuigi Rizzo 
1057a9e644cdSLuigi Rizzo 
1058a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
1059a9e644cdSLuigi Rizzo /* Called under core lock. */
1060a9e644cdSLuigi Rizzo static void
1061a9e644cdSLuigi Rizzo ptnet_tick(void *opaque)
1062a9e644cdSLuigi Rizzo {
1063a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = opaque;
1064a9e644cdSLuigi Rizzo 	int i;
1065a9e644cdSLuigi Rizzo 
1066a9e644cdSLuigi Rizzo 	for (i = 0; i < sc->num_rings; i++) {
1067a9e644cdSLuigi Rizzo 		struct ptnet_queue *pq = sc->queues + i;
1068a9e644cdSLuigi Rizzo 		struct ptnet_queue_stats cur = pq->stats;
1069a9e644cdSLuigi Rizzo 		struct timeval now;
1070a9e644cdSLuigi Rizzo 		unsigned int delta;
1071a9e644cdSLuigi Rizzo 
1072a9e644cdSLuigi Rizzo 		microtime(&now);
1073a9e644cdSLuigi Rizzo 		delta = now.tv_usec - sc->last_ts.tv_usec +
1074a9e644cdSLuigi Rizzo 			(now.tv_sec - sc->last_ts.tv_sec) * 1000000;
1075a9e644cdSLuigi Rizzo 		delta /= 1000; /* in milliseconds */
1076a9e644cdSLuigi Rizzo 
1077a9e644cdSLuigi Rizzo 		if (delta == 0)
1078a9e644cdSLuigi Rizzo 			continue;
1079a9e644cdSLuigi Rizzo 
1080a9e644cdSLuigi Rizzo 		device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, "
1081a9e644cdSLuigi Rizzo 			      "intr %lu\n", i, delta,
1082a9e644cdSLuigi Rizzo 			      (cur.packets - pq->last_stats.packets),
1083a9e644cdSLuigi Rizzo 			      (cur.kicks - pq->last_stats.kicks),
1084a9e644cdSLuigi Rizzo 			      (cur.intrs - pq->last_stats.intrs));
1085a9e644cdSLuigi Rizzo 		pq->last_stats = cur;
1086a9e644cdSLuigi Rizzo 	}
1087a9e644cdSLuigi Rizzo 	microtime(&sc->last_ts);
1088a9e644cdSLuigi Rizzo 	callout_schedule(&sc->tick, hz);
1089a9e644cdSLuigi Rizzo }
1090a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */
1091a9e644cdSLuigi Rizzo 
1092a9e644cdSLuigi Rizzo static void
1093a9e644cdSLuigi Rizzo ptnet_media_status(if_t ifp, struct ifmediareq *ifmr)
1094a9e644cdSLuigi Rizzo {
1095a9e644cdSLuigi Rizzo 	/* We are always active, as the backend netmap port is
1096a9e644cdSLuigi Rizzo 	 * always open in netmap mode. */
1097a9e644cdSLuigi Rizzo 	ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
1098a9e644cdSLuigi Rizzo 	ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
1099a9e644cdSLuigi Rizzo }
1100a9e644cdSLuigi Rizzo 
1101a9e644cdSLuigi Rizzo static uint32_t
1102b6e66be2SVincenzo Maffione ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd)
1103a9e644cdSLuigi Rizzo {
1104844a6f0cSLuigi Rizzo 	/*
1105844a6f0cSLuigi Rizzo 	 * Write a command and read back error status,
1106844a6f0cSLuigi Rizzo 	 * with zero meaning success.
1107844a6f0cSLuigi Rizzo 	 */
1108a9e644cdSLuigi Rizzo 	bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd);
1109844a6f0cSLuigi Rizzo 	return bus_read_4(sc->iomem, PTNET_IO_PTCTL);
1110a9e644cdSLuigi Rizzo }
1111a9e644cdSLuigi Rizzo 
1112a9e644cdSLuigi Rizzo static int
11132ff91c17SVincenzo Maffione ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info)
1114a9e644cdSLuigi Rizzo {
1115a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(na->ifp);
1116a9e644cdSLuigi Rizzo 
11172ff91c17SVincenzo Maffione 	info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
11182ff91c17SVincenzo Maffione 	info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
11192ff91c17SVincenzo Maffione 	info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
11202ff91c17SVincenzo Maffione 	info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
11212ff91c17SVincenzo Maffione 	info->rx_buf_maxsize = NETMAP_BUF_SIZE(na);
1122a9e644cdSLuigi Rizzo 
11232ff91c17SVincenzo Maffione 	device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n",
11242ff91c17SVincenzo Maffione 			info->num_tx_rings, info->num_rx_rings,
11252ff91c17SVincenzo Maffione 			info->num_tx_descs, info->num_rx_descs,
11262ff91c17SVincenzo Maffione 			info->rx_buf_maxsize);
1127a9e644cdSLuigi Rizzo 
1128a9e644cdSLuigi Rizzo 	return 0;
1129a9e644cdSLuigi Rizzo }
1130a9e644cdSLuigi Rizzo 
1131a9e644cdSLuigi Rizzo static void
1132a9e644cdSLuigi Rizzo ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na)
1133a9e644cdSLuigi Rizzo {
1134a9e644cdSLuigi Rizzo 	int i;
1135a9e644cdSLuigi Rizzo 
1136a9e644cdSLuigi Rizzo 	/* Sync krings from the host, reading from
1137a9e644cdSLuigi Rizzo 	 * CSB. */
1138a9e644cdSLuigi Rizzo 	for (i = 0; i < sc->num_rings; i++) {
1139b6e66be2SVincenzo Maffione 		struct nm_csb_atok *atok = sc->queues[i].atok;
1140b6e66be2SVincenzo Maffione 		struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa;
1141a9e644cdSLuigi Rizzo 		struct netmap_kring *kring;
1142a9e644cdSLuigi Rizzo 
1143a9e644cdSLuigi Rizzo 		if (i < na->num_tx_rings) {
11442ff91c17SVincenzo Maffione 			kring = na->tx_rings[i];
1145a9e644cdSLuigi Rizzo 		} else {
11462ff91c17SVincenzo Maffione 			kring = na->rx_rings[i - na->num_tx_rings];
1147a9e644cdSLuigi Rizzo 		}
1148b6e66be2SVincenzo Maffione 		kring->rhead = kring->ring->head = atok->head;
1149b6e66be2SVincenzo Maffione 		kring->rcur = kring->ring->cur = atok->cur;
1150b6e66be2SVincenzo Maffione 		kring->nr_hwcur = ktoa->hwcur;
1151a9e644cdSLuigi Rizzo 		kring->nr_hwtail = kring->rtail =
1152b6e66be2SVincenzo Maffione 			kring->ring->tail = ktoa->hwtail;
1153a9e644cdSLuigi Rizzo 
115475f4f3edSVincenzo Maffione 		nm_prdis("%d,%d: csb {hc %u h %u c %u ht %u}", t, i,
1155b6e66be2SVincenzo Maffione 		   ktoa->hwcur, atok->head, atok->cur,
1156b6e66be2SVincenzo Maffione 		   ktoa->hwtail);
115775f4f3edSVincenzo Maffione 		nm_prdis("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}",
1158a9e644cdSLuigi Rizzo 		   t, i, kring->nr_hwcur, kring->rhead, kring->rcur,
1159a9e644cdSLuigi Rizzo 		   kring->ring->head, kring->ring->cur, kring->nr_hwtail,
1160a9e644cdSLuigi Rizzo 		   kring->rtail, kring->ring->tail);
1161a9e644cdSLuigi Rizzo 	}
1162a9e644cdSLuigi Rizzo }
1163a9e644cdSLuigi Rizzo 
1164a9e644cdSLuigi Rizzo static void
1165a9e644cdSLuigi Rizzo ptnet_update_vnet_hdr(struct ptnet_softc *sc)
1166a9e644cdSLuigi Rizzo {
1167a2a74091SLuigi Rizzo 	unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0;
1168a2a74091SLuigi Rizzo 
1169a2a74091SLuigi Rizzo 	bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len);
1170a2a74091SLuigi Rizzo 	sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN);
1171a9e644cdSLuigi Rizzo 	sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len;
1172a9e644cdSLuigi Rizzo }
1173a9e644cdSLuigi Rizzo 
1174a9e644cdSLuigi Rizzo static int
1175a9e644cdSLuigi Rizzo ptnet_nm_register(struct netmap_adapter *na, int onoff)
1176a9e644cdSLuigi Rizzo {
1177a9e644cdSLuigi Rizzo 	/* device-specific */
1178a9e644cdSLuigi Rizzo 	if_t ifp = na->ifp;
1179a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(ifp);
1180a9e644cdSLuigi Rizzo 	int native = (na == &sc->ptna->hwup.up);
1181a9e644cdSLuigi Rizzo 	struct ptnet_queue *pq;
1182a9e644cdSLuigi Rizzo 	int ret = 0;
1183a9e644cdSLuigi Rizzo 	int i;
1184a9e644cdSLuigi Rizzo 
1185a9e644cdSLuigi Rizzo 	if (!onoff) {
1186b6e66be2SVincenzo Maffione 		sc->ptna->backend_users--;
1187a9e644cdSLuigi Rizzo 	}
1188a9e644cdSLuigi Rizzo 
1189a9e644cdSLuigi Rizzo 	/* If this is the last netmap client, guest interrupt enable flags may
1190a9e644cdSLuigi Rizzo 	 * be in arbitrary state. Since these flags are going to be used also
1191a9e644cdSLuigi Rizzo 	 * by the netdevice driver, we have to make sure to start with
1192a9e644cdSLuigi Rizzo 	 * notifications enabled. Also, schedule NAPI to flush pending packets
1193a9e644cdSLuigi Rizzo 	 * in the RX rings, since we will not receive further interrupts
1194a9e644cdSLuigi Rizzo 	 * until these will be processed. */
1195a9e644cdSLuigi Rizzo 	if (native && !onoff && na->active_fds == 0) {
119675f4f3edSVincenzo Maffione 		nm_prinf("Exit netmap mode, re-enable interrupts");
1197a9e644cdSLuigi Rizzo 		for (i = 0; i < sc->num_rings; i++) {
1198a9e644cdSLuigi Rizzo 			pq = sc->queues + i;
1199b6e66be2SVincenzo Maffione 			pq->atok->appl_need_kick = 1;
1200a9e644cdSLuigi Rizzo 		}
1201a9e644cdSLuigi Rizzo 	}
1202a9e644cdSLuigi Rizzo 
1203a9e644cdSLuigi Rizzo 	if (onoff) {
1204b6e66be2SVincenzo Maffione 		if (sc->ptna->backend_users == 0) {
1205a9e644cdSLuigi Rizzo 			/* Initialize notification enable fields in the CSB. */
1206a9e644cdSLuigi Rizzo 			for (i = 0; i < sc->num_rings; i++) {
1207a9e644cdSLuigi Rizzo 				pq = sc->queues + i;
1208b6e66be2SVincenzo Maffione 				pq->ktoa->kern_need_kick = 1;
1209b6e66be2SVincenzo Maffione 				pq->atok->appl_need_kick =
1210a9e644cdSLuigi Rizzo 					(!(ifp->if_capenable & IFCAP_POLLING)
1211a9e644cdSLuigi Rizzo 						&& i >= sc->num_tx_rings);
1212a9e644cdSLuigi Rizzo 			}
1213a9e644cdSLuigi Rizzo 
1214a9e644cdSLuigi Rizzo 			/* Set the virtio-net header length. */
1215a9e644cdSLuigi Rizzo 			ptnet_update_vnet_hdr(sc);
1216a9e644cdSLuigi Rizzo 
1217a9e644cdSLuigi Rizzo 			/* Make sure the host adapter passed through is ready
1218a9e644cdSLuigi Rizzo 			 * for txsync/rxsync. */
1219b6e66be2SVincenzo Maffione 			ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE);
1220a9e644cdSLuigi Rizzo 			if (ret) {
1221a9e644cdSLuigi Rizzo 				return ret;
1222a9e644cdSLuigi Rizzo 			}
1223a9e644cdSLuigi Rizzo 
1224b6e66be2SVincenzo Maffione 			/* Align the guest krings and rings to the state stored
1225b6e66be2SVincenzo Maffione 			 * in the CSB. */
1226a9e644cdSLuigi Rizzo 			ptnet_sync_from_csb(sc, na);
1227a9e644cdSLuigi Rizzo 		}
1228a9e644cdSLuigi Rizzo 
1229a9e644cdSLuigi Rizzo 		/* If not native, don't call nm_set_native_flags, since we don't want
1230a9e644cdSLuigi Rizzo 		 * to replace if_transmit method, nor set NAF_NETMAP_ON */
1231a9e644cdSLuigi Rizzo 		if (native) {
123275f4f3edSVincenzo Maffione 			netmap_krings_mode_commit(na, onoff);
1233a9e644cdSLuigi Rizzo 			nm_set_native_flags(na);
1234a9e644cdSLuigi Rizzo 		}
1235a9e644cdSLuigi Rizzo 
1236a9e644cdSLuigi Rizzo 	} else {
1237a9e644cdSLuigi Rizzo 		if (native) {
1238a9e644cdSLuigi Rizzo 			nm_clear_native_flags(na);
123975f4f3edSVincenzo Maffione 			netmap_krings_mode_commit(na, onoff);
1240a9e644cdSLuigi Rizzo 		}
1241a9e644cdSLuigi Rizzo 
1242b6e66be2SVincenzo Maffione 		if (sc->ptna->backend_users == 0) {
1243b6e66be2SVincenzo Maffione 			ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
1244a9e644cdSLuigi Rizzo 		}
1245a9e644cdSLuigi Rizzo 	}
1246a9e644cdSLuigi Rizzo 
1247a9e644cdSLuigi Rizzo 	if (onoff) {
1248b6e66be2SVincenzo Maffione 		sc->ptna->backend_users++;
1249a9e644cdSLuigi Rizzo 	}
1250a9e644cdSLuigi Rizzo 
1251a9e644cdSLuigi Rizzo 	return ret;
1252a9e644cdSLuigi Rizzo }
1253a9e644cdSLuigi Rizzo 
1254a9e644cdSLuigi Rizzo static int
1255a9e644cdSLuigi Rizzo ptnet_nm_txsync(struct netmap_kring *kring, int flags)
1256a9e644cdSLuigi Rizzo {
1257a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1258a9e644cdSLuigi Rizzo 	struct ptnet_queue *pq = sc->queues + kring->ring_id;
1259a9e644cdSLuigi Rizzo 	bool notify;
1260a9e644cdSLuigi Rizzo 
1261b6e66be2SVincenzo Maffione 	notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags);
1262a9e644cdSLuigi Rizzo 	if (notify) {
1263a9e644cdSLuigi Rizzo 		ptnet_kick(pq);
1264a9e644cdSLuigi Rizzo 	}
1265a9e644cdSLuigi Rizzo 
1266a9e644cdSLuigi Rizzo 	return 0;
1267a9e644cdSLuigi Rizzo }
1268a9e644cdSLuigi Rizzo 
1269a9e644cdSLuigi Rizzo static int
1270a9e644cdSLuigi Rizzo ptnet_nm_rxsync(struct netmap_kring *kring, int flags)
1271a9e644cdSLuigi Rizzo {
1272a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1273a9e644cdSLuigi Rizzo 	struct ptnet_queue *pq = sc->rxqueues + kring->ring_id;
1274a9e644cdSLuigi Rizzo 	bool notify;
1275a9e644cdSLuigi Rizzo 
1276b6e66be2SVincenzo Maffione 	notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags);
1277a9e644cdSLuigi Rizzo 	if (notify) {
1278a9e644cdSLuigi Rizzo 		ptnet_kick(pq);
1279a9e644cdSLuigi Rizzo 	}
1280a9e644cdSLuigi Rizzo 
1281a9e644cdSLuigi Rizzo 	return 0;
1282a9e644cdSLuigi Rizzo }
1283a9e644cdSLuigi Rizzo 
1284a9e644cdSLuigi Rizzo static void
12854f80b14cSVincenzo Maffione ptnet_nm_intr(struct netmap_adapter *na, int onoff)
12864f80b14cSVincenzo Maffione {
12874f80b14cSVincenzo Maffione 	struct ptnet_softc *sc = if_getsoftc(na->ifp);
12884f80b14cSVincenzo Maffione 	int i;
12894f80b14cSVincenzo Maffione 
12904f80b14cSVincenzo Maffione 	for (i = 0; i < sc->num_rings; i++) {
12914f80b14cSVincenzo Maffione 		struct ptnet_queue *pq = sc->queues + i;
1292b6e66be2SVincenzo Maffione 		pq->atok->appl_need_kick = onoff;
12934f80b14cSVincenzo Maffione 	}
12944f80b14cSVincenzo Maffione }
12954f80b14cSVincenzo Maffione 
12964f80b14cSVincenzo Maffione static void
1297a9e644cdSLuigi Rizzo ptnet_tx_intr(void *opaque)
1298a9e644cdSLuigi Rizzo {
1299a9e644cdSLuigi Rizzo 	struct ptnet_queue *pq = opaque;
1300a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = pq->sc;
1301a9e644cdSLuigi Rizzo 
1302a9e644cdSLuigi Rizzo 	DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id));
1303a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
1304a9e644cdSLuigi Rizzo 	pq->stats.intrs ++;
1305a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */
1306a9e644cdSLuigi Rizzo 
1307a9e644cdSLuigi Rizzo 	if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) {
1308a9e644cdSLuigi Rizzo 		return;
1309a9e644cdSLuigi Rizzo 	}
1310a9e644cdSLuigi Rizzo 
1311a9e644cdSLuigi Rizzo 	/* Schedule the tasqueue to flush process transmissions requests.
1312a9e644cdSLuigi Rizzo 	 * However, vtnet, if_em and if_igb just call ptnet_transmit() here,
1313a9e644cdSLuigi Rizzo 	 * at least when using MSI-X interrupts. The if_em driver, instead
1314a9e644cdSLuigi Rizzo 	 * schedule taskqueue when using legacy interrupts. */
1315a9e644cdSLuigi Rizzo 	taskqueue_enqueue(pq->taskq, &pq->task);
1316a9e644cdSLuigi Rizzo }
1317a9e644cdSLuigi Rizzo 
1318a9e644cdSLuigi Rizzo static void
1319a9e644cdSLuigi Rizzo ptnet_rx_intr(void *opaque)
1320a9e644cdSLuigi Rizzo {
1321a9e644cdSLuigi Rizzo 	struct ptnet_queue *pq = opaque;
1322a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = pq->sc;
1323a9e644cdSLuigi Rizzo 	unsigned int unused;
1324a9e644cdSLuigi Rizzo 
1325a9e644cdSLuigi Rizzo 	DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id));
1326a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS
1327a9e644cdSLuigi Rizzo 	pq->stats.intrs ++;
1328a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */
1329a9e644cdSLuigi Rizzo 
1330a9e644cdSLuigi Rizzo 	if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) {
1331a9e644cdSLuigi Rizzo 		return;
1332a9e644cdSLuigi Rizzo 	}
1333a9e644cdSLuigi Rizzo 
1334a9e644cdSLuigi Rizzo 	/* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts,
1335a9e644cdSLuigi Rizzo 	 * receive-side processing is executed directly in the interrupt
1336a9e644cdSLuigi Rizzo 	 * service routine. Alternatively, we may schedule the taskqueue. */
1337a9e644cdSLuigi Rizzo 	ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1338a9e644cdSLuigi Rizzo }
1339a9e644cdSLuigi Rizzo 
1340a9e644cdSLuigi Rizzo static void
1341a9e644cdSLuigi Rizzo ptnet_vlan_tag_remove(struct mbuf *m)
1342a9e644cdSLuigi Rizzo {
1343a9e644cdSLuigi Rizzo 	struct ether_vlan_header *evh;
1344a9e644cdSLuigi Rizzo 
1345a9e644cdSLuigi Rizzo 	evh = mtod(m, struct ether_vlan_header *);
1346a9e644cdSLuigi Rizzo 	m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
1347a9e644cdSLuigi Rizzo 	m->m_flags |= M_VLANTAG;
1348a9e644cdSLuigi Rizzo 
1349a9e644cdSLuigi Rizzo 	/* Strip the 802.1Q header. */
1350a9e644cdSLuigi Rizzo 	bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
1351a9e644cdSLuigi Rizzo 	    ETHER_HDR_LEN - ETHER_TYPE_LEN);
1352a9e644cdSLuigi Rizzo 	m_adj(m, ETHER_VLAN_ENCAP_LEN);
1353a9e644cdSLuigi Rizzo }
1354a9e644cdSLuigi Rizzo 
1355a9e644cdSLuigi Rizzo static void
1356a9e644cdSLuigi Rizzo ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring,
1357a9e644cdSLuigi Rizzo 		  unsigned int head, unsigned int sync_flags)
1358a9e644cdSLuigi Rizzo {
1359a9e644cdSLuigi Rizzo 	struct netmap_ring *ring = kring->ring;
1360b6e66be2SVincenzo Maffione 	struct nm_csb_atok *atok = pq->atok;
1361b6e66be2SVincenzo Maffione 	struct nm_csb_ktoa *ktoa = pq->ktoa;
1362a9e644cdSLuigi Rizzo 
1363a9e644cdSLuigi Rizzo 	/* Some packets have been pushed to the netmap ring. We have
1364a9e644cdSLuigi Rizzo 	 * to tell the host to process the new packets, updating cur
1365a9e644cdSLuigi Rizzo 	 * and head in the CSB. */
1366a9e644cdSLuigi Rizzo 	ring->head = ring->cur = head;
1367a9e644cdSLuigi Rizzo 
1368a9e644cdSLuigi Rizzo 	/* Mimic nm_txsync_prologue/nm_rxsync_prologue. */
1369a9e644cdSLuigi Rizzo 	kring->rcur = kring->rhead = head;
1370a9e644cdSLuigi Rizzo 
1371f79ba6d7SVincenzo Maffione 	nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
1372a9e644cdSLuigi Rizzo 
1373a9e644cdSLuigi Rizzo 	/* Kick the host if needed. */
1374b6e66be2SVincenzo Maffione 	if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
1375b6e66be2SVincenzo Maffione 		atok->sync_flags = sync_flags;
1376a9e644cdSLuigi Rizzo 		ptnet_kick(pq);
1377a9e644cdSLuigi Rizzo 	}
1378a9e644cdSLuigi Rizzo }
1379a9e644cdSLuigi Rizzo 
1380a9e644cdSLuigi Rizzo #define PTNET_TX_NOSPACE(_h, _k, _min)	\
1381a9e644cdSLuigi Rizzo 	((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \
1382a9e644cdSLuigi Rizzo 		(_k)->rtail - (_h)) < (_min)
1383a9e644cdSLuigi Rizzo 
1384a9e644cdSLuigi Rizzo /* This function may be called by the network stack, or by
1385a9e644cdSLuigi Rizzo  * by the taskqueue thread. */
1386a9e644cdSLuigi Rizzo static int
1387a9e644cdSLuigi Rizzo ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget,
1388a9e644cdSLuigi Rizzo 			   bool may_resched)
1389a9e644cdSLuigi Rizzo {
1390a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = pq->sc;
1391a9e644cdSLuigi Rizzo 	bool have_vnet_hdr = sc->vnet_hdr_len;
1392a9e644cdSLuigi Rizzo 	struct netmap_adapter *na = &sc->ptna->dr.up;
1393a9e644cdSLuigi Rizzo 	if_t ifp = sc->ifp;
1394a9e644cdSLuigi Rizzo 	unsigned int batch_count = 0;
1395b6e66be2SVincenzo Maffione 	struct nm_csb_atok *atok;
1396b6e66be2SVincenzo Maffione 	struct nm_csb_ktoa *ktoa;
1397a9e644cdSLuigi Rizzo 	struct netmap_kring *kring;
1398a9e644cdSLuigi Rizzo 	struct netmap_ring *ring;
1399a9e644cdSLuigi Rizzo 	struct netmap_slot *slot;
1400a9e644cdSLuigi Rizzo 	unsigned int count = 0;
1401a9e644cdSLuigi Rizzo 	unsigned int minspace;
1402a9e644cdSLuigi Rizzo 	unsigned int head;
1403a9e644cdSLuigi Rizzo 	unsigned int lim;
1404a9e644cdSLuigi Rizzo 	struct mbuf *mhead;
1405a9e644cdSLuigi Rizzo 	struct mbuf *mf;
1406a9e644cdSLuigi Rizzo 	int nmbuf_bytes;
1407a9e644cdSLuigi Rizzo 	uint8_t *nmbuf;
1408a9e644cdSLuigi Rizzo 
1409a9e644cdSLuigi Rizzo 	if (!PTNET_Q_TRYLOCK(pq)) {
1410a9e644cdSLuigi Rizzo 		/* We failed to acquire the lock, schedule the taskqueue. */
141175f4f3edSVincenzo Maffione 		nm_prlim(1, "Deferring TX work");
1412a9e644cdSLuigi Rizzo 		if (may_resched) {
1413a9e644cdSLuigi Rizzo 			taskqueue_enqueue(pq->taskq, &pq->task);
1414a9e644cdSLuigi Rizzo 		}
1415a9e644cdSLuigi Rizzo 
1416a9e644cdSLuigi Rizzo 		return 0;
1417a9e644cdSLuigi Rizzo 	}
1418a9e644cdSLuigi Rizzo 
1419a9e644cdSLuigi Rizzo 	if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1420a9e644cdSLuigi Rizzo 		PTNET_Q_UNLOCK(pq);
142175f4f3edSVincenzo Maffione 		nm_prlim(1, "Interface is down");
1422a9e644cdSLuigi Rizzo 		return ENETDOWN;
1423a9e644cdSLuigi Rizzo 	}
1424a9e644cdSLuigi Rizzo 
1425b6e66be2SVincenzo Maffione 	atok = pq->atok;
1426b6e66be2SVincenzo Maffione 	ktoa = pq->ktoa;
14272ff91c17SVincenzo Maffione 	kring = na->tx_rings[pq->kring_id];
1428a9e644cdSLuigi Rizzo 	ring = kring->ring;
1429a9e644cdSLuigi Rizzo 	lim = kring->nkr_num_slots - 1;
1430a9e644cdSLuigi Rizzo 	head = ring->head;
1431a9e644cdSLuigi Rizzo 	minspace = sc->min_tx_space;
1432a9e644cdSLuigi Rizzo 
1433a9e644cdSLuigi Rizzo 	while (count < budget) {
1434a9e644cdSLuigi Rizzo 		if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1435a9e644cdSLuigi Rizzo 			/* We ran out of slot, let's see if the host has
1436a9e644cdSLuigi Rizzo 			 * freed up some, by reading hwcur and hwtail from
1437a9e644cdSLuigi Rizzo 			 * the CSB. */
1438b6e66be2SVincenzo Maffione 			ptnet_sync_tail(ktoa, kring);
1439a9e644cdSLuigi Rizzo 
1440a9e644cdSLuigi Rizzo 			if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1441a9e644cdSLuigi Rizzo 				/* Still no slots available. Reactivate the
1442a9e644cdSLuigi Rizzo 				 * interrupts so that we can be notified
1443a9e644cdSLuigi Rizzo 				 * when some free slots are made available by
1444a9e644cdSLuigi Rizzo 				 * the host. */
1445b6e66be2SVincenzo Maffione 				atok->appl_need_kick = 1;
1446a9e644cdSLuigi Rizzo 
1447f79ba6d7SVincenzo Maffione 				/* Double check. We need a full barrier to
1448f79ba6d7SVincenzo Maffione 				 * prevent the store to atok->appl_need_kick
1449f79ba6d7SVincenzo Maffione 				 * to be reordered with the load from
1450f79ba6d7SVincenzo Maffione 				 * ktoa->hwcur and ktoa->hwtail (store-load
1451f79ba6d7SVincenzo Maffione 				 * barrier). */
1452f79ba6d7SVincenzo Maffione 				nm_stld_barrier();
1453b6e66be2SVincenzo Maffione 				ptnet_sync_tail(ktoa, kring);
1454a9e644cdSLuigi Rizzo 				if (likely(PTNET_TX_NOSPACE(head, kring,
1455a9e644cdSLuigi Rizzo 							    minspace))) {
1456a9e644cdSLuigi Rizzo 					break;
1457a9e644cdSLuigi Rizzo 				}
1458a9e644cdSLuigi Rizzo 
145975f4f3edSVincenzo Maffione 				nm_prlim(1, "Found more slots by doublecheck");
1460a9e644cdSLuigi Rizzo 				/* More slots were freed before reactivating
1461a9e644cdSLuigi Rizzo 				 * the interrupts. */
1462b6e66be2SVincenzo Maffione 				atok->appl_need_kick = 0;
1463a9e644cdSLuigi Rizzo 			}
1464a9e644cdSLuigi Rizzo 		}
1465a9e644cdSLuigi Rizzo 
1466a9e644cdSLuigi Rizzo 		mhead = drbr_peek(ifp, pq->bufring);
1467a9e644cdSLuigi Rizzo 		if (!mhead) {
1468a9e644cdSLuigi Rizzo 			break;
1469a9e644cdSLuigi Rizzo 		}
1470a9e644cdSLuigi Rizzo 
1471a9e644cdSLuigi Rizzo 		/* Initialize transmission state variables. */
1472a9e644cdSLuigi Rizzo 		slot = ring->slot + head;
1473a9e644cdSLuigi Rizzo 		nmbuf = NMB(na, slot);
1474a9e644cdSLuigi Rizzo 		nmbuf_bytes = 0;
1475a9e644cdSLuigi Rizzo 
1476a9e644cdSLuigi Rizzo 		/* If needed, prepare the virtio-net header at the beginning
1477a9e644cdSLuigi Rizzo 		 * of the first slot. */
1478a9e644cdSLuigi Rizzo 		if (have_vnet_hdr) {
1479a9e644cdSLuigi Rizzo 			struct virtio_net_hdr *vh =
1480a9e644cdSLuigi Rizzo 					(struct virtio_net_hdr *)nmbuf;
1481a9e644cdSLuigi Rizzo 
1482a9e644cdSLuigi Rizzo 			/* For performance, we could replace this memset() with
1483a9e644cdSLuigi Rizzo 			 * two 8-bytes-wide writes. */
1484a9e644cdSLuigi Rizzo 			memset(nmbuf, 0, PTNET_HDR_SIZE);
1485a9e644cdSLuigi Rizzo 			if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) {
1486f8bc74e2SVincenzo Maffione 				mhead = virtio_net_tx_offload(ifp, mhead, false,
1487a9e644cdSLuigi Rizzo 							 vh);
1488a9e644cdSLuigi Rizzo 				if (unlikely(!mhead)) {
1489a9e644cdSLuigi Rizzo 					/* Packet dropped because errors
1490a9e644cdSLuigi Rizzo 					 * occurred while preparing the vnet
1491a9e644cdSLuigi Rizzo 					 * header. Let's go ahead with the next
1492a9e644cdSLuigi Rizzo 					 * packet. */
1493a9e644cdSLuigi Rizzo 					pq->stats.errors ++;
1494a9e644cdSLuigi Rizzo 					drbr_advance(ifp, pq->bufring);
1495a9e644cdSLuigi Rizzo 					continue;
1496a9e644cdSLuigi Rizzo 				}
1497a9e644cdSLuigi Rizzo 			}
149875f4f3edSVincenzo Maffione 			nm_prdis(1, "%s: [csum_flags %lX] vnet hdr: flags %x "
1499a9e644cdSLuigi Rizzo 			      "csum_start %u csum_ofs %u hdr_len = %u "
1500a9e644cdSLuigi Rizzo 			      "gso_size %u gso_type %x", __func__,
1501a9e644cdSLuigi Rizzo 			      mhead->m_pkthdr.csum_flags, vh->flags,
1502a9e644cdSLuigi Rizzo 			      vh->csum_start, vh->csum_offset, vh->hdr_len,
1503a9e644cdSLuigi Rizzo 			      vh->gso_size, vh->gso_type);
1504a9e644cdSLuigi Rizzo 
1505a9e644cdSLuigi Rizzo 			nmbuf += PTNET_HDR_SIZE;
1506a9e644cdSLuigi Rizzo 			nmbuf_bytes += PTNET_HDR_SIZE;
1507a9e644cdSLuigi Rizzo 		}
1508a9e644cdSLuigi Rizzo 
1509a9e644cdSLuigi Rizzo 		for (mf = mhead; mf; mf = mf->m_next) {
1510a9e644cdSLuigi Rizzo 			uint8_t *mdata = mf->m_data;
1511a9e644cdSLuigi Rizzo 			int mlen = mf->m_len;
1512a9e644cdSLuigi Rizzo 
1513a9e644cdSLuigi Rizzo 			for (;;) {
1514a9e644cdSLuigi Rizzo 				int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes;
1515a9e644cdSLuigi Rizzo 
1516a9e644cdSLuigi Rizzo 				if (mlen < copy) {
1517a9e644cdSLuigi Rizzo 					copy = mlen;
1518a9e644cdSLuigi Rizzo 				}
1519a9e644cdSLuigi Rizzo 				memcpy(nmbuf, mdata, copy);
1520a9e644cdSLuigi Rizzo 
1521a9e644cdSLuigi Rizzo 				mdata += copy;
1522a9e644cdSLuigi Rizzo 				mlen -= copy;
1523a9e644cdSLuigi Rizzo 				nmbuf += copy;
1524a9e644cdSLuigi Rizzo 				nmbuf_bytes += copy;
1525a9e644cdSLuigi Rizzo 
1526a9e644cdSLuigi Rizzo 				if (!mlen) {
1527a9e644cdSLuigi Rizzo 					break;
1528a9e644cdSLuigi Rizzo 				}
1529a9e644cdSLuigi Rizzo 
1530a9e644cdSLuigi Rizzo 				slot->len = nmbuf_bytes;
1531a9e644cdSLuigi Rizzo 				slot->flags = NS_MOREFRAG;
1532a9e644cdSLuigi Rizzo 
1533a9e644cdSLuigi Rizzo 				head = nm_next(head, lim);
1534a9e644cdSLuigi Rizzo 				KASSERT(head != ring->tail,
1535a9e644cdSLuigi Rizzo 					("Unexpectedly run out of TX space"));
1536a9e644cdSLuigi Rizzo 				slot = ring->slot + head;
1537a9e644cdSLuigi Rizzo 				nmbuf = NMB(na, slot);
1538a9e644cdSLuigi Rizzo 				nmbuf_bytes = 0;
1539a9e644cdSLuigi Rizzo 			}
1540a9e644cdSLuigi Rizzo 		}
1541a9e644cdSLuigi Rizzo 
1542a9e644cdSLuigi Rizzo 		/* Complete last slot and update head. */
1543a9e644cdSLuigi Rizzo 		slot->len = nmbuf_bytes;
1544a9e644cdSLuigi Rizzo 		slot->flags = 0;
1545a9e644cdSLuigi Rizzo 		head = nm_next(head, lim);
1546a9e644cdSLuigi Rizzo 
1547a9e644cdSLuigi Rizzo 		/* Consume the packet just processed. */
1548a9e644cdSLuigi Rizzo 		drbr_advance(ifp, pq->bufring);
1549a9e644cdSLuigi Rizzo 
1550a9e644cdSLuigi Rizzo 		/* Copy the packet to listeners. */
1551a9e644cdSLuigi Rizzo 		ETHER_BPF_MTAP(ifp, mhead);
1552a9e644cdSLuigi Rizzo 
1553a9e644cdSLuigi Rizzo 		pq->stats.packets ++;
1554a9e644cdSLuigi Rizzo 		pq->stats.bytes += mhead->m_pkthdr.len;
1555a9e644cdSLuigi Rizzo 		if (mhead->m_flags & M_MCAST) {
1556a9e644cdSLuigi Rizzo 			pq->stats.mcasts ++;
1557a9e644cdSLuigi Rizzo 		}
1558a9e644cdSLuigi Rizzo 
1559a9e644cdSLuigi Rizzo 		m_freem(mhead);
1560a9e644cdSLuigi Rizzo 
1561a9e644cdSLuigi Rizzo 		count ++;
1562a9e644cdSLuigi Rizzo 		if (++batch_count == PTNET_TX_BATCH) {
1563a9e644cdSLuigi Rizzo 			ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1564a9e644cdSLuigi Rizzo 			batch_count = 0;
1565a9e644cdSLuigi Rizzo 		}
1566a9e644cdSLuigi Rizzo 	}
1567a9e644cdSLuigi Rizzo 
1568a9e644cdSLuigi Rizzo 	if (batch_count) {
1569a9e644cdSLuigi Rizzo 		ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1570a9e644cdSLuigi Rizzo 	}
1571a9e644cdSLuigi Rizzo 
1572a9e644cdSLuigi Rizzo 	if (count >= budget && may_resched) {
157375f4f3edSVincenzo Maffione 		DBG(nm_prlim(1, "out of budget: resched, %d mbufs pending\n",
1574a9e644cdSLuigi Rizzo 					drbr_inuse(ifp, pq->bufring)));
1575a9e644cdSLuigi Rizzo 		taskqueue_enqueue(pq->taskq, &pq->task);
1576a9e644cdSLuigi Rizzo 	}
1577a9e644cdSLuigi Rizzo 
1578a9e644cdSLuigi Rizzo 	PTNET_Q_UNLOCK(pq);
1579a9e644cdSLuigi Rizzo 
1580a9e644cdSLuigi Rizzo 	return count;
1581a9e644cdSLuigi Rizzo }
1582a9e644cdSLuigi Rizzo 
1583a9e644cdSLuigi Rizzo static int
1584a9e644cdSLuigi Rizzo ptnet_transmit(if_t ifp, struct mbuf *m)
1585a9e644cdSLuigi Rizzo {
1586a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(ifp);
1587a9e644cdSLuigi Rizzo 	struct ptnet_queue *pq;
1588a9e644cdSLuigi Rizzo 	unsigned int queue_idx;
1589a9e644cdSLuigi Rizzo 	int err;
1590a9e644cdSLuigi Rizzo 
1591a9e644cdSLuigi Rizzo 	DBG(device_printf(sc->dev, "transmit %p\n", m));
1592a9e644cdSLuigi Rizzo 
1593a9e644cdSLuigi Rizzo 	/* Insert 802.1Q header if needed. */
1594a9e644cdSLuigi Rizzo 	if (m->m_flags & M_VLANTAG) {
1595a9e644cdSLuigi Rizzo 		m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1596a9e644cdSLuigi Rizzo 		if (m == NULL) {
1597a9e644cdSLuigi Rizzo 			return ENOBUFS;
1598a9e644cdSLuigi Rizzo 		}
1599a9e644cdSLuigi Rizzo 		m->m_flags &= ~M_VLANTAG;
1600a9e644cdSLuigi Rizzo 	}
1601a9e644cdSLuigi Rizzo 
1602a9e644cdSLuigi Rizzo 	/* Get the flow-id if available. */
1603a9e644cdSLuigi Rizzo 	queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ?
1604a9e644cdSLuigi Rizzo 		    m->m_pkthdr.flowid : curcpu;
1605a9e644cdSLuigi Rizzo 
1606a9e644cdSLuigi Rizzo 	if (unlikely(queue_idx >= sc->num_tx_rings)) {
1607a9e644cdSLuigi Rizzo 		queue_idx %= sc->num_tx_rings;
1608a9e644cdSLuigi Rizzo 	}
1609a9e644cdSLuigi Rizzo 
1610a9e644cdSLuigi Rizzo 	pq = sc->queues + queue_idx;
1611a9e644cdSLuigi Rizzo 
1612a9e644cdSLuigi Rizzo 	err = drbr_enqueue(ifp, pq->bufring, m);
1613a9e644cdSLuigi Rizzo 	if (err) {
1614a9e644cdSLuigi Rizzo 		/* ENOBUFS when the bufring is full */
161575f4f3edSVincenzo Maffione 		nm_prlim(1, "%s: drbr_enqueue() failed %d\n",
1616a9e644cdSLuigi Rizzo 			__func__, err);
1617a9e644cdSLuigi Rizzo 		pq->stats.errors ++;
1618a9e644cdSLuigi Rizzo 		return err;
1619a9e644cdSLuigi Rizzo 	}
1620a9e644cdSLuigi Rizzo 
1621a9e644cdSLuigi Rizzo 	if (ifp->if_capenable & IFCAP_POLLING) {
1622a9e644cdSLuigi Rizzo 		/* If polling is on, the transmit queues will be
1623a9e644cdSLuigi Rizzo 		 * drained by the poller. */
1624a9e644cdSLuigi Rizzo 		return 0;
1625a9e644cdSLuigi Rizzo 	}
1626a9e644cdSLuigi Rizzo 
1627a9e644cdSLuigi Rizzo 	err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1628a9e644cdSLuigi Rizzo 
1629a9e644cdSLuigi Rizzo 	return (err < 0) ? err : 0;
1630a9e644cdSLuigi Rizzo }
1631a9e644cdSLuigi Rizzo 
1632a9e644cdSLuigi Rizzo static unsigned int
1633a9e644cdSLuigi Rizzo ptnet_rx_discard(struct netmap_kring *kring, unsigned int head)
1634a9e644cdSLuigi Rizzo {
1635a9e644cdSLuigi Rizzo 	struct netmap_ring *ring = kring->ring;
1636a9e644cdSLuigi Rizzo 	struct netmap_slot *slot = ring->slot + head;
1637a9e644cdSLuigi Rizzo 
1638a9e644cdSLuigi Rizzo 	for (;;) {
1639a9e644cdSLuigi Rizzo 		head = nm_next(head, kring->nkr_num_slots - 1);
1640a9e644cdSLuigi Rizzo 		if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) {
1641a9e644cdSLuigi Rizzo 			break;
1642a9e644cdSLuigi Rizzo 		}
1643a9e644cdSLuigi Rizzo 		slot = ring->slot + head;
1644a9e644cdSLuigi Rizzo 	}
1645a9e644cdSLuigi Rizzo 
1646a9e644cdSLuigi Rizzo 	return head;
1647a9e644cdSLuigi Rizzo }
1648a9e644cdSLuigi Rizzo 
1649a9e644cdSLuigi Rizzo static inline struct mbuf *
1650a9e644cdSLuigi Rizzo ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len)
1651a9e644cdSLuigi Rizzo {
1652a9e644cdSLuigi Rizzo 	uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len;
1653a9e644cdSLuigi Rizzo 
1654a9e644cdSLuigi Rizzo 	do {
1655a9e644cdSLuigi Rizzo 		unsigned int copy;
1656a9e644cdSLuigi Rizzo 
1657a9e644cdSLuigi Rizzo 		if (mtail->m_len == MCLBYTES) {
1658a9e644cdSLuigi Rizzo 			struct mbuf *mf;
1659a9e644cdSLuigi Rizzo 
1660a9e644cdSLuigi Rizzo 			mf = m_getcl(M_NOWAIT, MT_DATA, 0);
1661a9e644cdSLuigi Rizzo 			if (unlikely(!mf)) {
1662a9e644cdSLuigi Rizzo 				return NULL;
1663a9e644cdSLuigi Rizzo 			}
1664a9e644cdSLuigi Rizzo 
1665a9e644cdSLuigi Rizzo 			mtail->m_next = mf;
1666a9e644cdSLuigi Rizzo 			mtail = mf;
1667a9e644cdSLuigi Rizzo 			mdata = mtod(mtail, uint8_t *);
1668a9e644cdSLuigi Rizzo 			mtail->m_len = 0;
1669a9e644cdSLuigi Rizzo 		}
1670a9e644cdSLuigi Rizzo 
1671a9e644cdSLuigi Rizzo 		copy = MCLBYTES - mtail->m_len;
1672a9e644cdSLuigi Rizzo 		if (nmbuf_len < copy) {
1673a9e644cdSLuigi Rizzo 			copy = nmbuf_len;
1674a9e644cdSLuigi Rizzo 		}
1675a9e644cdSLuigi Rizzo 
1676a9e644cdSLuigi Rizzo 		memcpy(mdata, nmbuf, copy);
1677a9e644cdSLuigi Rizzo 
1678a9e644cdSLuigi Rizzo 		nmbuf += copy;
1679a9e644cdSLuigi Rizzo 		nmbuf_len -= copy;
1680a9e644cdSLuigi Rizzo 		mdata += copy;
1681a9e644cdSLuigi Rizzo 		mtail->m_len += copy;
1682a9e644cdSLuigi Rizzo 	} while (nmbuf_len);
1683a9e644cdSLuigi Rizzo 
1684a9e644cdSLuigi Rizzo 	return mtail;
1685a9e644cdSLuigi Rizzo }
1686a9e644cdSLuigi Rizzo 
1687a9e644cdSLuigi Rizzo static int
1688a9e644cdSLuigi Rizzo ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched)
1689a9e644cdSLuigi Rizzo {
1690a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = pq->sc;
1691a9e644cdSLuigi Rizzo 	bool have_vnet_hdr = sc->vnet_hdr_len;
1692b6e66be2SVincenzo Maffione 	struct nm_csb_atok *atok = pq->atok;
1693b6e66be2SVincenzo Maffione 	struct nm_csb_ktoa *ktoa = pq->ktoa;
1694a9e644cdSLuigi Rizzo 	struct netmap_adapter *na = &sc->ptna->dr.up;
16952ff91c17SVincenzo Maffione 	struct netmap_kring *kring = na->rx_rings[pq->kring_id];
1696a9e644cdSLuigi Rizzo 	struct netmap_ring *ring = kring->ring;
1697a9e644cdSLuigi Rizzo 	unsigned int const lim = kring->nkr_num_slots - 1;
1698a9e644cdSLuigi Rizzo 	unsigned int batch_count = 0;
1699a9e644cdSLuigi Rizzo 	if_t ifp = sc->ifp;
1700a9e644cdSLuigi Rizzo 	unsigned int count = 0;
170146023447SVincenzo Maffione 	uint32_t head;
1702a9e644cdSLuigi Rizzo 
1703a9e644cdSLuigi Rizzo 	PTNET_Q_LOCK(pq);
1704a9e644cdSLuigi Rizzo 
1705a9e644cdSLuigi Rizzo 	if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1706a9e644cdSLuigi Rizzo 		goto unlock;
1707a9e644cdSLuigi Rizzo 	}
1708a9e644cdSLuigi Rizzo 
1709a9e644cdSLuigi Rizzo 	kring->nr_kflags &= ~NKR_PENDINTR;
1710a9e644cdSLuigi Rizzo 
171146023447SVincenzo Maffione 	head = ring->head;
1712a9e644cdSLuigi Rizzo 	while (count < budget) {
171346023447SVincenzo Maffione 		uint32_t prev_head = head;
1714a9e644cdSLuigi Rizzo 		struct mbuf *mhead, *mtail;
1715a9e644cdSLuigi Rizzo 		struct virtio_net_hdr *vh;
1716a9e644cdSLuigi Rizzo 		struct netmap_slot *slot;
1717a9e644cdSLuigi Rizzo 		unsigned int nmbuf_len;
1718a9e644cdSLuigi Rizzo 		uint8_t *nmbuf;
171946023447SVincenzo Maffione 		int deliver = 1; /* the mbuf to the network stack. */
1720a9e644cdSLuigi Rizzo host_sync:
1721a9e644cdSLuigi Rizzo 		if (head == ring->tail) {
1722a9e644cdSLuigi Rizzo 			/* We ran out of slot, let's see if the host has
1723a9e644cdSLuigi Rizzo 			 * added some, by reading hwcur and hwtail from
1724a9e644cdSLuigi Rizzo 			 * the CSB. */
1725b6e66be2SVincenzo Maffione 			ptnet_sync_tail(ktoa, kring);
1726a9e644cdSLuigi Rizzo 
1727a9e644cdSLuigi Rizzo 			if (head == ring->tail) {
1728a9e644cdSLuigi Rizzo 				/* Still no slots available. Reactivate
1729a9e644cdSLuigi Rizzo 				 * interrupts as they were disabled by the
1730a9e644cdSLuigi Rizzo 				 * host thread right before issuing the
1731a9e644cdSLuigi Rizzo 				 * last interrupt. */
1732b6e66be2SVincenzo Maffione 				atok->appl_need_kick = 1;
1733a9e644cdSLuigi Rizzo 
1734f79ba6d7SVincenzo Maffione 				/* Double check for more completed RX slots.
1735f79ba6d7SVincenzo Maffione 				 * We need a full barrier to prevent the store
1736f79ba6d7SVincenzo Maffione 				 * to atok->appl_need_kick to be reordered with
1737f79ba6d7SVincenzo Maffione 				 * the load from ktoa->hwcur and ktoa->hwtail
1738f79ba6d7SVincenzo Maffione 				 * (store-load barrier). */
1739f79ba6d7SVincenzo Maffione 				nm_stld_barrier();
1740b6e66be2SVincenzo Maffione 				ptnet_sync_tail(ktoa, kring);
1741a9e644cdSLuigi Rizzo 				if (likely(head == ring->tail)) {
1742a9e644cdSLuigi Rizzo 					break;
1743a9e644cdSLuigi Rizzo 				}
1744b6e66be2SVincenzo Maffione 				atok->appl_need_kick = 0;
1745a9e644cdSLuigi Rizzo 			}
1746a9e644cdSLuigi Rizzo 		}
1747a9e644cdSLuigi Rizzo 
1748a9e644cdSLuigi Rizzo 		/* Initialize ring state variables, possibly grabbing the
1749a9e644cdSLuigi Rizzo 		 * virtio-net header. */
1750a9e644cdSLuigi Rizzo 		slot = ring->slot + head;
1751a9e644cdSLuigi Rizzo 		nmbuf = NMB(na, slot);
1752a9e644cdSLuigi Rizzo 		nmbuf_len = slot->len;
1753a9e644cdSLuigi Rizzo 
1754a9e644cdSLuigi Rizzo 		vh = (struct virtio_net_hdr *)nmbuf;
1755a9e644cdSLuigi Rizzo 		if (have_vnet_hdr) {
1756a9e644cdSLuigi Rizzo 			if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) {
1757a9e644cdSLuigi Rizzo 				/* There is no good reason why host should
1758a9e644cdSLuigi Rizzo 				 * put the header in multiple netmap slots.
1759a9e644cdSLuigi Rizzo 				 * If this is the case, discard. */
176075f4f3edSVincenzo Maffione 				nm_prlim(1, "Fragmented vnet-hdr: dropping");
1761a9e644cdSLuigi Rizzo 				head = ptnet_rx_discard(kring, head);
1762a9e644cdSLuigi Rizzo 				pq->stats.iqdrops ++;
176346023447SVincenzo Maffione 				deliver = 0;
1764a9e644cdSLuigi Rizzo 				goto skip;
1765a9e644cdSLuigi Rizzo 			}
176675f4f3edSVincenzo Maffione 			nm_prdis(1, "%s: vnet hdr: flags %x csum_start %u "
1767a9e644cdSLuigi Rizzo 			      "csum_ofs %u hdr_len = %u gso_size %u "
1768a9e644cdSLuigi Rizzo 			      "gso_type %x", __func__, vh->flags,
1769a9e644cdSLuigi Rizzo 			      vh->csum_start, vh->csum_offset, vh->hdr_len,
1770a9e644cdSLuigi Rizzo 			      vh->gso_size, vh->gso_type);
1771a9e644cdSLuigi Rizzo 			nmbuf += PTNET_HDR_SIZE;
1772a9e644cdSLuigi Rizzo 			nmbuf_len -= PTNET_HDR_SIZE;
1773a9e644cdSLuigi Rizzo 		}
1774a9e644cdSLuigi Rizzo 
1775a9e644cdSLuigi Rizzo 		/* Allocate the head of a new mbuf chain.
1776a9e644cdSLuigi Rizzo 		 * We use m_getcl() to allocate an mbuf with standard cluster
1777a9e644cdSLuigi Rizzo 		 * size (MCLBYTES). In the future we could use m_getjcl()
1778a9e644cdSLuigi Rizzo 		 * to choose different sizes. */
1779a9e644cdSLuigi Rizzo 		mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1780a9e644cdSLuigi Rizzo 		if (unlikely(mhead == NULL)) {
1781a9e644cdSLuigi Rizzo 			device_printf(sc->dev, "%s: failed to allocate mbuf "
1782a9e644cdSLuigi Rizzo 				      "head\n", __func__);
1783a9e644cdSLuigi Rizzo 			pq->stats.errors ++;
1784a9e644cdSLuigi Rizzo 			break;
1785a9e644cdSLuigi Rizzo 		}
1786a9e644cdSLuigi Rizzo 
1787a9e644cdSLuigi Rizzo 		/* Initialize the mbuf state variables. */
1788a9e644cdSLuigi Rizzo 		mhead->m_pkthdr.len = nmbuf_len;
1789a9e644cdSLuigi Rizzo 		mtail->m_len = 0;
1790a9e644cdSLuigi Rizzo 
1791a9e644cdSLuigi Rizzo 		/* Scan all the netmap slots containing the current packet. */
1792a9e644cdSLuigi Rizzo 		for (;;) {
1793a9e644cdSLuigi Rizzo 			DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag "
1794a9e644cdSLuigi Rizzo 					  "len %u, flags %u\n", __func__,
1795a9e644cdSLuigi Rizzo 					  head, ring->tail, slot->len,
1796a9e644cdSLuigi Rizzo 					  slot->flags));
1797a9e644cdSLuigi Rizzo 
1798a9e644cdSLuigi Rizzo 			mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len);
1799a9e644cdSLuigi Rizzo 			if (unlikely(!mtail)) {
1800a9e644cdSLuigi Rizzo 				/* Ouch. We ran out of memory while processing
1801a9e644cdSLuigi Rizzo 				 * a packet. We have to restore the previous
1802a9e644cdSLuigi Rizzo 				 * head position, free the mbuf chain, and
1803a9e644cdSLuigi Rizzo 				 * schedule the taskqueue to give the packet
1804a9e644cdSLuigi Rizzo 				 * another chance. */
1805a9e644cdSLuigi Rizzo 				device_printf(sc->dev, "%s: failed to allocate"
1806a9e644cdSLuigi Rizzo 					" mbuf frag, reset head %u --> %u\n",
1807a9e644cdSLuigi Rizzo 					__func__, head, prev_head);
1808a9e644cdSLuigi Rizzo 				head = prev_head;
1809a9e644cdSLuigi Rizzo 				m_freem(mhead);
1810a9e644cdSLuigi Rizzo 				pq->stats.errors ++;
1811a9e644cdSLuigi Rizzo 				if (may_resched) {
1812a9e644cdSLuigi Rizzo 					taskqueue_enqueue(pq->taskq,
1813a9e644cdSLuigi Rizzo 							  &pq->task);
1814a9e644cdSLuigi Rizzo 				}
1815a9e644cdSLuigi Rizzo 				goto escape;
1816a9e644cdSLuigi Rizzo 			}
1817a9e644cdSLuigi Rizzo 
1818a9e644cdSLuigi Rizzo 			/* We have to increment head irrespective of the
1819a9e644cdSLuigi Rizzo 			 * NS_MOREFRAG being set or not. */
1820a9e644cdSLuigi Rizzo 			head = nm_next(head, lim);
1821a9e644cdSLuigi Rizzo 
1822a9e644cdSLuigi Rizzo 			if (!(slot->flags & NS_MOREFRAG)) {
1823a9e644cdSLuigi Rizzo 				break;
1824a9e644cdSLuigi Rizzo 			}
1825a9e644cdSLuigi Rizzo 
1826a9e644cdSLuigi Rizzo 			if (unlikely(head == ring->tail)) {
1827a9e644cdSLuigi Rizzo 				/* The very last slot prepared by the host has
1828a9e644cdSLuigi Rizzo 				 * the NS_MOREFRAG set. Drop it and continue
1829a9e644cdSLuigi Rizzo 				 * the outer cycle (to do the double-check). */
183075f4f3edSVincenzo Maffione 				nm_prlim(1, "Incomplete packet: dropping");
1831a9e644cdSLuigi Rizzo 				m_freem(mhead);
1832a9e644cdSLuigi Rizzo 				pq->stats.iqdrops ++;
1833a9e644cdSLuigi Rizzo 				goto host_sync;
1834a9e644cdSLuigi Rizzo 			}
1835a9e644cdSLuigi Rizzo 
1836a9e644cdSLuigi Rizzo 			slot = ring->slot + head;
1837a9e644cdSLuigi Rizzo 			nmbuf = NMB(na, slot);
1838a9e644cdSLuigi Rizzo 			nmbuf_len = slot->len;
1839a9e644cdSLuigi Rizzo 			mhead->m_pkthdr.len += nmbuf_len;
1840a9e644cdSLuigi Rizzo 		}
1841a9e644cdSLuigi Rizzo 
1842a9e644cdSLuigi Rizzo 		mhead->m_pkthdr.rcvif = ifp;
1843a9e644cdSLuigi Rizzo 		mhead->m_pkthdr.csum_flags = 0;
1844a9e644cdSLuigi Rizzo 
1845a9e644cdSLuigi Rizzo 		/* Store the queue idx in the packet header. */
1846a9e644cdSLuigi Rizzo 		mhead->m_pkthdr.flowid = pq->kring_id;
1847a9e644cdSLuigi Rizzo 		M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE);
1848a9e644cdSLuigi Rizzo 
1849a9e644cdSLuigi Rizzo 		if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1850a9e644cdSLuigi Rizzo 			struct ether_header *eh;
1851a9e644cdSLuigi Rizzo 
1852a9e644cdSLuigi Rizzo 			eh = mtod(mhead, struct ether_header *);
1853a9e644cdSLuigi Rizzo 			if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1854a9e644cdSLuigi Rizzo 				ptnet_vlan_tag_remove(mhead);
1855a9e644cdSLuigi Rizzo 				/*
1856a9e644cdSLuigi Rizzo 				 * With the 802.1Q header removed, update the
1857a9e644cdSLuigi Rizzo 				 * checksum starting location accordingly.
1858a9e644cdSLuigi Rizzo 				 */
1859a9e644cdSLuigi Rizzo 				if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1860a9e644cdSLuigi Rizzo 					vh->csum_start -= ETHER_VLAN_ENCAP_LEN;
1861a9e644cdSLuigi Rizzo 			}
1862a9e644cdSLuigi Rizzo 		}
1863a9e644cdSLuigi Rizzo 
1864f8bc74e2SVincenzo Maffione 		if (unlikely(have_vnet_hdr && virtio_net_rx_csum(mhead, vh))) {
1865a9e644cdSLuigi Rizzo 			m_freem(mhead);
186675f4f3edSVincenzo Maffione 			nm_prlim(1, "Csum offload error: dropping");
1867a9e644cdSLuigi Rizzo 			pq->stats.iqdrops ++;
186846023447SVincenzo Maffione 			deliver = 0;
1869a9e644cdSLuigi Rizzo 		}
1870a9e644cdSLuigi Rizzo 
187146023447SVincenzo Maffione skip:
187246023447SVincenzo Maffione 		count ++;
187346023447SVincenzo Maffione 		if (++batch_count >= PTNET_RX_BATCH) {
187446023447SVincenzo Maffione 			/* Some packets have been (or will be) pushed to the network
187546023447SVincenzo Maffione 			 * stack. We need to update the CSB to tell the host about
187646023447SVincenzo Maffione 			 * the new ring->cur and ring->head (RX buffer refill). */
187746023447SVincenzo Maffione 			ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
187846023447SVincenzo Maffione 			batch_count = 0;
187946023447SVincenzo Maffione 		}
188046023447SVincenzo Maffione 
188146023447SVincenzo Maffione 		if (likely(deliver))  {
1882a9e644cdSLuigi Rizzo 			pq->stats.packets ++;
1883a9e644cdSLuigi Rizzo 			pq->stats.bytes += mhead->m_pkthdr.len;
1884a9e644cdSLuigi Rizzo 
1885a9e644cdSLuigi Rizzo 			PTNET_Q_UNLOCK(pq);
1886a9e644cdSLuigi Rizzo 			(*ifp->if_input)(ifp, mhead);
1887a9e644cdSLuigi Rizzo 			PTNET_Q_LOCK(pq);
188846023447SVincenzo Maffione 			/* The ring->head index (and related indices) are
188946023447SVincenzo Maffione 			 * updated under pq lock by ptnet_ring_update().
189046023447SVincenzo Maffione 			 * Since we dropped the lock to call if_input(), we
189146023447SVincenzo Maffione 			 * must reload ring->head and restart processing the
189246023447SVincenzo Maffione 			 * ring from there. */
189346023447SVincenzo Maffione 			head = ring->head;
1894a9e644cdSLuigi Rizzo 
1895a9e644cdSLuigi Rizzo 			if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1896a9e644cdSLuigi Rizzo 				/* The interface has gone down while we didn't
1897a9e644cdSLuigi Rizzo 				 * have the lock. Stop any processing and exit. */
1898a9e644cdSLuigi Rizzo 				goto unlock;
1899a9e644cdSLuigi Rizzo 			}
1900a9e644cdSLuigi Rizzo 		}
1901a9e644cdSLuigi Rizzo 	}
1902a9e644cdSLuigi Rizzo escape:
1903a9e644cdSLuigi Rizzo 	if (batch_count) {
1904a9e644cdSLuigi Rizzo 		ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
1905a9e644cdSLuigi Rizzo 
1906a9e644cdSLuigi Rizzo 	}
1907a9e644cdSLuigi Rizzo 
1908a9e644cdSLuigi Rizzo 	if (count >= budget && may_resched) {
1909a9e644cdSLuigi Rizzo 		/* If we ran out of budget or the double-check found new
1910a9e644cdSLuigi Rizzo 		 * slots to process, schedule the taskqueue. */
191175f4f3edSVincenzo Maffione 		DBG(nm_prlim(1, "out of budget: resched h %u t %u\n",
1912a9e644cdSLuigi Rizzo 					head, ring->tail));
1913a9e644cdSLuigi Rizzo 		taskqueue_enqueue(pq->taskq, &pq->task);
1914a9e644cdSLuigi Rizzo 	}
1915a9e644cdSLuigi Rizzo unlock:
1916a9e644cdSLuigi Rizzo 	PTNET_Q_UNLOCK(pq);
1917a9e644cdSLuigi Rizzo 
1918a9e644cdSLuigi Rizzo 	return count;
1919a9e644cdSLuigi Rizzo }
1920a9e644cdSLuigi Rizzo 
1921a9e644cdSLuigi Rizzo static void
1922a9e644cdSLuigi Rizzo ptnet_rx_task(void *context, int pending)
1923a9e644cdSLuigi Rizzo {
1924a9e644cdSLuigi Rizzo 	struct ptnet_queue *pq = context;
1925a9e644cdSLuigi Rizzo 
192675f4f3edSVincenzo Maffione 	DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
1927a9e644cdSLuigi Rizzo 	ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1928a9e644cdSLuigi Rizzo }
1929a9e644cdSLuigi Rizzo 
1930a9e644cdSLuigi Rizzo static void
1931a9e644cdSLuigi Rizzo ptnet_tx_task(void *context, int pending)
1932a9e644cdSLuigi Rizzo {
1933a9e644cdSLuigi Rizzo 	struct ptnet_queue *pq = context;
1934a9e644cdSLuigi Rizzo 
193575f4f3edSVincenzo Maffione 	DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
1936a9e644cdSLuigi Rizzo 	ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1937a9e644cdSLuigi Rizzo }
1938a9e644cdSLuigi Rizzo 
1939a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING
1940a9e644cdSLuigi Rizzo /* We don't need to handle differently POLL_AND_CHECK_STATUS and
1941a9e644cdSLuigi Rizzo  * POLL_ONLY, since we don't have an Interrupt Status Register. */
1942a9e644cdSLuigi Rizzo static int
1943a9e644cdSLuigi Rizzo ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget)
1944a9e644cdSLuigi Rizzo {
1945a9e644cdSLuigi Rizzo 	struct ptnet_softc *sc = if_getsoftc(ifp);
1946a9e644cdSLuigi Rizzo 	unsigned int queue_budget;
1947a9e644cdSLuigi Rizzo 	unsigned int count = 0;
1948a9e644cdSLuigi Rizzo 	bool borrow = false;
1949a9e644cdSLuigi Rizzo 	int i;
1950a9e644cdSLuigi Rizzo 
1951a9e644cdSLuigi Rizzo 	KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet"));
1952a9e644cdSLuigi Rizzo 	queue_budget = MAX(budget / sc->num_rings, 1);
195375f4f3edSVincenzo Maffione 	nm_prlim(1, "Per-queue budget is %d", queue_budget);
1954a9e644cdSLuigi Rizzo 
1955a9e644cdSLuigi Rizzo 	while (budget) {
1956a9e644cdSLuigi Rizzo 		unsigned int rcnt = 0;
1957a9e644cdSLuigi Rizzo 
1958a9e644cdSLuigi Rizzo 		for (i = 0; i < sc->num_rings; i++) {
1959a9e644cdSLuigi Rizzo 			struct ptnet_queue *pq = sc->queues + i;
1960a9e644cdSLuigi Rizzo 
1961a9e644cdSLuigi Rizzo 			if (borrow) {
1962a9e644cdSLuigi Rizzo 				queue_budget = MIN(queue_budget, budget);
1963a9e644cdSLuigi Rizzo 				if (queue_budget == 0) {
1964a9e644cdSLuigi Rizzo 					break;
1965a9e644cdSLuigi Rizzo 				}
1966a9e644cdSLuigi Rizzo 			}
1967a9e644cdSLuigi Rizzo 
1968a9e644cdSLuigi Rizzo 			if (i < sc->num_tx_rings) {
1969a9e644cdSLuigi Rizzo 				rcnt += ptnet_drain_transmit_queue(pq,
1970a9e644cdSLuigi Rizzo 						   queue_budget, false);
1971a9e644cdSLuigi Rizzo 			} else {
1972a9e644cdSLuigi Rizzo 				rcnt += ptnet_rx_eof(pq, queue_budget,
1973a9e644cdSLuigi Rizzo 						      false);
1974a9e644cdSLuigi Rizzo 			}
1975a9e644cdSLuigi Rizzo 		}
1976a9e644cdSLuigi Rizzo 
1977a9e644cdSLuigi Rizzo 		if (!rcnt) {
1978a9e644cdSLuigi Rizzo 			/* A scan of the queues gave no result, we can
1979a9e644cdSLuigi Rizzo 			 * stop here. */
1980a9e644cdSLuigi Rizzo 			break;
1981a9e644cdSLuigi Rizzo 		}
1982a9e644cdSLuigi Rizzo 
1983a9e644cdSLuigi Rizzo 		if (rcnt > budget) {
1984a9e644cdSLuigi Rizzo 			/* This may happen when initial budget < sc->num_rings,
1985a9e644cdSLuigi Rizzo 			 * since one packet budget is given to each queue
1986a9e644cdSLuigi Rizzo 			 * anyway. Just pretend we didn't eat "so much". */
1987a9e644cdSLuigi Rizzo 			rcnt = budget;
1988a9e644cdSLuigi Rizzo 		}
1989a9e644cdSLuigi Rizzo 		count += rcnt;
1990a9e644cdSLuigi Rizzo 		budget -= rcnt;
1991a9e644cdSLuigi Rizzo 		borrow = true;
1992a9e644cdSLuigi Rizzo 	}
1993a9e644cdSLuigi Rizzo 
1994a9e644cdSLuigi Rizzo 
1995a9e644cdSLuigi Rizzo 	return count;
1996a9e644cdSLuigi Rizzo }
1997a9e644cdSLuigi Rizzo #endif /* DEVICE_POLLING */
1998*2ec213abSVincenzo Maffione #endif /* WITH_PTNETMAP */
1999