xref: /freebsd/sys/dev/netmap/if_ptnet.c (revision 5956d97f4b3204318ceb6aa9c77bd0bc6ea87a41)
1 /*-
2  * Copyright (c) 2016, Vincenzo Maffione
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice unmodified, this list of conditions, and the following
10  *    disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * $FreeBSD$
27  */
28 
29 /* Driver for ptnet paravirtualized network device. */
30 
31 #include <sys/cdefs.h>
32 
33 #include <sys/types.h>
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/sockio.h>
38 #include <sys/mbuf.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/socket.h>
42 #include <sys/sysctl.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/taskqueue.h>
46 #include <sys/smp.h>
47 #include <sys/time.h>
48 #include <machine/smp.h>
49 
50 #include <vm/uma.h>
51 #include <vm/vm.h>
52 #include <vm/pmap.h>
53 
54 #include <net/ethernet.h>
55 #include <net/if.h>
56 #include <net/if_var.h>
57 #include <net/if_arp.h>
58 #include <net/if_dl.h>
59 #include <net/if_types.h>
60 #include <net/if_media.h>
61 #include <net/if_vlan_var.h>
62 #include <net/bpf.h>
63 
64 #include <netinet/in_systm.h>
65 #include <netinet/in.h>
66 #include <netinet/ip.h>
67 #include <netinet/ip6.h>
68 #include <netinet6/ip6_var.h>
69 #include <netinet/udp.h>
70 #include <netinet/tcp.h>
71 
72 #include <machine/bus.h>
73 #include <machine/resource.h>
74 #include <sys/bus.h>
75 #include <sys/rman.h>
76 
77 #include <dev/pci/pcivar.h>
78 #include <dev/pci/pcireg.h>
79 
80 #include "opt_inet.h"
81 #include "opt_inet6.h"
82 
83 #include <sys/selinfo.h>
84 #include <net/netmap.h>
85 #include <dev/netmap/netmap_kern.h>
86 #include <net/netmap_virt.h>
87 #include <dev/netmap/netmap_mem2.h>
88 #include <dev/virtio/network/virtio_net.h>
89 
90 #ifdef WITH_PTNETMAP
91 
92 #ifndef INET
93 #error "INET not defined, cannot support offloadings"
94 #endif
95 
96 #if __FreeBSD_version >= 1100000
97 static uint64_t	ptnet_get_counter(if_t, ift_counter);
98 #else
99 typedef struct ifnet *if_t;
100 #define if_getsoftc(_ifp)   (_ifp)->if_softc
101 #endif
102 
103 //#define PTNETMAP_STATS
104 //#define DEBUG
105 #ifdef DEBUG
106 #define DBG(x) x
107 #else   /* !DEBUG */
108 #define DBG(x)
109 #endif  /* !DEBUG */
110 
111 extern int ptnet_vnet_hdr; /* Tunable parameter */
112 
113 struct ptnet_softc;
114 
115 struct ptnet_queue_stats {
116 	uint64_t	packets; /* if_[io]packets */
117 	uint64_t	bytes;	 /* if_[io]bytes */
118 	uint64_t	errors;	 /* if_[io]errors */
119 	uint64_t	iqdrops; /* if_iqdrops */
120 	uint64_t	mcasts;  /* if_[io]mcasts */
121 #ifdef PTNETMAP_STATS
122 	uint64_t	intrs;
123 	uint64_t	kicks;
124 #endif /* PTNETMAP_STATS */
125 };
126 
127 struct ptnet_queue {
128 	struct ptnet_softc		*sc;
129 	struct				resource *irq;
130 	void				*cookie;
131 	int				kring_id;
132 	struct nm_csb_atok		*atok;
133 	struct nm_csb_ktoa		*ktoa;
134 	unsigned int			kick;
135 	struct mtx			lock;
136 	struct buf_ring			*bufring; /* for TX queues */
137 	struct ptnet_queue_stats	stats;
138 #ifdef PTNETMAP_STATS
139 	struct ptnet_queue_stats	last_stats;
140 #endif /* PTNETMAP_STATS */
141 	struct taskqueue		*taskq;
142 	struct task			task;
143 	char				lock_name[16];
144 };
145 
146 #define PTNET_Q_LOCK(_pq)	mtx_lock(&(_pq)->lock)
147 #define PTNET_Q_TRYLOCK(_pq)	mtx_trylock(&(_pq)->lock)
148 #define PTNET_Q_UNLOCK(_pq)	mtx_unlock(&(_pq)->lock)
149 
150 struct ptnet_softc {
151 	device_t		dev;
152 	if_t			ifp;
153 	struct ifmedia		media;
154 	struct mtx		lock;
155 	char			lock_name[16];
156 	char			hwaddr[ETHER_ADDR_LEN];
157 
158 	/* Mirror of PTFEAT register. */
159 	uint32_t		ptfeatures;
160 	unsigned int		vnet_hdr_len;
161 
162 	/* PCI BARs support. */
163 	struct resource		*iomem;
164 	struct resource		*msix_mem;
165 
166 	unsigned int		num_rings;
167 	unsigned int		num_tx_rings;
168 	struct ptnet_queue	*queues;
169 	struct ptnet_queue	*rxqueues;
170 	struct nm_csb_atok	*csb_gh;
171 	struct nm_csb_ktoa	*csb_hg;
172 
173 	unsigned int		min_tx_space;
174 
175 	struct netmap_pt_guest_adapter *ptna;
176 
177 	struct callout		tick;
178 #ifdef PTNETMAP_STATS
179 	struct timeval		last_ts;
180 #endif /* PTNETMAP_STATS */
181 };
182 
183 #define PTNET_CORE_LOCK(_sc)	mtx_lock(&(_sc)->lock)
184 #define PTNET_CORE_UNLOCK(_sc)	mtx_unlock(&(_sc)->lock)
185 
186 static int	ptnet_probe(device_t);
187 static int	ptnet_attach(device_t);
188 static int	ptnet_detach(device_t);
189 static int	ptnet_suspend(device_t);
190 static int	ptnet_resume(device_t);
191 static int	ptnet_shutdown(device_t);
192 
193 static void	ptnet_init(void *opaque);
194 static int	ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data);
195 static int	ptnet_init_locked(struct ptnet_softc *sc);
196 static int	ptnet_stop(struct ptnet_softc *sc);
197 static int	ptnet_transmit(if_t ifp, struct mbuf *m);
198 static int	ptnet_drain_transmit_queue(struct ptnet_queue *pq,
199 					   unsigned int budget,
200 					   bool may_resched);
201 static void	ptnet_qflush(if_t ifp);
202 static void	ptnet_tx_task(void *context, int pending);
203 
204 static int	ptnet_media_change(if_t ifp);
205 static void	ptnet_media_status(if_t ifp, struct ifmediareq *ifmr);
206 #ifdef PTNETMAP_STATS
207 static void	ptnet_tick(void *opaque);
208 #endif
209 
210 static int	ptnet_irqs_init(struct ptnet_softc *sc);
211 static void	ptnet_irqs_fini(struct ptnet_softc *sc);
212 
213 static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd);
214 static int      ptnet_nm_config(struct netmap_adapter *na,
215 				struct nm_config_info *info);
216 static void	ptnet_update_vnet_hdr(struct ptnet_softc *sc);
217 static int	ptnet_nm_register(struct netmap_adapter *na, int onoff);
218 static int	ptnet_nm_txsync(struct netmap_kring *kring, int flags);
219 static int	ptnet_nm_rxsync(struct netmap_kring *kring, int flags);
220 static void	ptnet_nm_intr(struct netmap_adapter *na, int onoff);
221 
222 static void	ptnet_tx_intr(void *opaque);
223 static void	ptnet_rx_intr(void *opaque);
224 
225 static unsigned	ptnet_rx_discard(struct netmap_kring *kring,
226 				 unsigned int head);
227 static int	ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget,
228 			     bool may_resched);
229 static void	ptnet_rx_task(void *context, int pending);
230 
231 #ifdef DEVICE_POLLING
232 static poll_handler_t ptnet_poll;
233 #endif
234 
235 static device_method_t ptnet_methods[] = {
236 	DEVMETHOD(device_probe,			ptnet_probe),
237 	DEVMETHOD(device_attach,		ptnet_attach),
238 	DEVMETHOD(device_detach,		ptnet_detach),
239 	DEVMETHOD(device_suspend,		ptnet_suspend),
240 	DEVMETHOD(device_resume,		ptnet_resume),
241 	DEVMETHOD(device_shutdown,		ptnet_shutdown),
242 	DEVMETHOD_END
243 };
244 
245 static driver_t ptnet_driver = {
246 	"ptnet",
247 	ptnet_methods,
248 	sizeof(struct ptnet_softc)
249 };
250 
251 /* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */
252 DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, NULL, NULL,
253 		      SI_ORDER_MIDDLE + 2);
254 
255 static int
256 ptnet_probe(device_t dev)
257 {
258 	if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID ||
259 		pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) {
260 		return (ENXIO);
261 	}
262 
263 	device_set_desc(dev, "ptnet network adapter");
264 
265 	return (BUS_PROBE_DEFAULT);
266 }
267 
268 static inline void ptnet_kick(struct ptnet_queue *pq)
269 {
270 #ifdef PTNETMAP_STATS
271 	pq->stats.kicks ++;
272 #endif /* PTNETMAP_STATS */
273 	bus_write_4(pq->sc->iomem, pq->kick, 0);
274 }
275 
276 #define PTNET_BUF_RING_SIZE	4096
277 #define PTNET_RX_BUDGET		512
278 #define PTNET_RX_BATCH		1
279 #define PTNET_TX_BUDGET		512
280 #define PTNET_TX_BATCH		64
281 #define PTNET_HDR_SIZE		sizeof(struct virtio_net_hdr_mrg_rxbuf)
282 #define PTNET_MAX_PKT_SIZE	65536
283 
284 #define PTNET_CSUM_OFFLOAD	(CSUM_TCP | CSUM_UDP)
285 #define PTNET_CSUM_OFFLOAD_IPV6	(CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
286 #define PTNET_ALL_OFFLOAD	(CSUM_TSO | PTNET_CSUM_OFFLOAD |\
287 				 PTNET_CSUM_OFFLOAD_IPV6)
288 
289 static int
290 ptnet_attach(device_t dev)
291 {
292 	uint32_t ptfeatures = 0;
293 	unsigned int num_rx_rings, num_tx_rings;
294 	struct netmap_adapter na_arg;
295 	unsigned int nifp_offset;
296 	struct ptnet_softc *sc;
297 	if_t ifp;
298 	uint32_t macreg;
299 	int err, rid;
300 	int i;
301 
302 	sc = device_get_softc(dev);
303 	sc->dev = dev;
304 
305 	/* Setup PCI resources. */
306 	pci_enable_busmaster(dev);
307 
308 	rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR);
309 	sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
310 					   RF_ACTIVE);
311 	if (sc->iomem == NULL) {
312 		device_printf(dev, "Failed to map I/O BAR\n");
313 		return (ENXIO);
314 	}
315 
316 	/* Negotiate features with the hypervisor. */
317 	if (ptnet_vnet_hdr) {
318 		ptfeatures |= PTNETMAP_F_VNET_HDR;
319 	}
320 	bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */
321 	ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */
322 	sc->ptfeatures = ptfeatures;
323 
324 	num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
325 	num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
326 	sc->num_rings = num_tx_rings + num_rx_rings;
327 	sc->num_tx_rings = num_tx_rings;
328 
329 	if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) {
330 		device_printf(dev, "CSB cannot handle that many rings (%u)\n",
331 				sc->num_rings);
332 		err = ENOMEM;
333 		goto err_path;
334 	}
335 
336 	/* Allocate CSB and carry out CSB allocation protocol. */
337 	sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO,
338 				  (size_t)0, -1UL, PAGE_SIZE, 0);
339 	if (sc->csb_gh == NULL) {
340 		device_printf(dev, "Failed to allocate CSB\n");
341 		err = ENOMEM;
342 		goto err_path;
343 	}
344 	sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE);
345 
346 	{
347 		/*
348 		 * We use uint64_t rather than vm_paddr_t since we
349 		 * need 64 bit addresses even on 32 bit platforms.
350 		 */
351 		uint64_t paddr = vtophys(sc->csb_gh);
352 
353 		/* CSB allocation protocol: write to BAH first, then
354 		 * to BAL (for both GH and HG sections). */
355 		bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH,
356 				(paddr >> 32) & 0xffffffff);
357 		bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL,
358 				paddr & 0xffffffff);
359 		paddr = vtophys(sc->csb_hg);
360 		bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH,
361 				(paddr >> 32) & 0xffffffff);
362 		bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL,
363 				paddr & 0xffffffff);
364 	}
365 
366 	/* Allocate and initialize per-queue data structures. */
367 	sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings,
368 			    M_DEVBUF, M_NOWAIT | M_ZERO);
369 	if (sc->queues == NULL) {
370 		err = ENOMEM;
371 		goto err_path;
372 	}
373 	sc->rxqueues = sc->queues + num_tx_rings;
374 
375 	for (i = 0; i < sc->num_rings; i++) {
376 		struct ptnet_queue *pq = sc->queues + i;
377 
378 		pq->sc = sc;
379 		pq->kring_id = i;
380 		pq->kick = PTNET_IO_KICK_BASE + 4 * i;
381 		pq->atok = sc->csb_gh + i;
382 		pq->ktoa = sc->csb_hg + i;
383 		snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d",
384 			 device_get_nameunit(dev), i);
385 		mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF);
386 		if (i >= num_tx_rings) {
387 			/* RX queue: fix kring_id. */
388 			pq->kring_id -= num_tx_rings;
389 		} else {
390 			/* TX queue: allocate buf_ring. */
391 			pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE,
392 						M_DEVBUF, M_NOWAIT, &pq->lock);
393 			if (pq->bufring == NULL) {
394 				err = ENOMEM;
395 				goto err_path;
396 			}
397 		}
398 	}
399 
400 	sc->min_tx_space = 64; /* Safe initial value. */
401 
402 	err = ptnet_irqs_init(sc);
403 	if (err) {
404 		goto err_path;
405 	}
406 
407 	/* Setup Ethernet interface. */
408 	sc->ifp = ifp = if_alloc(IFT_ETHER);
409 	if (ifp == NULL) {
410 		device_printf(dev, "Failed to allocate ifnet\n");
411 		err = ENOMEM;
412 		goto err_path;
413 	}
414 
415 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
416 	ifp->if_baudrate = IF_Gbps(10);
417 	ifp->if_softc = sc;
418 	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
419 	ifp->if_init = ptnet_init;
420 	ifp->if_ioctl = ptnet_ioctl;
421 #if __FreeBSD_version >= 1100000
422 	ifp->if_get_counter = ptnet_get_counter;
423 #endif
424 	ifp->if_transmit = ptnet_transmit;
425 	ifp->if_qflush = ptnet_qflush;
426 
427 	ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change,
428 		     ptnet_media_status);
429 	ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
430 	ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX);
431 
432 	macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI);
433 	sc->hwaddr[0] = (macreg >> 8) & 0xff;
434 	sc->hwaddr[1] = macreg & 0xff;
435 	macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO);
436 	sc->hwaddr[2] = (macreg >> 24) & 0xff;
437 	sc->hwaddr[3] = (macreg >> 16) & 0xff;
438 	sc->hwaddr[4] = (macreg >> 8) & 0xff;
439 	sc->hwaddr[5] = macreg & 0xff;
440 
441 	ether_ifattach(ifp, sc->hwaddr);
442 
443 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
444 	ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
445 
446 	if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) {
447 		/* Similarly to what the vtnet driver does, we can emulate
448 		 * VLAN offloadings by inserting and removing the 802.1Q
449 		 * header during transmit and receive. We are then able
450 		 * to do checksum offloading of VLAN frames. */
451 		ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6
452 					| IFCAP_VLAN_HWCSUM
453 					| IFCAP_TSO | IFCAP_LRO
454 					| IFCAP_VLAN_HWTSO
455 					| IFCAP_VLAN_HWTAGGING;
456 	}
457 
458 	ifp->if_capenable = ifp->if_capabilities;
459 #ifdef DEVICE_POLLING
460 	/* Don't enable polling by default. */
461 	ifp->if_capabilities |= IFCAP_POLLING;
462 #endif
463 	snprintf(sc->lock_name, sizeof(sc->lock_name),
464 		 "%s", device_get_nameunit(dev));
465 	mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF);
466 	callout_init_mtx(&sc->tick, &sc->lock, 0);
467 
468 	/* Prepare a netmap_adapter struct instance to do netmap_attach(). */
469 	nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS);
470 	memset(&na_arg, 0, sizeof(na_arg));
471 	na_arg.ifp = ifp;
472 	na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
473 	na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
474 	na_arg.num_tx_rings = num_tx_rings;
475 	na_arg.num_rx_rings = num_rx_rings;
476 	na_arg.nm_config = ptnet_nm_config;
477 	na_arg.nm_krings_create = ptnet_nm_krings_create;
478 	na_arg.nm_krings_delete = ptnet_nm_krings_delete;
479 	na_arg.nm_dtor = ptnet_nm_dtor;
480 	na_arg.nm_intr = ptnet_nm_intr;
481 	na_arg.nm_register = ptnet_nm_register;
482 	na_arg.nm_txsync = ptnet_nm_txsync;
483 	na_arg.nm_rxsync = ptnet_nm_rxsync;
484 
485 	netmap_pt_guest_attach(&na_arg, nifp_offset,
486                                 bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID));
487 
488 	/* Now a netmap adapter for this ifp has been allocated, and it
489 	 * can be accessed through NA(ifp). We also have to initialize the CSB
490 	 * pointer. */
491 	sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp);
492 
493 	/* If virtio-net header was negotiated, set the virt_hdr_len field in
494 	 * the netmap adapter, to inform users that this netmap adapter requires
495 	 * the application to deal with the headers. */
496 	ptnet_update_vnet_hdr(sc);
497 
498 	device_printf(dev, "%s() completed\n", __func__);
499 
500 	return (0);
501 
502 err_path:
503 	ptnet_detach(dev);
504 	return err;
505 }
506 
507 /* Stop host sync-kloop if it was running. */
508 static void
509 ptnet_device_shutdown(struct ptnet_softc *sc)
510 {
511 	ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
512 	bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0);
513 	bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0);
514 	bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0);
515 	bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0);
516 }
517 
518 static int
519 ptnet_detach(device_t dev)
520 {
521 	struct ptnet_softc *sc = device_get_softc(dev);
522 	int i;
523 
524 	ptnet_device_shutdown(sc);
525 
526 #ifdef DEVICE_POLLING
527 	if (sc->ifp->if_capenable & IFCAP_POLLING) {
528 		ether_poll_deregister(sc->ifp);
529 	}
530 #endif
531 	callout_drain(&sc->tick);
532 
533 	if (sc->queues) {
534 		/* Drain taskqueues before calling if_detach. */
535 		for (i = 0; i < sc->num_rings; i++) {
536 			struct ptnet_queue *pq = sc->queues + i;
537 
538 			if (pq->taskq) {
539 				taskqueue_drain(pq->taskq, &pq->task);
540 			}
541 		}
542 	}
543 
544 	if (sc->ifp) {
545 		ether_ifdetach(sc->ifp);
546 
547 		/* Uninitialize netmap adapters for this device. */
548 		netmap_detach(sc->ifp);
549 
550 		ifmedia_removeall(&sc->media);
551 		if_free(sc->ifp);
552 		sc->ifp = NULL;
553 	}
554 
555 	ptnet_irqs_fini(sc);
556 
557 	if (sc->csb_gh) {
558 		contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF);
559 		sc->csb_gh = NULL;
560 		sc->csb_hg = NULL;
561 	}
562 
563 	if (sc->queues) {
564 		for (i = 0; i < sc->num_rings; i++) {
565 			struct ptnet_queue *pq = sc->queues + i;
566 
567 			if (mtx_initialized(&pq->lock)) {
568 				mtx_destroy(&pq->lock);
569 			}
570 			if (pq->bufring != NULL) {
571 				buf_ring_free(pq->bufring, M_DEVBUF);
572 			}
573 		}
574 		free(sc->queues, M_DEVBUF);
575 		sc->queues = NULL;
576 	}
577 
578 	if (sc->iomem) {
579 		bus_release_resource(dev, SYS_RES_IOPORT,
580 				     PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem);
581 		sc->iomem = NULL;
582 	}
583 
584 	mtx_destroy(&sc->lock);
585 
586 	device_printf(dev, "%s() completed\n", __func__);
587 
588 	return (0);
589 }
590 
591 static int
592 ptnet_suspend(device_t dev)
593 {
594 	struct ptnet_softc *sc = device_get_softc(dev);
595 
596 	(void)sc;
597 
598 	return (0);
599 }
600 
601 static int
602 ptnet_resume(device_t dev)
603 {
604 	struct ptnet_softc *sc = device_get_softc(dev);
605 
606 	(void)sc;
607 
608 	return (0);
609 }
610 
611 static int
612 ptnet_shutdown(device_t dev)
613 {
614 	struct ptnet_softc *sc = device_get_softc(dev);
615 
616 	ptnet_device_shutdown(sc);
617 
618 	return (0);
619 }
620 
621 static int
622 ptnet_irqs_init(struct ptnet_softc *sc)
623 {
624 	int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR);
625 	int nvecs = sc->num_rings;
626 	device_t dev = sc->dev;
627 	int err = ENOSPC;
628 	int cpu_cur;
629 	int i;
630 
631 	if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0)  {
632 		device_printf(dev, "Could not find MSI-X capability\n");
633 		return (ENXIO);
634 	}
635 
636 	sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
637 					      &rid, RF_ACTIVE);
638 	if (sc->msix_mem == NULL) {
639 		device_printf(dev, "Failed to allocate MSIX PCI BAR\n");
640 		return (ENXIO);
641 	}
642 
643 	if (pci_msix_count(dev) < nvecs) {
644 		device_printf(dev, "Not enough MSI-X vectors\n");
645 		goto err_path;
646 	}
647 
648 	err = pci_alloc_msix(dev, &nvecs);
649 	if (err) {
650 		device_printf(dev, "Failed to allocate MSI-X vectors\n");
651 		goto err_path;
652 	}
653 
654 	for (i = 0; i < nvecs; i++) {
655 		struct ptnet_queue *pq = sc->queues + i;
656 
657 		rid = i + 1;
658 		pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
659 						 RF_ACTIVE);
660 		if (pq->irq == NULL) {
661 			device_printf(dev, "Failed to allocate interrupt "
662 					   "for queue #%d\n", i);
663 			err = ENOSPC;
664 			goto err_path;
665 		}
666 	}
667 
668 	cpu_cur = CPU_FIRST();
669 	for (i = 0; i < nvecs; i++) {
670 		struct ptnet_queue *pq = sc->queues + i;
671 		void (*handler)(void *) = ptnet_tx_intr;
672 
673 		if (i >= sc->num_tx_rings) {
674 			handler = ptnet_rx_intr;
675 		}
676 		err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE,
677 				     NULL /* intr_filter */, handler,
678 				     pq, &pq->cookie);
679 		if (err) {
680 			device_printf(dev, "Failed to register intr handler "
681 					   "for queue #%d\n", i);
682 			goto err_path;
683 		}
684 
685 		bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i);
686 #if 0
687 		bus_bind_intr(sc->dev, pq->irq, cpu_cur);
688 #endif
689 		cpu_cur = CPU_NEXT(cpu_cur);
690 	}
691 
692 	device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs);
693 
694 	cpu_cur = CPU_FIRST();
695 	for (i = 0; i < nvecs; i++) {
696 		struct ptnet_queue *pq = sc->queues + i;
697 
698 		if (i < sc->num_tx_rings)
699 			TASK_INIT(&pq->task, 0, ptnet_tx_task, pq);
700 		else
701 			NET_TASK_INIT(&pq->task, 0, ptnet_rx_task, pq);
702 
703 		pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT,
704 					taskqueue_thread_enqueue, &pq->taskq);
705 		taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d",
706 					device_get_nameunit(sc->dev), cpu_cur);
707 		cpu_cur = CPU_NEXT(cpu_cur);
708 	}
709 
710 	return 0;
711 err_path:
712 	ptnet_irqs_fini(sc);
713 	return err;
714 }
715 
716 static void
717 ptnet_irqs_fini(struct ptnet_softc *sc)
718 {
719 	device_t dev = sc->dev;
720 	int i;
721 
722 	for (i = 0; i < sc->num_rings; i++) {
723 		struct ptnet_queue *pq = sc->queues + i;
724 
725 		if (pq->taskq) {
726 			taskqueue_free(pq->taskq);
727 			pq->taskq = NULL;
728 		}
729 
730 		if (pq->cookie) {
731 			bus_teardown_intr(dev, pq->irq, pq->cookie);
732 			pq->cookie = NULL;
733 		}
734 
735 		if (pq->irq) {
736 			bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq);
737 			pq->irq = NULL;
738 		}
739 	}
740 
741 	if (sc->msix_mem) {
742 		pci_release_msi(dev);
743 
744 		bus_release_resource(dev, SYS_RES_MEMORY,
745 				     PCIR_BAR(PTNETMAP_MSIX_PCI_BAR),
746 				     sc->msix_mem);
747 		sc->msix_mem = NULL;
748 	}
749 }
750 
751 static void
752 ptnet_init(void *opaque)
753 {
754 	struct ptnet_softc *sc = opaque;
755 
756 	PTNET_CORE_LOCK(sc);
757 	ptnet_init_locked(sc);
758 	PTNET_CORE_UNLOCK(sc);
759 }
760 
761 static int
762 ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data)
763 {
764 	struct ptnet_softc *sc = if_getsoftc(ifp);
765 	device_t dev = sc->dev;
766 	struct ifreq *ifr = (struct ifreq *)data;
767 	int mask __unused, err = 0;
768 
769 	switch (cmd) {
770 	case SIOCSIFFLAGS:
771 		device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags);
772 		PTNET_CORE_LOCK(sc);
773 		if (ifp->if_flags & IFF_UP) {
774 			/* Network stack wants the iff to be up. */
775 			err = ptnet_init_locked(sc);
776 		} else {
777 			/* Network stack wants the iff to be down. */
778 			err = ptnet_stop(sc);
779 		}
780 		/* We don't need to do nothing to support IFF_PROMISC,
781 		 * since that is managed by the backend port. */
782 		PTNET_CORE_UNLOCK(sc);
783 		break;
784 
785 	case SIOCSIFCAP:
786 		device_printf(dev, "SIOCSIFCAP %x %x\n",
787 			      ifr->ifr_reqcap, ifp->if_capenable);
788 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
789 #ifdef DEVICE_POLLING
790 		if (mask & IFCAP_POLLING) {
791 			struct ptnet_queue *pq;
792 			int i;
793 
794 			if (ifr->ifr_reqcap & IFCAP_POLLING) {
795 				err = ether_poll_register(ptnet_poll, ifp);
796 				if (err) {
797 					break;
798 				}
799 				/* Stop queues and sync with taskqueues. */
800 				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
801 				for (i = 0; i < sc->num_rings; i++) {
802 					pq = sc-> queues + i;
803 					/* Make sure the worker sees the
804 					 * IFF_DRV_RUNNING down. */
805 					PTNET_Q_LOCK(pq);
806 					pq->atok->appl_need_kick = 0;
807 					PTNET_Q_UNLOCK(pq);
808 					/* Wait for rescheduling to finish. */
809 					if (pq->taskq) {
810 						taskqueue_drain(pq->taskq,
811 								&pq->task);
812 					}
813 				}
814 				ifp->if_drv_flags |= IFF_DRV_RUNNING;
815 			} else {
816 				err = ether_poll_deregister(ifp);
817 				for (i = 0; i < sc->num_rings; i++) {
818 					pq = sc-> queues + i;
819 					PTNET_Q_LOCK(pq);
820 					pq->atok->appl_need_kick = 1;
821 					PTNET_Q_UNLOCK(pq);
822 				}
823 			}
824 		}
825 #endif  /* DEVICE_POLLING */
826 		ifp->if_capenable = ifr->ifr_reqcap;
827 		break;
828 
829 	case SIOCSIFMTU:
830 		/* We support any reasonable MTU. */
831 		if (ifr->ifr_mtu < ETHERMIN ||
832 				ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) {
833 			err = EINVAL;
834 		} else {
835 			PTNET_CORE_LOCK(sc);
836 			ifp->if_mtu = ifr->ifr_mtu;
837 			PTNET_CORE_UNLOCK(sc);
838 		}
839 		break;
840 
841 	case SIOCSIFMEDIA:
842 	case SIOCGIFMEDIA:
843 		err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
844 		break;
845 
846 	default:
847 		err = ether_ioctl(ifp, cmd, data);
848 		break;
849 	}
850 
851 	return err;
852 }
853 
854 static int
855 ptnet_init_locked(struct ptnet_softc *sc)
856 {
857 	if_t ifp = sc->ifp;
858 	struct netmap_adapter *na_dr = &sc->ptna->dr.up;
859 	struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
860 	unsigned int nm_buf_size;
861 	int ret;
862 
863 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
864 		return 0; /* nothing to do */
865 	}
866 
867 	device_printf(sc->dev, "%s\n", __func__);
868 
869 	/* Translate offload capabilities according to if_capenable. */
870 	ifp->if_hwassist = 0;
871 	if (ifp->if_capenable & IFCAP_TXCSUM)
872 		ifp->if_hwassist |= PTNET_CSUM_OFFLOAD;
873 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
874 		ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6;
875 	if (ifp->if_capenable & IFCAP_TSO4)
876 		ifp->if_hwassist |= CSUM_IP_TSO;
877 	if (ifp->if_capenable & IFCAP_TSO6)
878 		ifp->if_hwassist |= CSUM_IP6_TSO;
879 
880 	/*
881 	 * Prepare the interface for netmap mode access.
882 	 */
883 	netmap_update_config(na_dr);
884 
885 	ret = netmap_mem_finalize(na_dr->nm_mem, na_dr);
886 	if (ret) {
887 		device_printf(sc->dev, "netmap_mem_finalize() failed\n");
888 		return ret;
889 	}
890 
891 	if (sc->ptna->backend_users == 0) {
892 		ret = ptnet_nm_krings_create(na_nm);
893 		if (ret) {
894 			device_printf(sc->dev, "ptnet_nm_krings_create() "
895 					       "failed\n");
896 			goto err_mem_finalize;
897 		}
898 
899 		ret = netmap_mem_rings_create(na_dr);
900 		if (ret) {
901 			device_printf(sc->dev, "netmap_mem_rings_create() "
902 					       "failed\n");
903 			goto err_rings_create;
904 		}
905 
906 		ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut);
907 		if (ret) {
908 			device_printf(sc->dev, "netmap_mem_get_lut() "
909 					       "failed\n");
910 			goto err_get_lut;
911 		}
912 	}
913 
914 	ret = ptnet_nm_register(na_dr, 1 /* on */);
915 	if (ret) {
916 		goto err_register;
917 	}
918 
919 	nm_buf_size = NETMAP_BUF_SIZE(na_dr);
920 
921 	KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size"));
922 	sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2;
923 	device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__,
924 		      sc->min_tx_space);
925 #ifdef PTNETMAP_STATS
926 	callout_reset(&sc->tick, hz, ptnet_tick, sc);
927 #endif
928 
929 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
930 
931 	return 0;
932 
933 err_register:
934 	memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut));
935 err_get_lut:
936 	netmap_mem_rings_delete(na_dr);
937 err_rings_create:
938 	ptnet_nm_krings_delete(na_nm);
939 err_mem_finalize:
940 	netmap_mem_deref(na_dr->nm_mem, na_dr);
941 
942 	return ret;
943 }
944 
945 /* To be called under core lock. */
946 static int
947 ptnet_stop(struct ptnet_softc *sc)
948 {
949 	if_t ifp = sc->ifp;
950 	struct netmap_adapter *na_dr = &sc->ptna->dr.up;
951 	struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
952 	int i;
953 
954 	device_printf(sc->dev, "%s\n", __func__);
955 
956 	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
957 		return 0; /* nothing to do */
958 	}
959 
960 	/* Clear the driver-ready flag, and synchronize with all the queues,
961 	 * so that after this loop we are sure nobody is working anymore with
962 	 * the device. This scheme is taken from the vtnet driver. */
963 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
964 	callout_stop(&sc->tick);
965 	for (i = 0; i < sc->num_rings; i++) {
966 		PTNET_Q_LOCK(sc->queues + i);
967 		PTNET_Q_UNLOCK(sc->queues + i);
968 	}
969 
970 	ptnet_nm_register(na_dr, 0 /* off */);
971 
972 	if (sc->ptna->backend_users == 0) {
973 		netmap_mem_rings_delete(na_dr);
974 		ptnet_nm_krings_delete(na_nm);
975 	}
976 	netmap_mem_deref(na_dr->nm_mem, na_dr);
977 
978 	return 0;
979 }
980 
981 static void
982 ptnet_qflush(if_t ifp)
983 {
984 	struct ptnet_softc *sc = if_getsoftc(ifp);
985 	int i;
986 
987 	/* Flush all the bufrings and do the interface flush. */
988 	for (i = 0; i < sc->num_rings; i++) {
989 		struct ptnet_queue *pq = sc->queues + i;
990 		struct mbuf *m;
991 
992 		PTNET_Q_LOCK(pq);
993 		if (pq->bufring) {
994 			while ((m = buf_ring_dequeue_sc(pq->bufring))) {
995 				m_freem(m);
996 			}
997 		}
998 		PTNET_Q_UNLOCK(pq);
999 	}
1000 
1001 	if_qflush(ifp);
1002 }
1003 
1004 static int
1005 ptnet_media_change(if_t ifp)
1006 {
1007 	struct ptnet_softc *sc = if_getsoftc(ifp);
1008 	struct ifmedia *ifm = &sc->media;
1009 
1010 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) {
1011 		return EINVAL;
1012 	}
1013 
1014 	return 0;
1015 }
1016 
1017 #if __FreeBSD_version >= 1100000
1018 static uint64_t
1019 ptnet_get_counter(if_t ifp, ift_counter cnt)
1020 {
1021 	struct ptnet_softc *sc = if_getsoftc(ifp);
1022 	struct ptnet_queue_stats stats[2];
1023 	int i;
1024 
1025 	/* Accumulate statistics over the queues. */
1026 	memset(stats, 0, sizeof(stats));
1027 	for (i = 0; i < sc->num_rings; i++) {
1028 		struct ptnet_queue *pq = sc->queues + i;
1029 		int idx = (i < sc->num_tx_rings) ? 0 : 1;
1030 
1031 		stats[idx].packets	+= pq->stats.packets;
1032 		stats[idx].bytes	+= pq->stats.bytes;
1033 		stats[idx].errors	+= pq->stats.errors;
1034 		stats[idx].iqdrops	+= pq->stats.iqdrops;
1035 		stats[idx].mcasts	+= pq->stats.mcasts;
1036 	}
1037 
1038 	switch (cnt) {
1039 	case IFCOUNTER_IPACKETS:
1040 		return (stats[1].packets);
1041 	case IFCOUNTER_IQDROPS:
1042 		return (stats[1].iqdrops);
1043 	case IFCOUNTER_IERRORS:
1044 		return (stats[1].errors);
1045 	case IFCOUNTER_OPACKETS:
1046 		return (stats[0].packets);
1047 	case IFCOUNTER_OBYTES:
1048 		return (stats[0].bytes);
1049 	case IFCOUNTER_OMCASTS:
1050 		return (stats[0].mcasts);
1051 	default:
1052 		return (if_get_counter_default(ifp, cnt));
1053 	}
1054 }
1055 #endif
1056 
1057 
1058 #ifdef PTNETMAP_STATS
1059 /* Called under core lock. */
1060 static void
1061 ptnet_tick(void *opaque)
1062 {
1063 	struct ptnet_softc *sc = opaque;
1064 	int i;
1065 
1066 	for (i = 0; i < sc->num_rings; i++) {
1067 		struct ptnet_queue *pq = sc->queues + i;
1068 		struct ptnet_queue_stats cur = pq->stats;
1069 		struct timeval now;
1070 		unsigned int delta;
1071 
1072 		microtime(&now);
1073 		delta = now.tv_usec - sc->last_ts.tv_usec +
1074 			(now.tv_sec - sc->last_ts.tv_sec) * 1000000;
1075 		delta /= 1000; /* in milliseconds */
1076 
1077 		if (delta == 0)
1078 			continue;
1079 
1080 		device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, "
1081 			      "intr %lu\n", i, delta,
1082 			      (cur.packets - pq->last_stats.packets),
1083 			      (cur.kicks - pq->last_stats.kicks),
1084 			      (cur.intrs - pq->last_stats.intrs));
1085 		pq->last_stats = cur;
1086 	}
1087 	microtime(&sc->last_ts);
1088 	callout_schedule(&sc->tick, hz);
1089 }
1090 #endif /* PTNETMAP_STATS */
1091 
1092 static void
1093 ptnet_media_status(if_t ifp, struct ifmediareq *ifmr)
1094 {
1095 	/* We are always active, as the backend netmap port is
1096 	 * always open in netmap mode. */
1097 	ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
1098 	ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
1099 }
1100 
1101 static uint32_t
1102 ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd)
1103 {
1104 	/*
1105 	 * Write a command and read back error status,
1106 	 * with zero meaning success.
1107 	 */
1108 	bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd);
1109 	return bus_read_4(sc->iomem, PTNET_IO_PTCTL);
1110 }
1111 
1112 static int
1113 ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info)
1114 {
1115 	struct ptnet_softc *sc = if_getsoftc(na->ifp);
1116 
1117 	info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
1118 	info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
1119 	info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
1120 	info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
1121 	info->rx_buf_maxsize = NETMAP_BUF_SIZE(na);
1122 
1123 	device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n",
1124 			info->num_tx_rings, info->num_rx_rings,
1125 			info->num_tx_descs, info->num_rx_descs,
1126 			info->rx_buf_maxsize);
1127 
1128 	return 0;
1129 }
1130 
1131 static void
1132 ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na)
1133 {
1134 	int i;
1135 
1136 	/* Sync krings from the host, reading from
1137 	 * CSB. */
1138 	for (i = 0; i < sc->num_rings; i++) {
1139 		struct nm_csb_atok *atok = sc->queues[i].atok;
1140 		struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa;
1141 		struct netmap_kring *kring;
1142 
1143 		if (i < na->num_tx_rings) {
1144 			kring = na->tx_rings[i];
1145 		} else {
1146 			kring = na->rx_rings[i - na->num_tx_rings];
1147 		}
1148 		kring->rhead = kring->ring->head = atok->head;
1149 		kring->rcur = kring->ring->cur = atok->cur;
1150 		kring->nr_hwcur = ktoa->hwcur;
1151 		kring->nr_hwtail = kring->rtail =
1152 			kring->ring->tail = ktoa->hwtail;
1153 
1154 		nm_prdis("%d,%d: csb {hc %u h %u c %u ht %u}", t, i,
1155 		   ktoa->hwcur, atok->head, atok->cur,
1156 		   ktoa->hwtail);
1157 		nm_prdis("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}",
1158 		   t, i, kring->nr_hwcur, kring->rhead, kring->rcur,
1159 		   kring->ring->head, kring->ring->cur, kring->nr_hwtail,
1160 		   kring->rtail, kring->ring->tail);
1161 	}
1162 }
1163 
1164 static void
1165 ptnet_update_vnet_hdr(struct ptnet_softc *sc)
1166 {
1167 	unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0;
1168 
1169 	bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len);
1170 	sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN);
1171 	sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len;
1172 }
1173 
1174 static int
1175 ptnet_nm_register(struct netmap_adapter *na, int onoff)
1176 {
1177 	/* device-specific */
1178 	if_t ifp = na->ifp;
1179 	struct ptnet_softc *sc = if_getsoftc(ifp);
1180 	int native = (na == &sc->ptna->hwup.up);
1181 	struct ptnet_queue *pq;
1182 	int ret = 0;
1183 	int i;
1184 
1185 	if (!onoff) {
1186 		sc->ptna->backend_users--;
1187 	}
1188 
1189 	/* If this is the last netmap client, guest interrupt enable flags may
1190 	 * be in arbitrary state. Since these flags are going to be used also
1191 	 * by the netdevice driver, we have to make sure to start with
1192 	 * notifications enabled. Also, schedule NAPI to flush pending packets
1193 	 * in the RX rings, since we will not receive further interrupts
1194 	 * until these will be processed. */
1195 	if (native && !onoff && na->active_fds == 0) {
1196 		nm_prinf("Exit netmap mode, re-enable interrupts");
1197 		for (i = 0; i < sc->num_rings; i++) {
1198 			pq = sc->queues + i;
1199 			pq->atok->appl_need_kick = 1;
1200 		}
1201 	}
1202 
1203 	if (onoff) {
1204 		if (sc->ptna->backend_users == 0) {
1205 			/* Initialize notification enable fields in the CSB. */
1206 			for (i = 0; i < sc->num_rings; i++) {
1207 				pq = sc->queues + i;
1208 				pq->ktoa->kern_need_kick = 1;
1209 				pq->atok->appl_need_kick =
1210 					(!(ifp->if_capenable & IFCAP_POLLING)
1211 						&& i >= sc->num_tx_rings);
1212 			}
1213 
1214 			/* Set the virtio-net header length. */
1215 			ptnet_update_vnet_hdr(sc);
1216 
1217 			/* Make sure the host adapter passed through is ready
1218 			 * for txsync/rxsync. */
1219 			ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE);
1220 			if (ret) {
1221 				return ret;
1222 			}
1223 
1224 			/* Align the guest krings and rings to the state stored
1225 			 * in the CSB. */
1226 			ptnet_sync_from_csb(sc, na);
1227 		}
1228 
1229 		/* If not native, don't call nm_set_native_flags, since we don't want
1230 		 * to replace if_transmit method, nor set NAF_NETMAP_ON */
1231 		if (native) {
1232 			netmap_krings_mode_commit(na, onoff);
1233 			nm_set_native_flags(na);
1234 		}
1235 
1236 	} else {
1237 		if (native) {
1238 			nm_clear_native_flags(na);
1239 			netmap_krings_mode_commit(na, onoff);
1240 		}
1241 
1242 		if (sc->ptna->backend_users == 0) {
1243 			ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
1244 		}
1245 	}
1246 
1247 	if (onoff) {
1248 		sc->ptna->backend_users++;
1249 	}
1250 
1251 	return ret;
1252 }
1253 
1254 static int
1255 ptnet_nm_txsync(struct netmap_kring *kring, int flags)
1256 {
1257 	struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1258 	struct ptnet_queue *pq = sc->queues + kring->ring_id;
1259 	bool notify;
1260 
1261 	notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags);
1262 	if (notify) {
1263 		ptnet_kick(pq);
1264 	}
1265 
1266 	return 0;
1267 }
1268 
1269 static int
1270 ptnet_nm_rxsync(struct netmap_kring *kring, int flags)
1271 {
1272 	struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1273 	struct ptnet_queue *pq = sc->rxqueues + kring->ring_id;
1274 	bool notify;
1275 
1276 	notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags);
1277 	if (notify) {
1278 		ptnet_kick(pq);
1279 	}
1280 
1281 	return 0;
1282 }
1283 
1284 static void
1285 ptnet_nm_intr(struct netmap_adapter *na, int onoff)
1286 {
1287 	struct ptnet_softc *sc = if_getsoftc(na->ifp);
1288 	int i;
1289 
1290 	for (i = 0; i < sc->num_rings; i++) {
1291 		struct ptnet_queue *pq = sc->queues + i;
1292 		pq->atok->appl_need_kick = onoff;
1293 	}
1294 }
1295 
1296 static void
1297 ptnet_tx_intr(void *opaque)
1298 {
1299 	struct ptnet_queue *pq = opaque;
1300 	struct ptnet_softc *sc = pq->sc;
1301 
1302 	DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id));
1303 #ifdef PTNETMAP_STATS
1304 	pq->stats.intrs ++;
1305 #endif /* PTNETMAP_STATS */
1306 
1307 	if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) {
1308 		return;
1309 	}
1310 
1311 	/* Schedule the tasqueue to flush process transmissions requests.
1312 	 * However, vtnet, if_em and if_igb just call ptnet_transmit() here,
1313 	 * at least when using MSI-X interrupts. The if_em driver, instead
1314 	 * schedule taskqueue when using legacy interrupts. */
1315 	taskqueue_enqueue(pq->taskq, &pq->task);
1316 }
1317 
1318 static void
1319 ptnet_rx_intr(void *opaque)
1320 {
1321 	struct ptnet_queue *pq = opaque;
1322 	struct ptnet_softc *sc = pq->sc;
1323 	unsigned int unused;
1324 
1325 	DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id));
1326 #ifdef PTNETMAP_STATS
1327 	pq->stats.intrs ++;
1328 #endif /* PTNETMAP_STATS */
1329 
1330 	if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) {
1331 		return;
1332 	}
1333 
1334 	/* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts,
1335 	 * receive-side processing is executed directly in the interrupt
1336 	 * service routine. Alternatively, we may schedule the taskqueue. */
1337 	ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1338 }
1339 
1340 static void
1341 ptnet_vlan_tag_remove(struct mbuf *m)
1342 {
1343 	struct ether_vlan_header *evh;
1344 
1345 	evh = mtod(m, struct ether_vlan_header *);
1346 	m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
1347 	m->m_flags |= M_VLANTAG;
1348 
1349 	/* Strip the 802.1Q header. */
1350 	bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
1351 	    ETHER_HDR_LEN - ETHER_TYPE_LEN);
1352 	m_adj(m, ETHER_VLAN_ENCAP_LEN);
1353 }
1354 
1355 static void
1356 ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring,
1357 		  unsigned int head, unsigned int sync_flags)
1358 {
1359 	struct netmap_ring *ring = kring->ring;
1360 	struct nm_csb_atok *atok = pq->atok;
1361 	struct nm_csb_ktoa *ktoa = pq->ktoa;
1362 
1363 	/* Some packets have been pushed to the netmap ring. We have
1364 	 * to tell the host to process the new packets, updating cur
1365 	 * and head in the CSB. */
1366 	ring->head = ring->cur = head;
1367 
1368 	/* Mimic nm_txsync_prologue/nm_rxsync_prologue. */
1369 	kring->rcur = kring->rhead = head;
1370 
1371 	nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
1372 
1373 	/* Kick the host if needed. */
1374 	if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
1375 		atok->sync_flags = sync_flags;
1376 		ptnet_kick(pq);
1377 	}
1378 }
1379 
1380 #define PTNET_TX_NOSPACE(_h, _k, _min)	\
1381 	((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \
1382 		(_k)->rtail - (_h)) < (_min)
1383 
1384 /* This function may be called by the network stack, or by
1385  * by the taskqueue thread. */
1386 static int
1387 ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget,
1388 			   bool may_resched)
1389 {
1390 	struct ptnet_softc *sc = pq->sc;
1391 	bool have_vnet_hdr = sc->vnet_hdr_len;
1392 	struct netmap_adapter *na = &sc->ptna->dr.up;
1393 	if_t ifp = sc->ifp;
1394 	unsigned int batch_count = 0;
1395 	struct nm_csb_atok *atok;
1396 	struct nm_csb_ktoa *ktoa;
1397 	struct netmap_kring *kring;
1398 	struct netmap_ring *ring;
1399 	struct netmap_slot *slot;
1400 	unsigned int count = 0;
1401 	unsigned int minspace;
1402 	unsigned int head;
1403 	unsigned int lim;
1404 	struct mbuf *mhead;
1405 	struct mbuf *mf;
1406 	int nmbuf_bytes;
1407 	uint8_t *nmbuf;
1408 
1409 	if (!PTNET_Q_TRYLOCK(pq)) {
1410 		/* We failed to acquire the lock, schedule the taskqueue. */
1411 		nm_prlim(1, "Deferring TX work");
1412 		if (may_resched) {
1413 			taskqueue_enqueue(pq->taskq, &pq->task);
1414 		}
1415 
1416 		return 0;
1417 	}
1418 
1419 	if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1420 		PTNET_Q_UNLOCK(pq);
1421 		nm_prlim(1, "Interface is down");
1422 		return ENETDOWN;
1423 	}
1424 
1425 	atok = pq->atok;
1426 	ktoa = pq->ktoa;
1427 	kring = na->tx_rings[pq->kring_id];
1428 	ring = kring->ring;
1429 	lim = kring->nkr_num_slots - 1;
1430 	head = ring->head;
1431 	minspace = sc->min_tx_space;
1432 
1433 	while (count < budget) {
1434 		if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1435 			/* We ran out of slot, let's see if the host has
1436 			 * freed up some, by reading hwcur and hwtail from
1437 			 * the CSB. */
1438 			ptnet_sync_tail(ktoa, kring);
1439 
1440 			if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1441 				/* Still no slots available. Reactivate the
1442 				 * interrupts so that we can be notified
1443 				 * when some free slots are made available by
1444 				 * the host. */
1445 				atok->appl_need_kick = 1;
1446 
1447 				/* Double check. We need a full barrier to
1448 				 * prevent the store to atok->appl_need_kick
1449 				 * to be reordered with the load from
1450 				 * ktoa->hwcur and ktoa->hwtail (store-load
1451 				 * barrier). */
1452 				nm_stld_barrier();
1453 				ptnet_sync_tail(ktoa, kring);
1454 				if (likely(PTNET_TX_NOSPACE(head, kring,
1455 							    minspace))) {
1456 					break;
1457 				}
1458 
1459 				nm_prlim(1, "Found more slots by doublecheck");
1460 				/* More slots were freed before reactivating
1461 				 * the interrupts. */
1462 				atok->appl_need_kick = 0;
1463 			}
1464 		}
1465 
1466 		mhead = drbr_peek(ifp, pq->bufring);
1467 		if (!mhead) {
1468 			break;
1469 		}
1470 
1471 		/* Initialize transmission state variables. */
1472 		slot = ring->slot + head;
1473 		nmbuf = NMB(na, slot);
1474 		nmbuf_bytes = 0;
1475 
1476 		/* If needed, prepare the virtio-net header at the beginning
1477 		 * of the first slot. */
1478 		if (have_vnet_hdr) {
1479 			struct virtio_net_hdr *vh =
1480 					(struct virtio_net_hdr *)nmbuf;
1481 
1482 			/* For performance, we could replace this memset() with
1483 			 * two 8-bytes-wide writes. */
1484 			memset(nmbuf, 0, PTNET_HDR_SIZE);
1485 			if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) {
1486 				mhead = virtio_net_tx_offload(ifp, mhead, false,
1487 							 vh);
1488 				if (unlikely(!mhead)) {
1489 					/* Packet dropped because errors
1490 					 * occurred while preparing the vnet
1491 					 * header. Let's go ahead with the next
1492 					 * packet. */
1493 					pq->stats.errors ++;
1494 					drbr_advance(ifp, pq->bufring);
1495 					continue;
1496 				}
1497 			}
1498 			nm_prdis(1, "%s: [csum_flags %lX] vnet hdr: flags %x "
1499 			      "csum_start %u csum_ofs %u hdr_len = %u "
1500 			      "gso_size %u gso_type %x", __func__,
1501 			      mhead->m_pkthdr.csum_flags, vh->flags,
1502 			      vh->csum_start, vh->csum_offset, vh->hdr_len,
1503 			      vh->gso_size, vh->gso_type);
1504 
1505 			nmbuf += PTNET_HDR_SIZE;
1506 			nmbuf_bytes += PTNET_HDR_SIZE;
1507 		}
1508 
1509 		for (mf = mhead; mf; mf = mf->m_next) {
1510 			uint8_t *mdata = mf->m_data;
1511 			int mlen = mf->m_len;
1512 
1513 			for (;;) {
1514 				int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes;
1515 
1516 				if (mlen < copy) {
1517 					copy = mlen;
1518 				}
1519 				memcpy(nmbuf, mdata, copy);
1520 
1521 				mdata += copy;
1522 				mlen -= copy;
1523 				nmbuf += copy;
1524 				nmbuf_bytes += copy;
1525 
1526 				if (!mlen) {
1527 					break;
1528 				}
1529 
1530 				slot->len = nmbuf_bytes;
1531 				slot->flags = NS_MOREFRAG;
1532 
1533 				head = nm_next(head, lim);
1534 				KASSERT(head != ring->tail,
1535 					("Unexpectedly run out of TX space"));
1536 				slot = ring->slot + head;
1537 				nmbuf = NMB(na, slot);
1538 				nmbuf_bytes = 0;
1539 			}
1540 		}
1541 
1542 		/* Complete last slot and update head. */
1543 		slot->len = nmbuf_bytes;
1544 		slot->flags = 0;
1545 		head = nm_next(head, lim);
1546 
1547 		/* Consume the packet just processed. */
1548 		drbr_advance(ifp, pq->bufring);
1549 
1550 		/* Copy the packet to listeners. */
1551 		ETHER_BPF_MTAP(ifp, mhead);
1552 
1553 		pq->stats.packets ++;
1554 		pq->stats.bytes += mhead->m_pkthdr.len;
1555 		if (mhead->m_flags & M_MCAST) {
1556 			pq->stats.mcasts ++;
1557 		}
1558 
1559 		m_freem(mhead);
1560 
1561 		count ++;
1562 		if (++batch_count == PTNET_TX_BATCH) {
1563 			ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1564 			batch_count = 0;
1565 		}
1566 	}
1567 
1568 	if (batch_count) {
1569 		ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1570 	}
1571 
1572 	if (count >= budget && may_resched) {
1573 		DBG(nm_prlim(1, "out of budget: resched, %d mbufs pending\n",
1574 					drbr_inuse(ifp, pq->bufring)));
1575 		taskqueue_enqueue(pq->taskq, &pq->task);
1576 	}
1577 
1578 	PTNET_Q_UNLOCK(pq);
1579 
1580 	return count;
1581 }
1582 
1583 static int
1584 ptnet_transmit(if_t ifp, struct mbuf *m)
1585 {
1586 	struct ptnet_softc *sc = if_getsoftc(ifp);
1587 	struct ptnet_queue *pq;
1588 	unsigned int queue_idx;
1589 	int err;
1590 
1591 	DBG(device_printf(sc->dev, "transmit %p\n", m));
1592 
1593 	/* Insert 802.1Q header if needed. */
1594 	if (m->m_flags & M_VLANTAG) {
1595 		m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1596 		if (m == NULL) {
1597 			return ENOBUFS;
1598 		}
1599 		m->m_flags &= ~M_VLANTAG;
1600 	}
1601 
1602 	/* Get the flow-id if available. */
1603 	queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ?
1604 		    m->m_pkthdr.flowid : curcpu;
1605 
1606 	if (unlikely(queue_idx >= sc->num_tx_rings)) {
1607 		queue_idx %= sc->num_tx_rings;
1608 	}
1609 
1610 	pq = sc->queues + queue_idx;
1611 
1612 	err = drbr_enqueue(ifp, pq->bufring, m);
1613 	if (err) {
1614 		/* ENOBUFS when the bufring is full */
1615 		nm_prlim(1, "%s: drbr_enqueue() failed %d\n",
1616 			__func__, err);
1617 		pq->stats.errors ++;
1618 		return err;
1619 	}
1620 
1621 	if (ifp->if_capenable & IFCAP_POLLING) {
1622 		/* If polling is on, the transmit queues will be
1623 		 * drained by the poller. */
1624 		return 0;
1625 	}
1626 
1627 	err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1628 
1629 	return (err < 0) ? err : 0;
1630 }
1631 
1632 static unsigned int
1633 ptnet_rx_discard(struct netmap_kring *kring, unsigned int head)
1634 {
1635 	struct netmap_ring *ring = kring->ring;
1636 	struct netmap_slot *slot = ring->slot + head;
1637 
1638 	for (;;) {
1639 		head = nm_next(head, kring->nkr_num_slots - 1);
1640 		if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) {
1641 			break;
1642 		}
1643 		slot = ring->slot + head;
1644 	}
1645 
1646 	return head;
1647 }
1648 
1649 static inline struct mbuf *
1650 ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len)
1651 {
1652 	uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len;
1653 
1654 	do {
1655 		unsigned int copy;
1656 
1657 		if (mtail->m_len == MCLBYTES) {
1658 			struct mbuf *mf;
1659 
1660 			mf = m_getcl(M_NOWAIT, MT_DATA, 0);
1661 			if (unlikely(!mf)) {
1662 				return NULL;
1663 			}
1664 
1665 			mtail->m_next = mf;
1666 			mtail = mf;
1667 			mdata = mtod(mtail, uint8_t *);
1668 			mtail->m_len = 0;
1669 		}
1670 
1671 		copy = MCLBYTES - mtail->m_len;
1672 		if (nmbuf_len < copy) {
1673 			copy = nmbuf_len;
1674 		}
1675 
1676 		memcpy(mdata, nmbuf, copy);
1677 
1678 		nmbuf += copy;
1679 		nmbuf_len -= copy;
1680 		mdata += copy;
1681 		mtail->m_len += copy;
1682 	} while (nmbuf_len);
1683 
1684 	return mtail;
1685 }
1686 
1687 static int
1688 ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched)
1689 {
1690 	struct ptnet_softc *sc = pq->sc;
1691 	bool have_vnet_hdr = sc->vnet_hdr_len;
1692 	struct nm_csb_atok *atok = pq->atok;
1693 	struct nm_csb_ktoa *ktoa = pq->ktoa;
1694 	struct netmap_adapter *na = &sc->ptna->dr.up;
1695 	struct netmap_kring *kring = na->rx_rings[pq->kring_id];
1696 	struct netmap_ring *ring = kring->ring;
1697 	unsigned int const lim = kring->nkr_num_slots - 1;
1698 	unsigned int batch_count = 0;
1699 	if_t ifp = sc->ifp;
1700 	unsigned int count = 0;
1701 	uint32_t head;
1702 
1703 	PTNET_Q_LOCK(pq);
1704 
1705 	if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1706 		goto unlock;
1707 	}
1708 
1709 	kring->nr_kflags &= ~NKR_PENDINTR;
1710 
1711 	head = ring->head;
1712 	while (count < budget) {
1713 		uint32_t prev_head = head;
1714 		struct mbuf *mhead, *mtail;
1715 		struct virtio_net_hdr *vh;
1716 		struct netmap_slot *slot;
1717 		unsigned int nmbuf_len;
1718 		uint8_t *nmbuf;
1719 		int deliver = 1; /* the mbuf to the network stack. */
1720 host_sync:
1721 		if (head == ring->tail) {
1722 			/* We ran out of slot, let's see if the host has
1723 			 * added some, by reading hwcur and hwtail from
1724 			 * the CSB. */
1725 			ptnet_sync_tail(ktoa, kring);
1726 
1727 			if (head == ring->tail) {
1728 				/* Still no slots available. Reactivate
1729 				 * interrupts as they were disabled by the
1730 				 * host thread right before issuing the
1731 				 * last interrupt. */
1732 				atok->appl_need_kick = 1;
1733 
1734 				/* Double check for more completed RX slots.
1735 				 * We need a full barrier to prevent the store
1736 				 * to atok->appl_need_kick to be reordered with
1737 				 * the load from ktoa->hwcur and ktoa->hwtail
1738 				 * (store-load barrier). */
1739 				nm_stld_barrier();
1740 				ptnet_sync_tail(ktoa, kring);
1741 				if (likely(head == ring->tail)) {
1742 					break;
1743 				}
1744 				atok->appl_need_kick = 0;
1745 			}
1746 		}
1747 
1748 		/* Initialize ring state variables, possibly grabbing the
1749 		 * virtio-net header. */
1750 		slot = ring->slot + head;
1751 		nmbuf = NMB(na, slot);
1752 		nmbuf_len = slot->len;
1753 
1754 		vh = (struct virtio_net_hdr *)nmbuf;
1755 		if (have_vnet_hdr) {
1756 			if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) {
1757 				/* There is no good reason why host should
1758 				 * put the header in multiple netmap slots.
1759 				 * If this is the case, discard. */
1760 				nm_prlim(1, "Fragmented vnet-hdr: dropping");
1761 				head = ptnet_rx_discard(kring, head);
1762 				pq->stats.iqdrops ++;
1763 				deliver = 0;
1764 				goto skip;
1765 			}
1766 			nm_prdis(1, "%s: vnet hdr: flags %x csum_start %u "
1767 			      "csum_ofs %u hdr_len = %u gso_size %u "
1768 			      "gso_type %x", __func__, vh->flags,
1769 			      vh->csum_start, vh->csum_offset, vh->hdr_len,
1770 			      vh->gso_size, vh->gso_type);
1771 			nmbuf += PTNET_HDR_SIZE;
1772 			nmbuf_len -= PTNET_HDR_SIZE;
1773 		}
1774 
1775 		/* Allocate the head of a new mbuf chain.
1776 		 * We use m_getcl() to allocate an mbuf with standard cluster
1777 		 * size (MCLBYTES). In the future we could use m_getjcl()
1778 		 * to choose different sizes. */
1779 		mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1780 		if (unlikely(mhead == NULL)) {
1781 			device_printf(sc->dev, "%s: failed to allocate mbuf "
1782 				      "head\n", __func__);
1783 			pq->stats.errors ++;
1784 			break;
1785 		}
1786 
1787 		/* Initialize the mbuf state variables. */
1788 		mhead->m_pkthdr.len = nmbuf_len;
1789 		mtail->m_len = 0;
1790 
1791 		/* Scan all the netmap slots containing the current packet. */
1792 		for (;;) {
1793 			DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag "
1794 					  "len %u, flags %u\n", __func__,
1795 					  head, ring->tail, slot->len,
1796 					  slot->flags));
1797 
1798 			mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len);
1799 			if (unlikely(!mtail)) {
1800 				/* Ouch. We ran out of memory while processing
1801 				 * a packet. We have to restore the previous
1802 				 * head position, free the mbuf chain, and
1803 				 * schedule the taskqueue to give the packet
1804 				 * another chance. */
1805 				device_printf(sc->dev, "%s: failed to allocate"
1806 					" mbuf frag, reset head %u --> %u\n",
1807 					__func__, head, prev_head);
1808 				head = prev_head;
1809 				m_freem(mhead);
1810 				pq->stats.errors ++;
1811 				if (may_resched) {
1812 					taskqueue_enqueue(pq->taskq,
1813 							  &pq->task);
1814 				}
1815 				goto escape;
1816 			}
1817 
1818 			/* We have to increment head irrespective of the
1819 			 * NS_MOREFRAG being set or not. */
1820 			head = nm_next(head, lim);
1821 
1822 			if (!(slot->flags & NS_MOREFRAG)) {
1823 				break;
1824 			}
1825 
1826 			if (unlikely(head == ring->tail)) {
1827 				/* The very last slot prepared by the host has
1828 				 * the NS_MOREFRAG set. Drop it and continue
1829 				 * the outer cycle (to do the double-check). */
1830 				nm_prlim(1, "Incomplete packet: dropping");
1831 				m_freem(mhead);
1832 				pq->stats.iqdrops ++;
1833 				goto host_sync;
1834 			}
1835 
1836 			slot = ring->slot + head;
1837 			nmbuf = NMB(na, slot);
1838 			nmbuf_len = slot->len;
1839 			mhead->m_pkthdr.len += nmbuf_len;
1840 		}
1841 
1842 		mhead->m_pkthdr.rcvif = ifp;
1843 		mhead->m_pkthdr.csum_flags = 0;
1844 
1845 		/* Store the queue idx in the packet header. */
1846 		mhead->m_pkthdr.flowid = pq->kring_id;
1847 		M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE);
1848 
1849 		if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1850 			struct ether_header *eh;
1851 
1852 			eh = mtod(mhead, struct ether_header *);
1853 			if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1854 				ptnet_vlan_tag_remove(mhead);
1855 				/*
1856 				 * With the 802.1Q header removed, update the
1857 				 * checksum starting location accordingly.
1858 				 */
1859 				if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1860 					vh->csum_start -= ETHER_VLAN_ENCAP_LEN;
1861 			}
1862 		}
1863 
1864 		if (unlikely(have_vnet_hdr && virtio_net_rx_csum(mhead, vh))) {
1865 			m_freem(mhead);
1866 			nm_prlim(1, "Csum offload error: dropping");
1867 			pq->stats.iqdrops ++;
1868 			deliver = 0;
1869 		}
1870 
1871 skip:
1872 		count ++;
1873 		if (++batch_count >= PTNET_RX_BATCH) {
1874 			/* Some packets have been (or will be) pushed to the network
1875 			 * stack. We need to update the CSB to tell the host about
1876 			 * the new ring->cur and ring->head (RX buffer refill). */
1877 			ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
1878 			batch_count = 0;
1879 		}
1880 
1881 		if (likely(deliver))  {
1882 			pq->stats.packets ++;
1883 			pq->stats.bytes += mhead->m_pkthdr.len;
1884 
1885 			PTNET_Q_UNLOCK(pq);
1886 			(*ifp->if_input)(ifp, mhead);
1887 			PTNET_Q_LOCK(pq);
1888 			/* The ring->head index (and related indices) are
1889 			 * updated under pq lock by ptnet_ring_update().
1890 			 * Since we dropped the lock to call if_input(), we
1891 			 * must reload ring->head and restart processing the
1892 			 * ring from there. */
1893 			head = ring->head;
1894 
1895 			if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1896 				/* The interface has gone down while we didn't
1897 				 * have the lock. Stop any processing and exit. */
1898 				goto unlock;
1899 			}
1900 		}
1901 	}
1902 escape:
1903 	if (batch_count) {
1904 		ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
1905 
1906 	}
1907 
1908 	if (count >= budget && may_resched) {
1909 		/* If we ran out of budget or the double-check found new
1910 		 * slots to process, schedule the taskqueue. */
1911 		DBG(nm_prlim(1, "out of budget: resched h %u t %u\n",
1912 					head, ring->tail));
1913 		taskqueue_enqueue(pq->taskq, &pq->task);
1914 	}
1915 unlock:
1916 	PTNET_Q_UNLOCK(pq);
1917 
1918 	return count;
1919 }
1920 
1921 static void
1922 ptnet_rx_task(void *context, int pending)
1923 {
1924 	struct ptnet_queue *pq = context;
1925 
1926 	DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
1927 	ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1928 }
1929 
1930 static void
1931 ptnet_tx_task(void *context, int pending)
1932 {
1933 	struct ptnet_queue *pq = context;
1934 
1935 	DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
1936 	ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1937 }
1938 
1939 #ifdef DEVICE_POLLING
1940 /* We don't need to handle differently POLL_AND_CHECK_STATUS and
1941  * POLL_ONLY, since we don't have an Interrupt Status Register. */
1942 static int
1943 ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget)
1944 {
1945 	struct ptnet_softc *sc = if_getsoftc(ifp);
1946 	unsigned int queue_budget;
1947 	unsigned int count = 0;
1948 	bool borrow = false;
1949 	int i;
1950 
1951 	KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet"));
1952 	queue_budget = MAX(budget / sc->num_rings, 1);
1953 	nm_prlim(1, "Per-queue budget is %d", queue_budget);
1954 
1955 	while (budget) {
1956 		unsigned int rcnt = 0;
1957 
1958 		for (i = 0; i < sc->num_rings; i++) {
1959 			struct ptnet_queue *pq = sc->queues + i;
1960 
1961 			if (borrow) {
1962 				queue_budget = MIN(queue_budget, budget);
1963 				if (queue_budget == 0) {
1964 					break;
1965 				}
1966 			}
1967 
1968 			if (i < sc->num_tx_rings) {
1969 				rcnt += ptnet_drain_transmit_queue(pq,
1970 						   queue_budget, false);
1971 			} else {
1972 				rcnt += ptnet_rx_eof(pq, queue_budget,
1973 						      false);
1974 			}
1975 		}
1976 
1977 		if (!rcnt) {
1978 			/* A scan of the queues gave no result, we can
1979 			 * stop here. */
1980 			break;
1981 		}
1982 
1983 		if (rcnt > budget) {
1984 			/* This may happen when initial budget < sc->num_rings,
1985 			 * since one packet budget is given to each queue
1986 			 * anyway. Just pretend we didn't eat "so much". */
1987 			rcnt = budget;
1988 		}
1989 		count += rcnt;
1990 		budget -= rcnt;
1991 		borrow = true;
1992 	}
1993 
1994 
1995 	return count;
1996 }
1997 #endif /* DEVICE_POLLING */
1998 #endif /* WITH_PTNETMAP */
1999