xref: /freebsd/sys/dev/virtio/network/if_vtnet.c (revision 069ac18495ad8fde2748bc94b0f80a50250bb01d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 /* Driver for VirtIO network devices. */
30 
31 #include <sys/param.h>
32 #include <sys/eventhandler.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/sockio.h>
36 #include <sys/malloc.h>
37 #include <sys/mbuf.h>
38 #include <sys/module.h>
39 #include <sys/msan.h>
40 #include <sys/socket.h>
41 #include <sys/sysctl.h>
42 #include <sys/random.h>
43 #include <sys/sglist.h>
44 #include <sys/lock.h>
45 #include <sys/mutex.h>
46 #include <sys/taskqueue.h>
47 #include <sys/smp.h>
48 #include <machine/smp.h>
49 
50 #include <vm/uma.h>
51 
52 #include <net/debugnet.h>
53 #include <net/ethernet.h>
54 #include <net/pfil.h>
55 #include <net/if.h>
56 #include <net/if_var.h>
57 #include <net/if_arp.h>
58 #include <net/if_dl.h>
59 #include <net/if_types.h>
60 #include <net/if_media.h>
61 #include <net/if_vlan_var.h>
62 
63 #include <net/bpf.h>
64 
65 #include <netinet/in_systm.h>
66 #include <netinet/in.h>
67 #include <netinet/ip.h>
68 #include <netinet/ip6.h>
69 #include <netinet6/ip6_var.h>
70 #include <netinet/udp.h>
71 #include <netinet/tcp.h>
72 #include <netinet/tcp_lro.h>
73 
74 #include <machine/bus.h>
75 #include <machine/resource.h>
76 #include <sys/bus.h>
77 #include <sys/rman.h>
78 
79 #include <dev/virtio/virtio.h>
80 #include <dev/virtio/virtqueue.h>
81 #include <dev/virtio/network/virtio_net.h>
82 #include <dev/virtio/network/if_vtnetvar.h>
83 #include "virtio_if.h"
84 
85 #include "opt_inet.h"
86 #include "opt_inet6.h"
87 
88 #if defined(INET) || defined(INET6)
89 #include <machine/in_cksum.h>
90 #endif
91 
92 static int	vtnet_modevent(module_t, int, void *);
93 
94 static int	vtnet_probe(device_t);
95 static int	vtnet_attach(device_t);
96 static int	vtnet_detach(device_t);
97 static int	vtnet_suspend(device_t);
98 static int	vtnet_resume(device_t);
99 static int	vtnet_shutdown(device_t);
100 static int	vtnet_attach_completed(device_t);
101 static int	vtnet_config_change(device_t);
102 
103 static int	vtnet_negotiate_features(struct vtnet_softc *);
104 static int	vtnet_setup_features(struct vtnet_softc *);
105 static int	vtnet_init_rxq(struct vtnet_softc *, int);
106 static int	vtnet_init_txq(struct vtnet_softc *, int);
107 static int	vtnet_alloc_rxtx_queues(struct vtnet_softc *);
108 static void	vtnet_free_rxtx_queues(struct vtnet_softc *);
109 static int	vtnet_alloc_rx_filters(struct vtnet_softc *);
110 static void	vtnet_free_rx_filters(struct vtnet_softc *);
111 static int	vtnet_alloc_virtqueues(struct vtnet_softc *);
112 static int	vtnet_alloc_interface(struct vtnet_softc *);
113 static int	vtnet_setup_interface(struct vtnet_softc *);
114 static int	vtnet_ioctl_mtu(struct vtnet_softc *, u_int);
115 static int	vtnet_ioctl_ifflags(struct vtnet_softc *);
116 static int	vtnet_ioctl_multi(struct vtnet_softc *);
117 static int	vtnet_ioctl_ifcap(struct vtnet_softc *, struct ifreq *);
118 static int	vtnet_ioctl(if_t, u_long, caddr_t);
119 static uint64_t	vtnet_get_counter(if_t, ift_counter);
120 
121 static int	vtnet_rxq_populate(struct vtnet_rxq *);
122 static void	vtnet_rxq_free_mbufs(struct vtnet_rxq *);
123 static struct mbuf *
124 		vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **);
125 static int	vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *,
126 		    struct mbuf *, int);
127 static int	vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int);
128 static int	vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *);
129 static int	vtnet_rxq_new_buf(struct vtnet_rxq *);
130 static int	vtnet_rxq_csum_needs_csum(struct vtnet_rxq *, struct mbuf *,
131 		     uint16_t, int, struct virtio_net_hdr *);
132 static int	vtnet_rxq_csum_data_valid(struct vtnet_rxq *, struct mbuf *,
133 		     uint16_t, int, struct virtio_net_hdr *);
134 static int	vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *,
135 		     struct virtio_net_hdr *);
136 static void	vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int);
137 static void	vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *);
138 static int	vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int);
139 static void	vtnet_rxq_input(struct vtnet_rxq *, struct mbuf *,
140 		    struct virtio_net_hdr *);
141 static int	vtnet_rxq_eof(struct vtnet_rxq *);
142 static void	vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries);
143 static void	vtnet_rx_vq_intr(void *);
144 static void	vtnet_rxq_tq_intr(void *, int);
145 
146 static int	vtnet_txq_intr_threshold(struct vtnet_txq *);
147 static int	vtnet_txq_below_threshold(struct vtnet_txq *);
148 static int	vtnet_txq_notify(struct vtnet_txq *);
149 static void	vtnet_txq_free_mbufs(struct vtnet_txq *);
150 static int	vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *,
151 		    int *, int *, int *);
152 static int	vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int,
153 		    int, struct virtio_net_hdr *);
154 static struct mbuf *
155 		vtnet_txq_offload(struct vtnet_txq *, struct mbuf *,
156 		    struct virtio_net_hdr *);
157 static int	vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **,
158 		    struct vtnet_tx_header *);
159 static int	vtnet_txq_encap(struct vtnet_txq *, struct mbuf **, int);
160 #ifdef VTNET_LEGACY_TX
161 static void	vtnet_start_locked(struct vtnet_txq *, if_t);
162 static void	vtnet_start(if_t);
163 #else
164 static int	vtnet_txq_mq_start_locked(struct vtnet_txq *, struct mbuf *);
165 static int	vtnet_txq_mq_start(if_t, struct mbuf *);
166 static void	vtnet_txq_tq_deferred(void *, int);
167 #endif
168 static void	vtnet_txq_start(struct vtnet_txq *);
169 static void	vtnet_txq_tq_intr(void *, int);
170 static int	vtnet_txq_eof(struct vtnet_txq *);
171 static void	vtnet_tx_vq_intr(void *);
172 static void	vtnet_tx_start_all(struct vtnet_softc *);
173 
174 #ifndef VTNET_LEGACY_TX
175 static void	vtnet_qflush(if_t);
176 #endif
177 
178 static int	vtnet_watchdog(struct vtnet_txq *);
179 static void	vtnet_accum_stats(struct vtnet_softc *,
180 		    struct vtnet_rxq_stats *, struct vtnet_txq_stats *);
181 static void	vtnet_tick(void *);
182 
183 static void	vtnet_start_taskqueues(struct vtnet_softc *);
184 static void	vtnet_free_taskqueues(struct vtnet_softc *);
185 static void	vtnet_drain_taskqueues(struct vtnet_softc *);
186 
187 static void	vtnet_drain_rxtx_queues(struct vtnet_softc *);
188 static void	vtnet_stop_rendezvous(struct vtnet_softc *);
189 static void	vtnet_stop(struct vtnet_softc *);
190 static int	vtnet_virtio_reinit(struct vtnet_softc *);
191 static void	vtnet_init_rx_filters(struct vtnet_softc *);
192 static int	vtnet_init_rx_queues(struct vtnet_softc *);
193 static int	vtnet_init_tx_queues(struct vtnet_softc *);
194 static int	vtnet_init_rxtx_queues(struct vtnet_softc *);
195 static void	vtnet_set_active_vq_pairs(struct vtnet_softc *);
196 static void	vtnet_update_rx_offloads(struct vtnet_softc *);
197 static int	vtnet_reinit(struct vtnet_softc *);
198 static void	vtnet_init_locked(struct vtnet_softc *, int);
199 static void	vtnet_init(void *);
200 
201 static void	vtnet_free_ctrl_vq(struct vtnet_softc *);
202 static void	vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
203 		    struct sglist *, int, int);
204 static int	vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *);
205 static int	vtnet_ctrl_guest_offloads(struct vtnet_softc *, uint64_t);
206 static int	vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t);
207 static int	vtnet_ctrl_rx_cmd(struct vtnet_softc *, uint8_t, bool);
208 static int	vtnet_set_promisc(struct vtnet_softc *, bool);
209 static int	vtnet_set_allmulti(struct vtnet_softc *, bool);
210 static void	vtnet_rx_filter(struct vtnet_softc *);
211 static void	vtnet_rx_filter_mac(struct vtnet_softc *);
212 static int	vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
213 static void	vtnet_rx_filter_vlan(struct vtnet_softc *);
214 static void	vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t);
215 static void	vtnet_register_vlan(void *, if_t, uint16_t);
216 static void	vtnet_unregister_vlan(void *, if_t, uint16_t);
217 
218 static void	vtnet_update_speed_duplex(struct vtnet_softc *);
219 static int	vtnet_is_link_up(struct vtnet_softc *);
220 static void	vtnet_update_link_status(struct vtnet_softc *);
221 static int	vtnet_ifmedia_upd(if_t);
222 static void	vtnet_ifmedia_sts(if_t, struct ifmediareq *);
223 static void	vtnet_get_macaddr(struct vtnet_softc *);
224 static void	vtnet_set_macaddr(struct vtnet_softc *);
225 static void	vtnet_attached_set_macaddr(struct vtnet_softc *);
226 static void	vtnet_vlan_tag_remove(struct mbuf *);
227 static void	vtnet_set_rx_process_limit(struct vtnet_softc *);
228 
229 static void	vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *,
230 		    struct sysctl_oid_list *, struct vtnet_rxq *);
231 static void	vtnet_setup_txq_sysctl(struct sysctl_ctx_list *,
232 		    struct sysctl_oid_list *, struct vtnet_txq *);
233 static void	vtnet_setup_queue_sysctl(struct vtnet_softc *);
234 static void	vtnet_load_tunables(struct vtnet_softc *);
235 static void	vtnet_setup_sysctl(struct vtnet_softc *);
236 
237 static int	vtnet_rxq_enable_intr(struct vtnet_rxq *);
238 static void	vtnet_rxq_disable_intr(struct vtnet_rxq *);
239 static int	vtnet_txq_enable_intr(struct vtnet_txq *);
240 static void	vtnet_txq_disable_intr(struct vtnet_txq *);
241 static void	vtnet_enable_rx_interrupts(struct vtnet_softc *);
242 static void	vtnet_enable_tx_interrupts(struct vtnet_softc *);
243 static void	vtnet_enable_interrupts(struct vtnet_softc *);
244 static void	vtnet_disable_rx_interrupts(struct vtnet_softc *);
245 static void	vtnet_disable_tx_interrupts(struct vtnet_softc *);
246 static void	vtnet_disable_interrupts(struct vtnet_softc *);
247 
248 static int	vtnet_tunable_int(struct vtnet_softc *, const char *, int);
249 
250 DEBUGNET_DEFINE(vtnet);
251 
252 #define vtnet_htog16(_sc, _val)	virtio_htog16(vtnet_modern(_sc), _val)
253 #define vtnet_htog32(_sc, _val)	virtio_htog32(vtnet_modern(_sc), _val)
254 #define vtnet_htog64(_sc, _val)	virtio_htog64(vtnet_modern(_sc), _val)
255 #define vtnet_gtoh16(_sc, _val)	virtio_gtoh16(vtnet_modern(_sc), _val)
256 #define vtnet_gtoh32(_sc, _val)	virtio_gtoh32(vtnet_modern(_sc), _val)
257 #define vtnet_gtoh64(_sc, _val)	virtio_gtoh64(vtnet_modern(_sc), _val)
258 
259 /* Tunables. */
260 static SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
261     "VirtIO Net driver parameters");
262 
263 static int vtnet_csum_disable = 0;
264 SYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN,
265     &vtnet_csum_disable, 0, "Disables receive and send checksum offload");
266 
267 static int vtnet_fixup_needs_csum = 0;
268 SYSCTL_INT(_hw_vtnet, OID_AUTO, fixup_needs_csum, CTLFLAG_RDTUN,
269     &vtnet_fixup_needs_csum, 0,
270     "Calculate valid checksum for NEEDS_CSUM packets");
271 
272 static int vtnet_tso_disable = 0;
273 SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN,
274     &vtnet_tso_disable, 0, "Disables TSO");
275 
276 static int vtnet_lro_disable = 0;
277 SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_disable, CTLFLAG_RDTUN,
278     &vtnet_lro_disable, 0, "Disables hardware LRO");
279 
280 static int vtnet_mq_disable = 0;
281 SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_disable, CTLFLAG_RDTUN,
282     &vtnet_mq_disable, 0, "Disables multiqueue support");
283 
284 static int vtnet_mq_max_pairs = VTNET_MAX_QUEUE_PAIRS;
285 SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_max_pairs, CTLFLAG_RDTUN,
286     &vtnet_mq_max_pairs, 0, "Maximum number of multiqueue pairs");
287 
288 static int vtnet_tso_maxlen = IP_MAXPACKET;
289 SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN,
290     &vtnet_tso_maxlen, 0, "TSO burst limit");
291 
292 static int vtnet_rx_process_limit = 1024;
293 SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
294     &vtnet_rx_process_limit, 0,
295     "Number of RX segments processed in one pass");
296 
297 static int vtnet_lro_entry_count = 128;
298 SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN,
299     &vtnet_lro_entry_count, 0, "Software LRO entry count");
300 
301 /* Enable sorted LRO, and the depth of the mbuf queue. */
302 static int vtnet_lro_mbufq_depth = 0;
303 SYSCTL_UINT(_hw_vtnet, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN,
304     &vtnet_lro_mbufq_depth, 0, "Depth of software LRO mbuf queue");
305 
306 static uma_zone_t vtnet_tx_header_zone;
307 
308 static struct virtio_feature_desc vtnet_feature_desc[] = {
309 	{ VIRTIO_NET_F_CSUM,			"TxChecksum"		},
310 	{ VIRTIO_NET_F_GUEST_CSUM,		"RxChecksum"		},
311 	{ VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,	"CtrlRxOffloads"	},
312 	{ VIRTIO_NET_F_MAC,			"MAC"			},
313 	{ VIRTIO_NET_F_GSO,			"TxGSO"			},
314 	{ VIRTIO_NET_F_GUEST_TSO4,		"RxLROv4"		},
315 	{ VIRTIO_NET_F_GUEST_TSO6,		"RxLROv6"		},
316 	{ VIRTIO_NET_F_GUEST_ECN,		"RxLROECN"		},
317 	{ VIRTIO_NET_F_GUEST_UFO,		"RxUFO"			},
318 	{ VIRTIO_NET_F_HOST_TSO4,		"TxTSOv4"		},
319 	{ VIRTIO_NET_F_HOST_TSO6,		"TxTSOv6"		},
320 	{ VIRTIO_NET_F_HOST_ECN,		"TxTSOECN"		},
321 	{ VIRTIO_NET_F_HOST_UFO,		"TxUFO"			},
322 	{ VIRTIO_NET_F_MRG_RXBUF,		"MrgRxBuf"		},
323 	{ VIRTIO_NET_F_STATUS,			"Status"		},
324 	{ VIRTIO_NET_F_CTRL_VQ,			"CtrlVq"		},
325 	{ VIRTIO_NET_F_CTRL_RX,			"CtrlRxMode"		},
326 	{ VIRTIO_NET_F_CTRL_VLAN,		"CtrlVLANFilter"	},
327 	{ VIRTIO_NET_F_CTRL_RX_EXTRA,		"CtrlRxModeExtra"	},
328 	{ VIRTIO_NET_F_GUEST_ANNOUNCE,		"GuestAnnounce"		},
329 	{ VIRTIO_NET_F_MQ,			"Multiqueue"		},
330 	{ VIRTIO_NET_F_CTRL_MAC_ADDR,		"CtrlMacAddr"		},
331 	{ VIRTIO_NET_F_SPEED_DUPLEX,		"SpeedDuplex"		},
332 
333 	{ 0, NULL }
334 };
335 
336 static device_method_t vtnet_methods[] = {
337 	/* Device methods. */
338 	DEVMETHOD(device_probe,			vtnet_probe),
339 	DEVMETHOD(device_attach,		vtnet_attach),
340 	DEVMETHOD(device_detach,		vtnet_detach),
341 	DEVMETHOD(device_suspend,		vtnet_suspend),
342 	DEVMETHOD(device_resume,		vtnet_resume),
343 	DEVMETHOD(device_shutdown,		vtnet_shutdown),
344 
345 	/* VirtIO methods. */
346 	DEVMETHOD(virtio_attach_completed,	vtnet_attach_completed),
347 	DEVMETHOD(virtio_config_change,		vtnet_config_change),
348 
349 	DEVMETHOD_END
350 };
351 
352 #ifdef DEV_NETMAP
353 #include <dev/netmap/if_vtnet_netmap.h>
354 #endif
355 
356 static driver_t vtnet_driver = {
357     .name = "vtnet",
358     .methods = vtnet_methods,
359     .size = sizeof(struct vtnet_softc)
360 };
361 VIRTIO_DRIVER_MODULE(vtnet, vtnet_driver, vtnet_modevent, NULL);
362 MODULE_VERSION(vtnet, 1);
363 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
364 #ifdef DEV_NETMAP
365 MODULE_DEPEND(vtnet, netmap, 1, 1, 1);
366 #endif
367 
368 VIRTIO_SIMPLE_PNPINFO(vtnet, VIRTIO_ID_NETWORK, "VirtIO Networking Adapter");
369 
370 static int
371 vtnet_modevent(module_t mod __unused, int type, void *unused __unused)
372 {
373 	int error = 0;
374 	static int loaded = 0;
375 
376 	switch (type) {
377 	case MOD_LOAD:
378 		if (loaded++ == 0) {
379 			vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr",
380 				sizeof(struct vtnet_tx_header),
381 				NULL, NULL, NULL, NULL, 0, 0);
382 #ifdef DEBUGNET
383 			/*
384 			 * We need to allocate from this zone in the transmit path, so ensure
385 			 * that we have at least one item per header available.
386 			 * XXX add a separate zone like we do for mbufs? otherwise we may alloc
387 			 * buckets
388 			 */
389 			uma_zone_reserve(vtnet_tx_header_zone, DEBUGNET_MAX_IN_FLIGHT * 2);
390 			uma_prealloc(vtnet_tx_header_zone, DEBUGNET_MAX_IN_FLIGHT * 2);
391 #endif
392 		}
393 		break;
394 	case MOD_QUIESCE:
395 		if (uma_zone_get_cur(vtnet_tx_header_zone) > 0)
396 			error = EBUSY;
397 		break;
398 	case MOD_UNLOAD:
399 		if (--loaded == 0) {
400 			uma_zdestroy(vtnet_tx_header_zone);
401 			vtnet_tx_header_zone = NULL;
402 		}
403 		break;
404 	case MOD_SHUTDOWN:
405 		break;
406 	default:
407 		error = EOPNOTSUPP;
408 		break;
409 	}
410 
411 	return (error);
412 }
413 
414 static int
415 vtnet_probe(device_t dev)
416 {
417 	return (VIRTIO_SIMPLE_PROBE(dev, vtnet));
418 }
419 
420 static int
421 vtnet_attach(device_t dev)
422 {
423 	struct vtnet_softc *sc;
424 	int error;
425 
426 	sc = device_get_softc(dev);
427 	sc->vtnet_dev = dev;
428 	virtio_set_feature_desc(dev, vtnet_feature_desc);
429 
430 	VTNET_CORE_LOCK_INIT(sc);
431 	callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0);
432 	vtnet_load_tunables(sc);
433 
434 	error = vtnet_alloc_interface(sc);
435 	if (error) {
436 		device_printf(dev, "cannot allocate interface\n");
437 		goto fail;
438 	}
439 
440 	vtnet_setup_sysctl(sc);
441 
442 	error = vtnet_setup_features(sc);
443 	if (error) {
444 		device_printf(dev, "cannot setup features\n");
445 		goto fail;
446 	}
447 
448 	error = vtnet_alloc_rx_filters(sc);
449 	if (error) {
450 		device_printf(dev, "cannot allocate Rx filters\n");
451 		goto fail;
452 	}
453 
454 	error = vtnet_alloc_rxtx_queues(sc);
455 	if (error) {
456 		device_printf(dev, "cannot allocate queues\n");
457 		goto fail;
458 	}
459 
460 	error = vtnet_alloc_virtqueues(sc);
461 	if (error) {
462 		device_printf(dev, "cannot allocate virtqueues\n");
463 		goto fail;
464 	}
465 
466 	error = vtnet_setup_interface(sc);
467 	if (error) {
468 		device_printf(dev, "cannot setup interface\n");
469 		goto fail;
470 	}
471 
472 	error = virtio_setup_intr(dev, INTR_TYPE_NET);
473 	if (error) {
474 		device_printf(dev, "cannot setup interrupts\n");
475 		ether_ifdetach(sc->vtnet_ifp);
476 		goto fail;
477 	}
478 
479 #ifdef DEV_NETMAP
480 	vtnet_netmap_attach(sc);
481 #endif
482 	vtnet_start_taskqueues(sc);
483 
484 fail:
485 	if (error)
486 		vtnet_detach(dev);
487 
488 	return (error);
489 }
490 
491 static int
492 vtnet_detach(device_t dev)
493 {
494 	struct vtnet_softc *sc;
495 	if_t ifp;
496 
497 	sc = device_get_softc(dev);
498 	ifp = sc->vtnet_ifp;
499 
500 	if (device_is_attached(dev)) {
501 		VTNET_CORE_LOCK(sc);
502 		vtnet_stop(sc);
503 		VTNET_CORE_UNLOCK(sc);
504 
505 		callout_drain(&sc->vtnet_tick_ch);
506 		vtnet_drain_taskqueues(sc);
507 
508 		ether_ifdetach(ifp);
509 	}
510 
511 #ifdef DEV_NETMAP
512 	netmap_detach(ifp);
513 #endif
514 
515 	if (sc->vtnet_pfil != NULL) {
516 		pfil_head_unregister(sc->vtnet_pfil);
517 		sc->vtnet_pfil = NULL;
518 	}
519 
520 	vtnet_free_taskqueues(sc);
521 
522 	if (sc->vtnet_vlan_attach != NULL) {
523 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
524 		sc->vtnet_vlan_attach = NULL;
525 	}
526 	if (sc->vtnet_vlan_detach != NULL) {
527 		EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach);
528 		sc->vtnet_vlan_detach = NULL;
529 	}
530 
531 	ifmedia_removeall(&sc->vtnet_media);
532 
533 	if (ifp != NULL) {
534 		if_free(ifp);
535 		sc->vtnet_ifp = NULL;
536 	}
537 
538 	vtnet_free_rxtx_queues(sc);
539 	vtnet_free_rx_filters(sc);
540 
541 	if (sc->vtnet_ctrl_vq != NULL)
542 		vtnet_free_ctrl_vq(sc);
543 
544 	VTNET_CORE_LOCK_DESTROY(sc);
545 
546 	return (0);
547 }
548 
549 static int
550 vtnet_suspend(device_t dev)
551 {
552 	struct vtnet_softc *sc;
553 
554 	sc = device_get_softc(dev);
555 
556 	VTNET_CORE_LOCK(sc);
557 	vtnet_stop(sc);
558 	sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
559 	VTNET_CORE_UNLOCK(sc);
560 
561 	return (0);
562 }
563 
564 static int
565 vtnet_resume(device_t dev)
566 {
567 	struct vtnet_softc *sc;
568 	if_t ifp;
569 
570 	sc = device_get_softc(dev);
571 	ifp = sc->vtnet_ifp;
572 
573 	VTNET_CORE_LOCK(sc);
574 	if (if_getflags(ifp) & IFF_UP)
575 		vtnet_init_locked(sc, 0);
576 	sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
577 	VTNET_CORE_UNLOCK(sc);
578 
579 	return (0);
580 }
581 
582 static int
583 vtnet_shutdown(device_t dev)
584 {
585 	/*
586 	 * Suspend already does all of what we need to
587 	 * do here; we just never expect to be resumed.
588 	 */
589 	return (vtnet_suspend(dev));
590 }
591 
592 static int
593 vtnet_attach_completed(device_t dev)
594 {
595 	struct vtnet_softc *sc;
596 
597 	sc = device_get_softc(dev);
598 
599 	VTNET_CORE_LOCK(sc);
600 	vtnet_attached_set_macaddr(sc);
601 	VTNET_CORE_UNLOCK(sc);
602 
603 	return (0);
604 }
605 
606 static int
607 vtnet_config_change(device_t dev)
608 {
609 	struct vtnet_softc *sc;
610 
611 	sc = device_get_softc(dev);
612 
613 	VTNET_CORE_LOCK(sc);
614 	vtnet_update_link_status(sc);
615 	if (sc->vtnet_link_active != 0)
616 		vtnet_tx_start_all(sc);
617 	VTNET_CORE_UNLOCK(sc);
618 
619 	return (0);
620 }
621 
622 static int
623 vtnet_negotiate_features(struct vtnet_softc *sc)
624 {
625 	device_t dev;
626 	uint64_t features, negotiated_features;
627 	int no_csum;
628 
629 	dev = sc->vtnet_dev;
630 	features = virtio_bus_is_modern(dev) ? VTNET_MODERN_FEATURES :
631 	    VTNET_LEGACY_FEATURES;
632 
633 	/*
634 	 * TSO and LRO are only available when their corresponding checksum
635 	 * offload feature is also negotiated.
636 	 */
637 	no_csum = vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable);
638 	if (no_csum)
639 		features &= ~(VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM);
640 	if (no_csum || vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable))
641 		features &= ~VTNET_TSO_FEATURES;
642 	if (no_csum || vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable))
643 		features &= ~VTNET_LRO_FEATURES;
644 
645 #ifndef VTNET_LEGACY_TX
646 	if (vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable))
647 		features &= ~VIRTIO_NET_F_MQ;
648 #else
649 	features &= ~VIRTIO_NET_F_MQ;
650 #endif
651 
652 	negotiated_features = virtio_negotiate_features(dev, features);
653 
654 	if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) {
655 		uint16_t mtu;
656 
657 		mtu = virtio_read_dev_config_2(dev,
658 		    offsetof(struct virtio_net_config, mtu));
659 		if (mtu < VTNET_MIN_MTU /* || mtu > VTNET_MAX_MTU */) {
660 			device_printf(dev, "Invalid MTU value: %d. "
661 			    "MTU feature disabled.\n", mtu);
662 			features &= ~VIRTIO_NET_F_MTU;
663 			negotiated_features =
664 			    virtio_negotiate_features(dev, features);
665 		}
666 	}
667 
668 	if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) {
669 		uint16_t npairs;
670 
671 		npairs = virtio_read_dev_config_2(dev,
672 		    offsetof(struct virtio_net_config, max_virtqueue_pairs));
673 		if (npairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
674 		    npairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) {
675 			device_printf(dev, "Invalid max_virtqueue_pairs value: "
676 			    "%d. Multiqueue feature disabled.\n", npairs);
677 			features &= ~VIRTIO_NET_F_MQ;
678 			negotiated_features =
679 			    virtio_negotiate_features(dev, features);
680 		}
681 	}
682 
683 	if (virtio_with_feature(dev, VTNET_LRO_FEATURES) &&
684 	    virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) {
685 		/*
686 		 * LRO without mergeable buffers requires special care. This
687 		 * is not ideal because every receive buffer must be large
688 		 * enough to hold the maximum TCP packet, the Ethernet header,
689 		 * and the header. This requires up to 34 descriptors with
690 		 * MCLBYTES clusters. If we do not have indirect descriptors,
691 		 * LRO is disabled since the virtqueue will not contain very
692 		 * many receive buffers.
693 		 */
694 		if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) {
695 			device_printf(dev,
696 			    "Host LRO disabled since both mergeable buffers "
697 			    "and indirect descriptors were not negotiated\n");
698 			features &= ~VTNET_LRO_FEATURES;
699 			negotiated_features =
700 			    virtio_negotiate_features(dev, features);
701 		} else
702 			sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
703 	}
704 
705 	sc->vtnet_features = negotiated_features;
706 	sc->vtnet_negotiated_features = negotiated_features;
707 
708 	return (virtio_finalize_features(dev));
709 }
710 
711 static int
712 vtnet_setup_features(struct vtnet_softc *sc)
713 {
714 	device_t dev;
715 	int error;
716 
717 	dev = sc->vtnet_dev;
718 
719 	error = vtnet_negotiate_features(sc);
720 	if (error)
721 		return (error);
722 
723 	if (virtio_with_feature(dev, VIRTIO_F_VERSION_1))
724 		sc->vtnet_flags |= VTNET_FLAG_MODERN;
725 	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
726 		sc->vtnet_flags |= VTNET_FLAG_INDIRECT;
727 	if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX))
728 		sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX;
729 
730 	if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
731 		/* This feature should always be negotiated. */
732 		sc->vtnet_flags |= VTNET_FLAG_MAC;
733 	}
734 
735 	if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) {
736 		sc->vtnet_max_mtu = virtio_read_dev_config_2(dev,
737 		    offsetof(struct virtio_net_config, mtu));
738 	} else
739 		sc->vtnet_max_mtu = VTNET_MAX_MTU;
740 
741 	if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
742 		sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
743 		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
744 	} else if (vtnet_modern(sc)) {
745 		/* This is identical to the mergeable header. */
746 		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_v1);
747 	} else
748 		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
749 
750 	if (vtnet_modern(sc) || sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS)
751 		sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_INLINE;
752 	else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
753 		sc->vtnet_rx_nsegs = VTNET_RX_SEGS_LRO_NOMRG;
754 	else
755 		sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_SEPARATE;
756 
757 	/*
758 	 * Favor "hardware" LRO if negotiated, but support software LRO as
759 	 * a fallback; there is usually little benefit (or worse) with both.
760 	 */
761 	if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) == 0 &&
762 	    virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) == 0)
763 		sc->vtnet_flags |= VTNET_FLAG_SW_LRO;
764 
765 	if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) ||
766 	    virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) ||
767 	    virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
768 		sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MAX;
769 	else
770 		sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MIN;
771 
772 	sc->vtnet_req_vq_pairs = 1;
773 	sc->vtnet_max_vq_pairs = 1;
774 
775 	if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
776 		sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
777 
778 		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
779 			sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
780 		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
781 			sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
782 		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR))
783 			sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC;
784 
785 		if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) {
786 			sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev,
787 			    offsetof(struct virtio_net_config,
788 			    max_virtqueue_pairs));
789 		}
790 	}
791 
792 	if (sc->vtnet_max_vq_pairs > 1) {
793 		int req;
794 
795 		/*
796 		 * Limit the maximum number of requested queue pairs to the
797 		 * number of CPUs and the configured maximum.
798 		 */
799 		req = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs);
800 		if (req < 0)
801 			req = 1;
802 		if (req == 0)
803 			req = mp_ncpus;
804 		if (req > sc->vtnet_max_vq_pairs)
805 			req = sc->vtnet_max_vq_pairs;
806 		if (req > mp_ncpus)
807 			req = mp_ncpus;
808 		if (req > 1) {
809 			sc->vtnet_req_vq_pairs = req;
810 			sc->vtnet_flags |= VTNET_FLAG_MQ;
811 		}
812 	}
813 
814 	return (0);
815 }
816 
817 static int
818 vtnet_init_rxq(struct vtnet_softc *sc, int id)
819 {
820 	struct vtnet_rxq *rxq;
821 
822 	rxq = &sc->vtnet_rxqs[id];
823 
824 	snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d",
825 	    device_get_nameunit(sc->vtnet_dev), id);
826 	mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF);
827 
828 	rxq->vtnrx_sc = sc;
829 	rxq->vtnrx_id = id;
830 
831 	rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT);
832 	if (rxq->vtnrx_sg == NULL)
833 		return (ENOMEM);
834 
835 #if defined(INET) || defined(INET6)
836 	if (vtnet_software_lro(sc)) {
837 		if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp,
838 		    sc->vtnet_lro_entry_count, sc->vtnet_lro_mbufq_depth) != 0)
839 			return (ENOMEM);
840 	}
841 #endif
842 
843 	NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq);
844 	rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT,
845 	    taskqueue_thread_enqueue, &rxq->vtnrx_tq);
846 
847 	return (rxq->vtnrx_tq == NULL ? ENOMEM : 0);
848 }
849 
850 static int
851 vtnet_init_txq(struct vtnet_softc *sc, int id)
852 {
853 	struct vtnet_txq *txq;
854 
855 	txq = &sc->vtnet_txqs[id];
856 
857 	snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d",
858 	    device_get_nameunit(sc->vtnet_dev), id);
859 	mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF);
860 
861 	txq->vtntx_sc = sc;
862 	txq->vtntx_id = id;
863 
864 	txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT);
865 	if (txq->vtntx_sg == NULL)
866 		return (ENOMEM);
867 
868 #ifndef VTNET_LEGACY_TX
869 	txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF,
870 	    M_NOWAIT, &txq->vtntx_mtx);
871 	if (txq->vtntx_br == NULL)
872 		return (ENOMEM);
873 
874 	TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq);
875 #endif
876 	TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq);
877 	txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT,
878 	    taskqueue_thread_enqueue, &txq->vtntx_tq);
879 	if (txq->vtntx_tq == NULL)
880 		return (ENOMEM);
881 
882 	return (0);
883 }
884 
885 static int
886 vtnet_alloc_rxtx_queues(struct vtnet_softc *sc)
887 {
888 	int i, npairs, error;
889 
890 	npairs = sc->vtnet_max_vq_pairs;
891 
892 	sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF,
893 	    M_NOWAIT | M_ZERO);
894 	sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF,
895 	    M_NOWAIT | M_ZERO);
896 	if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL)
897 		return (ENOMEM);
898 
899 	for (i = 0; i < npairs; i++) {
900 		error = vtnet_init_rxq(sc, i);
901 		if (error)
902 			return (error);
903 		error = vtnet_init_txq(sc, i);
904 		if (error)
905 			return (error);
906 	}
907 
908 	vtnet_set_rx_process_limit(sc);
909 	vtnet_setup_queue_sysctl(sc);
910 
911 	return (0);
912 }
913 
914 static void
915 vtnet_destroy_rxq(struct vtnet_rxq *rxq)
916 {
917 
918 	rxq->vtnrx_sc = NULL;
919 	rxq->vtnrx_id = -1;
920 
921 #if defined(INET) || defined(INET6)
922 	tcp_lro_free(&rxq->vtnrx_lro);
923 #endif
924 
925 	if (rxq->vtnrx_sg != NULL) {
926 		sglist_free(rxq->vtnrx_sg);
927 		rxq->vtnrx_sg = NULL;
928 	}
929 
930 	if (mtx_initialized(&rxq->vtnrx_mtx) != 0)
931 		mtx_destroy(&rxq->vtnrx_mtx);
932 }
933 
934 static void
935 vtnet_destroy_txq(struct vtnet_txq *txq)
936 {
937 
938 	txq->vtntx_sc = NULL;
939 	txq->vtntx_id = -1;
940 
941 	if (txq->vtntx_sg != NULL) {
942 		sglist_free(txq->vtntx_sg);
943 		txq->vtntx_sg = NULL;
944 	}
945 
946 #ifndef VTNET_LEGACY_TX
947 	if (txq->vtntx_br != NULL) {
948 		buf_ring_free(txq->vtntx_br, M_DEVBUF);
949 		txq->vtntx_br = NULL;
950 	}
951 #endif
952 
953 	if (mtx_initialized(&txq->vtntx_mtx) != 0)
954 		mtx_destroy(&txq->vtntx_mtx);
955 }
956 
957 static void
958 vtnet_free_rxtx_queues(struct vtnet_softc *sc)
959 {
960 	int i;
961 
962 	if (sc->vtnet_rxqs != NULL) {
963 		for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
964 			vtnet_destroy_rxq(&sc->vtnet_rxqs[i]);
965 		free(sc->vtnet_rxqs, M_DEVBUF);
966 		sc->vtnet_rxqs = NULL;
967 	}
968 
969 	if (sc->vtnet_txqs != NULL) {
970 		for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
971 			vtnet_destroy_txq(&sc->vtnet_txqs[i]);
972 		free(sc->vtnet_txqs, M_DEVBUF);
973 		sc->vtnet_txqs = NULL;
974 	}
975 }
976 
977 static int
978 vtnet_alloc_rx_filters(struct vtnet_softc *sc)
979 {
980 
981 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
982 		sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter),
983 		    M_DEVBUF, M_NOWAIT | M_ZERO);
984 		if (sc->vtnet_mac_filter == NULL)
985 			return (ENOMEM);
986 	}
987 
988 	if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
989 		sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) *
990 		    VTNET_VLAN_FILTER_NWORDS, M_DEVBUF, M_NOWAIT | M_ZERO);
991 		if (sc->vtnet_vlan_filter == NULL)
992 			return (ENOMEM);
993 	}
994 
995 	return (0);
996 }
997 
998 static void
999 vtnet_free_rx_filters(struct vtnet_softc *sc)
1000 {
1001 
1002 	if (sc->vtnet_mac_filter != NULL) {
1003 		free(sc->vtnet_mac_filter, M_DEVBUF);
1004 		sc->vtnet_mac_filter = NULL;
1005 	}
1006 
1007 	if (sc->vtnet_vlan_filter != NULL) {
1008 		free(sc->vtnet_vlan_filter, M_DEVBUF);
1009 		sc->vtnet_vlan_filter = NULL;
1010 	}
1011 }
1012 
1013 static int
1014 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
1015 {
1016 	device_t dev;
1017 	struct vq_alloc_info *info;
1018 	struct vtnet_rxq *rxq;
1019 	struct vtnet_txq *txq;
1020 	int i, idx, nvqs, error;
1021 
1022 	dev = sc->vtnet_dev;
1023 
1024 	nvqs = sc->vtnet_max_vq_pairs * 2;
1025 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ)
1026 		nvqs++;
1027 
1028 	info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT);
1029 	if (info == NULL)
1030 		return (ENOMEM);
1031 
1032 	for (i = 0, idx = 0; i < sc->vtnet_req_vq_pairs; i++, idx += 2) {
1033 		rxq = &sc->vtnet_rxqs[i];
1034 		VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs,
1035 		    vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq,
1036 		    "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id);
1037 
1038 		txq = &sc->vtnet_txqs[i];
1039 		VQ_ALLOC_INFO_INIT(&info[idx+1], sc->vtnet_tx_nsegs,
1040 		    vtnet_tx_vq_intr, txq, &txq->vtntx_vq,
1041 		    "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id);
1042 	}
1043 
1044 	/* These queues will not be used so allocate the minimum resources. */
1045 	for (/**/; i < sc->vtnet_max_vq_pairs; i++, idx += 2) {
1046 		rxq = &sc->vtnet_rxqs[i];
1047 		VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, rxq, &rxq->vtnrx_vq,
1048 		    "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id);
1049 
1050 		txq = &sc->vtnet_txqs[i];
1051 		VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL, txq, &txq->vtntx_vq,
1052 		    "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id);
1053 	}
1054 
1055 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
1056 		VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL,
1057 		    &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev));
1058 	}
1059 
1060 	error = virtio_alloc_virtqueues(dev, nvqs, info);
1061 	free(info, M_TEMP);
1062 
1063 	return (error);
1064 }
1065 
1066 static int
1067 vtnet_alloc_interface(struct vtnet_softc *sc)
1068 {
1069 	device_t dev;
1070 	if_t ifp;
1071 
1072 	dev = sc->vtnet_dev;
1073 
1074 	ifp = if_alloc(IFT_ETHER);
1075 	if (ifp == NULL)
1076 		return (ENOMEM);
1077 
1078 	sc->vtnet_ifp = ifp;
1079 	if_setsoftc(ifp, sc);
1080 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1081 
1082 	return (0);
1083 }
1084 
1085 static int
1086 vtnet_setup_interface(struct vtnet_softc *sc)
1087 {
1088 	device_t dev;
1089 	struct pfil_head_args pa;
1090 	if_t ifp;
1091 
1092 	dev = sc->vtnet_dev;
1093 	ifp = sc->vtnet_ifp;
1094 
1095 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
1096 	if_setbaudrate(ifp, IF_Gbps(10));
1097 	if_setinitfn(ifp, vtnet_init);
1098 	if_setioctlfn(ifp, vtnet_ioctl);
1099 	if_setgetcounterfn(ifp, vtnet_get_counter);
1100 #ifndef VTNET_LEGACY_TX
1101 	if_settransmitfn(ifp, vtnet_txq_mq_start);
1102 	if_setqflushfn(ifp, vtnet_qflush);
1103 #else
1104 	struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq;
1105 	if_setstartfn(ifp, vtnet_start);
1106 	if_setsendqlen(ifp, virtqueue_size(vq) - 1);
1107 	if_setsendqready(ifp);
1108 #endif
1109 
1110 	vtnet_get_macaddr(sc);
1111 
1112 	if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS))
1113 		if_setcapabilitiesbit(ifp, IFCAP_LINKSTATE, 0);
1114 
1115 	ifmedia_init(&sc->vtnet_media, 0, vtnet_ifmedia_upd, vtnet_ifmedia_sts);
1116 	ifmedia_add(&sc->vtnet_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1117 	ifmedia_set(&sc->vtnet_media, IFM_ETHER | IFM_AUTO);
1118 
1119 	if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
1120 		int gso;
1121 
1122 		if_setcapabilitiesbit(ifp, IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6, 0);
1123 
1124 		gso = virtio_with_feature(dev, VIRTIO_NET_F_GSO);
1125 		if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
1126 			if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0);
1127 		if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
1128 			if_setcapabilitiesbit(ifp, IFCAP_TSO6, 0);
1129 		if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
1130 			sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
1131 
1132 		if (if_getcapabilities(ifp) & (IFCAP_TSO4 | IFCAP_TSO6)) {
1133 			int tso_maxlen;
1134 
1135 			if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTSO, 0);
1136 
1137 			tso_maxlen = vtnet_tunable_int(sc, "tso_maxlen",
1138 			    vtnet_tso_maxlen);
1139 			if_sethwtsomax(ifp, tso_maxlen -
1140 			    (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
1141 			if_sethwtsomaxsegcount(ifp, sc->vtnet_tx_nsegs - 1);
1142 			if_sethwtsomaxsegsize(ifp, PAGE_SIZE);
1143 		}
1144 	}
1145 
1146 	if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
1147 		if_setcapabilitiesbit(ifp, IFCAP_RXCSUM, 0);
1148 #ifdef notyet
1149 		/* BMV: Rx checksums not distinguished between IPv4 and IPv6. */
1150 		if_setcapabilitiesbit(ifp, IFCAP_RXCSUM_IPV6, 0);
1151 #endif
1152 
1153 		if (vtnet_tunable_int(sc, "fixup_needs_csum",
1154 		    vtnet_fixup_needs_csum) != 0)
1155 			sc->vtnet_flags |= VTNET_FLAG_FIXUP_NEEDS_CSUM;
1156 
1157 		/* Support either "hardware" or software LRO. */
1158 		if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
1159 	}
1160 
1161 	if (if_getcapabilities(ifp) & (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6)) {
1162 		/*
1163 		 * VirtIO does not support VLAN tagging, but we can fake
1164 		 * it by inserting and removing the 802.1Q header during
1165 		 * transmit and receive. We are then able to do checksum
1166 		 * offloading of VLAN frames.
1167 		 */
1168 		if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM, 0);
1169 	}
1170 
1171 	if (sc->vtnet_max_mtu >= ETHERMTU_JUMBO)
1172 		if_setcapabilitiesbit(ifp, IFCAP_JUMBO_MTU, 0);
1173 	if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU, 0);
1174 
1175 	/*
1176 	 * Capabilities after here are not enabled by default.
1177 	 */
1178 	if_setcapenable(ifp, if_getcapabilities(ifp));
1179 
1180 	if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
1181 		if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0);
1182 
1183 		sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
1184 		    vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
1185 		sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
1186 		    vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
1187 	}
1188 
1189 	ether_ifattach(ifp, sc->vtnet_hwaddr);
1190 
1191 	/* Tell the upper layer(s) we support long frames. */
1192 	if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
1193 
1194 	DEBUGNET_SET(ifp, vtnet);
1195 
1196 	pa.pa_version = PFIL_VERSION;
1197 	pa.pa_flags = PFIL_IN;
1198 	pa.pa_type = PFIL_TYPE_ETHERNET;
1199 	pa.pa_headname = if_name(ifp);
1200 	sc->vtnet_pfil = pfil_head_register(&pa);
1201 
1202 	return (0);
1203 }
1204 
1205 static int
1206 vtnet_rx_cluster_size(struct vtnet_softc *sc, int mtu)
1207 {
1208 	int framesz;
1209 
1210 	if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS)
1211 		return (MJUMPAGESIZE);
1212 	else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
1213 		return (MCLBYTES);
1214 
1215 	/*
1216 	 * Try to scale the receive mbuf cluster size from the MTU. We
1217 	 * could also use the VQ size to influence the selected size,
1218 	 * but that would only matter for very small queues.
1219 	 */
1220 	if (vtnet_modern(sc)) {
1221 		MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr_v1));
1222 		framesz = sizeof(struct virtio_net_hdr_v1);
1223 	} else
1224 		framesz = sizeof(struct vtnet_rx_header);
1225 	framesz += sizeof(struct ether_vlan_header) + mtu;
1226 
1227 	if (framesz <= MCLBYTES)
1228 		return (MCLBYTES);
1229 	else if (framesz <= MJUMPAGESIZE)
1230 		return (MJUMPAGESIZE);
1231 	else if (framesz <= MJUM9BYTES)
1232 		return (MJUM9BYTES);
1233 
1234 	/* Sane default; avoid 16KB clusters. */
1235 	return (MCLBYTES);
1236 }
1237 
1238 static int
1239 vtnet_ioctl_mtu(struct vtnet_softc *sc, u_int mtu)
1240 {
1241 	if_t ifp;
1242 	int clustersz;
1243 
1244 	ifp = sc->vtnet_ifp;
1245 	VTNET_CORE_LOCK_ASSERT(sc);
1246 
1247 	if (if_getmtu(ifp) == mtu)
1248 		return (0);
1249 	else if (mtu < ETHERMIN || mtu > sc->vtnet_max_mtu)
1250 		return (EINVAL);
1251 
1252 	if_setmtu(ifp, mtu);
1253 	clustersz = vtnet_rx_cluster_size(sc, mtu);
1254 
1255 	if (clustersz != sc->vtnet_rx_clustersz &&
1256 	    if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1257 		if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1258 		vtnet_init_locked(sc, 0);
1259 	}
1260 
1261 	return (0);
1262 }
1263 
1264 static int
1265 vtnet_ioctl_ifflags(struct vtnet_softc *sc)
1266 {
1267 	if_t ifp;
1268 	int drv_running;
1269 
1270 	ifp = sc->vtnet_ifp;
1271 	drv_running = (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0;
1272 
1273 	VTNET_CORE_LOCK_ASSERT(sc);
1274 
1275 	if ((if_getflags(ifp) & IFF_UP) == 0) {
1276 		if (drv_running)
1277 			vtnet_stop(sc);
1278 		goto out;
1279 	}
1280 
1281 	if (!drv_running) {
1282 		vtnet_init_locked(sc, 0);
1283 		goto out;
1284 	}
1285 
1286 	if ((if_getflags(ifp) ^ sc->vtnet_if_flags) &
1287 	    (IFF_PROMISC | IFF_ALLMULTI)) {
1288 		if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
1289 			vtnet_rx_filter(sc);
1290 		else {
1291 			/*
1292 			 * We don't support filtering out multicast, so
1293 			 * ALLMULTI is always set.
1294 			 */
1295 			if_setflagbits(ifp, IFF_ALLMULTI, 0);
1296 			if_setflagbits(ifp, IFF_PROMISC, 0);
1297 		}
1298 	}
1299 
1300 out:
1301 	sc->vtnet_if_flags = if_getflags(ifp);
1302 	return (0);
1303 }
1304 
1305 static int
1306 vtnet_ioctl_multi(struct vtnet_softc *sc)
1307 {
1308 	if_t ifp;
1309 
1310 	ifp = sc->vtnet_ifp;
1311 
1312 	VTNET_CORE_LOCK_ASSERT(sc);
1313 
1314 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX &&
1315 	    if_getdrvflags(ifp) & IFF_DRV_RUNNING)
1316 		vtnet_rx_filter_mac(sc);
1317 
1318 	return (0);
1319 }
1320 
1321 static int
1322 vtnet_ioctl_ifcap(struct vtnet_softc *sc, struct ifreq *ifr)
1323 {
1324 	if_t ifp;
1325 	int mask, reinit, update;
1326 
1327 	ifp = sc->vtnet_ifp;
1328 	mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^ if_getcapenable(ifp);
1329 	reinit = update = 0;
1330 
1331 	VTNET_CORE_LOCK_ASSERT(sc);
1332 
1333 	if (mask & IFCAP_TXCSUM)
1334 		if_togglecapenable(ifp, IFCAP_TXCSUM);
1335 	if (mask & IFCAP_TXCSUM_IPV6)
1336 		if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6);
1337 	if (mask & IFCAP_TSO4)
1338 		if_togglecapenable(ifp, IFCAP_TSO4);
1339 	if (mask & IFCAP_TSO6)
1340 		if_togglecapenable(ifp, IFCAP_TSO6);
1341 
1342 	if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) {
1343 		/*
1344 		 * These Rx features require the negotiated features to
1345 		 * be updated. Avoid a full reinit if possible.
1346 		 */
1347 		if (sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
1348 			update = 1;
1349 		else
1350 			reinit = 1;
1351 
1352 		/* BMV: Avoid needless renegotiation for just software LRO. */
1353 		if ((mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) ==
1354 		    IFCAP_LRO && vtnet_software_lro(sc))
1355 			reinit = update = 0;
1356 
1357 		if (mask & IFCAP_RXCSUM)
1358 			if_togglecapenable(ifp, IFCAP_RXCSUM);
1359 		if (mask & IFCAP_RXCSUM_IPV6)
1360 			if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6);
1361 		if (mask & IFCAP_LRO)
1362 			if_togglecapenable(ifp, IFCAP_LRO);
1363 
1364 		/*
1365 		 * VirtIO does not distinguish between IPv4 and IPv6 checksums
1366 		 * so treat them as a pair. Guest TSO (LRO) requires receive
1367 		 * checksums.
1368 		 */
1369 		if (if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
1370 			if_setcapenablebit(ifp, IFCAP_RXCSUM, 0);
1371 #ifdef notyet
1372 			if_setcapenablebit(ifp, IFCAP_RXCSUM_IPV6, 0);
1373 #endif
1374 		} else
1375 			if_setcapenablebit(ifp, 0,
1376 			    (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO));
1377 	}
1378 
1379 	if (mask & IFCAP_VLAN_HWFILTER) {
1380 		/* These Rx features require renegotiation. */
1381 		reinit = 1;
1382 
1383 		if (mask & IFCAP_VLAN_HWFILTER)
1384 			if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
1385 	}
1386 
1387 	if (mask & IFCAP_VLAN_HWTSO)
1388 		if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
1389 	if (mask & IFCAP_VLAN_HWTAGGING)
1390 		if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
1391 
1392 	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1393 		if (reinit) {
1394 			if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1395 			vtnet_init_locked(sc, 0);
1396 		} else if (update)
1397 			vtnet_update_rx_offloads(sc);
1398 	}
1399 
1400 	return (0);
1401 }
1402 
1403 static int
1404 vtnet_ioctl(if_t ifp, u_long cmd, caddr_t data)
1405 {
1406 	struct vtnet_softc *sc;
1407 	struct ifreq *ifr;
1408 	int error;
1409 
1410 	sc = if_getsoftc(ifp);
1411 	ifr = (struct ifreq *) data;
1412 	error = 0;
1413 
1414 	switch (cmd) {
1415 	case SIOCSIFMTU:
1416 		VTNET_CORE_LOCK(sc);
1417 		error = vtnet_ioctl_mtu(sc, ifr->ifr_mtu);
1418 		VTNET_CORE_UNLOCK(sc);
1419 		break;
1420 
1421 	case SIOCSIFFLAGS:
1422 		VTNET_CORE_LOCK(sc);
1423 		error = vtnet_ioctl_ifflags(sc);
1424 		VTNET_CORE_UNLOCK(sc);
1425 		break;
1426 
1427 	case SIOCADDMULTI:
1428 	case SIOCDELMULTI:
1429 		VTNET_CORE_LOCK(sc);
1430 		error = vtnet_ioctl_multi(sc);
1431 		VTNET_CORE_UNLOCK(sc);
1432 		break;
1433 
1434 	case SIOCSIFMEDIA:
1435 	case SIOCGIFMEDIA:
1436 		error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
1437 		break;
1438 
1439 	case SIOCSIFCAP:
1440 		VTNET_CORE_LOCK(sc);
1441 		error = vtnet_ioctl_ifcap(sc, ifr);
1442 		VTNET_CORE_UNLOCK(sc);
1443 		VLAN_CAPABILITIES(ifp);
1444 		break;
1445 
1446 	default:
1447 		error = ether_ioctl(ifp, cmd, data);
1448 		break;
1449 	}
1450 
1451 	VTNET_CORE_LOCK_ASSERT_NOTOWNED(sc);
1452 
1453 	return (error);
1454 }
1455 
1456 static int
1457 vtnet_rxq_populate(struct vtnet_rxq *rxq)
1458 {
1459 	struct virtqueue *vq;
1460 	int nbufs, error;
1461 
1462 #ifdef DEV_NETMAP
1463 	error = vtnet_netmap_rxq_populate(rxq);
1464 	if (error >= 0)
1465 		return (error);
1466 #endif  /* DEV_NETMAP */
1467 
1468 	vq = rxq->vtnrx_vq;
1469 	error = ENOSPC;
1470 
1471 	for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
1472 		error = vtnet_rxq_new_buf(rxq);
1473 		if (error)
1474 			break;
1475 	}
1476 
1477 	if (nbufs > 0) {
1478 		virtqueue_notify(vq);
1479 		/*
1480 		 * EMSGSIZE signifies the virtqueue did not have enough
1481 		 * entries available to hold the last mbuf. This is not
1482 		 * an error.
1483 		 */
1484 		if (error == EMSGSIZE)
1485 			error = 0;
1486 	}
1487 
1488 	return (error);
1489 }
1490 
1491 static void
1492 vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq)
1493 {
1494 	struct virtqueue *vq;
1495 	struct mbuf *m;
1496 	int last;
1497 #ifdef DEV_NETMAP
1498 	struct netmap_kring *kring = netmap_kring_on(NA(rxq->vtnrx_sc->vtnet_ifp),
1499 							rxq->vtnrx_id, NR_RX);
1500 #else  /* !DEV_NETMAP */
1501 	void *kring = NULL;
1502 #endif /* !DEV_NETMAP */
1503 
1504 	vq = rxq->vtnrx_vq;
1505 	last = 0;
1506 
1507 	while ((m = virtqueue_drain(vq, &last)) != NULL) {
1508 		if (kring == NULL)
1509 			m_freem(m);
1510 	}
1511 
1512 	KASSERT(virtqueue_empty(vq),
1513 	    ("%s: mbufs remaining in rx queue %p", __func__, rxq));
1514 }
1515 
1516 static struct mbuf *
1517 vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1518 {
1519 	struct mbuf *m_head, *m_tail, *m;
1520 	int i, size;
1521 
1522 	m_head = NULL;
1523 	size = sc->vtnet_rx_clustersz;
1524 
1525 	KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1526 	    ("%s: mbuf %d chain requested without LRO_NOMRG", __func__, nbufs));
1527 
1528 	for (i = 0; i < nbufs; i++) {
1529 		m = m_getjcl(M_NOWAIT, MT_DATA, i == 0 ? M_PKTHDR : 0, size);
1530 		if (m == NULL) {
1531 			sc->vtnet_stats.mbuf_alloc_failed++;
1532 			m_freem(m_head);
1533 			return (NULL);
1534 		}
1535 
1536 		m->m_len = size;
1537 		if (m_head != NULL) {
1538 			m_tail->m_next = m;
1539 			m_tail = m;
1540 		} else
1541 			m_head = m_tail = m;
1542 	}
1543 
1544 	if (m_tailp != NULL)
1545 		*m_tailp = m_tail;
1546 
1547 	return (m_head);
1548 }
1549 
1550 /*
1551  * Slow path for when LRO without mergeable buffers is negotiated.
1552  */
1553 static int
1554 vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *rxq, struct mbuf *m0,
1555     int len0)
1556 {
1557 	struct vtnet_softc *sc;
1558 	struct mbuf *m, *m_prev, *m_new, *m_tail;
1559 	int len, clustersz, nreplace, error;
1560 
1561 	sc = rxq->vtnrx_sc;
1562 	clustersz = sc->vtnet_rx_clustersz;
1563 
1564 	m_prev = NULL;
1565 	m_tail = NULL;
1566 	nreplace = 0;
1567 
1568 	m = m0;
1569 	len = len0;
1570 
1571 	/*
1572 	 * Since these mbuf chains are so large, avoid allocating a complete
1573 	 * replacement when the received frame did not consume the entire
1574 	 * chain. Unused mbufs are moved to the tail of the replacement mbuf.
1575 	 */
1576 	while (len > 0) {
1577 		if (m == NULL) {
1578 			sc->vtnet_stats.rx_frame_too_large++;
1579 			return (EMSGSIZE);
1580 		}
1581 
1582 		/*
1583 		 * Every mbuf should have the expected cluster size since that
1584 		 * is also used to allocate the replacements.
1585 		 */
1586 		KASSERT(m->m_len == clustersz,
1587 		    ("%s: mbuf size %d not expected cluster size %d", __func__,
1588 		    m->m_len, clustersz));
1589 
1590 		m->m_len = MIN(m->m_len, len);
1591 		len -= m->m_len;
1592 
1593 		m_prev = m;
1594 		m = m->m_next;
1595 		nreplace++;
1596 	}
1597 
1598 	KASSERT(nreplace > 0 && nreplace <= sc->vtnet_rx_nmbufs,
1599 	    ("%s: invalid replacement mbuf count %d max %d", __func__,
1600 	    nreplace, sc->vtnet_rx_nmbufs));
1601 
1602 	m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail);
1603 	if (m_new == NULL) {
1604 		m_prev->m_len = clustersz;
1605 		return (ENOBUFS);
1606 	}
1607 
1608 	/*
1609 	 * Move any unused mbufs from the received mbuf chain onto the
1610 	 * end of the replacement chain.
1611 	 */
1612 	if (m_prev->m_next != NULL) {
1613 		m_tail->m_next = m_prev->m_next;
1614 		m_prev->m_next = NULL;
1615 	}
1616 
1617 	error = vtnet_rxq_enqueue_buf(rxq, m_new);
1618 	if (error) {
1619 		/*
1620 		 * The replacement is suppose to be an copy of the one
1621 		 * dequeued so this is a very unexpected error.
1622 		 *
1623 		 * Restore the m0 chain to the original state if it was
1624 		 * modified so we can then discard it.
1625 		 */
1626 		if (m_tail->m_next != NULL) {
1627 			m_prev->m_next = m_tail->m_next;
1628 			m_tail->m_next = NULL;
1629 		}
1630 		m_prev->m_len = clustersz;
1631 		sc->vtnet_stats.rx_enq_replacement_failed++;
1632 		m_freem(m_new);
1633 	}
1634 
1635 	return (error);
1636 }
1637 
1638 static int
1639 vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len)
1640 {
1641 	struct vtnet_softc *sc;
1642 	struct mbuf *m_new;
1643 	int error;
1644 
1645 	sc = rxq->vtnrx_sc;
1646 
1647 	if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
1648 		return (vtnet_rxq_replace_lro_nomrg_buf(rxq, m, len));
1649 
1650 	MPASS(m->m_next == NULL);
1651 	if (m->m_len < len)
1652 		return (EMSGSIZE);
1653 
1654 	m_new = vtnet_rx_alloc_buf(sc, 1, NULL);
1655 	if (m_new == NULL)
1656 		return (ENOBUFS);
1657 
1658 	error = vtnet_rxq_enqueue_buf(rxq, m_new);
1659 	if (error) {
1660 		sc->vtnet_stats.rx_enq_replacement_failed++;
1661 		m_freem(m_new);
1662 	} else
1663 		m->m_len = len;
1664 
1665 	return (error);
1666 }
1667 
1668 static int
1669 vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m)
1670 {
1671 	struct vtnet_softc *sc;
1672 	struct sglist *sg;
1673 	int header_inlined, error;
1674 
1675 	sc = rxq->vtnrx_sc;
1676 	sg = rxq->vtnrx_sg;
1677 
1678 	KASSERT(m->m_next == NULL || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1679 	    ("%s: mbuf chain without LRO_NOMRG", __func__));
1680 	VTNET_RXQ_LOCK_ASSERT(rxq);
1681 
1682 	sglist_reset(sg);
1683 	header_inlined = vtnet_modern(sc) ||
1684 	    (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) != 0; /* TODO: ANY_LAYOUT */
1685 
1686 	if (header_inlined)
1687 		error = sglist_append_mbuf(sg, m);
1688 	else {
1689 		struct vtnet_rx_header *rxhdr =
1690 		    mtod(m, struct vtnet_rx_header *);
1691 		MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr));
1692 
1693 		/* Append the header and remaining mbuf data. */
1694 		error = sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size);
1695 		if (error)
1696 			return (error);
1697 		error = sglist_append(sg, &rxhdr[1],
1698 		    m->m_len - sizeof(struct vtnet_rx_header));
1699 		if (error)
1700 			return (error);
1701 
1702 		if (m->m_next != NULL)
1703 			error = sglist_append_mbuf(sg, m->m_next);
1704 	}
1705 
1706 	if (error)
1707 		return (error);
1708 
1709 	return (virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg));
1710 }
1711 
1712 static int
1713 vtnet_rxq_new_buf(struct vtnet_rxq *rxq)
1714 {
1715 	struct vtnet_softc *sc;
1716 	struct mbuf *m;
1717 	int error;
1718 
1719 	sc = rxq->vtnrx_sc;
1720 
1721 	m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL);
1722 	if (m == NULL)
1723 		return (ENOBUFS);
1724 
1725 	error = vtnet_rxq_enqueue_buf(rxq, m);
1726 	if (error)
1727 		m_freem(m);
1728 
1729 	return (error);
1730 }
1731 
1732 static int
1733 vtnet_rxq_csum_needs_csum(struct vtnet_rxq *rxq, struct mbuf *m, uint16_t etype,
1734     int hoff, struct virtio_net_hdr *hdr)
1735 {
1736 	struct vtnet_softc *sc;
1737 	int error;
1738 
1739 	sc = rxq->vtnrx_sc;
1740 
1741 	/*
1742 	 * NEEDS_CSUM corresponds to Linux's CHECKSUM_PARTIAL, but FreeBSD does
1743 	 * not have an analogous CSUM flag. The checksum has been validated,
1744 	 * but is incomplete (TCP/UDP pseudo header).
1745 	 *
1746 	 * The packet is likely from another VM on the same host that itself
1747 	 * performed checksum offloading so Tx/Rx is basically a memcpy and
1748 	 * the checksum has little value.
1749 	 *
1750 	 * Default to receiving the packet as-is for performance reasons, but
1751 	 * this can cause issues if the packet is to be forwarded because it
1752 	 * does not contain a valid checksum. This patch may be helpful:
1753 	 * https://reviews.freebsd.org/D6611. In the meantime, have the driver
1754 	 * compute the checksum if requested.
1755 	 *
1756 	 * BMV: Need to add an CSUM_PARTIAL flag?
1757 	 */
1758 	if ((sc->vtnet_flags & VTNET_FLAG_FIXUP_NEEDS_CSUM) == 0) {
1759 		error = vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr);
1760 		return (error);
1761 	}
1762 
1763 	/*
1764 	 * Compute the checksum in the driver so the packet will contain a
1765 	 * valid checksum. The checksum is at csum_offset from csum_start.
1766 	 */
1767 	switch (etype) {
1768 #if defined(INET) || defined(INET6)
1769 	case ETHERTYPE_IP:
1770 	case ETHERTYPE_IPV6: {
1771 		int csum_off, csum_end;
1772 		uint16_t csum;
1773 
1774 		csum_off = hdr->csum_start + hdr->csum_offset;
1775 		csum_end = csum_off + sizeof(uint16_t);
1776 
1777 		/* Assume checksum will be in the first mbuf. */
1778 		if (m->m_len < csum_end || m->m_pkthdr.len < csum_end)
1779 			return (1);
1780 
1781 		/*
1782 		 * Like in_delayed_cksum()/in6_delayed_cksum(), compute the
1783 		 * checksum and write it at the specified offset. We could
1784 		 * try to verify the packet: csum_start should probably
1785 		 * correspond to the start of the TCP/UDP header.
1786 		 *
1787 		 * BMV: Need to properly handle UDP with zero checksum. Is
1788 		 * the IPv4 header checksum implicitly validated?
1789 		 */
1790 		csum = in_cksum_skip(m, m->m_pkthdr.len, hdr->csum_start);
1791 		*(uint16_t *)(mtodo(m, csum_off)) = csum;
1792 		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1793 		m->m_pkthdr.csum_data = 0xFFFF;
1794 		break;
1795 	}
1796 #endif
1797 	default:
1798 		sc->vtnet_stats.rx_csum_bad_ethtype++;
1799 		return (1);
1800 	}
1801 
1802 	return (0);
1803 }
1804 
1805 static int
1806 vtnet_rxq_csum_data_valid(struct vtnet_rxq *rxq, struct mbuf *m,
1807     uint16_t etype, int hoff, struct virtio_net_hdr *hdr __unused)
1808 {
1809 #if 0
1810 	struct vtnet_softc *sc;
1811 #endif
1812 	int protocol;
1813 
1814 #if 0
1815 	sc = rxq->vtnrx_sc;
1816 #endif
1817 
1818 	switch (etype) {
1819 #if defined(INET)
1820 	case ETHERTYPE_IP:
1821 		if (__predict_false(m->m_len < hoff + sizeof(struct ip)))
1822 			protocol = IPPROTO_DONE;
1823 		else {
1824 			struct ip *ip = (struct ip *)(m->m_data + hoff);
1825 			protocol = ip->ip_p;
1826 		}
1827 		break;
1828 #endif
1829 #if defined(INET6)
1830 	case ETHERTYPE_IPV6:
1831 		if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr))
1832 		    || ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0)
1833 			protocol = IPPROTO_DONE;
1834 		break;
1835 #endif
1836 	default:
1837 		protocol = IPPROTO_DONE;
1838 		break;
1839 	}
1840 
1841 	switch (protocol) {
1842 	case IPPROTO_TCP:
1843 	case IPPROTO_UDP:
1844 		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1845 		m->m_pkthdr.csum_data = 0xFFFF;
1846 		break;
1847 	default:
1848 		/*
1849 		 * FreeBSD does not support checksum offloading of this
1850 		 * protocol. Let the stack re-verify the checksum later
1851 		 * if the protocol is supported.
1852 		 */
1853 #if 0
1854 		if_printf(sc->vtnet_ifp,
1855 		    "%s: checksum offload of unsupported protocol "
1856 		    "etype=%#x protocol=%d csum_start=%d csum_offset=%d\n",
1857 		    __func__, etype, protocol, hdr->csum_start,
1858 		    hdr->csum_offset);
1859 #endif
1860 		break;
1861 	}
1862 
1863 	return (0);
1864 }
1865 
1866 static int
1867 vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m,
1868     struct virtio_net_hdr *hdr)
1869 {
1870 	const struct ether_header *eh;
1871 	int hoff;
1872 	uint16_t etype;
1873 
1874 	eh = mtod(m, const struct ether_header *);
1875 	etype = ntohs(eh->ether_type);
1876 	if (etype == ETHERTYPE_VLAN) {
1877 		/* TODO BMV: Handle QinQ. */
1878 		const struct ether_vlan_header *evh =
1879 		    mtod(m, const struct ether_vlan_header *);
1880 		etype = ntohs(evh->evl_proto);
1881 		hoff = sizeof(struct ether_vlan_header);
1882 	} else
1883 		hoff = sizeof(struct ether_header);
1884 
1885 	if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1886 		return (vtnet_rxq_csum_needs_csum(rxq, m, etype, hoff, hdr));
1887 	else /* VIRTIO_NET_HDR_F_DATA_VALID */
1888 		return (vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr));
1889 }
1890 
1891 static void
1892 vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs)
1893 {
1894 	struct mbuf *m;
1895 
1896 	while (--nbufs > 0) {
1897 		m = virtqueue_dequeue(rxq->vtnrx_vq, NULL);
1898 		if (m == NULL)
1899 			break;
1900 		vtnet_rxq_discard_buf(rxq, m);
1901 	}
1902 }
1903 
1904 static void
1905 vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m)
1906 {
1907 	int error __diagused;
1908 
1909 	/*
1910 	 * Requeue the discarded mbuf. This should always be successful
1911 	 * since it was just dequeued.
1912 	 */
1913 	error = vtnet_rxq_enqueue_buf(rxq, m);
1914 	KASSERT(error == 0,
1915 	    ("%s: cannot requeue discarded mbuf %d", __func__, error));
1916 }
1917 
1918 static int
1919 vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs)
1920 {
1921 	struct vtnet_softc *sc;
1922 	struct virtqueue *vq;
1923 	struct mbuf *m_tail;
1924 
1925 	sc = rxq->vtnrx_sc;
1926 	vq = rxq->vtnrx_vq;
1927 	m_tail = m_head;
1928 
1929 	while (--nbufs > 0) {
1930 		struct mbuf *m;
1931 		uint32_t len;
1932 
1933 		m = virtqueue_dequeue(vq, &len);
1934 		if (m == NULL) {
1935 			rxq->vtnrx_stats.vrxs_ierrors++;
1936 			goto fail;
1937 		}
1938 
1939 		if (vtnet_rxq_new_buf(rxq) != 0) {
1940 			rxq->vtnrx_stats.vrxs_iqdrops++;
1941 			vtnet_rxq_discard_buf(rxq, m);
1942 			if (nbufs > 1)
1943 				vtnet_rxq_discard_merged_bufs(rxq, nbufs);
1944 			goto fail;
1945 		}
1946 
1947 		if (m->m_len < len)
1948 			len = m->m_len;
1949 
1950 		m->m_len = len;
1951 		m->m_flags &= ~M_PKTHDR;
1952 
1953 		m_head->m_pkthdr.len += len;
1954 		m_tail->m_next = m;
1955 		m_tail = m;
1956 	}
1957 
1958 	return (0);
1959 
1960 fail:
1961 	sc->vtnet_stats.rx_mergeable_failed++;
1962 	m_freem(m_head);
1963 
1964 	return (1);
1965 }
1966 
1967 #if defined(INET) || defined(INET6)
1968 static int
1969 vtnet_lro_rx(struct vtnet_rxq *rxq, struct mbuf *m)
1970 {
1971 	struct lro_ctrl *lro;
1972 
1973 	lro = &rxq->vtnrx_lro;
1974 
1975 	if (lro->lro_mbuf_max != 0) {
1976 		tcp_lro_queue_mbuf(lro, m);
1977 		return (0);
1978 	}
1979 
1980 	return (tcp_lro_rx(lro, m, 0));
1981 }
1982 #endif
1983 
1984 static void
1985 vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m,
1986     struct virtio_net_hdr *hdr)
1987 {
1988 	struct vtnet_softc *sc;
1989 	if_t ifp;
1990 
1991 	sc = rxq->vtnrx_sc;
1992 	ifp = sc->vtnet_ifp;
1993 
1994 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
1995 		struct ether_header *eh = mtod(m, struct ether_header *);
1996 		if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1997 			vtnet_vlan_tag_remove(m);
1998 			/*
1999 			 * With the 802.1Q header removed, update the
2000 			 * checksum starting location accordingly.
2001 			 */
2002 			if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
2003 				hdr->csum_start -= ETHER_VLAN_ENCAP_LEN;
2004 		}
2005 	}
2006 
2007 	m->m_pkthdr.flowid = rxq->vtnrx_id;
2008 	M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2009 
2010 	if (hdr->flags &
2011 	    (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID)) {
2012 		if (vtnet_rxq_csum(rxq, m, hdr) == 0)
2013 			rxq->vtnrx_stats.vrxs_csum++;
2014 		else
2015 			rxq->vtnrx_stats.vrxs_csum_failed++;
2016 	}
2017 
2018 	if (hdr->gso_size != 0) {
2019 		switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2020 		case VIRTIO_NET_HDR_GSO_TCPV4:
2021 		case VIRTIO_NET_HDR_GSO_TCPV6:
2022 			m->m_pkthdr.lro_nsegs =
2023 			    howmany(m->m_pkthdr.len, hdr->gso_size);
2024 			rxq->vtnrx_stats.vrxs_host_lro++;
2025 			break;
2026 		}
2027 	}
2028 
2029 	rxq->vtnrx_stats.vrxs_ipackets++;
2030 	rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len;
2031 
2032 #if defined(INET) || defined(INET6)
2033 	if (vtnet_software_lro(sc) && if_getcapenable(ifp) & IFCAP_LRO) {
2034 		if (vtnet_lro_rx(rxq, m) == 0)
2035 			return;
2036 	}
2037 #endif
2038 
2039 	if_input(ifp, m);
2040 }
2041 
2042 static int
2043 vtnet_rxq_eof(struct vtnet_rxq *rxq)
2044 {
2045 	struct virtio_net_hdr lhdr, *hdr;
2046 	struct vtnet_softc *sc;
2047 	if_t ifp;
2048 	struct virtqueue *vq;
2049 	int deq, count;
2050 
2051 	sc = rxq->vtnrx_sc;
2052 	vq = rxq->vtnrx_vq;
2053 	ifp = sc->vtnet_ifp;
2054 	deq = 0;
2055 	count = sc->vtnet_rx_process_limit;
2056 
2057 	VTNET_RXQ_LOCK_ASSERT(rxq);
2058 
2059 	while (count-- > 0) {
2060 		struct mbuf *m;
2061 		uint32_t len, nbufs, adjsz;
2062 
2063 		m = virtqueue_dequeue(vq, &len);
2064 		if (m == NULL)
2065 			break;
2066 		deq++;
2067 
2068 		if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
2069 			rxq->vtnrx_stats.vrxs_ierrors++;
2070 			vtnet_rxq_discard_buf(rxq, m);
2071 			continue;
2072 		}
2073 
2074 		if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) {
2075 			struct virtio_net_hdr_mrg_rxbuf *mhdr =
2076 			    mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
2077 			kmsan_mark(mhdr, sizeof(*mhdr), KMSAN_STATE_INITED);
2078 			nbufs = vtnet_htog16(sc, mhdr->num_buffers);
2079 			adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2080 		} else if (vtnet_modern(sc)) {
2081 			nbufs = 1; /* num_buffers is always 1 */
2082 			adjsz = sizeof(struct virtio_net_hdr_v1);
2083 		} else {
2084 			nbufs = 1;
2085 			adjsz = sizeof(struct vtnet_rx_header);
2086 			/*
2087 			 * Account for our gap between the header and start of
2088 			 * data to keep the segments separated.
2089 			 */
2090 			len += VTNET_RX_HEADER_PAD;
2091 		}
2092 
2093 		if (vtnet_rxq_replace_buf(rxq, m, len) != 0) {
2094 			rxq->vtnrx_stats.vrxs_iqdrops++;
2095 			vtnet_rxq_discard_buf(rxq, m);
2096 			if (nbufs > 1)
2097 				vtnet_rxq_discard_merged_bufs(rxq, nbufs);
2098 			continue;
2099 		}
2100 
2101 		m->m_pkthdr.len = len;
2102 		m->m_pkthdr.rcvif = ifp;
2103 		m->m_pkthdr.csum_flags = 0;
2104 
2105 		if (nbufs > 1) {
2106 			/* Dequeue the rest of chain. */
2107 			if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0)
2108 				continue;
2109 		}
2110 
2111 		kmsan_mark_mbuf(m, KMSAN_STATE_INITED);
2112 
2113 		/*
2114 		 * Save an endian swapped version of the header prior to it
2115 		 * being stripped. The header is always at the start of the
2116 		 * mbuf data. num_buffers was already saved (and not needed)
2117 		 * so use the standard header.
2118 		 */
2119 		hdr = mtod(m, struct virtio_net_hdr *);
2120 		lhdr.flags = hdr->flags;
2121 		lhdr.gso_type = hdr->gso_type;
2122 		lhdr.hdr_len = vtnet_htog16(sc, hdr->hdr_len);
2123 		lhdr.gso_size = vtnet_htog16(sc, hdr->gso_size);
2124 		lhdr.csum_start = vtnet_htog16(sc, hdr->csum_start);
2125 		lhdr.csum_offset = vtnet_htog16(sc, hdr->csum_offset);
2126 		m_adj(m, adjsz);
2127 
2128 		if (PFIL_HOOKED_IN(sc->vtnet_pfil)) {
2129 			pfil_return_t pfil;
2130 
2131 			pfil = pfil_mbuf_in(sc->vtnet_pfil, &m, ifp, NULL);
2132 			switch (pfil) {
2133 			case PFIL_DROPPED:
2134 			case PFIL_CONSUMED:
2135 				continue;
2136 			default:
2137 				KASSERT(pfil == PFIL_PASS,
2138 				    ("Filter returned %d!", pfil));
2139 			}
2140 		}
2141 
2142 		vtnet_rxq_input(rxq, m, &lhdr);
2143 	}
2144 
2145 	if (deq > 0) {
2146 #if defined(INET) || defined(INET6)
2147 		if (vtnet_software_lro(sc))
2148 			tcp_lro_flush_all(&rxq->vtnrx_lro);
2149 #endif
2150 		virtqueue_notify(vq);
2151 	}
2152 
2153 	return (count > 0 ? 0 : EAGAIN);
2154 }
2155 
2156 static void
2157 vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries)
2158 {
2159 	struct vtnet_softc *sc;
2160 	if_t ifp;
2161 	u_int more;
2162 #ifdef DEV_NETMAP
2163 	int nmirq;
2164 #endif /* DEV_NETMAP */
2165 
2166 	sc = rxq->vtnrx_sc;
2167 	ifp = sc->vtnet_ifp;
2168 
2169 	if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) {
2170 		/*
2171 		 * Ignore this interrupt. Either this is a spurious interrupt
2172 		 * or multiqueue without per-VQ MSIX so every queue needs to
2173 		 * be polled (a brain dead configuration we could try harder
2174 		 * to avoid).
2175 		 */
2176 		vtnet_rxq_disable_intr(rxq);
2177 		return;
2178 	}
2179 
2180 	VTNET_RXQ_LOCK(rxq);
2181 
2182 #ifdef DEV_NETMAP
2183 	/*
2184 	 * We call netmap_rx_irq() under lock to prevent concurrent calls.
2185 	 * This is not necessary to serialize the access to the RX vq, but
2186 	 * rather to avoid races that may happen if this interface is
2187 	 * attached to a VALE switch, which would cause received packets
2188 	 * to stall in the RX queue (nm_kr_tryget() could find the kring
2189 	 * busy when called from netmap_bwrap_intr_notify()).
2190 	 */
2191 	nmirq = netmap_rx_irq(ifp, rxq->vtnrx_id, &more);
2192 	if (nmirq != NM_IRQ_PASS) {
2193 		VTNET_RXQ_UNLOCK(rxq);
2194 		if (nmirq == NM_IRQ_RESCHED) {
2195 			taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
2196 		}
2197 		return;
2198 	}
2199 #endif /* DEV_NETMAP */
2200 
2201 again:
2202 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
2203 		VTNET_RXQ_UNLOCK(rxq);
2204 		return;
2205 	}
2206 
2207 	more = vtnet_rxq_eof(rxq);
2208 	if (more || vtnet_rxq_enable_intr(rxq) != 0) {
2209 		if (!more)
2210 			vtnet_rxq_disable_intr(rxq);
2211 		/*
2212 		 * This is an occasional condition or race (when !more),
2213 		 * so retry a few times before scheduling the taskqueue.
2214 		 */
2215 		if (tries-- > 0)
2216 			goto again;
2217 
2218 		rxq->vtnrx_stats.vrxs_rescheduled++;
2219 		VTNET_RXQ_UNLOCK(rxq);
2220 		taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
2221 	} else
2222 		VTNET_RXQ_UNLOCK(rxq);
2223 }
2224 
2225 static void
2226 vtnet_rx_vq_intr(void *xrxq)
2227 {
2228 	struct vtnet_rxq *rxq;
2229 
2230 	rxq = xrxq;
2231 	vtnet_rx_vq_process(rxq, VTNET_INTR_DISABLE_RETRIES);
2232 }
2233 
2234 static void
2235 vtnet_rxq_tq_intr(void *xrxq, int pending __unused)
2236 {
2237 	struct vtnet_rxq *rxq;
2238 
2239 	rxq = xrxq;
2240 	vtnet_rx_vq_process(rxq, 0);
2241 }
2242 
2243 static int
2244 vtnet_txq_intr_threshold(struct vtnet_txq *txq)
2245 {
2246 	struct vtnet_softc *sc;
2247 	int threshold;
2248 
2249 	sc = txq->vtntx_sc;
2250 
2251 	/*
2252 	 * The Tx interrupt is disabled until the queue free count falls
2253 	 * below our threshold. Completed frames are drained from the Tx
2254 	 * virtqueue before transmitting new frames and in the watchdog
2255 	 * callout, so the frequency of Tx interrupts is greatly reduced,
2256 	 * at the cost of not freeing mbufs as quickly as they otherwise
2257 	 * would be.
2258 	 */
2259 	threshold = virtqueue_size(txq->vtntx_vq) / 4;
2260 
2261 	/*
2262 	 * Without indirect descriptors, leave enough room for the most
2263 	 * segments we handle.
2264 	 */
2265 	if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 &&
2266 	    threshold < sc->vtnet_tx_nsegs)
2267 		threshold = sc->vtnet_tx_nsegs;
2268 
2269 	return (threshold);
2270 }
2271 
2272 static int
2273 vtnet_txq_below_threshold(struct vtnet_txq *txq)
2274 {
2275 	struct virtqueue *vq;
2276 
2277 	vq = txq->vtntx_vq;
2278 
2279 	return (virtqueue_nfree(vq) <= txq->vtntx_intr_threshold);
2280 }
2281 
2282 static int
2283 vtnet_txq_notify(struct vtnet_txq *txq)
2284 {
2285 	struct virtqueue *vq;
2286 
2287 	vq = txq->vtntx_vq;
2288 
2289 	txq->vtntx_watchdog = VTNET_TX_TIMEOUT;
2290 	virtqueue_notify(vq);
2291 
2292 	if (vtnet_txq_enable_intr(txq) == 0)
2293 		return (0);
2294 
2295 	/*
2296 	 * Drain frames that were completed since last checked. If this
2297 	 * causes the queue to go above the threshold, the caller should
2298 	 * continue transmitting.
2299 	 */
2300 	if (vtnet_txq_eof(txq) != 0 && vtnet_txq_below_threshold(txq) == 0) {
2301 		virtqueue_disable_intr(vq);
2302 		return (1);
2303 	}
2304 
2305 	return (0);
2306 }
2307 
2308 static void
2309 vtnet_txq_free_mbufs(struct vtnet_txq *txq)
2310 {
2311 	struct virtqueue *vq;
2312 	struct vtnet_tx_header *txhdr;
2313 	int last;
2314 #ifdef DEV_NETMAP
2315 	struct netmap_kring *kring = netmap_kring_on(NA(txq->vtntx_sc->vtnet_ifp),
2316 							txq->vtntx_id, NR_TX);
2317 #else  /* !DEV_NETMAP */
2318 	void *kring = NULL;
2319 #endif /* !DEV_NETMAP */
2320 
2321 	vq = txq->vtntx_vq;
2322 	last = 0;
2323 
2324 	while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
2325 		if (kring == NULL) {
2326 			m_freem(txhdr->vth_mbuf);
2327 			uma_zfree(vtnet_tx_header_zone, txhdr);
2328 		}
2329 	}
2330 
2331 	KASSERT(virtqueue_empty(vq),
2332 	    ("%s: mbufs remaining in tx queue %p", __func__, txq));
2333 }
2334 
2335 /*
2336  * BMV: This can go away once we finally have offsets in the mbuf header.
2337  */
2338 static int
2339 vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m, int *etype,
2340     int *proto, int *start)
2341 {
2342 	struct vtnet_softc *sc;
2343 	struct ether_vlan_header *evh;
2344 #if defined(INET) || defined(INET6)
2345 	int offset;
2346 #endif
2347 
2348 	sc = txq->vtntx_sc;
2349 
2350 	evh = mtod(m, struct ether_vlan_header *);
2351 	if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2352 		/* BMV: We should handle nested VLAN tags too. */
2353 		*etype = ntohs(evh->evl_proto);
2354 #if defined(INET) || defined(INET6)
2355 		offset = sizeof(struct ether_vlan_header);
2356 #endif
2357 	} else {
2358 		*etype = ntohs(evh->evl_encap_proto);
2359 #if defined(INET) || defined(INET6)
2360 		offset = sizeof(struct ether_header);
2361 #endif
2362 	}
2363 
2364 	switch (*etype) {
2365 #if defined(INET)
2366 	case ETHERTYPE_IP: {
2367 		struct ip *ip, iphdr;
2368 		if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
2369 			m_copydata(m, offset, sizeof(struct ip),
2370 			    (caddr_t) &iphdr);
2371 			ip = &iphdr;
2372 		} else
2373 			ip = (struct ip *)(m->m_data + offset);
2374 		*proto = ip->ip_p;
2375 		*start = offset + (ip->ip_hl << 2);
2376 		break;
2377 	}
2378 #endif
2379 #if defined(INET6)
2380 	case ETHERTYPE_IPV6:
2381 		*proto = -1;
2382 		*start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
2383 		/* Assert the network stack sent us a valid packet. */
2384 		KASSERT(*start > offset,
2385 		    ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
2386 		    *start, offset, *proto));
2387 		break;
2388 #endif
2389 	default:
2390 		sc->vtnet_stats.tx_csum_unknown_ethtype++;
2391 		return (EINVAL);
2392 	}
2393 
2394 	return (0);
2395 }
2396 
2397 static int
2398 vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type,
2399     int offset, struct virtio_net_hdr *hdr)
2400 {
2401 	static struct timeval lastecn;
2402 	static int curecn;
2403 	struct vtnet_softc *sc;
2404 	struct tcphdr *tcp, tcphdr;
2405 
2406 	sc = txq->vtntx_sc;
2407 
2408 	if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) {
2409 		m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr);
2410 		tcp = &tcphdr;
2411 	} else
2412 		tcp = (struct tcphdr *)(m->m_data + offset);
2413 
2414 	hdr->hdr_len = vtnet_gtoh16(sc, offset + (tcp->th_off << 2));
2415 	hdr->gso_size = vtnet_gtoh16(sc, m->m_pkthdr.tso_segsz);
2416 	hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
2417 	    VIRTIO_NET_HDR_GSO_TCPV6;
2418 
2419 	if (__predict_false(tcp->th_flags & TH_CWR)) {
2420 		/*
2421 		 * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In
2422 		 * FreeBSD, ECN support is not on a per-interface basis,
2423 		 * but globally via the net.inet.tcp.ecn.enable sysctl
2424 		 * knob. The default is off.
2425 		 */
2426 		if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
2427 			if (ppsratecheck(&lastecn, &curecn, 1))
2428 				if_printf(sc->vtnet_ifp,
2429 				    "TSO with ECN not negotiated with host\n");
2430 			return (ENOTSUP);
2431 		}
2432 		hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2433 	}
2434 
2435 	txq->vtntx_stats.vtxs_tso++;
2436 
2437 	return (0);
2438 }
2439 
2440 static struct mbuf *
2441 vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m,
2442     struct virtio_net_hdr *hdr)
2443 {
2444 	struct vtnet_softc *sc;
2445 	int flags, etype, csum_start, proto, error;
2446 
2447 	sc = txq->vtntx_sc;
2448 	flags = m->m_pkthdr.csum_flags;
2449 
2450 	error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start);
2451 	if (error)
2452 		goto drop;
2453 
2454 	if (flags & (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6)) {
2455 		/* Sanity check the parsed mbuf matches the offload flags. */
2456 		if (__predict_false((flags & VTNET_CSUM_OFFLOAD &&
2457 		    etype != ETHERTYPE_IP) || (flags & VTNET_CSUM_OFFLOAD_IPV6
2458 		    && etype != ETHERTYPE_IPV6))) {
2459 			sc->vtnet_stats.tx_csum_proto_mismatch++;
2460 			goto drop;
2461 		}
2462 
2463 		hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
2464 		hdr->csum_start = vtnet_gtoh16(sc, csum_start);
2465 		hdr->csum_offset = vtnet_gtoh16(sc, m->m_pkthdr.csum_data);
2466 		txq->vtntx_stats.vtxs_csum++;
2467 	}
2468 
2469 	if (flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) {
2470 		/*
2471 		 * Sanity check the parsed mbuf IP protocol is TCP, and
2472 		 * VirtIO TSO reqires the checksum offloading above.
2473 		 */
2474 		if (__predict_false(proto != IPPROTO_TCP)) {
2475 			sc->vtnet_stats.tx_tso_not_tcp++;
2476 			goto drop;
2477 		} else if (__predict_false((hdr->flags &
2478 		    VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) {
2479 			sc->vtnet_stats.tx_tso_without_csum++;
2480 			goto drop;
2481 		}
2482 
2483 		error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr);
2484 		if (error)
2485 			goto drop;
2486 	}
2487 
2488 	return (m);
2489 
2490 drop:
2491 	m_freem(m);
2492 	return (NULL);
2493 }
2494 
2495 static int
2496 vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head,
2497     struct vtnet_tx_header *txhdr)
2498 {
2499 	struct vtnet_softc *sc;
2500 	struct virtqueue *vq;
2501 	struct sglist *sg;
2502 	struct mbuf *m;
2503 	int error;
2504 
2505 	sc = txq->vtntx_sc;
2506 	vq = txq->vtntx_vq;
2507 	sg = txq->vtntx_sg;
2508 	m = *m_head;
2509 
2510 	sglist_reset(sg);
2511 	error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
2512 	if (error != 0 || sg->sg_nseg != 1) {
2513 		KASSERT(0, ("%s: cannot add header to sglist error %d nseg %d",
2514 		    __func__, error, sg->sg_nseg));
2515 		goto fail;
2516 	}
2517 
2518 	error = sglist_append_mbuf(sg, m);
2519 	if (error) {
2520 		m = m_defrag(m, M_NOWAIT);
2521 		if (m == NULL)
2522 			goto fail;
2523 
2524 		*m_head = m;
2525 		sc->vtnet_stats.tx_defragged++;
2526 
2527 		error = sglist_append_mbuf(sg, m);
2528 		if (error)
2529 			goto fail;
2530 	}
2531 
2532 	txhdr->vth_mbuf = m;
2533 	error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0);
2534 
2535 	return (error);
2536 
2537 fail:
2538 	sc->vtnet_stats.tx_defrag_failed++;
2539 	m_freem(*m_head);
2540 	*m_head = NULL;
2541 
2542 	return (ENOBUFS);
2543 }
2544 
2545 static int
2546 vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head, int flags)
2547 {
2548 	struct vtnet_tx_header *txhdr;
2549 	struct virtio_net_hdr *hdr;
2550 	struct mbuf *m;
2551 	int error;
2552 
2553 	m = *m_head;
2554 	M_ASSERTPKTHDR(m);
2555 
2556 	txhdr = uma_zalloc(vtnet_tx_header_zone, flags | M_ZERO);
2557 	if (txhdr == NULL) {
2558 		m_freem(m);
2559 		*m_head = NULL;
2560 		return (ENOMEM);
2561 	}
2562 
2563 	/*
2564 	 * Always use the non-mergeable header, regardless if mergable headers
2565 	 * were negotiated, because for transmit num_buffers is always zero.
2566 	 * The vtnet_hdr_size is used to enqueue the right header size segment.
2567 	 */
2568 	hdr = &txhdr->vth_uhdr.hdr;
2569 
2570 	if (m->m_flags & M_VLANTAG) {
2571 		m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
2572 		if ((*m_head = m) == NULL) {
2573 			error = ENOBUFS;
2574 			goto fail;
2575 		}
2576 		m->m_flags &= ~M_VLANTAG;
2577 	}
2578 
2579 	if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) {
2580 		m = vtnet_txq_offload(txq, m, hdr);
2581 		if ((*m_head = m) == NULL) {
2582 			error = ENOBUFS;
2583 			goto fail;
2584 		}
2585 	}
2586 
2587 	error = vtnet_txq_enqueue_buf(txq, m_head, txhdr);
2588 fail:
2589 	if (error)
2590 		uma_zfree(vtnet_tx_header_zone, txhdr);
2591 
2592 	return (error);
2593 }
2594 
2595 #ifdef VTNET_LEGACY_TX
2596 
2597 static void
2598 vtnet_start_locked(struct vtnet_txq *txq, if_t ifp)
2599 {
2600 	struct vtnet_softc *sc;
2601 	struct virtqueue *vq;
2602 	struct mbuf *m0;
2603 	int tries, enq;
2604 
2605 	sc = txq->vtntx_sc;
2606 	vq = txq->vtntx_vq;
2607 	tries = 0;
2608 
2609 	VTNET_TXQ_LOCK_ASSERT(txq);
2610 
2611 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 ||
2612 	    sc->vtnet_link_active == 0)
2613 		return;
2614 
2615 	vtnet_txq_eof(txq);
2616 
2617 again:
2618 	enq = 0;
2619 
2620 	while (!if_sendq_empty(ifp)) {
2621 		if (virtqueue_full(vq))
2622 			break;
2623 
2624 		m0 = if_dequeue(ifp);
2625 		if (m0 == NULL)
2626 			break;
2627 
2628 		if (vtnet_txq_encap(txq, &m0, M_NOWAIT) != 0) {
2629 			if (m0 != NULL)
2630 				if_sendq_prepend(ifp, m0);
2631 			break;
2632 		}
2633 
2634 		enq++;
2635 		ETHER_BPF_MTAP(ifp, m0);
2636 	}
2637 
2638 	if (enq > 0 && vtnet_txq_notify(txq) != 0) {
2639 		if (tries++ < VTNET_NOTIFY_RETRIES)
2640 			goto again;
2641 
2642 		txq->vtntx_stats.vtxs_rescheduled++;
2643 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask);
2644 	}
2645 }
2646 
2647 static void
2648 vtnet_start(if_t ifp)
2649 {
2650 	struct vtnet_softc *sc;
2651 	struct vtnet_txq *txq;
2652 
2653 	sc = if_getsoftc(ifp);
2654 	txq = &sc->vtnet_txqs[0];
2655 
2656 	VTNET_TXQ_LOCK(txq);
2657 	vtnet_start_locked(txq, ifp);
2658 	VTNET_TXQ_UNLOCK(txq);
2659 }
2660 
2661 #else /* !VTNET_LEGACY_TX */
2662 
2663 static int
2664 vtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m)
2665 {
2666 	struct vtnet_softc *sc;
2667 	struct virtqueue *vq;
2668 	struct buf_ring *br;
2669 	if_t ifp;
2670 	int enq, tries, error;
2671 
2672 	sc = txq->vtntx_sc;
2673 	vq = txq->vtntx_vq;
2674 	br = txq->vtntx_br;
2675 	ifp = sc->vtnet_ifp;
2676 	tries = 0;
2677 	error = 0;
2678 
2679 	VTNET_TXQ_LOCK_ASSERT(txq);
2680 
2681 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 ||
2682 	    sc->vtnet_link_active == 0) {
2683 		if (m != NULL)
2684 			error = drbr_enqueue(ifp, br, m);
2685 		return (error);
2686 	}
2687 
2688 	if (m != NULL) {
2689 		error = drbr_enqueue(ifp, br, m);
2690 		if (error)
2691 			return (error);
2692 	}
2693 
2694 	vtnet_txq_eof(txq);
2695 
2696 again:
2697 	enq = 0;
2698 
2699 	while ((m = drbr_peek(ifp, br)) != NULL) {
2700 		if (virtqueue_full(vq)) {
2701 			drbr_putback(ifp, br, m);
2702 			break;
2703 		}
2704 
2705 		if (vtnet_txq_encap(txq, &m, M_NOWAIT) != 0) {
2706 			if (m != NULL)
2707 				drbr_putback(ifp, br, m);
2708 			else
2709 				drbr_advance(ifp, br);
2710 			break;
2711 		}
2712 		drbr_advance(ifp, br);
2713 
2714 		enq++;
2715 		ETHER_BPF_MTAP(ifp, m);
2716 	}
2717 
2718 	if (enq > 0 && vtnet_txq_notify(txq) != 0) {
2719 		if (tries++ < VTNET_NOTIFY_RETRIES)
2720 			goto again;
2721 
2722 		txq->vtntx_stats.vtxs_rescheduled++;
2723 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask);
2724 	}
2725 
2726 	return (0);
2727 }
2728 
2729 static int
2730 vtnet_txq_mq_start(if_t ifp, struct mbuf *m)
2731 {
2732 	struct vtnet_softc *sc;
2733 	struct vtnet_txq *txq;
2734 	int i, npairs, error;
2735 
2736 	sc = if_getsoftc(ifp);
2737 	npairs = sc->vtnet_act_vq_pairs;
2738 
2739 	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
2740 		i = m->m_pkthdr.flowid % npairs;
2741 	else
2742 		i = curcpu % npairs;
2743 
2744 	txq = &sc->vtnet_txqs[i];
2745 
2746 	if (VTNET_TXQ_TRYLOCK(txq) != 0) {
2747 		error = vtnet_txq_mq_start_locked(txq, m);
2748 		VTNET_TXQ_UNLOCK(txq);
2749 	} else {
2750 		error = drbr_enqueue(ifp, txq->vtntx_br, m);
2751 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask);
2752 	}
2753 
2754 	return (error);
2755 }
2756 
2757 static void
2758 vtnet_txq_tq_deferred(void *xtxq, int pending __unused)
2759 {
2760 	struct vtnet_softc *sc;
2761 	struct vtnet_txq *txq;
2762 
2763 	txq = xtxq;
2764 	sc = txq->vtntx_sc;
2765 
2766 	VTNET_TXQ_LOCK(txq);
2767 	if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br))
2768 		vtnet_txq_mq_start_locked(txq, NULL);
2769 	VTNET_TXQ_UNLOCK(txq);
2770 }
2771 
2772 #endif /* VTNET_LEGACY_TX */
2773 
2774 static void
2775 vtnet_txq_start(struct vtnet_txq *txq)
2776 {
2777 	struct vtnet_softc *sc;
2778 	if_t ifp;
2779 
2780 	sc = txq->vtntx_sc;
2781 	ifp = sc->vtnet_ifp;
2782 
2783 #ifdef VTNET_LEGACY_TX
2784 	if (!if_sendq_empty(ifp))
2785 		vtnet_start_locked(txq, ifp);
2786 #else
2787 	if (!drbr_empty(ifp, txq->vtntx_br))
2788 		vtnet_txq_mq_start_locked(txq, NULL);
2789 #endif
2790 }
2791 
2792 static void
2793 vtnet_txq_tq_intr(void *xtxq, int pending __unused)
2794 {
2795 	struct vtnet_softc *sc;
2796 	struct vtnet_txq *txq;
2797 	if_t ifp;
2798 
2799 	txq = xtxq;
2800 	sc = txq->vtntx_sc;
2801 	ifp = sc->vtnet_ifp;
2802 
2803 	VTNET_TXQ_LOCK(txq);
2804 
2805 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
2806 		VTNET_TXQ_UNLOCK(txq);
2807 		return;
2808 	}
2809 
2810 	vtnet_txq_eof(txq);
2811 	vtnet_txq_start(txq);
2812 
2813 	VTNET_TXQ_UNLOCK(txq);
2814 }
2815 
2816 static int
2817 vtnet_txq_eof(struct vtnet_txq *txq)
2818 {
2819 	struct virtqueue *vq;
2820 	struct vtnet_tx_header *txhdr;
2821 	struct mbuf *m;
2822 	int deq;
2823 
2824 	vq = txq->vtntx_vq;
2825 	deq = 0;
2826 	VTNET_TXQ_LOCK_ASSERT(txq);
2827 
2828 	while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
2829 		m = txhdr->vth_mbuf;
2830 		deq++;
2831 
2832 		txq->vtntx_stats.vtxs_opackets++;
2833 		txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len;
2834 		if (m->m_flags & M_MCAST)
2835 			txq->vtntx_stats.vtxs_omcasts++;
2836 
2837 		m_freem(m);
2838 		uma_zfree(vtnet_tx_header_zone, txhdr);
2839 	}
2840 
2841 	if (virtqueue_empty(vq))
2842 		txq->vtntx_watchdog = 0;
2843 
2844 	return (deq);
2845 }
2846 
2847 static void
2848 vtnet_tx_vq_intr(void *xtxq)
2849 {
2850 	struct vtnet_softc *sc;
2851 	struct vtnet_txq *txq;
2852 	if_t ifp;
2853 
2854 	txq = xtxq;
2855 	sc = txq->vtntx_sc;
2856 	ifp = sc->vtnet_ifp;
2857 
2858 	if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) {
2859 		/*
2860 		 * Ignore this interrupt. Either this is a spurious interrupt
2861 		 * or multiqueue without per-VQ MSIX so every queue needs to
2862 		 * be polled (a brain dead configuration we could try harder
2863 		 * to avoid).
2864 		 */
2865 		vtnet_txq_disable_intr(txq);
2866 		return;
2867 	}
2868 
2869 #ifdef DEV_NETMAP
2870 	if (netmap_tx_irq(ifp, txq->vtntx_id) != NM_IRQ_PASS)
2871 		return;
2872 #endif /* DEV_NETMAP */
2873 
2874 	VTNET_TXQ_LOCK(txq);
2875 
2876 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
2877 		VTNET_TXQ_UNLOCK(txq);
2878 		return;
2879 	}
2880 
2881 	vtnet_txq_eof(txq);
2882 	vtnet_txq_start(txq);
2883 
2884 	VTNET_TXQ_UNLOCK(txq);
2885 }
2886 
2887 static void
2888 vtnet_tx_start_all(struct vtnet_softc *sc)
2889 {
2890 	struct vtnet_txq *txq;
2891 	int i;
2892 
2893 	VTNET_CORE_LOCK_ASSERT(sc);
2894 
2895 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
2896 		txq = &sc->vtnet_txqs[i];
2897 
2898 		VTNET_TXQ_LOCK(txq);
2899 		vtnet_txq_start(txq);
2900 		VTNET_TXQ_UNLOCK(txq);
2901 	}
2902 }
2903 
2904 #ifndef VTNET_LEGACY_TX
2905 static void
2906 vtnet_qflush(if_t ifp)
2907 {
2908 	struct vtnet_softc *sc;
2909 	struct vtnet_txq *txq;
2910 	struct mbuf *m;
2911 	int i;
2912 
2913 	sc = if_getsoftc(ifp);
2914 
2915 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
2916 		txq = &sc->vtnet_txqs[i];
2917 
2918 		VTNET_TXQ_LOCK(txq);
2919 		while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL)
2920 			m_freem(m);
2921 		VTNET_TXQ_UNLOCK(txq);
2922 	}
2923 
2924 	if_qflush(ifp);
2925 }
2926 #endif
2927 
2928 static int
2929 vtnet_watchdog(struct vtnet_txq *txq)
2930 {
2931 	if_t ifp;
2932 
2933 	ifp = txq->vtntx_sc->vtnet_ifp;
2934 
2935 	VTNET_TXQ_LOCK(txq);
2936 	if (txq->vtntx_watchdog == 1) {
2937 		/*
2938 		 * Only drain completed frames if the watchdog is about to
2939 		 * expire. If any frames were drained, there may be enough
2940 		 * free descriptors now available to transmit queued frames.
2941 		 * In that case, the timer will immediately be decremented
2942 		 * below, but the timeout is generous enough that should not
2943 		 * be a problem.
2944 		 */
2945 		if (vtnet_txq_eof(txq) != 0)
2946 			vtnet_txq_start(txq);
2947 	}
2948 
2949 	if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) {
2950 		VTNET_TXQ_UNLOCK(txq);
2951 		return (0);
2952 	}
2953 	VTNET_TXQ_UNLOCK(txq);
2954 
2955 	if_printf(ifp, "watchdog timeout on queue %d\n", txq->vtntx_id);
2956 	return (1);
2957 }
2958 
2959 static void
2960 vtnet_accum_stats(struct vtnet_softc *sc, struct vtnet_rxq_stats *rxacc,
2961     struct vtnet_txq_stats *txacc)
2962 {
2963 
2964 	bzero(rxacc, sizeof(struct vtnet_rxq_stats));
2965 	bzero(txacc, sizeof(struct vtnet_txq_stats));
2966 
2967 	for (int i = 0; i < sc->vtnet_max_vq_pairs; i++) {
2968 		struct vtnet_rxq_stats *rxst;
2969 		struct vtnet_txq_stats *txst;
2970 
2971 		rxst = &sc->vtnet_rxqs[i].vtnrx_stats;
2972 		rxacc->vrxs_ipackets += rxst->vrxs_ipackets;
2973 		rxacc->vrxs_ibytes += rxst->vrxs_ibytes;
2974 		rxacc->vrxs_iqdrops += rxst->vrxs_iqdrops;
2975 		rxacc->vrxs_csum += rxst->vrxs_csum;
2976 		rxacc->vrxs_csum_failed += rxst->vrxs_csum_failed;
2977 		rxacc->vrxs_rescheduled += rxst->vrxs_rescheduled;
2978 
2979 		txst = &sc->vtnet_txqs[i].vtntx_stats;
2980 		txacc->vtxs_opackets += txst->vtxs_opackets;
2981 		txacc->vtxs_obytes += txst->vtxs_obytes;
2982 		txacc->vtxs_csum += txst->vtxs_csum;
2983 		txacc->vtxs_tso += txst->vtxs_tso;
2984 		txacc->vtxs_rescheduled += txst->vtxs_rescheduled;
2985 	}
2986 }
2987 
2988 static uint64_t
2989 vtnet_get_counter(if_t ifp, ift_counter cnt)
2990 {
2991 	struct vtnet_softc *sc;
2992 	struct vtnet_rxq_stats rxaccum;
2993 	struct vtnet_txq_stats txaccum;
2994 
2995 	sc = if_getsoftc(ifp);
2996 	vtnet_accum_stats(sc, &rxaccum, &txaccum);
2997 
2998 	switch (cnt) {
2999 	case IFCOUNTER_IPACKETS:
3000 		return (rxaccum.vrxs_ipackets);
3001 	case IFCOUNTER_IQDROPS:
3002 		return (rxaccum.vrxs_iqdrops);
3003 	case IFCOUNTER_IERRORS:
3004 		return (rxaccum.vrxs_ierrors);
3005 	case IFCOUNTER_OPACKETS:
3006 		return (txaccum.vtxs_opackets);
3007 #ifndef VTNET_LEGACY_TX
3008 	case IFCOUNTER_OBYTES:
3009 		return (txaccum.vtxs_obytes);
3010 	case IFCOUNTER_OMCASTS:
3011 		return (txaccum.vtxs_omcasts);
3012 #endif
3013 	default:
3014 		return (if_get_counter_default(ifp, cnt));
3015 	}
3016 }
3017 
3018 static void
3019 vtnet_tick(void *xsc)
3020 {
3021 	struct vtnet_softc *sc;
3022 	if_t ifp;
3023 	int i, timedout;
3024 
3025 	sc = xsc;
3026 	ifp = sc->vtnet_ifp;
3027 	timedout = 0;
3028 
3029 	VTNET_CORE_LOCK_ASSERT(sc);
3030 
3031 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
3032 		timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]);
3033 
3034 	if (timedout != 0) {
3035 		if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
3036 		vtnet_init_locked(sc, 0);
3037 	} else
3038 		callout_schedule(&sc->vtnet_tick_ch, hz);
3039 }
3040 
3041 static void
3042 vtnet_start_taskqueues(struct vtnet_softc *sc)
3043 {
3044 	device_t dev;
3045 	struct vtnet_rxq *rxq;
3046 	struct vtnet_txq *txq;
3047 	int i, error;
3048 
3049 	dev = sc->vtnet_dev;
3050 
3051 	/*
3052 	 * Errors here are very difficult to recover from - we cannot
3053 	 * easily fail because, if this is during boot, we will hang
3054 	 * when freeing any successfully started taskqueues because
3055 	 * the scheduler isn't up yet.
3056 	 *
3057 	 * Most drivers just ignore the return value - it only fails
3058 	 * with ENOMEM so an error is not likely.
3059 	 */
3060 	for (i = 0; i < sc->vtnet_req_vq_pairs; i++) {
3061 		rxq = &sc->vtnet_rxqs[i];
3062 		error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET,
3063 		    "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id);
3064 		if (error) {
3065 			device_printf(dev, "failed to start rx taskq %d\n",
3066 			    rxq->vtnrx_id);
3067 		}
3068 
3069 		txq = &sc->vtnet_txqs[i];
3070 		error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET,
3071 		    "%s txq %d", device_get_nameunit(dev), txq->vtntx_id);
3072 		if (error) {
3073 			device_printf(dev, "failed to start tx taskq %d\n",
3074 			    txq->vtntx_id);
3075 		}
3076 	}
3077 }
3078 
3079 static void
3080 vtnet_free_taskqueues(struct vtnet_softc *sc)
3081 {
3082 	struct vtnet_rxq *rxq;
3083 	struct vtnet_txq *txq;
3084 	int i;
3085 
3086 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
3087 		rxq = &sc->vtnet_rxqs[i];
3088 		if (rxq->vtnrx_tq != NULL) {
3089 			taskqueue_free(rxq->vtnrx_tq);
3090 			rxq->vtnrx_tq = NULL;
3091 		}
3092 
3093 		txq = &sc->vtnet_txqs[i];
3094 		if (txq->vtntx_tq != NULL) {
3095 			taskqueue_free(txq->vtntx_tq);
3096 			txq->vtntx_tq = NULL;
3097 		}
3098 	}
3099 }
3100 
3101 static void
3102 vtnet_drain_taskqueues(struct vtnet_softc *sc)
3103 {
3104 	struct vtnet_rxq *rxq;
3105 	struct vtnet_txq *txq;
3106 	int i;
3107 
3108 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
3109 		rxq = &sc->vtnet_rxqs[i];
3110 		if (rxq->vtnrx_tq != NULL)
3111 			taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
3112 
3113 		txq = &sc->vtnet_txqs[i];
3114 		if (txq->vtntx_tq != NULL) {
3115 			taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask);
3116 #ifndef VTNET_LEGACY_TX
3117 			taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask);
3118 #endif
3119 		}
3120 	}
3121 }
3122 
3123 static void
3124 vtnet_drain_rxtx_queues(struct vtnet_softc *sc)
3125 {
3126 	struct vtnet_rxq *rxq;
3127 	struct vtnet_txq *txq;
3128 	int i;
3129 
3130 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
3131 		rxq = &sc->vtnet_rxqs[i];
3132 		vtnet_rxq_free_mbufs(rxq);
3133 
3134 		txq = &sc->vtnet_txqs[i];
3135 		vtnet_txq_free_mbufs(txq);
3136 	}
3137 }
3138 
3139 static void
3140 vtnet_stop_rendezvous(struct vtnet_softc *sc)
3141 {
3142 	struct vtnet_rxq *rxq;
3143 	struct vtnet_txq *txq;
3144 	int i;
3145 
3146 	VTNET_CORE_LOCK_ASSERT(sc);
3147 
3148 	/*
3149 	 * Lock and unlock the per-queue mutex so we known the stop
3150 	 * state is visible. Doing only the active queues should be
3151 	 * sufficient, but it does not cost much extra to do all the
3152 	 * queues.
3153 	 */
3154 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
3155 		rxq = &sc->vtnet_rxqs[i];
3156 		VTNET_RXQ_LOCK(rxq);
3157 		VTNET_RXQ_UNLOCK(rxq);
3158 
3159 		txq = &sc->vtnet_txqs[i];
3160 		VTNET_TXQ_LOCK(txq);
3161 		VTNET_TXQ_UNLOCK(txq);
3162 	}
3163 }
3164 
3165 static void
3166 vtnet_stop(struct vtnet_softc *sc)
3167 {
3168 	device_t dev;
3169 	if_t ifp;
3170 
3171 	dev = sc->vtnet_dev;
3172 	ifp = sc->vtnet_ifp;
3173 
3174 	VTNET_CORE_LOCK_ASSERT(sc);
3175 
3176 	if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
3177 	sc->vtnet_link_active = 0;
3178 	callout_stop(&sc->vtnet_tick_ch);
3179 
3180 	/* Only advisory. */
3181 	vtnet_disable_interrupts(sc);
3182 
3183 #ifdef DEV_NETMAP
3184 	/* Stop any pending txsync/rxsync and disable them. */
3185 	netmap_disable_all_rings(ifp);
3186 #endif /* DEV_NETMAP */
3187 
3188 	/*
3189 	 * Stop the host adapter. This resets it to the pre-initialized
3190 	 * state. It will not generate any interrupts until after it is
3191 	 * reinitialized.
3192 	 */
3193 	virtio_stop(dev);
3194 	vtnet_stop_rendezvous(sc);
3195 
3196 	vtnet_drain_rxtx_queues(sc);
3197 	sc->vtnet_act_vq_pairs = 1;
3198 }
3199 
3200 static int
3201 vtnet_virtio_reinit(struct vtnet_softc *sc)
3202 {
3203 	device_t dev;
3204 	if_t ifp;
3205 	uint64_t features;
3206 	int error;
3207 
3208 	dev = sc->vtnet_dev;
3209 	ifp = sc->vtnet_ifp;
3210 	features = sc->vtnet_negotiated_features;
3211 
3212 	/*
3213 	 * Re-negotiate with the host, removing any disabled receive
3214 	 * features. Transmit features are disabled only on our side
3215 	 * via if_capenable and if_hwassist.
3216 	 */
3217 
3218 	if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) == 0)
3219 		features &= ~(VIRTIO_NET_F_GUEST_CSUM | VTNET_LRO_FEATURES);
3220 
3221 	if ((if_getcapenable(ifp) & IFCAP_LRO) == 0)
3222 		features &= ~VTNET_LRO_FEATURES;
3223 
3224 	if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) == 0)
3225 		features &= ~VIRTIO_NET_F_CTRL_VLAN;
3226 
3227 	error = virtio_reinit(dev, features);
3228 	if (error) {
3229 		device_printf(dev, "virtio reinit error %d\n", error);
3230 		return (error);
3231 	}
3232 
3233 	sc->vtnet_features = features;
3234 	virtio_reinit_complete(dev);
3235 
3236 	return (0);
3237 }
3238 
3239 static void
3240 vtnet_init_rx_filters(struct vtnet_softc *sc)
3241 {
3242 	if_t ifp;
3243 
3244 	ifp = sc->vtnet_ifp;
3245 
3246 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
3247 		vtnet_rx_filter(sc);
3248 		vtnet_rx_filter_mac(sc);
3249 	}
3250 
3251 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3252 		vtnet_rx_filter_vlan(sc);
3253 }
3254 
3255 static int
3256 vtnet_init_rx_queues(struct vtnet_softc *sc)
3257 {
3258 	device_t dev;
3259 	if_t ifp;
3260 	struct vtnet_rxq *rxq;
3261 	int i, clustersz, error;
3262 
3263 	dev = sc->vtnet_dev;
3264 	ifp = sc->vtnet_ifp;
3265 
3266 	clustersz = vtnet_rx_cluster_size(sc, if_getmtu(ifp));
3267 	sc->vtnet_rx_clustersz = clustersz;
3268 
3269 	if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) {
3270 		sc->vtnet_rx_nmbufs = howmany(sizeof(struct vtnet_rx_header) +
3271 		    VTNET_MAX_RX_SIZE, clustersz);
3272 		KASSERT(sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs,
3273 		    ("%s: too many rx mbufs %d for %d segments", __func__,
3274 		    sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs));
3275 	} else
3276 		sc->vtnet_rx_nmbufs = 1;
3277 
3278 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
3279 		rxq = &sc->vtnet_rxqs[i];
3280 
3281 		/* Hold the lock to satisfy asserts. */
3282 		VTNET_RXQ_LOCK(rxq);
3283 		error = vtnet_rxq_populate(rxq);
3284 		VTNET_RXQ_UNLOCK(rxq);
3285 
3286 		if (error) {
3287 			device_printf(dev, "cannot populate Rx queue %d\n", i);
3288 			return (error);
3289 		}
3290 	}
3291 
3292 	return (0);
3293 }
3294 
3295 static int
3296 vtnet_init_tx_queues(struct vtnet_softc *sc)
3297 {
3298 	struct vtnet_txq *txq;
3299 	int i;
3300 
3301 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
3302 		txq = &sc->vtnet_txqs[i];
3303 		txq->vtntx_watchdog = 0;
3304 		txq->vtntx_intr_threshold = vtnet_txq_intr_threshold(txq);
3305 #ifdef DEV_NETMAP
3306 		netmap_reset(NA(sc->vtnet_ifp), NR_TX, i, 0);
3307 #endif /* DEV_NETMAP */
3308 	}
3309 
3310 	return (0);
3311 }
3312 
3313 static int
3314 vtnet_init_rxtx_queues(struct vtnet_softc *sc)
3315 {
3316 	int error;
3317 
3318 	error = vtnet_init_rx_queues(sc);
3319 	if (error)
3320 		return (error);
3321 
3322 	error = vtnet_init_tx_queues(sc);
3323 	if (error)
3324 		return (error);
3325 
3326 	return (0);
3327 }
3328 
3329 static void
3330 vtnet_set_active_vq_pairs(struct vtnet_softc *sc)
3331 {
3332 	device_t dev;
3333 	int npairs;
3334 
3335 	dev = sc->vtnet_dev;
3336 
3337 	if ((sc->vtnet_flags & VTNET_FLAG_MQ) == 0) {
3338 		sc->vtnet_act_vq_pairs = 1;
3339 		return;
3340 	}
3341 
3342 	npairs = sc->vtnet_req_vq_pairs;
3343 
3344 	if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) {
3345 		device_printf(dev, "cannot set active queue pairs to %d, "
3346 		    "falling back to 1 queue pair\n", npairs);
3347 		npairs = 1;
3348 	}
3349 
3350 	sc->vtnet_act_vq_pairs = npairs;
3351 }
3352 
3353 static void
3354 vtnet_update_rx_offloads(struct vtnet_softc *sc)
3355 {
3356 	if_t ifp;
3357 	uint64_t features;
3358 	int error;
3359 
3360 	ifp = sc->vtnet_ifp;
3361 	features = sc->vtnet_features;
3362 
3363 	VTNET_CORE_LOCK_ASSERT(sc);
3364 
3365 	if (if_getcapabilities(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
3366 		if (if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
3367 			features |= VIRTIO_NET_F_GUEST_CSUM;
3368 		else
3369 			features &= ~VIRTIO_NET_F_GUEST_CSUM;
3370 	}
3371 
3372 	if (if_getcapabilities(ifp) & IFCAP_LRO && !vtnet_software_lro(sc)) {
3373 		if (if_getcapenable(ifp) & IFCAP_LRO)
3374 			features |= VTNET_LRO_FEATURES;
3375 		else
3376 			features &= ~VTNET_LRO_FEATURES;
3377 	}
3378 
3379 	error = vtnet_ctrl_guest_offloads(sc,
3380 	    features & (VIRTIO_NET_F_GUEST_CSUM | VIRTIO_NET_F_GUEST_TSO4 |
3381 		        VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN  |
3382 			VIRTIO_NET_F_GUEST_UFO));
3383 	if (error) {
3384 		device_printf(sc->vtnet_dev,
3385 		    "%s: cannot update Rx features\n", __func__);
3386 		if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3387 			if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
3388 			vtnet_init_locked(sc, 0);
3389 		}
3390 	} else
3391 		sc->vtnet_features = features;
3392 }
3393 
3394 static int
3395 vtnet_reinit(struct vtnet_softc *sc)
3396 {
3397 	if_t ifp;
3398 	int error;
3399 
3400 	ifp = sc->vtnet_ifp;
3401 
3402 	bcopy(if_getlladdr(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
3403 
3404 	error = vtnet_virtio_reinit(sc);
3405 	if (error)
3406 		return (error);
3407 
3408 	vtnet_set_macaddr(sc);
3409 	vtnet_set_active_vq_pairs(sc);
3410 
3411 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ)
3412 		vtnet_init_rx_filters(sc);
3413 
3414 	if_sethwassist(ifp, 0);
3415 	if (if_getcapenable(ifp) & IFCAP_TXCSUM)
3416 		if_sethwassistbits(ifp, VTNET_CSUM_OFFLOAD, 0);
3417 	if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
3418 		if_sethwassistbits(ifp, VTNET_CSUM_OFFLOAD_IPV6, 0);
3419 	if (if_getcapenable(ifp) & IFCAP_TSO4)
3420 		if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
3421 	if (if_getcapenable(ifp) & IFCAP_TSO6)
3422 		if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
3423 
3424 	error = vtnet_init_rxtx_queues(sc);
3425 	if (error)
3426 		return (error);
3427 
3428 	return (0);
3429 }
3430 
3431 static void
3432 vtnet_init_locked(struct vtnet_softc *sc, int init_mode)
3433 {
3434 	if_t ifp;
3435 
3436 	ifp = sc->vtnet_ifp;
3437 
3438 	VTNET_CORE_LOCK_ASSERT(sc);
3439 
3440 	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
3441 		return;
3442 
3443 	vtnet_stop(sc);
3444 
3445 #ifdef DEV_NETMAP
3446 	/* Once stopped we can update the netmap flags, if necessary. */
3447 	switch (init_mode) {
3448 	case VTNET_INIT_NETMAP_ENTER:
3449 		nm_set_native_flags(NA(ifp));
3450 		break;
3451 	case VTNET_INIT_NETMAP_EXIT:
3452 		nm_clear_native_flags(NA(ifp));
3453 		break;
3454 	}
3455 #endif /* DEV_NETMAP */
3456 
3457 	if (vtnet_reinit(sc) != 0) {
3458 		vtnet_stop(sc);
3459 		return;
3460 	}
3461 
3462 	if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
3463 	vtnet_update_link_status(sc);
3464 	vtnet_enable_interrupts(sc);
3465 	callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
3466 
3467 #ifdef DEV_NETMAP
3468 	/* Re-enable txsync/rxsync. */
3469 	netmap_enable_all_rings(ifp);
3470 #endif /* DEV_NETMAP */
3471 }
3472 
3473 static void
3474 vtnet_init(void *xsc)
3475 {
3476 	struct vtnet_softc *sc;
3477 
3478 	sc = xsc;
3479 
3480 	VTNET_CORE_LOCK(sc);
3481 	vtnet_init_locked(sc, 0);
3482 	VTNET_CORE_UNLOCK(sc);
3483 }
3484 
3485 static void
3486 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
3487 {
3488 
3489 	/*
3490 	 * The control virtqueue is only polled and therefore it should
3491 	 * already be empty.
3492 	 */
3493 	KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
3494 	    ("%s: ctrl vq %p not empty", __func__, sc->vtnet_ctrl_vq));
3495 }
3496 
3497 static void
3498 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
3499     struct sglist *sg, int readable, int writable)
3500 {
3501 	struct virtqueue *vq;
3502 
3503 	vq = sc->vtnet_ctrl_vq;
3504 
3505 	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ);
3506 	VTNET_CORE_LOCK_ASSERT(sc);
3507 
3508 	if (!virtqueue_empty(vq))
3509 		return;
3510 
3511 	/*
3512 	 * Poll for the response, but the command is likely completed before
3513 	 * returning from the notify.
3514 	 */
3515 	if (virtqueue_enqueue(vq, cookie, sg, readable, writable) == 0)  {
3516 		virtqueue_notify(vq);
3517 		virtqueue_poll(vq, NULL);
3518 	}
3519 }
3520 
3521 static int
3522 vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr)
3523 {
3524 	struct sglist_seg segs[3];
3525 	struct sglist sg;
3526 	struct {
3527 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3528 		uint8_t pad1;
3529 		uint8_t addr[ETHER_ADDR_LEN] __aligned(8);
3530 		uint8_t pad2;
3531 		uint8_t ack;
3532 	} s;
3533 	int error;
3534 
3535 	error = 0;
3536 	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_MAC);
3537 
3538 	s.hdr.class = VIRTIO_NET_CTRL_MAC;
3539 	s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
3540 	bcopy(hwaddr, &s.addr[0], ETHER_ADDR_LEN);
3541 	s.ack = VIRTIO_NET_ERR;
3542 
3543 	sglist_init(&sg, nitems(segs), segs);
3544 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3545 	error |= sglist_append(&sg, &s.addr[0], ETHER_ADDR_LEN);
3546 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3547 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3548 
3549 	if (error == 0)
3550 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3551 
3552 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3553 }
3554 
3555 static int
3556 vtnet_ctrl_guest_offloads(struct vtnet_softc *sc, uint64_t offloads)
3557 {
3558 	struct sglist_seg segs[3];
3559 	struct sglist sg;
3560 	struct {
3561 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3562 		uint8_t pad1;
3563 		uint64_t offloads __aligned(8);
3564 		uint8_t pad2;
3565 		uint8_t ack;
3566 	} s;
3567 	int error;
3568 
3569 	error = 0;
3570 	MPASS(sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS);
3571 
3572 	s.hdr.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
3573 	s.hdr.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
3574 	s.offloads = vtnet_gtoh64(sc, offloads);
3575 	s.ack = VIRTIO_NET_ERR;
3576 
3577 	sglist_init(&sg, nitems(segs), segs);
3578 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3579 	error |= sglist_append(&sg, &s.offloads, sizeof(uint64_t));
3580 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3581 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3582 
3583 	if (error == 0)
3584 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3585 
3586 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3587 }
3588 
3589 static int
3590 vtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs)
3591 {
3592 	struct sglist_seg segs[3];
3593 	struct sglist sg;
3594 	struct {
3595 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3596 		uint8_t pad1;
3597 		struct virtio_net_ctrl_mq mq __aligned(2);
3598 		uint8_t pad2;
3599 		uint8_t ack;
3600 	} s;
3601 	int error;
3602 
3603 	error = 0;
3604 	MPASS(sc->vtnet_flags & VTNET_FLAG_MQ);
3605 
3606 	s.hdr.class = VIRTIO_NET_CTRL_MQ;
3607 	s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
3608 	s.mq.virtqueue_pairs = vtnet_gtoh16(sc, npairs);
3609 	s.ack = VIRTIO_NET_ERR;
3610 
3611 	sglist_init(&sg, nitems(segs), segs);
3612 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3613 	error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq));
3614 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3615 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3616 
3617 	if (error == 0)
3618 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3619 
3620 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3621 }
3622 
3623 static int
3624 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, uint8_t cmd, bool on)
3625 {
3626 	struct sglist_seg segs[3];
3627 	struct sglist sg;
3628 	struct {
3629 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3630 		uint8_t pad1;
3631 		uint8_t onoff;
3632 		uint8_t pad2;
3633 		uint8_t ack;
3634 	} s;
3635 	int error;
3636 
3637 	error = 0;
3638 	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX);
3639 
3640 	s.hdr.class = VIRTIO_NET_CTRL_RX;
3641 	s.hdr.cmd = cmd;
3642 	s.onoff = on;
3643 	s.ack = VIRTIO_NET_ERR;
3644 
3645 	sglist_init(&sg, nitems(segs), segs);
3646 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3647 	error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t));
3648 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3649 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3650 
3651 	if (error == 0)
3652 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3653 
3654 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3655 }
3656 
3657 static int
3658 vtnet_set_promisc(struct vtnet_softc *sc, bool on)
3659 {
3660 	return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
3661 }
3662 
3663 static int
3664 vtnet_set_allmulti(struct vtnet_softc *sc, bool on)
3665 {
3666 	return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
3667 }
3668 
3669 static void
3670 vtnet_rx_filter(struct vtnet_softc *sc)
3671 {
3672 	device_t dev;
3673 	if_t ifp;
3674 
3675 	dev = sc->vtnet_dev;
3676 	ifp = sc->vtnet_ifp;
3677 
3678 	VTNET_CORE_LOCK_ASSERT(sc);
3679 
3680 	if (vtnet_set_promisc(sc, if_getflags(ifp) & IFF_PROMISC) != 0) {
3681 		device_printf(dev, "cannot %s promiscuous mode\n",
3682 		    if_getflags(ifp) & IFF_PROMISC ? "enable" : "disable");
3683 	}
3684 
3685 	if (vtnet_set_allmulti(sc, if_getflags(ifp) & IFF_ALLMULTI) != 0) {
3686 		device_printf(dev, "cannot %s all-multicast mode\n",
3687 		    if_getflags(ifp) & IFF_ALLMULTI ? "enable" : "disable");
3688 	}
3689 }
3690 
3691 static u_int
3692 vtnet_copy_ifaddr(void *arg, struct sockaddr_dl *sdl, u_int ucnt)
3693 {
3694 	struct vtnet_softc *sc = arg;
3695 
3696 	if (memcmp(LLADDR(sdl), sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0)
3697 		return (0);
3698 
3699 	if (ucnt < VTNET_MAX_MAC_ENTRIES)
3700 		bcopy(LLADDR(sdl),
3701 		    &sc->vtnet_mac_filter->vmf_unicast.macs[ucnt],
3702 		    ETHER_ADDR_LEN);
3703 
3704 	return (1);
3705 }
3706 
3707 static u_int
3708 vtnet_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int mcnt)
3709 {
3710 	struct vtnet_mac_filter *filter = arg;
3711 
3712 	if (mcnt < VTNET_MAX_MAC_ENTRIES)
3713 		bcopy(LLADDR(sdl), &filter->vmf_multicast.macs[mcnt],
3714 		    ETHER_ADDR_LEN);
3715 
3716 	return (1);
3717 }
3718 
3719 static void
3720 vtnet_rx_filter_mac(struct vtnet_softc *sc)
3721 {
3722 	struct virtio_net_ctrl_hdr hdr __aligned(2);
3723 	struct vtnet_mac_filter *filter;
3724 	struct sglist_seg segs[4];
3725 	struct sglist sg;
3726 	if_t ifp;
3727 	bool promisc, allmulti;
3728 	u_int ucnt, mcnt;
3729 	int error;
3730 	uint8_t ack;
3731 
3732 	ifp = sc->vtnet_ifp;
3733 	filter = sc->vtnet_mac_filter;
3734 	error = 0;
3735 
3736 	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX);
3737 	VTNET_CORE_LOCK_ASSERT(sc);
3738 
3739 	/* Unicast MAC addresses: */
3740 	ucnt = if_foreach_lladdr(ifp, vtnet_copy_ifaddr, sc);
3741 	promisc = (ucnt > VTNET_MAX_MAC_ENTRIES);
3742 
3743 	if (promisc) {
3744 		ucnt = 0;
3745 		if_printf(ifp, "more than %d MAC addresses assigned, "
3746 		    "falling back to promiscuous mode\n",
3747 		    VTNET_MAX_MAC_ENTRIES);
3748 	}
3749 
3750 	/* Multicast MAC addresses: */
3751 	mcnt = if_foreach_llmaddr(ifp, vtnet_copy_maddr, filter);
3752 	allmulti = (mcnt > VTNET_MAX_MAC_ENTRIES);
3753 
3754 	if (allmulti) {
3755 		mcnt = 0;
3756 		if_printf(ifp, "more than %d multicast MAC addresses "
3757 		    "assigned, falling back to all-multicast mode\n",
3758 		    VTNET_MAX_MAC_ENTRIES);
3759 	}
3760 
3761 	if (promisc && allmulti)
3762 		goto out;
3763 
3764 	filter->vmf_unicast.nentries = vtnet_gtoh32(sc, ucnt);
3765 	filter->vmf_multicast.nentries = vtnet_gtoh32(sc, mcnt);
3766 
3767 	hdr.class = VIRTIO_NET_CTRL_MAC;
3768 	hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
3769 	ack = VIRTIO_NET_ERR;
3770 
3771 	sglist_init(&sg, nitems(segs), segs);
3772 	error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
3773 	error |= sglist_append(&sg, &filter->vmf_unicast,
3774 	    sizeof(uint32_t) + ucnt * ETHER_ADDR_LEN);
3775 	error |= sglist_append(&sg, &filter->vmf_multicast,
3776 	    sizeof(uint32_t) + mcnt * ETHER_ADDR_LEN);
3777 	error |= sglist_append(&sg, &ack, sizeof(uint8_t));
3778 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3779 
3780 	if (error == 0)
3781 		vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
3782 	if (ack != VIRTIO_NET_OK)
3783 		if_printf(ifp, "error setting host MAC filter table\n");
3784 
3785 out:
3786 	if (promisc != 0 && vtnet_set_promisc(sc, true) != 0)
3787 		if_printf(ifp, "cannot enable promiscuous mode\n");
3788 	if (allmulti != 0 && vtnet_set_allmulti(sc, true) != 0)
3789 		if_printf(ifp, "cannot enable all-multicast mode\n");
3790 }
3791 
3792 static int
3793 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
3794 {
3795 	struct sglist_seg segs[3];
3796 	struct sglist sg;
3797 	struct {
3798 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3799 		uint8_t pad1;
3800 		uint16_t tag __aligned(2);
3801 		uint8_t pad2;
3802 		uint8_t ack;
3803 	} s;
3804 	int error;
3805 
3806 	error = 0;
3807 	MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER);
3808 
3809 	s.hdr.class = VIRTIO_NET_CTRL_VLAN;
3810 	s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
3811 	s.tag = vtnet_gtoh16(sc, tag);
3812 	s.ack = VIRTIO_NET_ERR;
3813 
3814 	sglist_init(&sg, nitems(segs), segs);
3815 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3816 	error |= sglist_append(&sg, &s.tag, sizeof(uint16_t));
3817 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3818 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3819 
3820 	if (error == 0)
3821 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3822 
3823 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3824 }
3825 
3826 static void
3827 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
3828 {
3829 	int i, bit;
3830 	uint32_t w;
3831 	uint16_t tag;
3832 
3833 	MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER);
3834 	VTNET_CORE_LOCK_ASSERT(sc);
3835 
3836 	/* Enable the filter for each configured VLAN. */
3837 	for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) {
3838 		w = sc->vtnet_vlan_filter[i];
3839 
3840 		while ((bit = ffs(w) - 1) != -1) {
3841 			w &= ~(1 << bit);
3842 			tag = sizeof(w) * CHAR_BIT * i + bit;
3843 
3844 			if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) {
3845 				device_printf(sc->vtnet_dev,
3846 				    "cannot enable VLAN %d filter\n", tag);
3847 			}
3848 		}
3849 	}
3850 }
3851 
3852 static void
3853 vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
3854 {
3855 	if_t ifp;
3856 	int idx, bit;
3857 
3858 	ifp = sc->vtnet_ifp;
3859 	idx = (tag >> 5) & 0x7F;
3860 	bit = tag & 0x1F;
3861 
3862 	if (tag == 0 || tag > 4095)
3863 		return;
3864 
3865 	VTNET_CORE_LOCK(sc);
3866 
3867 	if (add)
3868 		sc->vtnet_vlan_filter[idx] |= (1 << bit);
3869 	else
3870 		sc->vtnet_vlan_filter[idx] &= ~(1 << bit);
3871 
3872 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER &&
3873 	    if_getdrvflags(ifp) & IFF_DRV_RUNNING &&
3874 	    vtnet_exec_vlan_filter(sc, add, tag) != 0) {
3875 		device_printf(sc->vtnet_dev,
3876 		    "cannot %s VLAN %d %s the host filter table\n",
3877 		    add ? "add" : "remove", tag, add ? "to" : "from");
3878 	}
3879 
3880 	VTNET_CORE_UNLOCK(sc);
3881 }
3882 
3883 static void
3884 vtnet_register_vlan(void *arg, if_t ifp, uint16_t tag)
3885 {
3886 
3887 	if (if_getsoftc(ifp) != arg)
3888 		return;
3889 
3890 	vtnet_update_vlan_filter(arg, 1, tag);
3891 }
3892 
3893 static void
3894 vtnet_unregister_vlan(void *arg, if_t ifp, uint16_t tag)
3895 {
3896 
3897 	if (if_getsoftc(ifp) != arg)
3898 		return;
3899 
3900 	vtnet_update_vlan_filter(arg, 0, tag);
3901 }
3902 
3903 static void
3904 vtnet_update_speed_duplex(struct vtnet_softc *sc)
3905 {
3906 	if_t ifp;
3907 	uint32_t speed;
3908 
3909 	ifp = sc->vtnet_ifp;
3910 
3911 	if ((sc->vtnet_features & VIRTIO_NET_F_SPEED_DUPLEX) == 0)
3912 		return;
3913 
3914 	/* BMV: Ignore duplex. */
3915 	speed = virtio_read_dev_config_4(sc->vtnet_dev,
3916 	    offsetof(struct virtio_net_config, speed));
3917 	if (speed != UINT32_MAX)
3918 		if_setbaudrate(ifp, IF_Mbps(speed));
3919 }
3920 
3921 static int
3922 vtnet_is_link_up(struct vtnet_softc *sc)
3923 {
3924 	uint16_t status;
3925 
3926 	if ((sc->vtnet_features & VIRTIO_NET_F_STATUS) == 0)
3927 		return (1);
3928 
3929 	status = virtio_read_dev_config_2(sc->vtnet_dev,
3930 	    offsetof(struct virtio_net_config, status));
3931 
3932 	return ((status & VIRTIO_NET_S_LINK_UP) != 0);
3933 }
3934 
3935 static void
3936 vtnet_update_link_status(struct vtnet_softc *sc)
3937 {
3938 	if_t ifp;
3939 	int link;
3940 
3941 	ifp = sc->vtnet_ifp;
3942 	VTNET_CORE_LOCK_ASSERT(sc);
3943 	link = vtnet_is_link_up(sc);
3944 
3945 	/* Notify if the link status has changed. */
3946 	if (link != 0 && sc->vtnet_link_active == 0) {
3947 		vtnet_update_speed_duplex(sc);
3948 		sc->vtnet_link_active = 1;
3949 		if_link_state_change(ifp, LINK_STATE_UP);
3950 	} else if (link == 0 && sc->vtnet_link_active != 0) {
3951 		sc->vtnet_link_active = 0;
3952 		if_link_state_change(ifp, LINK_STATE_DOWN);
3953 	}
3954 }
3955 
3956 static int
3957 vtnet_ifmedia_upd(if_t ifp __unused)
3958 {
3959 	return (EOPNOTSUPP);
3960 }
3961 
3962 static void
3963 vtnet_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr)
3964 {
3965 	struct vtnet_softc *sc;
3966 
3967 	sc = if_getsoftc(ifp);
3968 
3969 	ifmr->ifm_status = IFM_AVALID;
3970 	ifmr->ifm_active = IFM_ETHER;
3971 
3972 	VTNET_CORE_LOCK(sc);
3973 	if (vtnet_is_link_up(sc) != 0) {
3974 		ifmr->ifm_status |= IFM_ACTIVE;
3975 		ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
3976 	} else
3977 		ifmr->ifm_active |= IFM_NONE;
3978 	VTNET_CORE_UNLOCK(sc);
3979 }
3980 
3981 static void
3982 vtnet_get_macaddr(struct vtnet_softc *sc)
3983 {
3984 
3985 	if (sc->vtnet_flags & VTNET_FLAG_MAC) {
3986 		virtio_read_device_config_array(sc->vtnet_dev,
3987 		    offsetof(struct virtio_net_config, mac),
3988 		    &sc->vtnet_hwaddr[0], sizeof(uint8_t), ETHER_ADDR_LEN);
3989 	} else {
3990 		/* Generate a random locally administered unicast address. */
3991 		sc->vtnet_hwaddr[0] = 0xB2;
3992 		arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0);
3993 	}
3994 }
3995 
3996 static void
3997 vtnet_set_macaddr(struct vtnet_softc *sc)
3998 {
3999 	device_t dev;
4000 	int error;
4001 
4002 	dev = sc->vtnet_dev;
4003 
4004 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) {
4005 		error = vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr);
4006 		if (error)
4007 			device_printf(dev, "unable to set MAC address\n");
4008 		return;
4009 	}
4010 
4011 	/* MAC in config is read-only in modern VirtIO. */
4012 	if (!vtnet_modern(sc) && sc->vtnet_flags & VTNET_FLAG_MAC) {
4013 		for (int i = 0; i < ETHER_ADDR_LEN; i++) {
4014 			virtio_write_dev_config_1(dev,
4015 			    offsetof(struct virtio_net_config, mac) + i,
4016 			    sc->vtnet_hwaddr[i]);
4017 		}
4018 	}
4019 }
4020 
4021 static void
4022 vtnet_attached_set_macaddr(struct vtnet_softc *sc)
4023 {
4024 
4025 	/* Assign MAC address if it was generated. */
4026 	if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0)
4027 		vtnet_set_macaddr(sc);
4028 }
4029 
4030 static void
4031 vtnet_vlan_tag_remove(struct mbuf *m)
4032 {
4033 	struct ether_vlan_header *evh;
4034 
4035 	evh = mtod(m, struct ether_vlan_header *);
4036 	m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
4037 	m->m_flags |= M_VLANTAG;
4038 
4039 	/* Strip the 802.1Q header. */
4040 	bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
4041 	    ETHER_HDR_LEN - ETHER_TYPE_LEN);
4042 	m_adj(m, ETHER_VLAN_ENCAP_LEN);
4043 }
4044 
4045 static void
4046 vtnet_set_rx_process_limit(struct vtnet_softc *sc)
4047 {
4048 	int limit;
4049 
4050 	limit = vtnet_tunable_int(sc, "rx_process_limit",
4051 	    vtnet_rx_process_limit);
4052 	if (limit < 0)
4053 		limit = INT_MAX;
4054 	sc->vtnet_rx_process_limit = limit;
4055 }
4056 
4057 static void
4058 vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx,
4059     struct sysctl_oid_list *child, struct vtnet_rxq *rxq)
4060 {
4061 	struct sysctl_oid *node;
4062 	struct sysctl_oid_list *list;
4063 	struct vtnet_rxq_stats *stats;
4064 	char namebuf[16];
4065 
4066 	snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id);
4067 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
4068 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Receive Queue");
4069 	list = SYSCTL_CHILDREN(node);
4070 
4071 	stats = &rxq->vtnrx_stats;
4072 
4073 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD,
4074 	    &stats->vrxs_ipackets, "Receive packets");
4075 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD,
4076 	    &stats->vrxs_ibytes, "Receive bytes");
4077 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD,
4078 	    &stats->vrxs_iqdrops, "Receive drops");
4079 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD,
4080 	    &stats->vrxs_ierrors, "Receive errors");
4081 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
4082 	    &stats->vrxs_csum, "Receive checksum offloaded");
4083 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD,
4084 	    &stats->vrxs_csum_failed, "Receive checksum offload failed");
4085 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "host_lro", CTLFLAG_RD,
4086 	    &stats->vrxs_host_lro, "Receive host segmentation offloaded");
4087 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD,
4088 	    &stats->vrxs_rescheduled,
4089 	    "Receive interrupt handler rescheduled");
4090 }
4091 
4092 static void
4093 vtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx,
4094     struct sysctl_oid_list *child, struct vtnet_txq *txq)
4095 {
4096 	struct sysctl_oid *node;
4097 	struct sysctl_oid_list *list;
4098 	struct vtnet_txq_stats *stats;
4099 	char namebuf[16];
4100 
4101 	snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id);
4102 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
4103 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Transmit Queue");
4104 	list = SYSCTL_CHILDREN(node);
4105 
4106 	stats = &txq->vtntx_stats;
4107 
4108 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD,
4109 	    &stats->vtxs_opackets, "Transmit packets");
4110 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD,
4111 	    &stats->vtxs_obytes, "Transmit bytes");
4112 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD,
4113 	    &stats->vtxs_omcasts, "Transmit multicasts");
4114 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
4115 	    &stats->vtxs_csum, "Transmit checksum offloaded");
4116 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
4117 	    &stats->vtxs_tso, "Transmit TCP segmentation offloaded");
4118 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD,
4119 	    &stats->vtxs_rescheduled,
4120 	    "Transmit interrupt handler rescheduled");
4121 }
4122 
4123 static void
4124 vtnet_setup_queue_sysctl(struct vtnet_softc *sc)
4125 {
4126 	device_t dev;
4127 	struct sysctl_ctx_list *ctx;
4128 	struct sysctl_oid *tree;
4129 	struct sysctl_oid_list *child;
4130 	int i;
4131 
4132 	dev = sc->vtnet_dev;
4133 	ctx = device_get_sysctl_ctx(dev);
4134 	tree = device_get_sysctl_tree(dev);
4135 	child = SYSCTL_CHILDREN(tree);
4136 
4137 	for (i = 0; i < sc->vtnet_req_vq_pairs; i++) {
4138 		vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]);
4139 		vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]);
4140 	}
4141 }
4142 
4143 static void
4144 vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx,
4145     struct sysctl_oid_list *child, struct vtnet_softc *sc)
4146 {
4147 	struct vtnet_statistics *stats;
4148 	struct vtnet_rxq_stats rxaccum;
4149 	struct vtnet_txq_stats txaccum;
4150 
4151 	vtnet_accum_stats(sc, &rxaccum, &txaccum);
4152 
4153 	stats = &sc->vtnet_stats;
4154 	stats->rx_csum_offloaded = rxaccum.vrxs_csum;
4155 	stats->rx_csum_failed = rxaccum.vrxs_csum_failed;
4156 	stats->rx_task_rescheduled = rxaccum.vrxs_rescheduled;
4157 	stats->tx_csum_offloaded = txaccum.vtxs_csum;
4158 	stats->tx_tso_offloaded = txaccum.vtxs_tso;
4159 	stats->tx_task_rescheduled = txaccum.vtxs_rescheduled;
4160 
4161 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed",
4162 	    CTLFLAG_RD, &stats->mbuf_alloc_failed,
4163 	    "Mbuf cluster allocation failures");
4164 
4165 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large",
4166 	    CTLFLAG_RD, &stats->rx_frame_too_large,
4167 	    "Received frame larger than the mbuf chain");
4168 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
4169 	    CTLFLAG_RD, &stats->rx_enq_replacement_failed,
4170 	    "Enqueuing the replacement receive mbuf failed");
4171 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed",
4172 	    CTLFLAG_RD, &stats->rx_mergeable_failed,
4173 	    "Mergeable buffers receive failures");
4174 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
4175 	    CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
4176 	    "Received checksum offloaded buffer with unsupported "
4177 	    "Ethernet type");
4178 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
4179 	    CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
4180 	    "Received checksum offloaded buffer with incorrect IP protocol");
4181 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset",
4182 	    CTLFLAG_RD, &stats->rx_csum_bad_offset,
4183 	    "Received checksum offloaded buffer with incorrect offset");
4184 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_proto",
4185 	    CTLFLAG_RD, &stats->rx_csum_bad_proto,
4186 	    "Received checksum offloaded buffer with incorrect protocol");
4187 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed",
4188 	    CTLFLAG_RD, &stats->rx_csum_failed,
4189 	    "Received buffer checksum offload failed");
4190 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded",
4191 	    CTLFLAG_RD, &stats->rx_csum_offloaded,
4192 	    "Received buffer checksum offload succeeded");
4193 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled",
4194 	    CTLFLAG_RD, &stats->rx_task_rescheduled,
4195 	    "Times the receive interrupt task rescheduled itself");
4196 
4197 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_unknown_ethtype",
4198 	    CTLFLAG_RD, &stats->tx_csum_unknown_ethtype,
4199 	    "Aborted transmit of checksum offloaded buffer with unknown "
4200 	    "Ethernet type");
4201 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_proto_mismatch",
4202 	    CTLFLAG_RD, &stats->tx_csum_proto_mismatch,
4203 	    "Aborted transmit of checksum offloaded buffer because mismatched "
4204 	    "protocols");
4205 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp",
4206 	    CTLFLAG_RD, &stats->tx_tso_not_tcp,
4207 	    "Aborted transmit of TSO buffer with non TCP protocol");
4208 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_without_csum",
4209 	    CTLFLAG_RD, &stats->tx_tso_without_csum,
4210 	    "Aborted transmit of TSO buffer without TCP checksum offload");
4211 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged",
4212 	    CTLFLAG_RD, &stats->tx_defragged,
4213 	    "Transmit mbufs defragged");
4214 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed",
4215 	    CTLFLAG_RD, &stats->tx_defrag_failed,
4216 	    "Aborted transmit of buffer because defrag failed");
4217 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded",
4218 	    CTLFLAG_RD, &stats->tx_csum_offloaded,
4219 	    "Offloaded checksum of transmitted buffer");
4220 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded",
4221 	    CTLFLAG_RD, &stats->tx_tso_offloaded,
4222 	    "Segmentation offload of transmitted buffer");
4223 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled",
4224 	    CTLFLAG_RD, &stats->tx_task_rescheduled,
4225 	    "Times the transmit interrupt task rescheduled itself");
4226 }
4227 
4228 static void
4229 vtnet_setup_sysctl(struct vtnet_softc *sc)
4230 {
4231 	device_t dev;
4232 	struct sysctl_ctx_list *ctx;
4233 	struct sysctl_oid *tree;
4234 	struct sysctl_oid_list *child;
4235 
4236 	dev = sc->vtnet_dev;
4237 	ctx = device_get_sysctl_ctx(dev);
4238 	tree = device_get_sysctl_tree(dev);
4239 	child = SYSCTL_CHILDREN(tree);
4240 
4241 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs",
4242 	    CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0,
4243 	    "Number of maximum supported virtqueue pairs");
4244 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "req_vq_pairs",
4245 	    CTLFLAG_RD, &sc->vtnet_req_vq_pairs, 0,
4246 	    "Number of requested virtqueue pairs");
4247 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs",
4248 	    CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0,
4249 	    "Number of active virtqueue pairs");
4250 
4251 	vtnet_setup_stat_sysctl(ctx, child, sc);
4252 }
4253 
4254 static void
4255 vtnet_load_tunables(struct vtnet_softc *sc)
4256 {
4257 
4258 	sc->vtnet_lro_entry_count = vtnet_tunable_int(sc,
4259 	    "lro_entry_count", vtnet_lro_entry_count);
4260 	if (sc->vtnet_lro_entry_count < TCP_LRO_ENTRIES)
4261 		sc->vtnet_lro_entry_count = TCP_LRO_ENTRIES;
4262 
4263 	sc->vtnet_lro_mbufq_depth = vtnet_tunable_int(sc,
4264 	    "lro_mbufq_depth", vtnet_lro_mbufq_depth);
4265 }
4266 
4267 static int
4268 vtnet_rxq_enable_intr(struct vtnet_rxq *rxq)
4269 {
4270 
4271 	return (virtqueue_enable_intr(rxq->vtnrx_vq));
4272 }
4273 
4274 static void
4275 vtnet_rxq_disable_intr(struct vtnet_rxq *rxq)
4276 {
4277 
4278 	virtqueue_disable_intr(rxq->vtnrx_vq);
4279 }
4280 
4281 static int
4282 vtnet_txq_enable_intr(struct vtnet_txq *txq)
4283 {
4284 	struct virtqueue *vq;
4285 
4286 	vq = txq->vtntx_vq;
4287 
4288 	if (vtnet_txq_below_threshold(txq) != 0)
4289 		return (virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG));
4290 
4291 	/*
4292 	 * The free count is above our threshold. Keep the Tx interrupt
4293 	 * disabled until the queue is fuller.
4294 	 */
4295 	return (0);
4296 }
4297 
4298 static void
4299 vtnet_txq_disable_intr(struct vtnet_txq *txq)
4300 {
4301 
4302 	virtqueue_disable_intr(txq->vtntx_vq);
4303 }
4304 
4305 static void
4306 vtnet_enable_rx_interrupts(struct vtnet_softc *sc)
4307 {
4308 	struct vtnet_rxq *rxq;
4309 	int i;
4310 
4311 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
4312 		rxq = &sc->vtnet_rxqs[i];
4313 		if (vtnet_rxq_enable_intr(rxq) != 0)
4314 			taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
4315 	}
4316 }
4317 
4318 static void
4319 vtnet_enable_tx_interrupts(struct vtnet_softc *sc)
4320 {
4321 	int i;
4322 
4323 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
4324 		vtnet_txq_enable_intr(&sc->vtnet_txqs[i]);
4325 }
4326 
4327 static void
4328 vtnet_enable_interrupts(struct vtnet_softc *sc)
4329 {
4330 
4331 	vtnet_enable_rx_interrupts(sc);
4332 	vtnet_enable_tx_interrupts(sc);
4333 }
4334 
4335 static void
4336 vtnet_disable_rx_interrupts(struct vtnet_softc *sc)
4337 {
4338 	int i;
4339 
4340 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
4341 		vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]);
4342 }
4343 
4344 static void
4345 vtnet_disable_tx_interrupts(struct vtnet_softc *sc)
4346 {
4347 	int i;
4348 
4349 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
4350 		vtnet_txq_disable_intr(&sc->vtnet_txqs[i]);
4351 }
4352 
4353 static void
4354 vtnet_disable_interrupts(struct vtnet_softc *sc)
4355 {
4356 
4357 	vtnet_disable_rx_interrupts(sc);
4358 	vtnet_disable_tx_interrupts(sc);
4359 }
4360 
4361 static int
4362 vtnet_tunable_int(struct vtnet_softc *sc, const char *knob, int def)
4363 {
4364 	char path[64];
4365 
4366 	snprintf(path, sizeof(path),
4367 	    "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob);
4368 	TUNABLE_INT_FETCH(path, &def);
4369 
4370 	return (def);
4371 }
4372 
4373 #ifdef DEBUGNET
4374 static void
4375 vtnet_debugnet_init(if_t ifp, int *nrxr, int *ncl, int *clsize)
4376 {
4377 	struct vtnet_softc *sc;
4378 
4379 	sc = if_getsoftc(ifp);
4380 
4381 	VTNET_CORE_LOCK(sc);
4382 	*nrxr = sc->vtnet_req_vq_pairs;
4383 	*ncl = DEBUGNET_MAX_IN_FLIGHT;
4384 	*clsize = sc->vtnet_rx_clustersz;
4385 	VTNET_CORE_UNLOCK(sc);
4386 }
4387 
4388 static void
4389 vtnet_debugnet_event(if_t ifp __unused, enum debugnet_ev event)
4390 {
4391 	struct vtnet_softc *sc;
4392 	static bool sw_lro_enabled = false;
4393 
4394 	/*
4395 	 * Disable software LRO, since it would require entering the network
4396 	 * epoch when calling vtnet_txq_eof() in vtnet_debugnet_poll().
4397 	 */
4398 	sc = if_getsoftc(ifp);
4399 	switch (event) {
4400 	case DEBUGNET_START:
4401 		sw_lro_enabled = (sc->vtnet_flags & VTNET_FLAG_SW_LRO) != 0;
4402 		if (sw_lro_enabled)
4403 			sc->vtnet_flags &= ~VTNET_FLAG_SW_LRO;
4404 		break;
4405 	case DEBUGNET_END:
4406 		if (sw_lro_enabled)
4407 			sc->vtnet_flags |= VTNET_FLAG_SW_LRO;
4408 		break;
4409 	}
4410 }
4411 
4412 static int
4413 vtnet_debugnet_transmit(if_t ifp, struct mbuf *m)
4414 {
4415 	struct vtnet_softc *sc;
4416 	struct vtnet_txq *txq;
4417 	int error;
4418 
4419 	sc = if_getsoftc(ifp);
4420 	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
4421 	    IFF_DRV_RUNNING)
4422 		return (EBUSY);
4423 
4424 	txq = &sc->vtnet_txqs[0];
4425 	error = vtnet_txq_encap(txq, &m, M_NOWAIT | M_USE_RESERVE);
4426 	if (error == 0)
4427 		(void)vtnet_txq_notify(txq);
4428 	return (error);
4429 }
4430 
4431 static int
4432 vtnet_debugnet_poll(if_t ifp, int count)
4433 {
4434 	struct vtnet_softc *sc;
4435 	int i;
4436 
4437 	sc = if_getsoftc(ifp);
4438 	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
4439 	    IFF_DRV_RUNNING)
4440 		return (EBUSY);
4441 
4442 	(void)vtnet_txq_eof(&sc->vtnet_txqs[0]);
4443 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
4444 		(void)vtnet_rxq_eof(&sc->vtnet_rxqs[i]);
4445 	return (0);
4446 }
4447 #endif /* DEBUGNET */
4448