xref: /freebsd/sys/dev/virtio/network/if_vtnet.c (revision e92ffd9b626833ebdbf2742c8ffddc6cd94b963e)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 /* Driver for VirtIO network devices. */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/eventhandler.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/sockio.h>
39 #include <sys/malloc.h>
40 #include <sys/mbuf.h>
41 #include <sys/module.h>
42 #include <sys/msan.h>
43 #include <sys/socket.h>
44 #include <sys/sysctl.h>
45 #include <sys/random.h>
46 #include <sys/sglist.h>
47 #include <sys/lock.h>
48 #include <sys/mutex.h>
49 #include <sys/taskqueue.h>
50 #include <sys/smp.h>
51 #include <machine/smp.h>
52 
53 #include <vm/uma.h>
54 
55 #include <net/debugnet.h>
56 #include <net/ethernet.h>
57 #include <net/pfil.h>
58 #include <net/if.h>
59 #include <net/if_var.h>
60 #include <net/if_arp.h>
61 #include <net/if_dl.h>
62 #include <net/if_types.h>
63 #include <net/if_media.h>
64 #include <net/if_vlan_var.h>
65 
66 #include <net/bpf.h>
67 
68 #include <netinet/in_systm.h>
69 #include <netinet/in.h>
70 #include <netinet/ip.h>
71 #include <netinet/ip6.h>
72 #include <netinet6/ip6_var.h>
73 #include <netinet/udp.h>
74 #include <netinet/tcp.h>
75 #include <netinet/tcp_lro.h>
76 
77 #include <machine/bus.h>
78 #include <machine/resource.h>
79 #include <sys/bus.h>
80 #include <sys/rman.h>
81 
82 #include <dev/virtio/virtio.h>
83 #include <dev/virtio/virtqueue.h>
84 #include <dev/virtio/network/virtio_net.h>
85 #include <dev/virtio/network/if_vtnetvar.h>
86 #include "virtio_if.h"
87 
88 #include "opt_inet.h"
89 #include "opt_inet6.h"
90 
91 #if defined(INET) || defined(INET6)
92 #include <machine/in_cksum.h>
93 #endif
94 
95 static int	vtnet_modevent(module_t, int, void *);
96 
97 static int	vtnet_probe(device_t);
98 static int	vtnet_attach(device_t);
99 static int	vtnet_detach(device_t);
100 static int	vtnet_suspend(device_t);
101 static int	vtnet_resume(device_t);
102 static int	vtnet_shutdown(device_t);
103 static int	vtnet_attach_completed(device_t);
104 static int	vtnet_config_change(device_t);
105 
106 static int	vtnet_negotiate_features(struct vtnet_softc *);
107 static int	vtnet_setup_features(struct vtnet_softc *);
108 static int	vtnet_init_rxq(struct vtnet_softc *, int);
109 static int	vtnet_init_txq(struct vtnet_softc *, int);
110 static int	vtnet_alloc_rxtx_queues(struct vtnet_softc *);
111 static void	vtnet_free_rxtx_queues(struct vtnet_softc *);
112 static int	vtnet_alloc_rx_filters(struct vtnet_softc *);
113 static void	vtnet_free_rx_filters(struct vtnet_softc *);
114 static int	vtnet_alloc_virtqueues(struct vtnet_softc *);
115 static int	vtnet_alloc_interface(struct vtnet_softc *);
116 static int	vtnet_setup_interface(struct vtnet_softc *);
117 static int	vtnet_ioctl_mtu(struct vtnet_softc *, u_int);
118 static int	vtnet_ioctl_ifflags(struct vtnet_softc *);
119 static int	vtnet_ioctl_multi(struct vtnet_softc *);
120 static int	vtnet_ioctl_ifcap(struct vtnet_softc *, struct ifreq *);
121 static int	vtnet_ioctl(struct ifnet *, u_long, caddr_t);
122 static uint64_t	vtnet_get_counter(struct ifnet *, ift_counter);
123 
124 static int	vtnet_rxq_populate(struct vtnet_rxq *);
125 static void	vtnet_rxq_free_mbufs(struct vtnet_rxq *);
126 static struct mbuf *
127 		vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **);
128 static int	vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *,
129 		    struct mbuf *, int);
130 static int	vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int);
131 static int	vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *);
132 static int	vtnet_rxq_new_buf(struct vtnet_rxq *);
133 static int	vtnet_rxq_csum_needs_csum(struct vtnet_rxq *, struct mbuf *,
134 		     uint16_t, int, struct virtio_net_hdr *);
135 static int	vtnet_rxq_csum_data_valid(struct vtnet_rxq *, struct mbuf *,
136 		     uint16_t, int, struct virtio_net_hdr *);
137 static int	vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *,
138 		     struct virtio_net_hdr *);
139 static void	vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int);
140 static void	vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *);
141 static int	vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int);
142 static void	vtnet_rxq_input(struct vtnet_rxq *, struct mbuf *,
143 		    struct virtio_net_hdr *);
144 static int	vtnet_rxq_eof(struct vtnet_rxq *);
145 static void	vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries);
146 static void	vtnet_rx_vq_intr(void *);
147 static void	vtnet_rxq_tq_intr(void *, int);
148 
149 static int	vtnet_txq_intr_threshold(struct vtnet_txq *);
150 static int	vtnet_txq_below_threshold(struct vtnet_txq *);
151 static int	vtnet_txq_notify(struct vtnet_txq *);
152 static void	vtnet_txq_free_mbufs(struct vtnet_txq *);
153 static int	vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *,
154 		    int *, int *, int *);
155 static int	vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int,
156 		    int, struct virtio_net_hdr *);
157 static struct mbuf *
158 		vtnet_txq_offload(struct vtnet_txq *, struct mbuf *,
159 		    struct virtio_net_hdr *);
160 static int	vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **,
161 		    struct vtnet_tx_header *);
162 static int	vtnet_txq_encap(struct vtnet_txq *, struct mbuf **, int);
163 #ifdef VTNET_LEGACY_TX
164 static void	vtnet_start_locked(struct vtnet_txq *, struct ifnet *);
165 static void	vtnet_start(struct ifnet *);
166 #else
167 static int	vtnet_txq_mq_start_locked(struct vtnet_txq *, struct mbuf *);
168 static int	vtnet_txq_mq_start(struct ifnet *, struct mbuf *);
169 static void	vtnet_txq_tq_deferred(void *, int);
170 #endif
171 static void	vtnet_txq_start(struct vtnet_txq *);
172 static void	vtnet_txq_tq_intr(void *, int);
173 static int	vtnet_txq_eof(struct vtnet_txq *);
174 static void	vtnet_tx_vq_intr(void *);
175 static void	vtnet_tx_start_all(struct vtnet_softc *);
176 
177 #ifndef VTNET_LEGACY_TX
178 static void	vtnet_qflush(struct ifnet *);
179 #endif
180 
181 static int	vtnet_watchdog(struct vtnet_txq *);
182 static void	vtnet_accum_stats(struct vtnet_softc *,
183 		    struct vtnet_rxq_stats *, struct vtnet_txq_stats *);
184 static void	vtnet_tick(void *);
185 
186 static void	vtnet_start_taskqueues(struct vtnet_softc *);
187 static void	vtnet_free_taskqueues(struct vtnet_softc *);
188 static void	vtnet_drain_taskqueues(struct vtnet_softc *);
189 
190 static void	vtnet_drain_rxtx_queues(struct vtnet_softc *);
191 static void	vtnet_stop_rendezvous(struct vtnet_softc *);
192 static void	vtnet_stop(struct vtnet_softc *);
193 static int	vtnet_virtio_reinit(struct vtnet_softc *);
194 static void	vtnet_init_rx_filters(struct vtnet_softc *);
195 static int	vtnet_init_rx_queues(struct vtnet_softc *);
196 static int	vtnet_init_tx_queues(struct vtnet_softc *);
197 static int	vtnet_init_rxtx_queues(struct vtnet_softc *);
198 static void	vtnet_set_active_vq_pairs(struct vtnet_softc *);
199 static void	vtnet_update_rx_offloads(struct vtnet_softc *);
200 static int	vtnet_reinit(struct vtnet_softc *);
201 static void	vtnet_init_locked(struct vtnet_softc *, int);
202 static void	vtnet_init(void *);
203 
204 static void	vtnet_free_ctrl_vq(struct vtnet_softc *);
205 static void	vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
206 		    struct sglist *, int, int);
207 static int	vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *);
208 static int	vtnet_ctrl_guest_offloads(struct vtnet_softc *, uint64_t);
209 static int	vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t);
210 static int	vtnet_ctrl_rx_cmd(struct vtnet_softc *, uint8_t, bool);
211 static int	vtnet_set_promisc(struct vtnet_softc *, bool);
212 static int	vtnet_set_allmulti(struct vtnet_softc *, bool);
213 static void	vtnet_rx_filter(struct vtnet_softc *);
214 static void	vtnet_rx_filter_mac(struct vtnet_softc *);
215 static int	vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
216 static void	vtnet_rx_filter_vlan(struct vtnet_softc *);
217 static void	vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t);
218 static void	vtnet_register_vlan(void *, struct ifnet *, uint16_t);
219 static void	vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
220 
221 static void	vtnet_update_speed_duplex(struct vtnet_softc *);
222 static int	vtnet_is_link_up(struct vtnet_softc *);
223 static void	vtnet_update_link_status(struct vtnet_softc *);
224 static int	vtnet_ifmedia_upd(struct ifnet *);
225 static void	vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
226 static void	vtnet_get_macaddr(struct vtnet_softc *);
227 static void	vtnet_set_macaddr(struct vtnet_softc *);
228 static void	vtnet_attached_set_macaddr(struct vtnet_softc *);
229 static void	vtnet_vlan_tag_remove(struct mbuf *);
230 static void	vtnet_set_rx_process_limit(struct vtnet_softc *);
231 
232 static void	vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *,
233 		    struct sysctl_oid_list *, struct vtnet_rxq *);
234 static void	vtnet_setup_txq_sysctl(struct sysctl_ctx_list *,
235 		    struct sysctl_oid_list *, struct vtnet_txq *);
236 static void	vtnet_setup_queue_sysctl(struct vtnet_softc *);
237 static void	vtnet_load_tunables(struct vtnet_softc *);
238 static void	vtnet_setup_sysctl(struct vtnet_softc *);
239 
240 static int	vtnet_rxq_enable_intr(struct vtnet_rxq *);
241 static void	vtnet_rxq_disable_intr(struct vtnet_rxq *);
242 static int	vtnet_txq_enable_intr(struct vtnet_txq *);
243 static void	vtnet_txq_disable_intr(struct vtnet_txq *);
244 static void	vtnet_enable_rx_interrupts(struct vtnet_softc *);
245 static void	vtnet_enable_tx_interrupts(struct vtnet_softc *);
246 static void	vtnet_enable_interrupts(struct vtnet_softc *);
247 static void	vtnet_disable_rx_interrupts(struct vtnet_softc *);
248 static void	vtnet_disable_tx_interrupts(struct vtnet_softc *);
249 static void	vtnet_disable_interrupts(struct vtnet_softc *);
250 
251 static int	vtnet_tunable_int(struct vtnet_softc *, const char *, int);
252 
253 DEBUGNET_DEFINE(vtnet);
254 
255 #define vtnet_htog16(_sc, _val)	virtio_htog16(vtnet_modern(_sc), _val)
256 #define vtnet_htog32(_sc, _val)	virtio_htog32(vtnet_modern(_sc), _val)
257 #define vtnet_htog64(_sc, _val)	virtio_htog64(vtnet_modern(_sc), _val)
258 #define vtnet_gtoh16(_sc, _val)	virtio_gtoh16(vtnet_modern(_sc), _val)
259 #define vtnet_gtoh32(_sc, _val)	virtio_gtoh32(vtnet_modern(_sc), _val)
260 #define vtnet_gtoh64(_sc, _val)	virtio_gtoh64(vtnet_modern(_sc), _val)
261 
262 /* Tunables. */
263 static SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
264     "VirtIO Net driver parameters");
265 
266 static int vtnet_csum_disable = 0;
267 SYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN,
268     &vtnet_csum_disable, 0, "Disables receive and send checksum offload");
269 
270 static int vtnet_fixup_needs_csum = 0;
271 SYSCTL_INT(_hw_vtnet, OID_AUTO, fixup_needs_csum, CTLFLAG_RDTUN,
272     &vtnet_fixup_needs_csum, 0,
273     "Calculate valid checksum for NEEDS_CSUM packets");
274 
275 static int vtnet_tso_disable = 0;
276 SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN,
277     &vtnet_tso_disable, 0, "Disables TSO");
278 
279 static int vtnet_lro_disable = 0;
280 SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_disable, CTLFLAG_RDTUN,
281     &vtnet_lro_disable, 0, "Disables hardware LRO");
282 
283 static int vtnet_mq_disable = 0;
284 SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_disable, CTLFLAG_RDTUN,
285     &vtnet_mq_disable, 0, "Disables multiqueue support");
286 
287 static int vtnet_mq_max_pairs = VTNET_MAX_QUEUE_PAIRS;
288 SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_max_pairs, CTLFLAG_RDTUN,
289     &vtnet_mq_max_pairs, 0, "Maximum number of multiqueue pairs");
290 
291 static int vtnet_tso_maxlen = IP_MAXPACKET;
292 SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN,
293     &vtnet_tso_maxlen, 0, "TSO burst limit");
294 
295 static int vtnet_rx_process_limit = 1024;
296 SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
297     &vtnet_rx_process_limit, 0,
298     "Number of RX segments processed in one pass");
299 
300 static int vtnet_lro_entry_count = 128;
301 SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN,
302     &vtnet_lro_entry_count, 0, "Software LRO entry count");
303 
304 /* Enable sorted LRO, and the depth of the mbuf queue. */
305 static int vtnet_lro_mbufq_depth = 0;
306 SYSCTL_UINT(_hw_vtnet, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN,
307     &vtnet_lro_mbufq_depth, 0, "Depth of software LRO mbuf queue");
308 
309 static uma_zone_t vtnet_tx_header_zone;
310 
311 static struct virtio_feature_desc vtnet_feature_desc[] = {
312 	{ VIRTIO_NET_F_CSUM,			"TxChecksum"		},
313 	{ VIRTIO_NET_F_GUEST_CSUM,		"RxChecksum"		},
314 	{ VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,	"CtrlRxOffloads"	},
315 	{ VIRTIO_NET_F_MAC,			"MAC"			},
316 	{ VIRTIO_NET_F_GSO,			"TxGSO"			},
317 	{ VIRTIO_NET_F_GUEST_TSO4,		"RxLROv4"		},
318 	{ VIRTIO_NET_F_GUEST_TSO6,		"RxLROv6"		},
319 	{ VIRTIO_NET_F_GUEST_ECN,		"RxLROECN"		},
320 	{ VIRTIO_NET_F_GUEST_UFO,		"RxUFO"			},
321 	{ VIRTIO_NET_F_HOST_TSO4,		"TxTSOv4"		},
322 	{ VIRTIO_NET_F_HOST_TSO6,		"TxTSOv6"		},
323 	{ VIRTIO_NET_F_HOST_ECN,		"TxTSOECN"		},
324 	{ VIRTIO_NET_F_HOST_UFO,		"TxUFO"			},
325 	{ VIRTIO_NET_F_MRG_RXBUF,		"MrgRxBuf"		},
326 	{ VIRTIO_NET_F_STATUS,			"Status"		},
327 	{ VIRTIO_NET_F_CTRL_VQ,			"CtrlVq"		},
328 	{ VIRTIO_NET_F_CTRL_RX,			"CtrlRxMode"		},
329 	{ VIRTIO_NET_F_CTRL_VLAN,		"CtrlVLANFilter"	},
330 	{ VIRTIO_NET_F_CTRL_RX_EXTRA,		"CtrlRxModeExtra"	},
331 	{ VIRTIO_NET_F_GUEST_ANNOUNCE,		"GuestAnnounce"		},
332 	{ VIRTIO_NET_F_MQ,			"Multiqueue"		},
333 	{ VIRTIO_NET_F_CTRL_MAC_ADDR,		"CtrlMacAddr"		},
334 	{ VIRTIO_NET_F_SPEED_DUPLEX,		"SpeedDuplex"		},
335 
336 	{ 0, NULL }
337 };
338 
339 static device_method_t vtnet_methods[] = {
340 	/* Device methods. */
341 	DEVMETHOD(device_probe,			vtnet_probe),
342 	DEVMETHOD(device_attach,		vtnet_attach),
343 	DEVMETHOD(device_detach,		vtnet_detach),
344 	DEVMETHOD(device_suspend,		vtnet_suspend),
345 	DEVMETHOD(device_resume,		vtnet_resume),
346 	DEVMETHOD(device_shutdown,		vtnet_shutdown),
347 
348 	/* VirtIO methods. */
349 	DEVMETHOD(virtio_attach_completed,	vtnet_attach_completed),
350 	DEVMETHOD(virtio_config_change,		vtnet_config_change),
351 
352 	DEVMETHOD_END
353 };
354 
355 #ifdef DEV_NETMAP
356 #include <dev/netmap/if_vtnet_netmap.h>
357 #endif
358 
359 static driver_t vtnet_driver = {
360     .name = "vtnet",
361     .methods = vtnet_methods,
362     .size = sizeof(struct vtnet_softc)
363 };
364 static devclass_t vtnet_devclass;
365 
366 VIRTIO_DRIVER_MODULE(vtnet, vtnet_driver, vtnet_devclass,
367     vtnet_modevent, 0);
368 MODULE_VERSION(vtnet, 1);
369 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
370 #ifdef DEV_NETMAP
371 MODULE_DEPEND(vtnet, netmap, 1, 1, 1);
372 #endif
373 
374 VIRTIO_SIMPLE_PNPINFO(vtnet, VIRTIO_ID_NETWORK, "VirtIO Networking Adapter");
375 
376 static int
377 vtnet_modevent(module_t mod __unused, int type, void *unused __unused)
378 {
379 	int error = 0;
380 	static int loaded = 0;
381 
382 	switch (type) {
383 	case MOD_LOAD:
384 		if (loaded++ == 0) {
385 			vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr",
386 				sizeof(struct vtnet_tx_header),
387 				NULL, NULL, NULL, NULL, 0, 0);
388 #ifdef DEBUGNET
389 			/*
390 			 * We need to allocate from this zone in the transmit path, so ensure
391 			 * that we have at least one item per header available.
392 			 * XXX add a separate zone like we do for mbufs? otherwise we may alloc
393 			 * buckets
394 			 */
395 			uma_zone_reserve(vtnet_tx_header_zone, DEBUGNET_MAX_IN_FLIGHT * 2);
396 			uma_prealloc(vtnet_tx_header_zone, DEBUGNET_MAX_IN_FLIGHT * 2);
397 #endif
398 		}
399 		break;
400 	case MOD_QUIESCE:
401 		if (uma_zone_get_cur(vtnet_tx_header_zone) > 0)
402 			error = EBUSY;
403 		break;
404 	case MOD_UNLOAD:
405 		if (--loaded == 0) {
406 			uma_zdestroy(vtnet_tx_header_zone);
407 			vtnet_tx_header_zone = NULL;
408 		}
409 		break;
410 	case MOD_SHUTDOWN:
411 		break;
412 	default:
413 		error = EOPNOTSUPP;
414 		break;
415 	}
416 
417 	return (error);
418 }
419 
420 static int
421 vtnet_probe(device_t dev)
422 {
423 	return (VIRTIO_SIMPLE_PROBE(dev, vtnet));
424 }
425 
426 static int
427 vtnet_attach(device_t dev)
428 {
429 	struct vtnet_softc *sc;
430 	int error;
431 
432 	sc = device_get_softc(dev);
433 	sc->vtnet_dev = dev;
434 	virtio_set_feature_desc(dev, vtnet_feature_desc);
435 
436 	VTNET_CORE_LOCK_INIT(sc);
437 	callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0);
438 	vtnet_load_tunables(sc);
439 
440 	error = vtnet_alloc_interface(sc);
441 	if (error) {
442 		device_printf(dev, "cannot allocate interface\n");
443 		goto fail;
444 	}
445 
446 	vtnet_setup_sysctl(sc);
447 
448 	error = vtnet_setup_features(sc);
449 	if (error) {
450 		device_printf(dev, "cannot setup features\n");
451 		goto fail;
452 	}
453 
454 	error = vtnet_alloc_rx_filters(sc);
455 	if (error) {
456 		device_printf(dev, "cannot allocate Rx filters\n");
457 		goto fail;
458 	}
459 
460 	error = vtnet_alloc_rxtx_queues(sc);
461 	if (error) {
462 		device_printf(dev, "cannot allocate queues\n");
463 		goto fail;
464 	}
465 
466 	error = vtnet_alloc_virtqueues(sc);
467 	if (error) {
468 		device_printf(dev, "cannot allocate virtqueues\n");
469 		goto fail;
470 	}
471 
472 	error = vtnet_setup_interface(sc);
473 	if (error) {
474 		device_printf(dev, "cannot setup interface\n");
475 		goto fail;
476 	}
477 
478 	error = virtio_setup_intr(dev, INTR_TYPE_NET);
479 	if (error) {
480 		device_printf(dev, "cannot setup interrupts\n");
481 		ether_ifdetach(sc->vtnet_ifp);
482 		goto fail;
483 	}
484 
485 #ifdef DEV_NETMAP
486 	vtnet_netmap_attach(sc);
487 #endif
488 	vtnet_start_taskqueues(sc);
489 
490 fail:
491 	if (error)
492 		vtnet_detach(dev);
493 
494 	return (error);
495 }
496 
497 static int
498 vtnet_detach(device_t dev)
499 {
500 	struct vtnet_softc *sc;
501 	struct ifnet *ifp;
502 
503 	sc = device_get_softc(dev);
504 	ifp = sc->vtnet_ifp;
505 
506 	if (device_is_attached(dev)) {
507 		VTNET_CORE_LOCK(sc);
508 		vtnet_stop(sc);
509 		VTNET_CORE_UNLOCK(sc);
510 
511 		callout_drain(&sc->vtnet_tick_ch);
512 		vtnet_drain_taskqueues(sc);
513 
514 		ether_ifdetach(ifp);
515 	}
516 
517 #ifdef DEV_NETMAP
518 	netmap_detach(ifp);
519 #endif
520 
521 	if (sc->vtnet_pfil != NULL) {
522 		pfil_head_unregister(sc->vtnet_pfil);
523 		sc->vtnet_pfil = NULL;
524 	}
525 
526 	vtnet_free_taskqueues(sc);
527 
528 	if (sc->vtnet_vlan_attach != NULL) {
529 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
530 		sc->vtnet_vlan_attach = NULL;
531 	}
532 	if (sc->vtnet_vlan_detach != NULL) {
533 		EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach);
534 		sc->vtnet_vlan_detach = NULL;
535 	}
536 
537 	ifmedia_removeall(&sc->vtnet_media);
538 
539 	if (ifp != NULL) {
540 		if_free(ifp);
541 		sc->vtnet_ifp = NULL;
542 	}
543 
544 	vtnet_free_rxtx_queues(sc);
545 	vtnet_free_rx_filters(sc);
546 
547 	if (sc->vtnet_ctrl_vq != NULL)
548 		vtnet_free_ctrl_vq(sc);
549 
550 	VTNET_CORE_LOCK_DESTROY(sc);
551 
552 	return (0);
553 }
554 
555 static int
556 vtnet_suspend(device_t dev)
557 {
558 	struct vtnet_softc *sc;
559 
560 	sc = device_get_softc(dev);
561 
562 	VTNET_CORE_LOCK(sc);
563 	vtnet_stop(sc);
564 	sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
565 	VTNET_CORE_UNLOCK(sc);
566 
567 	return (0);
568 }
569 
570 static int
571 vtnet_resume(device_t dev)
572 {
573 	struct vtnet_softc *sc;
574 	struct ifnet *ifp;
575 
576 	sc = device_get_softc(dev);
577 	ifp = sc->vtnet_ifp;
578 
579 	VTNET_CORE_LOCK(sc);
580 	if (ifp->if_flags & IFF_UP)
581 		vtnet_init_locked(sc, 0);
582 	sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
583 	VTNET_CORE_UNLOCK(sc);
584 
585 	return (0);
586 }
587 
588 static int
589 vtnet_shutdown(device_t dev)
590 {
591 	/*
592 	 * Suspend already does all of what we need to
593 	 * do here; we just never expect to be resumed.
594 	 */
595 	return (vtnet_suspend(dev));
596 }
597 
598 static int
599 vtnet_attach_completed(device_t dev)
600 {
601 	struct vtnet_softc *sc;
602 
603 	sc = device_get_softc(dev);
604 
605 	VTNET_CORE_LOCK(sc);
606 	vtnet_attached_set_macaddr(sc);
607 	VTNET_CORE_UNLOCK(sc);
608 
609 	return (0);
610 }
611 
612 static int
613 vtnet_config_change(device_t dev)
614 {
615 	struct vtnet_softc *sc;
616 
617 	sc = device_get_softc(dev);
618 
619 	VTNET_CORE_LOCK(sc);
620 	vtnet_update_link_status(sc);
621 	if (sc->vtnet_link_active != 0)
622 		vtnet_tx_start_all(sc);
623 	VTNET_CORE_UNLOCK(sc);
624 
625 	return (0);
626 }
627 
628 static int
629 vtnet_negotiate_features(struct vtnet_softc *sc)
630 {
631 	device_t dev;
632 	uint64_t features, negotiated_features;
633 	int no_csum;
634 
635 	dev = sc->vtnet_dev;
636 	features = virtio_bus_is_modern(dev) ? VTNET_MODERN_FEATURES :
637 	    VTNET_LEGACY_FEATURES;
638 
639 	/*
640 	 * TSO and LRO are only available when their corresponding checksum
641 	 * offload feature is also negotiated.
642 	 */
643 	no_csum = vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable);
644 	if (no_csum)
645 		features &= ~(VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM);
646 	if (no_csum || vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable))
647 		features &= ~VTNET_TSO_FEATURES;
648 	if (no_csum || vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable))
649 		features &= ~VTNET_LRO_FEATURES;
650 
651 #ifndef VTNET_LEGACY_TX
652 	if (vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable))
653 		features &= ~VIRTIO_NET_F_MQ;
654 #else
655 	features &= ~VIRTIO_NET_F_MQ;
656 #endif
657 
658 	negotiated_features = virtio_negotiate_features(dev, features);
659 
660 	if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) {
661 		uint16_t mtu;
662 
663 		mtu = virtio_read_dev_config_2(dev,
664 		    offsetof(struct virtio_net_config, mtu));
665 		if (mtu < VTNET_MIN_MTU /* || mtu > VTNET_MAX_MTU */) {
666 			device_printf(dev, "Invalid MTU value: %d. "
667 			    "MTU feature disabled.\n", mtu);
668 			features &= ~VIRTIO_NET_F_MTU;
669 			negotiated_features =
670 			    virtio_negotiate_features(dev, features);
671 		}
672 	}
673 
674 	if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) {
675 		uint16_t npairs;
676 
677 		npairs = virtio_read_dev_config_2(dev,
678 		    offsetof(struct virtio_net_config, max_virtqueue_pairs));
679 		if (npairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
680 		    npairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) {
681 			device_printf(dev, "Invalid max_virtqueue_pairs value: "
682 			    "%d. Multiqueue feature disabled.\n", npairs);
683 			features &= ~VIRTIO_NET_F_MQ;
684 			negotiated_features =
685 			    virtio_negotiate_features(dev, features);
686 		}
687 	}
688 
689 	if (virtio_with_feature(dev, VTNET_LRO_FEATURES) &&
690 	    virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) {
691 		/*
692 		 * LRO without mergeable buffers requires special care. This
693 		 * is not ideal because every receive buffer must be large
694 		 * enough to hold the maximum TCP packet, the Ethernet header,
695 		 * and the header. This requires up to 34 descriptors with
696 		 * MCLBYTES clusters. If we do not have indirect descriptors,
697 		 * LRO is disabled since the virtqueue will not contain very
698 		 * many receive buffers.
699 		 */
700 		if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) {
701 			device_printf(dev,
702 			    "Host LRO disabled since both mergeable buffers "
703 			    "and indirect descriptors were not negotiated\n");
704 			features &= ~VTNET_LRO_FEATURES;
705 			negotiated_features =
706 			    virtio_negotiate_features(dev, features);
707 		} else
708 			sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
709 	}
710 
711 	sc->vtnet_features = negotiated_features;
712 	sc->vtnet_negotiated_features = negotiated_features;
713 
714 	return (virtio_finalize_features(dev));
715 }
716 
717 static int
718 vtnet_setup_features(struct vtnet_softc *sc)
719 {
720 	device_t dev;
721 	int error;
722 
723 	dev = sc->vtnet_dev;
724 
725 	error = vtnet_negotiate_features(sc);
726 	if (error)
727 		return (error);
728 
729 	if (virtio_with_feature(dev, VIRTIO_F_VERSION_1))
730 		sc->vtnet_flags |= VTNET_FLAG_MODERN;
731 	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
732 		sc->vtnet_flags |= VTNET_FLAG_INDIRECT;
733 	if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX))
734 		sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX;
735 
736 	if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
737 		/* This feature should always be negotiated. */
738 		sc->vtnet_flags |= VTNET_FLAG_MAC;
739 	}
740 
741 	if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) {
742 		sc->vtnet_max_mtu = virtio_read_dev_config_2(dev,
743 		    offsetof(struct virtio_net_config, mtu));
744 	} else
745 		sc->vtnet_max_mtu = VTNET_MAX_MTU;
746 
747 	if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
748 		sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
749 		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
750 	} else if (vtnet_modern(sc)) {
751 		/* This is identical to the mergeable header. */
752 		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_v1);
753 	} else
754 		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
755 
756 	if (vtnet_modern(sc) || sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS)
757 		sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_INLINE;
758 	else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
759 		sc->vtnet_rx_nsegs = VTNET_RX_SEGS_LRO_NOMRG;
760 	else
761 		sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_SEPARATE;
762 
763 	/*
764 	 * Favor "hardware" LRO if negotiated, but support software LRO as
765 	 * a fallback; there is usually little benefit (or worse) with both.
766 	 */
767 	if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) == 0 &&
768 	    virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) == 0)
769 		sc->vtnet_flags |= VTNET_FLAG_SW_LRO;
770 
771 	if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) ||
772 	    virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) ||
773 	    virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
774 		sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MAX;
775 	else
776 		sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MIN;
777 
778 	sc->vtnet_req_vq_pairs = 1;
779 	sc->vtnet_max_vq_pairs = 1;
780 
781 	if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
782 		sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
783 
784 		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
785 			sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
786 		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
787 			sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
788 		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR))
789 			sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC;
790 
791 		if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) {
792 			sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev,
793 			    offsetof(struct virtio_net_config,
794 			    max_virtqueue_pairs));
795 		}
796 	}
797 
798 	if (sc->vtnet_max_vq_pairs > 1) {
799 		int req;
800 
801 		/*
802 		 * Limit the maximum number of requested queue pairs to the
803 		 * number of CPUs and the configured maximum.
804 		 */
805 		req = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs);
806 		if (req < 0)
807 			req = 1;
808 		if (req == 0)
809 			req = mp_ncpus;
810 		if (req > sc->vtnet_max_vq_pairs)
811 			req = sc->vtnet_max_vq_pairs;
812 		if (req > mp_ncpus)
813 			req = mp_ncpus;
814 		if (req > 1) {
815 			sc->vtnet_req_vq_pairs = req;
816 			sc->vtnet_flags |= VTNET_FLAG_MQ;
817 		}
818 	}
819 
820 	return (0);
821 }
822 
823 static int
824 vtnet_init_rxq(struct vtnet_softc *sc, int id)
825 {
826 	struct vtnet_rxq *rxq;
827 
828 	rxq = &sc->vtnet_rxqs[id];
829 
830 	snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d",
831 	    device_get_nameunit(sc->vtnet_dev), id);
832 	mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF);
833 
834 	rxq->vtnrx_sc = sc;
835 	rxq->vtnrx_id = id;
836 
837 	rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT);
838 	if (rxq->vtnrx_sg == NULL)
839 		return (ENOMEM);
840 
841 #if defined(INET) || defined(INET6)
842 	if (vtnet_software_lro(sc)) {
843 		if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp,
844 		    sc->vtnet_lro_entry_count, sc->vtnet_lro_mbufq_depth) != 0)
845 			return (ENOMEM);
846 	}
847 #endif
848 
849 	NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq);
850 	rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT,
851 	    taskqueue_thread_enqueue, &rxq->vtnrx_tq);
852 
853 	return (rxq->vtnrx_tq == NULL ? ENOMEM : 0);
854 }
855 
856 static int
857 vtnet_init_txq(struct vtnet_softc *sc, int id)
858 {
859 	struct vtnet_txq *txq;
860 
861 	txq = &sc->vtnet_txqs[id];
862 
863 	snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d",
864 	    device_get_nameunit(sc->vtnet_dev), id);
865 	mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF);
866 
867 	txq->vtntx_sc = sc;
868 	txq->vtntx_id = id;
869 
870 	txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT);
871 	if (txq->vtntx_sg == NULL)
872 		return (ENOMEM);
873 
874 #ifndef VTNET_LEGACY_TX
875 	txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF,
876 	    M_NOWAIT, &txq->vtntx_mtx);
877 	if (txq->vtntx_br == NULL)
878 		return (ENOMEM);
879 
880 	TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq);
881 #endif
882 	TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq);
883 	txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT,
884 	    taskqueue_thread_enqueue, &txq->vtntx_tq);
885 	if (txq->vtntx_tq == NULL)
886 		return (ENOMEM);
887 
888 	return (0);
889 }
890 
891 static int
892 vtnet_alloc_rxtx_queues(struct vtnet_softc *sc)
893 {
894 	int i, npairs, error;
895 
896 	npairs = sc->vtnet_max_vq_pairs;
897 
898 	sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF,
899 	    M_NOWAIT | M_ZERO);
900 	sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF,
901 	    M_NOWAIT | M_ZERO);
902 	if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL)
903 		return (ENOMEM);
904 
905 	for (i = 0; i < npairs; i++) {
906 		error = vtnet_init_rxq(sc, i);
907 		if (error)
908 			return (error);
909 		error = vtnet_init_txq(sc, i);
910 		if (error)
911 			return (error);
912 	}
913 
914 	vtnet_set_rx_process_limit(sc);
915 	vtnet_setup_queue_sysctl(sc);
916 
917 	return (0);
918 }
919 
920 static void
921 vtnet_destroy_rxq(struct vtnet_rxq *rxq)
922 {
923 
924 	rxq->vtnrx_sc = NULL;
925 	rxq->vtnrx_id = -1;
926 
927 #if defined(INET) || defined(INET6)
928 	tcp_lro_free(&rxq->vtnrx_lro);
929 #endif
930 
931 	if (rxq->vtnrx_sg != NULL) {
932 		sglist_free(rxq->vtnrx_sg);
933 		rxq->vtnrx_sg = NULL;
934 	}
935 
936 	if (mtx_initialized(&rxq->vtnrx_mtx) != 0)
937 		mtx_destroy(&rxq->vtnrx_mtx);
938 }
939 
940 static void
941 vtnet_destroy_txq(struct vtnet_txq *txq)
942 {
943 
944 	txq->vtntx_sc = NULL;
945 	txq->vtntx_id = -1;
946 
947 	if (txq->vtntx_sg != NULL) {
948 		sglist_free(txq->vtntx_sg);
949 		txq->vtntx_sg = NULL;
950 	}
951 
952 #ifndef VTNET_LEGACY_TX
953 	if (txq->vtntx_br != NULL) {
954 		buf_ring_free(txq->vtntx_br, M_DEVBUF);
955 		txq->vtntx_br = NULL;
956 	}
957 #endif
958 
959 	if (mtx_initialized(&txq->vtntx_mtx) != 0)
960 		mtx_destroy(&txq->vtntx_mtx);
961 }
962 
963 static void
964 vtnet_free_rxtx_queues(struct vtnet_softc *sc)
965 {
966 	int i;
967 
968 	if (sc->vtnet_rxqs != NULL) {
969 		for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
970 			vtnet_destroy_rxq(&sc->vtnet_rxqs[i]);
971 		free(sc->vtnet_rxqs, M_DEVBUF);
972 		sc->vtnet_rxqs = NULL;
973 	}
974 
975 	if (sc->vtnet_txqs != NULL) {
976 		for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
977 			vtnet_destroy_txq(&sc->vtnet_txqs[i]);
978 		free(sc->vtnet_txqs, M_DEVBUF);
979 		sc->vtnet_txqs = NULL;
980 	}
981 }
982 
983 static int
984 vtnet_alloc_rx_filters(struct vtnet_softc *sc)
985 {
986 
987 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
988 		sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter),
989 		    M_DEVBUF, M_NOWAIT | M_ZERO);
990 		if (sc->vtnet_mac_filter == NULL)
991 			return (ENOMEM);
992 	}
993 
994 	if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
995 		sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) *
996 		    VTNET_VLAN_FILTER_NWORDS, M_DEVBUF, M_NOWAIT | M_ZERO);
997 		if (sc->vtnet_vlan_filter == NULL)
998 			return (ENOMEM);
999 	}
1000 
1001 	return (0);
1002 }
1003 
1004 static void
1005 vtnet_free_rx_filters(struct vtnet_softc *sc)
1006 {
1007 
1008 	if (sc->vtnet_mac_filter != NULL) {
1009 		free(sc->vtnet_mac_filter, M_DEVBUF);
1010 		sc->vtnet_mac_filter = NULL;
1011 	}
1012 
1013 	if (sc->vtnet_vlan_filter != NULL) {
1014 		free(sc->vtnet_vlan_filter, M_DEVBUF);
1015 		sc->vtnet_vlan_filter = NULL;
1016 	}
1017 }
1018 
1019 static int
1020 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
1021 {
1022 	device_t dev;
1023 	struct vq_alloc_info *info;
1024 	struct vtnet_rxq *rxq;
1025 	struct vtnet_txq *txq;
1026 	int i, idx, flags, nvqs, error;
1027 
1028 	dev = sc->vtnet_dev;
1029 	flags = 0;
1030 
1031 	nvqs = sc->vtnet_max_vq_pairs * 2;
1032 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ)
1033 		nvqs++;
1034 
1035 	info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT);
1036 	if (info == NULL)
1037 		return (ENOMEM);
1038 
1039 	for (i = 0, idx = 0; i < sc->vtnet_req_vq_pairs; i++, idx += 2) {
1040 		rxq = &sc->vtnet_rxqs[i];
1041 		VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs,
1042 		    vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq,
1043 		    "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id);
1044 
1045 		txq = &sc->vtnet_txqs[i];
1046 		VQ_ALLOC_INFO_INIT(&info[idx+1], sc->vtnet_tx_nsegs,
1047 		    vtnet_tx_vq_intr, txq, &txq->vtntx_vq,
1048 		    "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id);
1049 	}
1050 
1051 	/* These queues will not be used so allocate the minimum resources. */
1052 	for (/**/; i < sc->vtnet_max_vq_pairs; i++, idx += 2) {
1053 		rxq = &sc->vtnet_rxqs[i];
1054 		VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, rxq, &rxq->vtnrx_vq,
1055 		    "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id);
1056 
1057 		txq = &sc->vtnet_txqs[i];
1058 		VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL, txq, &txq->vtntx_vq,
1059 		    "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id);
1060 	}
1061 
1062 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
1063 		VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL,
1064 		    &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev));
1065 	}
1066 
1067 	/*
1068 	 * TODO: Enable interrupt binding if this is multiqueue. This will
1069 	 * only matter when per-virtqueue MSIX is available.
1070 	 */
1071 	if (sc->vtnet_flags & VTNET_FLAG_MQ)
1072 		flags |= 0;
1073 
1074 	error = virtio_alloc_virtqueues(dev, flags, nvqs, info);
1075 	free(info, M_TEMP);
1076 
1077 	return (error);
1078 }
1079 
1080 static int
1081 vtnet_alloc_interface(struct vtnet_softc *sc)
1082 {
1083 	device_t dev;
1084 	struct ifnet *ifp;
1085 
1086 	dev = sc->vtnet_dev;
1087 
1088 	ifp = if_alloc(IFT_ETHER);
1089 	if (ifp == NULL)
1090 		return (ENOMEM);
1091 
1092 	sc->vtnet_ifp = ifp;
1093 	ifp->if_softc = sc;
1094 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1095 
1096 	return (0);
1097 }
1098 
1099 static int
1100 vtnet_setup_interface(struct vtnet_softc *sc)
1101 {
1102 	device_t dev;
1103 	struct pfil_head_args pa;
1104 	struct ifnet *ifp;
1105 
1106 	dev = sc->vtnet_dev;
1107 	ifp = sc->vtnet_ifp;
1108 
1109 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST |
1110 	    IFF_KNOWSEPOCH;
1111 	ifp->if_baudrate = IF_Gbps(10);
1112 	ifp->if_init = vtnet_init;
1113 	ifp->if_ioctl = vtnet_ioctl;
1114 	ifp->if_get_counter = vtnet_get_counter;
1115 #ifndef VTNET_LEGACY_TX
1116 	ifp->if_transmit = vtnet_txq_mq_start;
1117 	ifp->if_qflush = vtnet_qflush;
1118 #else
1119 	struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq;
1120 	ifp->if_start = vtnet_start;
1121 	IFQ_SET_MAXLEN(&ifp->if_snd, virtqueue_size(vq) - 1);
1122 	ifp->if_snd.ifq_drv_maxlen = virtqueue_size(vq) - 1;
1123 	IFQ_SET_READY(&ifp->if_snd);
1124 #endif
1125 
1126 	vtnet_get_macaddr(sc);
1127 
1128 	if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS))
1129 		ifp->if_capabilities |= IFCAP_LINKSTATE;
1130 
1131 	ifmedia_init(&sc->vtnet_media, 0, vtnet_ifmedia_upd, vtnet_ifmedia_sts);
1132 	ifmedia_add(&sc->vtnet_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1133 	ifmedia_set(&sc->vtnet_media, IFM_ETHER | IFM_AUTO);
1134 
1135 	if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
1136 		int gso;
1137 
1138 		ifp->if_capabilities |= IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6;
1139 
1140 		gso = virtio_with_feature(dev, VIRTIO_NET_F_GSO);
1141 		if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
1142 			ifp->if_capabilities |= IFCAP_TSO4;
1143 		if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
1144 			ifp->if_capabilities |= IFCAP_TSO6;
1145 		if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
1146 			sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
1147 
1148 		if (ifp->if_capabilities & (IFCAP_TSO4 | IFCAP_TSO6)) {
1149 			int tso_maxlen;
1150 
1151 			ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
1152 
1153 			tso_maxlen = vtnet_tunable_int(sc, "tso_maxlen",
1154 			    vtnet_tso_maxlen);
1155 			ifp->if_hw_tsomax = tso_maxlen -
1156 			    (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1157 			ifp->if_hw_tsomaxsegcount = sc->vtnet_tx_nsegs - 1;
1158 			ifp->if_hw_tsomaxsegsize = PAGE_SIZE;
1159 		}
1160 	}
1161 
1162 	if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
1163 		ifp->if_capabilities |= IFCAP_RXCSUM;
1164 #ifdef notyet
1165 		/* BMV: Rx checksums not distinguished between IPv4 and IPv6. */
1166 		ifp->if_capabilities |= IFCAP_RXCSUM_IPV6;
1167 #endif
1168 
1169 		if (vtnet_tunable_int(sc, "fixup_needs_csum",
1170 		    vtnet_fixup_needs_csum) != 0)
1171 			sc->vtnet_flags |= VTNET_FLAG_FIXUP_NEEDS_CSUM;
1172 
1173 		/* Support either "hardware" or software LRO. */
1174 		ifp->if_capabilities |= IFCAP_LRO;
1175 	}
1176 
1177 	if (ifp->if_capabilities & (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6)) {
1178 		/*
1179 		 * VirtIO does not support VLAN tagging, but we can fake
1180 		 * it by inserting and removing the 802.1Q header during
1181 		 * transmit and receive. We are then able to do checksum
1182 		 * offloading of VLAN frames.
1183 		 */
1184 		ifp->if_capabilities |=
1185 		    IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
1186 	}
1187 
1188 	if (sc->vtnet_max_mtu >= ETHERMTU_JUMBO)
1189 		ifp->if_capabilities |= IFCAP_JUMBO_MTU;
1190 	ifp->if_capabilities |= IFCAP_VLAN_MTU;
1191 
1192 	/*
1193 	 * Capabilities after here are not enabled by default.
1194 	 */
1195 	ifp->if_capenable = ifp->if_capabilities;
1196 
1197 	if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
1198 		ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
1199 
1200 		sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
1201 		    vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
1202 		sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
1203 		    vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
1204 	}
1205 
1206 	ether_ifattach(ifp, sc->vtnet_hwaddr);
1207 
1208 	/* Tell the upper layer(s) we support long frames. */
1209 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
1210 
1211 	DEBUGNET_SET(ifp, vtnet);
1212 
1213 	pa.pa_version = PFIL_VERSION;
1214 	pa.pa_flags = PFIL_IN;
1215 	pa.pa_type = PFIL_TYPE_ETHERNET;
1216 	pa.pa_headname = ifp->if_xname;
1217 	sc->vtnet_pfil = pfil_head_register(&pa);
1218 
1219 	return (0);
1220 }
1221 
1222 static int
1223 vtnet_rx_cluster_size(struct vtnet_softc *sc, int mtu)
1224 {
1225 	int framesz;
1226 
1227 	if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS)
1228 		return (MJUMPAGESIZE);
1229 	else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
1230 		return (MCLBYTES);
1231 
1232 	/*
1233 	 * Try to scale the receive mbuf cluster size from the MTU. We
1234 	 * could also use the VQ size to influence the selected size,
1235 	 * but that would only matter for very small queues.
1236 	 */
1237 	if (vtnet_modern(sc)) {
1238 		MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr_v1));
1239 		framesz = sizeof(struct virtio_net_hdr_v1);
1240 	} else
1241 		framesz = sizeof(struct vtnet_rx_header);
1242 	framesz += sizeof(struct ether_vlan_header) + mtu;
1243 
1244 	if (framesz <= MCLBYTES)
1245 		return (MCLBYTES);
1246 	else if (framesz <= MJUMPAGESIZE)
1247 		return (MJUMPAGESIZE);
1248 	else if (framesz <= MJUM9BYTES)
1249 		return (MJUM9BYTES);
1250 
1251 	/* Sane default; avoid 16KB clusters. */
1252 	return (MCLBYTES);
1253 }
1254 
1255 static int
1256 vtnet_ioctl_mtu(struct vtnet_softc *sc, u_int mtu)
1257 {
1258 	struct ifnet *ifp;
1259 	int clustersz;
1260 
1261 	ifp = sc->vtnet_ifp;
1262 	VTNET_CORE_LOCK_ASSERT(sc);
1263 
1264 	if (ifp->if_mtu == mtu)
1265 		return (0);
1266 	else if (mtu < ETHERMIN || mtu > sc->vtnet_max_mtu)
1267 		return (EINVAL);
1268 
1269 	ifp->if_mtu = mtu;
1270 	clustersz = vtnet_rx_cluster_size(sc, mtu);
1271 
1272 	if (clustersz != sc->vtnet_rx_clustersz &&
1273 	    ifp->if_drv_flags & IFF_DRV_RUNNING) {
1274 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1275 		vtnet_init_locked(sc, 0);
1276 	}
1277 
1278 	return (0);
1279 }
1280 
1281 static int
1282 vtnet_ioctl_ifflags(struct vtnet_softc *sc)
1283 {
1284 	struct ifnet *ifp;
1285 	int drv_running;
1286 
1287 	ifp = sc->vtnet_ifp;
1288 	drv_running = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
1289 
1290 	VTNET_CORE_LOCK_ASSERT(sc);
1291 
1292 	if ((ifp->if_flags & IFF_UP) == 0) {
1293 		if (drv_running)
1294 			vtnet_stop(sc);
1295 		goto out;
1296 	}
1297 
1298 	if (!drv_running) {
1299 		vtnet_init_locked(sc, 0);
1300 		goto out;
1301 	}
1302 
1303 	if ((ifp->if_flags ^ sc->vtnet_if_flags) &
1304 	    (IFF_PROMISC | IFF_ALLMULTI)) {
1305 		if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
1306 			return (ENOTSUP);
1307 		vtnet_rx_filter(sc);
1308 	}
1309 
1310 out:
1311 	sc->vtnet_if_flags = ifp->if_flags;
1312 	return (0);
1313 }
1314 
1315 static int
1316 vtnet_ioctl_multi(struct vtnet_softc *sc)
1317 {
1318 	struct ifnet *ifp;
1319 
1320 	ifp = sc->vtnet_ifp;
1321 
1322 	VTNET_CORE_LOCK_ASSERT(sc);
1323 
1324 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX &&
1325 	    ifp->if_drv_flags & IFF_DRV_RUNNING)
1326 		vtnet_rx_filter_mac(sc);
1327 
1328 	return (0);
1329 }
1330 
1331 static int
1332 vtnet_ioctl_ifcap(struct vtnet_softc *sc, struct ifreq *ifr)
1333 {
1334 	struct ifnet *ifp;
1335 	int mask, reinit, update;
1336 
1337 	ifp = sc->vtnet_ifp;
1338 	mask = (ifr->ifr_reqcap & ifp->if_capabilities) ^ ifp->if_capenable;
1339 	reinit = update = 0;
1340 
1341 	VTNET_CORE_LOCK_ASSERT(sc);
1342 
1343 	if (mask & IFCAP_TXCSUM)
1344 		ifp->if_capenable ^= IFCAP_TXCSUM;
1345 	if (mask & IFCAP_TXCSUM_IPV6)
1346 		ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1347 	if (mask & IFCAP_TSO4)
1348 		ifp->if_capenable ^= IFCAP_TSO4;
1349 	if (mask & IFCAP_TSO6)
1350 		ifp->if_capenable ^= IFCAP_TSO6;
1351 
1352 	if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) {
1353 		/*
1354 		 * These Rx features require the negotiated features to
1355 		 * be updated. Avoid a full reinit if possible.
1356 		 */
1357 		if (sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
1358 			update = 1;
1359 		else
1360 			reinit = 1;
1361 
1362 		/* BMV: Avoid needless renegotiation for just software LRO. */
1363 		if ((mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) ==
1364 		    IFCAP_LRO && vtnet_software_lro(sc))
1365 			reinit = update = 0;
1366 
1367 		if (mask & IFCAP_RXCSUM)
1368 			ifp->if_capenable ^= IFCAP_RXCSUM;
1369 		if (mask & IFCAP_RXCSUM_IPV6)
1370 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1371 		if (mask & IFCAP_LRO)
1372 			ifp->if_capenable ^= IFCAP_LRO;
1373 
1374 		/*
1375 		 * VirtIO does not distinguish between IPv4 and IPv6 checksums
1376 		 * so treat them as a pair. Guest TSO (LRO) requires receive
1377 		 * checksums.
1378 		 */
1379 		if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
1380 			ifp->if_capenable |= IFCAP_RXCSUM;
1381 #ifdef notyet
1382 			ifp->if_capenable |= IFCAP_RXCSUM_IPV6;
1383 #endif
1384 		} else
1385 			ifp->if_capenable &=
1386 			    ~(IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO);
1387 	}
1388 
1389 	if (mask & IFCAP_VLAN_HWFILTER) {
1390 		/* These Rx features require renegotiation. */
1391 		reinit = 1;
1392 
1393 		if (mask & IFCAP_VLAN_HWFILTER)
1394 			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1395 	}
1396 
1397 	if (mask & IFCAP_VLAN_HWTSO)
1398 		ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1399 	if (mask & IFCAP_VLAN_HWTAGGING)
1400 		ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1401 
1402 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1403 		if (reinit) {
1404 			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1405 			vtnet_init_locked(sc, 0);
1406 		} else if (update)
1407 			vtnet_update_rx_offloads(sc);
1408 	}
1409 
1410 	return (0);
1411 }
1412 
1413 static int
1414 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1415 {
1416 	struct vtnet_softc *sc;
1417 	struct ifreq *ifr;
1418 	int error;
1419 
1420 	sc = ifp->if_softc;
1421 	ifr = (struct ifreq *) data;
1422 	error = 0;
1423 
1424 	switch (cmd) {
1425 	case SIOCSIFMTU:
1426 		VTNET_CORE_LOCK(sc);
1427 		error = vtnet_ioctl_mtu(sc, ifr->ifr_mtu);
1428 		VTNET_CORE_UNLOCK(sc);
1429 		break;
1430 
1431 	case SIOCSIFFLAGS:
1432 		VTNET_CORE_LOCK(sc);
1433 		error = vtnet_ioctl_ifflags(sc);
1434 		VTNET_CORE_UNLOCK(sc);
1435 		break;
1436 
1437 	case SIOCADDMULTI:
1438 	case SIOCDELMULTI:
1439 		VTNET_CORE_LOCK(sc);
1440 		error = vtnet_ioctl_multi(sc);
1441 		VTNET_CORE_UNLOCK(sc);
1442 		break;
1443 
1444 	case SIOCSIFMEDIA:
1445 	case SIOCGIFMEDIA:
1446 		error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
1447 		break;
1448 
1449 	case SIOCSIFCAP:
1450 		VTNET_CORE_LOCK(sc);
1451 		error = vtnet_ioctl_ifcap(sc, ifr);
1452 		VTNET_CORE_UNLOCK(sc);
1453 		VLAN_CAPABILITIES(ifp);
1454 		break;
1455 
1456 	default:
1457 		error = ether_ioctl(ifp, cmd, data);
1458 		break;
1459 	}
1460 
1461 	VTNET_CORE_LOCK_ASSERT_NOTOWNED(sc);
1462 
1463 	return (error);
1464 }
1465 
1466 static int
1467 vtnet_rxq_populate(struct vtnet_rxq *rxq)
1468 {
1469 	struct virtqueue *vq;
1470 	int nbufs, error;
1471 
1472 #ifdef DEV_NETMAP
1473 	error = vtnet_netmap_rxq_populate(rxq);
1474 	if (error >= 0)
1475 		return (error);
1476 #endif  /* DEV_NETMAP */
1477 
1478 	vq = rxq->vtnrx_vq;
1479 	error = ENOSPC;
1480 
1481 	for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
1482 		error = vtnet_rxq_new_buf(rxq);
1483 		if (error)
1484 			break;
1485 	}
1486 
1487 	if (nbufs > 0) {
1488 		virtqueue_notify(vq);
1489 		/*
1490 		 * EMSGSIZE signifies the virtqueue did not have enough
1491 		 * entries available to hold the last mbuf. This is not
1492 		 * an error.
1493 		 */
1494 		if (error == EMSGSIZE)
1495 			error = 0;
1496 	}
1497 
1498 	return (error);
1499 }
1500 
1501 static void
1502 vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq)
1503 {
1504 	struct virtqueue *vq;
1505 	struct mbuf *m;
1506 	int last;
1507 #ifdef DEV_NETMAP
1508 	struct netmap_kring *kring = netmap_kring_on(NA(rxq->vtnrx_sc->vtnet_ifp),
1509 							rxq->vtnrx_id, NR_RX);
1510 #else  /* !DEV_NETMAP */
1511 	void *kring = NULL;
1512 #endif /* !DEV_NETMAP */
1513 
1514 	vq = rxq->vtnrx_vq;
1515 	last = 0;
1516 
1517 	while ((m = virtqueue_drain(vq, &last)) != NULL) {
1518 		if (kring == NULL)
1519 			m_freem(m);
1520 	}
1521 
1522 	KASSERT(virtqueue_empty(vq),
1523 	    ("%s: mbufs remaining in rx queue %p", __func__, rxq));
1524 }
1525 
1526 static struct mbuf *
1527 vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1528 {
1529 	struct mbuf *m_head, *m_tail, *m;
1530 	int i, size;
1531 
1532 	m_head = NULL;
1533 	size = sc->vtnet_rx_clustersz;
1534 
1535 	KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1536 	    ("%s: mbuf %d chain requested without LRO_NOMRG", __func__, nbufs));
1537 
1538 	for (i = 0; i < nbufs; i++) {
1539 		m = m_getjcl(M_NOWAIT, MT_DATA, i == 0 ? M_PKTHDR : 0, size);
1540 		if (m == NULL) {
1541 			sc->vtnet_stats.mbuf_alloc_failed++;
1542 			m_freem(m_head);
1543 			return (NULL);
1544 		}
1545 
1546 		m->m_len = size;
1547 		if (m_head != NULL) {
1548 			m_tail->m_next = m;
1549 			m_tail = m;
1550 		} else
1551 			m_head = m_tail = m;
1552 	}
1553 
1554 	if (m_tailp != NULL)
1555 		*m_tailp = m_tail;
1556 
1557 	return (m_head);
1558 }
1559 
1560 /*
1561  * Slow path for when LRO without mergeable buffers is negotiated.
1562  */
1563 static int
1564 vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *rxq, struct mbuf *m0,
1565     int len0)
1566 {
1567 	struct vtnet_softc *sc;
1568 	struct mbuf *m, *m_prev, *m_new, *m_tail;
1569 	int len, clustersz, nreplace, error;
1570 
1571 	sc = rxq->vtnrx_sc;
1572 	clustersz = sc->vtnet_rx_clustersz;
1573 
1574 	m_prev = NULL;
1575 	m_tail = NULL;
1576 	nreplace = 0;
1577 
1578 	m = m0;
1579 	len = len0;
1580 
1581 	/*
1582 	 * Since these mbuf chains are so large, avoid allocating a complete
1583 	 * replacement when the received frame did not consume the entire
1584 	 * chain. Unused mbufs are moved to the tail of the replacement mbuf.
1585 	 */
1586 	while (len > 0) {
1587 		if (m == NULL) {
1588 			sc->vtnet_stats.rx_frame_too_large++;
1589 			return (EMSGSIZE);
1590 		}
1591 
1592 		/*
1593 		 * Every mbuf should have the expected cluster size since that
1594 		 * is also used to allocate the replacements.
1595 		 */
1596 		KASSERT(m->m_len == clustersz,
1597 		    ("%s: mbuf size %d not expected cluster size %d", __func__,
1598 		    m->m_len, clustersz));
1599 
1600 		m->m_len = MIN(m->m_len, len);
1601 		len -= m->m_len;
1602 
1603 		m_prev = m;
1604 		m = m->m_next;
1605 		nreplace++;
1606 	}
1607 
1608 	KASSERT(nreplace > 0 && nreplace <= sc->vtnet_rx_nmbufs,
1609 	    ("%s: invalid replacement mbuf count %d max %d", __func__,
1610 	    nreplace, sc->vtnet_rx_nmbufs));
1611 
1612 	m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail);
1613 	if (m_new == NULL) {
1614 		m_prev->m_len = clustersz;
1615 		return (ENOBUFS);
1616 	}
1617 
1618 	/*
1619 	 * Move any unused mbufs from the received mbuf chain onto the
1620 	 * end of the replacement chain.
1621 	 */
1622 	if (m_prev->m_next != NULL) {
1623 		m_tail->m_next = m_prev->m_next;
1624 		m_prev->m_next = NULL;
1625 	}
1626 
1627 	error = vtnet_rxq_enqueue_buf(rxq, m_new);
1628 	if (error) {
1629 		/*
1630 		 * The replacement is suppose to be an copy of the one
1631 		 * dequeued so this is a very unexpected error.
1632 		 *
1633 		 * Restore the m0 chain to the original state if it was
1634 		 * modified so we can then discard it.
1635 		 */
1636 		if (m_tail->m_next != NULL) {
1637 			m_prev->m_next = m_tail->m_next;
1638 			m_tail->m_next = NULL;
1639 		}
1640 		m_prev->m_len = clustersz;
1641 		sc->vtnet_stats.rx_enq_replacement_failed++;
1642 		m_freem(m_new);
1643 	}
1644 
1645 	return (error);
1646 }
1647 
1648 static int
1649 vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len)
1650 {
1651 	struct vtnet_softc *sc;
1652 	struct mbuf *m_new;
1653 	int error;
1654 
1655 	sc = rxq->vtnrx_sc;
1656 
1657 	if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
1658 		return (vtnet_rxq_replace_lro_nomrg_buf(rxq, m, len));
1659 
1660 	MPASS(m->m_next == NULL);
1661 	if (m->m_len < len)
1662 		return (EMSGSIZE);
1663 
1664 	m_new = vtnet_rx_alloc_buf(sc, 1, NULL);
1665 	if (m_new == NULL)
1666 		return (ENOBUFS);
1667 
1668 	error = vtnet_rxq_enqueue_buf(rxq, m_new);
1669 	if (error) {
1670 		sc->vtnet_stats.rx_enq_replacement_failed++;
1671 		m_freem(m_new);
1672 	} else
1673 		m->m_len = len;
1674 
1675 	return (error);
1676 }
1677 
1678 static int
1679 vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m)
1680 {
1681 	struct vtnet_softc *sc;
1682 	struct sglist *sg;
1683 	int header_inlined, error;
1684 
1685 	sc = rxq->vtnrx_sc;
1686 	sg = rxq->vtnrx_sg;
1687 
1688 	KASSERT(m->m_next == NULL || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1689 	    ("%s: mbuf chain without LRO_NOMRG", __func__));
1690 	VTNET_RXQ_LOCK_ASSERT(rxq);
1691 
1692 	sglist_reset(sg);
1693 	header_inlined = vtnet_modern(sc) ||
1694 	    (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) != 0; /* TODO: ANY_LAYOUT */
1695 
1696 	if (header_inlined)
1697 		error = sglist_append_mbuf(sg, m);
1698 	else {
1699 		struct vtnet_rx_header *rxhdr =
1700 		    mtod(m, struct vtnet_rx_header *);
1701 		MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr));
1702 
1703 		/* Append the header and remaining mbuf data. */
1704 		error = sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size);
1705 		if (error)
1706 			return (error);
1707 		error = sglist_append(sg, &rxhdr[1],
1708 		    m->m_len - sizeof(struct vtnet_rx_header));
1709 		if (error)
1710 			return (error);
1711 
1712 		if (m->m_next != NULL)
1713 			error = sglist_append_mbuf(sg, m->m_next);
1714 	}
1715 
1716 	if (error)
1717 		return (error);
1718 
1719 	return (virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg));
1720 }
1721 
1722 static int
1723 vtnet_rxq_new_buf(struct vtnet_rxq *rxq)
1724 {
1725 	struct vtnet_softc *sc;
1726 	struct mbuf *m;
1727 	int error;
1728 
1729 	sc = rxq->vtnrx_sc;
1730 
1731 	m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL);
1732 	if (m == NULL)
1733 		return (ENOBUFS);
1734 
1735 	error = vtnet_rxq_enqueue_buf(rxq, m);
1736 	if (error)
1737 		m_freem(m);
1738 
1739 	return (error);
1740 }
1741 
1742 static int
1743 vtnet_rxq_csum_needs_csum(struct vtnet_rxq *rxq, struct mbuf *m, uint16_t etype,
1744     int hoff, struct virtio_net_hdr *hdr)
1745 {
1746 	struct vtnet_softc *sc;
1747 	int error;
1748 
1749 	sc = rxq->vtnrx_sc;
1750 
1751 	/*
1752 	 * NEEDS_CSUM corresponds to Linux's CHECKSUM_PARTIAL, but FreeBSD does
1753 	 * not have an analogous CSUM flag. The checksum has been validated,
1754 	 * but is incomplete (TCP/UDP pseudo header).
1755 	 *
1756 	 * The packet is likely from another VM on the same host that itself
1757 	 * performed checksum offloading so Tx/Rx is basically a memcpy and
1758 	 * the checksum has little value.
1759 	 *
1760 	 * Default to receiving the packet as-is for performance reasons, but
1761 	 * this can cause issues if the packet is to be forwarded because it
1762 	 * does not contain a valid checksum. This patch may be helpful:
1763 	 * https://reviews.freebsd.org/D6611. In the meantime, have the driver
1764 	 * compute the checksum if requested.
1765 	 *
1766 	 * BMV: Need to add an CSUM_PARTIAL flag?
1767 	 */
1768 	if ((sc->vtnet_flags & VTNET_FLAG_FIXUP_NEEDS_CSUM) == 0) {
1769 		error = vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr);
1770 		return (error);
1771 	}
1772 
1773 	/*
1774 	 * Compute the checksum in the driver so the packet will contain a
1775 	 * valid checksum. The checksum is at csum_offset from csum_start.
1776 	 */
1777 	switch (etype) {
1778 #if defined(INET) || defined(INET6)
1779 	case ETHERTYPE_IP:
1780 	case ETHERTYPE_IPV6: {
1781 		int csum_off, csum_end;
1782 		uint16_t csum;
1783 
1784 		csum_off = hdr->csum_start + hdr->csum_offset;
1785 		csum_end = csum_off + sizeof(uint16_t);
1786 
1787 		/* Assume checksum will be in the first mbuf. */
1788 		if (m->m_len < csum_end || m->m_pkthdr.len < csum_end)
1789 			return (1);
1790 
1791 		/*
1792 		 * Like in_delayed_cksum()/in6_delayed_cksum(), compute the
1793 		 * checksum and write it at the specified offset. We could
1794 		 * try to verify the packet: csum_start should probably
1795 		 * correspond to the start of the TCP/UDP header.
1796 		 *
1797 		 * BMV: Need to properly handle UDP with zero checksum. Is
1798 		 * the IPv4 header checksum implicitly validated?
1799 		 */
1800 		csum = in_cksum_skip(m, m->m_pkthdr.len, hdr->csum_start);
1801 		*(uint16_t *)(mtodo(m, csum_off)) = csum;
1802 		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1803 		m->m_pkthdr.csum_data = 0xFFFF;
1804 		break;
1805 	}
1806 #endif
1807 	default:
1808 		sc->vtnet_stats.rx_csum_bad_ethtype++;
1809 		return (1);
1810 	}
1811 
1812 	return (0);
1813 }
1814 
1815 static int
1816 vtnet_rxq_csum_data_valid(struct vtnet_rxq *rxq, struct mbuf *m,
1817     uint16_t etype, int hoff, struct virtio_net_hdr *hdr __unused)
1818 {
1819 #if 0
1820 	struct vtnet_softc *sc;
1821 #endif
1822 	int protocol;
1823 
1824 #if 0
1825 	sc = rxq->vtnrx_sc;
1826 #endif
1827 
1828 	switch (etype) {
1829 #if defined(INET)
1830 	case ETHERTYPE_IP:
1831 		if (__predict_false(m->m_len < hoff + sizeof(struct ip)))
1832 			protocol = IPPROTO_DONE;
1833 		else {
1834 			struct ip *ip = (struct ip *)(m->m_data + hoff);
1835 			protocol = ip->ip_p;
1836 		}
1837 		break;
1838 #endif
1839 #if defined(INET6)
1840 	case ETHERTYPE_IPV6:
1841 		if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr))
1842 		    || ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0)
1843 			protocol = IPPROTO_DONE;
1844 		break;
1845 #endif
1846 	default:
1847 		protocol = IPPROTO_DONE;
1848 		break;
1849 	}
1850 
1851 	switch (protocol) {
1852 	case IPPROTO_TCP:
1853 	case IPPROTO_UDP:
1854 		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1855 		m->m_pkthdr.csum_data = 0xFFFF;
1856 		break;
1857 	default:
1858 		/*
1859 		 * FreeBSD does not support checksum offloading of this
1860 		 * protocol. Let the stack re-verify the checksum later
1861 		 * if the protocol is supported.
1862 		 */
1863 #if 0
1864 		if_printf(sc->vtnet_ifp,
1865 		    "%s: checksum offload of unsupported protocol "
1866 		    "etype=%#x protocol=%d csum_start=%d csum_offset=%d\n",
1867 		    __func__, etype, protocol, hdr->csum_start,
1868 		    hdr->csum_offset);
1869 #endif
1870 		break;
1871 	}
1872 
1873 	return (0);
1874 }
1875 
1876 static int
1877 vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m,
1878     struct virtio_net_hdr *hdr)
1879 {
1880 	const struct ether_header *eh;
1881 	int hoff;
1882 	uint16_t etype;
1883 
1884 	eh = mtod(m, const struct ether_header *);
1885 	etype = ntohs(eh->ether_type);
1886 	if (etype == ETHERTYPE_VLAN) {
1887 		/* TODO BMV: Handle QinQ. */
1888 		const struct ether_vlan_header *evh =
1889 		    mtod(m, const struct ether_vlan_header *);
1890 		etype = ntohs(evh->evl_proto);
1891 		hoff = sizeof(struct ether_vlan_header);
1892 	} else
1893 		hoff = sizeof(struct ether_header);
1894 
1895 	if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1896 		return (vtnet_rxq_csum_needs_csum(rxq, m, etype, hoff, hdr));
1897 	else /* VIRTIO_NET_HDR_F_DATA_VALID */
1898 		return (vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr));
1899 }
1900 
1901 static void
1902 vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs)
1903 {
1904 	struct mbuf *m;
1905 
1906 	while (--nbufs > 0) {
1907 		m = virtqueue_dequeue(rxq->vtnrx_vq, NULL);
1908 		if (m == NULL)
1909 			break;
1910 		vtnet_rxq_discard_buf(rxq, m);
1911 	}
1912 }
1913 
1914 static void
1915 vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m)
1916 {
1917 	int error __diagused;
1918 
1919 	/*
1920 	 * Requeue the discarded mbuf. This should always be successful
1921 	 * since it was just dequeued.
1922 	 */
1923 	error = vtnet_rxq_enqueue_buf(rxq, m);
1924 	KASSERT(error == 0,
1925 	    ("%s: cannot requeue discarded mbuf %d", __func__, error));
1926 }
1927 
1928 static int
1929 vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs)
1930 {
1931 	struct vtnet_softc *sc;
1932 	struct virtqueue *vq;
1933 	struct mbuf *m_tail;
1934 
1935 	sc = rxq->vtnrx_sc;
1936 	vq = rxq->vtnrx_vq;
1937 	m_tail = m_head;
1938 
1939 	while (--nbufs > 0) {
1940 		struct mbuf *m;
1941 		uint32_t len;
1942 
1943 		m = virtqueue_dequeue(vq, &len);
1944 		if (m == NULL) {
1945 			rxq->vtnrx_stats.vrxs_ierrors++;
1946 			goto fail;
1947 		}
1948 
1949 		if (vtnet_rxq_new_buf(rxq) != 0) {
1950 			rxq->vtnrx_stats.vrxs_iqdrops++;
1951 			vtnet_rxq_discard_buf(rxq, m);
1952 			if (nbufs > 1)
1953 				vtnet_rxq_discard_merged_bufs(rxq, nbufs);
1954 			goto fail;
1955 		}
1956 
1957 		if (m->m_len < len)
1958 			len = m->m_len;
1959 
1960 		m->m_len = len;
1961 		m->m_flags &= ~M_PKTHDR;
1962 
1963 		m_head->m_pkthdr.len += len;
1964 		m_tail->m_next = m;
1965 		m_tail = m;
1966 	}
1967 
1968 	return (0);
1969 
1970 fail:
1971 	sc->vtnet_stats.rx_mergeable_failed++;
1972 	m_freem(m_head);
1973 
1974 	return (1);
1975 }
1976 
1977 #if defined(INET) || defined(INET6)
1978 static int
1979 vtnet_lro_rx(struct vtnet_rxq *rxq, struct mbuf *m)
1980 {
1981 	struct lro_ctrl *lro;
1982 
1983 	lro = &rxq->vtnrx_lro;
1984 
1985 	if (lro->lro_mbuf_max != 0) {
1986 		tcp_lro_queue_mbuf(lro, m);
1987 		return (0);
1988 	}
1989 
1990 	return (tcp_lro_rx(lro, m, 0));
1991 }
1992 #endif
1993 
1994 static void
1995 vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m,
1996     struct virtio_net_hdr *hdr)
1997 {
1998 	struct vtnet_softc *sc;
1999 	struct ifnet *ifp;
2000 
2001 	sc = rxq->vtnrx_sc;
2002 	ifp = sc->vtnet_ifp;
2003 
2004 	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
2005 		struct ether_header *eh = mtod(m, struct ether_header *);
2006 		if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2007 			vtnet_vlan_tag_remove(m);
2008 			/*
2009 			 * With the 802.1Q header removed, update the
2010 			 * checksum starting location accordingly.
2011 			 */
2012 			if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
2013 				hdr->csum_start -= ETHER_VLAN_ENCAP_LEN;
2014 		}
2015 	}
2016 
2017 	m->m_pkthdr.flowid = rxq->vtnrx_id;
2018 	M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2019 
2020 	if (hdr->flags &
2021 	    (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID)) {
2022 		if (vtnet_rxq_csum(rxq, m, hdr) == 0)
2023 			rxq->vtnrx_stats.vrxs_csum++;
2024 		else
2025 			rxq->vtnrx_stats.vrxs_csum_failed++;
2026 	}
2027 
2028 	if (hdr->gso_size != 0) {
2029 		switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2030 		case VIRTIO_NET_HDR_GSO_TCPV4:
2031 		case VIRTIO_NET_HDR_GSO_TCPV6:
2032 			m->m_pkthdr.lro_nsegs =
2033 			    howmany(m->m_pkthdr.len, hdr->gso_size);
2034 			rxq->vtnrx_stats.vrxs_host_lro++;
2035 			break;
2036 		}
2037 	}
2038 
2039 	rxq->vtnrx_stats.vrxs_ipackets++;
2040 	rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len;
2041 
2042 #if defined(INET) || defined(INET6)
2043 	if (vtnet_software_lro(sc) && ifp->if_capenable & IFCAP_LRO) {
2044 		if (vtnet_lro_rx(rxq, m) == 0)
2045 			return;
2046 	}
2047 #endif
2048 
2049 	(*ifp->if_input)(ifp, m);
2050 }
2051 
2052 static int
2053 vtnet_rxq_eof(struct vtnet_rxq *rxq)
2054 {
2055 	struct virtio_net_hdr lhdr, *hdr;
2056 	struct vtnet_softc *sc;
2057 	struct ifnet *ifp;
2058 	struct virtqueue *vq;
2059 	int deq, count;
2060 
2061 	sc = rxq->vtnrx_sc;
2062 	vq = rxq->vtnrx_vq;
2063 	ifp = sc->vtnet_ifp;
2064 	deq = 0;
2065 	count = sc->vtnet_rx_process_limit;
2066 
2067 	VTNET_RXQ_LOCK_ASSERT(rxq);
2068 
2069 	while (count-- > 0) {
2070 		struct mbuf *m;
2071 		uint32_t len, nbufs, adjsz;
2072 
2073 		m = virtqueue_dequeue(vq, &len);
2074 		if (m == NULL)
2075 			break;
2076 		deq++;
2077 
2078 		if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
2079 			rxq->vtnrx_stats.vrxs_ierrors++;
2080 			vtnet_rxq_discard_buf(rxq, m);
2081 			continue;
2082 		}
2083 
2084 		if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) {
2085 			struct virtio_net_hdr_mrg_rxbuf *mhdr =
2086 			    mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
2087 			kmsan_mark(mhdr, sizeof(*mhdr), KMSAN_STATE_INITED);
2088 			nbufs = vtnet_htog16(sc, mhdr->num_buffers);
2089 			adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2090 		} else if (vtnet_modern(sc)) {
2091 			nbufs = 1; /* num_buffers is always 1 */
2092 			adjsz = sizeof(struct virtio_net_hdr_v1);
2093 		} else {
2094 			nbufs = 1;
2095 			adjsz = sizeof(struct vtnet_rx_header);
2096 			/*
2097 			 * Account for our gap between the header and start of
2098 			 * data to keep the segments separated.
2099 			 */
2100 			len += VTNET_RX_HEADER_PAD;
2101 		}
2102 
2103 		if (vtnet_rxq_replace_buf(rxq, m, len) != 0) {
2104 			rxq->vtnrx_stats.vrxs_iqdrops++;
2105 			vtnet_rxq_discard_buf(rxq, m);
2106 			if (nbufs > 1)
2107 				vtnet_rxq_discard_merged_bufs(rxq, nbufs);
2108 			continue;
2109 		}
2110 
2111 		m->m_pkthdr.len = len;
2112 		m->m_pkthdr.rcvif = ifp;
2113 		m->m_pkthdr.csum_flags = 0;
2114 
2115 		if (nbufs > 1) {
2116 			/* Dequeue the rest of chain. */
2117 			if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0)
2118 				continue;
2119 		}
2120 
2121 		kmsan_mark_mbuf(m, KMSAN_STATE_INITED);
2122 
2123 		/*
2124 		 * Save an endian swapped version of the header prior to it
2125 		 * being stripped. The header is always at the start of the
2126 		 * mbuf data. num_buffers was already saved (and not needed)
2127 		 * so use the standard header.
2128 		 */
2129 		hdr = mtod(m, struct virtio_net_hdr *);
2130 		lhdr.flags = hdr->flags;
2131 		lhdr.gso_type = hdr->gso_type;
2132 		lhdr.hdr_len = vtnet_htog16(sc, hdr->hdr_len);
2133 		lhdr.gso_size = vtnet_htog16(sc, hdr->gso_size);
2134 		lhdr.csum_start = vtnet_htog16(sc, hdr->csum_start);
2135 		lhdr.csum_offset = vtnet_htog16(sc, hdr->csum_offset);
2136 		m_adj(m, adjsz);
2137 
2138 		if (PFIL_HOOKED_IN(sc->vtnet_pfil)) {
2139 			pfil_return_t pfil;
2140 
2141 			pfil = pfil_run_hooks(sc->vtnet_pfil, &m, ifp, PFIL_IN,
2142 			    NULL);
2143 			switch (pfil) {
2144 			case PFIL_REALLOCED:
2145 				m = pfil_mem2mbuf(m->m_data);
2146 				break;
2147 			case PFIL_DROPPED:
2148 			case PFIL_CONSUMED:
2149 				continue;
2150 			default:
2151 				KASSERT(pfil == PFIL_PASS,
2152 				    ("Filter returned %d!", pfil));
2153 			}
2154 		}
2155 
2156 		vtnet_rxq_input(rxq, m, &lhdr);
2157 	}
2158 
2159 	if (deq > 0) {
2160 #if defined(INET) || defined(INET6)
2161 		if (vtnet_software_lro(sc))
2162 			tcp_lro_flush_all(&rxq->vtnrx_lro);
2163 #endif
2164 		virtqueue_notify(vq);
2165 	}
2166 
2167 	return (count > 0 ? 0 : EAGAIN);
2168 }
2169 
2170 static void
2171 vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries)
2172 {
2173 	struct vtnet_softc *sc;
2174 	struct ifnet *ifp;
2175 	u_int more;
2176 #ifdef DEV_NETMAP
2177 	int nmirq;
2178 #endif /* DEV_NETMAP */
2179 
2180 	sc = rxq->vtnrx_sc;
2181 	ifp = sc->vtnet_ifp;
2182 
2183 	if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) {
2184 		/*
2185 		 * Ignore this interrupt. Either this is a spurious interrupt
2186 		 * or multiqueue without per-VQ MSIX so every queue needs to
2187 		 * be polled (a brain dead configuration we could try harder
2188 		 * to avoid).
2189 		 */
2190 		vtnet_rxq_disable_intr(rxq);
2191 		return;
2192 	}
2193 
2194 	VTNET_RXQ_LOCK(rxq);
2195 
2196 #ifdef DEV_NETMAP
2197 	/*
2198 	 * We call netmap_rx_irq() under lock to prevent concurrent calls.
2199 	 * This is not necessary to serialize the access to the RX vq, but
2200 	 * rather to avoid races that may happen if this interface is
2201 	 * attached to a VALE switch, which would cause received packets
2202 	 * to stall in the RX queue (nm_kr_tryget() could find the kring
2203 	 * busy when called from netmap_bwrap_intr_notify()).
2204 	 */
2205 	nmirq = netmap_rx_irq(ifp, rxq->vtnrx_id, &more);
2206 	if (nmirq != NM_IRQ_PASS) {
2207 		VTNET_RXQ_UNLOCK(rxq);
2208 		if (nmirq == NM_IRQ_RESCHED) {
2209 			taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
2210 		}
2211 		return;
2212 	}
2213 #endif /* DEV_NETMAP */
2214 
2215 again:
2216 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2217 		VTNET_RXQ_UNLOCK(rxq);
2218 		return;
2219 	}
2220 
2221 	more = vtnet_rxq_eof(rxq);
2222 	if (more || vtnet_rxq_enable_intr(rxq) != 0) {
2223 		if (!more)
2224 			vtnet_rxq_disable_intr(rxq);
2225 		/*
2226 		 * This is an occasional condition or race (when !more),
2227 		 * so retry a few times before scheduling the taskqueue.
2228 		 */
2229 		if (tries-- > 0)
2230 			goto again;
2231 
2232 		rxq->vtnrx_stats.vrxs_rescheduled++;
2233 		VTNET_RXQ_UNLOCK(rxq);
2234 		taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
2235 	} else
2236 		VTNET_RXQ_UNLOCK(rxq);
2237 }
2238 
2239 static void
2240 vtnet_rx_vq_intr(void *xrxq)
2241 {
2242 	struct vtnet_rxq *rxq;
2243 
2244 	rxq = xrxq;
2245 	vtnet_rx_vq_process(rxq, VTNET_INTR_DISABLE_RETRIES);
2246 }
2247 
2248 static void
2249 vtnet_rxq_tq_intr(void *xrxq, int pending __unused)
2250 {
2251 	struct vtnet_rxq *rxq;
2252 
2253 	rxq = xrxq;
2254 	vtnet_rx_vq_process(rxq, 0);
2255 }
2256 
2257 static int
2258 vtnet_txq_intr_threshold(struct vtnet_txq *txq)
2259 {
2260 	struct vtnet_softc *sc;
2261 	int threshold;
2262 
2263 	sc = txq->vtntx_sc;
2264 
2265 	/*
2266 	 * The Tx interrupt is disabled until the queue free count falls
2267 	 * below our threshold. Completed frames are drained from the Tx
2268 	 * virtqueue before transmitting new frames and in the watchdog
2269 	 * callout, so the frequency of Tx interrupts is greatly reduced,
2270 	 * at the cost of not freeing mbufs as quickly as they otherwise
2271 	 * would be.
2272 	 */
2273 	threshold = virtqueue_size(txq->vtntx_vq) / 4;
2274 
2275 	/*
2276 	 * Without indirect descriptors, leave enough room for the most
2277 	 * segments we handle.
2278 	 */
2279 	if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 &&
2280 	    threshold < sc->vtnet_tx_nsegs)
2281 		threshold = sc->vtnet_tx_nsegs;
2282 
2283 	return (threshold);
2284 }
2285 
2286 static int
2287 vtnet_txq_below_threshold(struct vtnet_txq *txq)
2288 {
2289 	struct virtqueue *vq;
2290 
2291 	vq = txq->vtntx_vq;
2292 
2293 	return (virtqueue_nfree(vq) <= txq->vtntx_intr_threshold);
2294 }
2295 
2296 static int
2297 vtnet_txq_notify(struct vtnet_txq *txq)
2298 {
2299 	struct virtqueue *vq;
2300 
2301 	vq = txq->vtntx_vq;
2302 
2303 	txq->vtntx_watchdog = VTNET_TX_TIMEOUT;
2304 	virtqueue_notify(vq);
2305 
2306 	if (vtnet_txq_enable_intr(txq) == 0)
2307 		return (0);
2308 
2309 	/*
2310 	 * Drain frames that were completed since last checked. If this
2311 	 * causes the queue to go above the threshold, the caller should
2312 	 * continue transmitting.
2313 	 */
2314 	if (vtnet_txq_eof(txq) != 0 && vtnet_txq_below_threshold(txq) == 0) {
2315 		virtqueue_disable_intr(vq);
2316 		return (1);
2317 	}
2318 
2319 	return (0);
2320 }
2321 
2322 static void
2323 vtnet_txq_free_mbufs(struct vtnet_txq *txq)
2324 {
2325 	struct virtqueue *vq;
2326 	struct vtnet_tx_header *txhdr;
2327 	int last;
2328 #ifdef DEV_NETMAP
2329 	struct netmap_kring *kring = netmap_kring_on(NA(txq->vtntx_sc->vtnet_ifp),
2330 							txq->vtntx_id, NR_TX);
2331 #else  /* !DEV_NETMAP */
2332 	void *kring = NULL;
2333 #endif /* !DEV_NETMAP */
2334 
2335 	vq = txq->vtntx_vq;
2336 	last = 0;
2337 
2338 	while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
2339 		if (kring == NULL) {
2340 			m_freem(txhdr->vth_mbuf);
2341 			uma_zfree(vtnet_tx_header_zone, txhdr);
2342 		}
2343 	}
2344 
2345 	KASSERT(virtqueue_empty(vq),
2346 	    ("%s: mbufs remaining in tx queue %p", __func__, txq));
2347 }
2348 
2349 /*
2350  * BMV: This can go away once we finally have offsets in the mbuf header.
2351  */
2352 static int
2353 vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m, int *etype,
2354     int *proto, int *start)
2355 {
2356 	struct vtnet_softc *sc;
2357 	struct ether_vlan_header *evh;
2358 	int offset;
2359 
2360 	sc = txq->vtntx_sc;
2361 
2362 	evh = mtod(m, struct ether_vlan_header *);
2363 	if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2364 		/* BMV: We should handle nested VLAN tags too. */
2365 		*etype = ntohs(evh->evl_proto);
2366 		offset = sizeof(struct ether_vlan_header);
2367 	} else {
2368 		*etype = ntohs(evh->evl_encap_proto);
2369 		offset = sizeof(struct ether_header);
2370 	}
2371 
2372 	switch (*etype) {
2373 #if defined(INET)
2374 	case ETHERTYPE_IP: {
2375 		struct ip *ip, iphdr;
2376 		if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
2377 			m_copydata(m, offset, sizeof(struct ip),
2378 			    (caddr_t) &iphdr);
2379 			ip = &iphdr;
2380 		} else
2381 			ip = (struct ip *)(m->m_data + offset);
2382 		*proto = ip->ip_p;
2383 		*start = offset + (ip->ip_hl << 2);
2384 		break;
2385 	}
2386 #endif
2387 #if defined(INET6)
2388 	case ETHERTYPE_IPV6:
2389 		*proto = -1;
2390 		*start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
2391 		/* Assert the network stack sent us a valid packet. */
2392 		KASSERT(*start > offset,
2393 		    ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
2394 		    *start, offset, *proto));
2395 		break;
2396 #endif
2397 	default:
2398 		sc->vtnet_stats.tx_csum_unknown_ethtype++;
2399 		return (EINVAL);
2400 	}
2401 
2402 	return (0);
2403 }
2404 
2405 static int
2406 vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type,
2407     int offset, struct virtio_net_hdr *hdr)
2408 {
2409 	static struct timeval lastecn;
2410 	static int curecn;
2411 	struct vtnet_softc *sc;
2412 	struct tcphdr *tcp, tcphdr;
2413 
2414 	sc = txq->vtntx_sc;
2415 
2416 	if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) {
2417 		m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr);
2418 		tcp = &tcphdr;
2419 	} else
2420 		tcp = (struct tcphdr *)(m->m_data + offset);
2421 
2422 	hdr->hdr_len = vtnet_gtoh16(sc, offset + (tcp->th_off << 2));
2423 	hdr->gso_size = vtnet_gtoh16(sc, m->m_pkthdr.tso_segsz);
2424 	hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
2425 	    VIRTIO_NET_HDR_GSO_TCPV6;
2426 
2427 	if (__predict_false(tcp->th_flags & TH_CWR)) {
2428 		/*
2429 		 * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In
2430 		 * FreeBSD, ECN support is not on a per-interface basis,
2431 		 * but globally via the net.inet.tcp.ecn.enable sysctl
2432 		 * knob. The default is off.
2433 		 */
2434 		if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
2435 			if (ppsratecheck(&lastecn, &curecn, 1))
2436 				if_printf(sc->vtnet_ifp,
2437 				    "TSO with ECN not negotiated with host\n");
2438 			return (ENOTSUP);
2439 		}
2440 		hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2441 	}
2442 
2443 	txq->vtntx_stats.vtxs_tso++;
2444 
2445 	return (0);
2446 }
2447 
2448 static struct mbuf *
2449 vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m,
2450     struct virtio_net_hdr *hdr)
2451 {
2452 	struct vtnet_softc *sc;
2453 	int flags, etype, csum_start, proto, error;
2454 
2455 	sc = txq->vtntx_sc;
2456 	flags = m->m_pkthdr.csum_flags;
2457 
2458 	error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start);
2459 	if (error)
2460 		goto drop;
2461 
2462 	if (flags & (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6)) {
2463 		/* Sanity check the parsed mbuf matches the offload flags. */
2464 		if (__predict_false((flags & VTNET_CSUM_OFFLOAD &&
2465 		    etype != ETHERTYPE_IP) || (flags & VTNET_CSUM_OFFLOAD_IPV6
2466 		    && etype != ETHERTYPE_IPV6))) {
2467 			sc->vtnet_stats.tx_csum_proto_mismatch++;
2468 			goto drop;
2469 		}
2470 
2471 		hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
2472 		hdr->csum_start = vtnet_gtoh16(sc, csum_start);
2473 		hdr->csum_offset = vtnet_gtoh16(sc, m->m_pkthdr.csum_data);
2474 		txq->vtntx_stats.vtxs_csum++;
2475 	}
2476 
2477 	if (flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) {
2478 		/*
2479 		 * Sanity check the parsed mbuf IP protocol is TCP, and
2480 		 * VirtIO TSO reqires the checksum offloading above.
2481 		 */
2482 		if (__predict_false(proto != IPPROTO_TCP)) {
2483 			sc->vtnet_stats.tx_tso_not_tcp++;
2484 			goto drop;
2485 		} else if (__predict_false((hdr->flags &
2486 		    VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) {
2487 			sc->vtnet_stats.tx_tso_without_csum++;
2488 			goto drop;
2489 		}
2490 
2491 		error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr);
2492 		if (error)
2493 			goto drop;
2494 	}
2495 
2496 	return (m);
2497 
2498 drop:
2499 	m_freem(m);
2500 	return (NULL);
2501 }
2502 
2503 static int
2504 vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head,
2505     struct vtnet_tx_header *txhdr)
2506 {
2507 	struct vtnet_softc *sc;
2508 	struct virtqueue *vq;
2509 	struct sglist *sg;
2510 	struct mbuf *m;
2511 	int error;
2512 
2513 	sc = txq->vtntx_sc;
2514 	vq = txq->vtntx_vq;
2515 	sg = txq->vtntx_sg;
2516 	m = *m_head;
2517 
2518 	sglist_reset(sg);
2519 	error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
2520 	if (error != 0 || sg->sg_nseg != 1) {
2521 		KASSERT(0, ("%s: cannot add header to sglist error %d nseg %d",
2522 		    __func__, error, sg->sg_nseg));
2523 		goto fail;
2524 	}
2525 
2526 	error = sglist_append_mbuf(sg, m);
2527 	if (error) {
2528 		m = m_defrag(m, M_NOWAIT);
2529 		if (m == NULL)
2530 			goto fail;
2531 
2532 		*m_head = m;
2533 		sc->vtnet_stats.tx_defragged++;
2534 
2535 		error = sglist_append_mbuf(sg, m);
2536 		if (error)
2537 			goto fail;
2538 	}
2539 
2540 	txhdr->vth_mbuf = m;
2541 	error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0);
2542 
2543 	return (error);
2544 
2545 fail:
2546 	sc->vtnet_stats.tx_defrag_failed++;
2547 	m_freem(*m_head);
2548 	*m_head = NULL;
2549 
2550 	return (ENOBUFS);
2551 }
2552 
2553 static int
2554 vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head, int flags)
2555 {
2556 	struct vtnet_tx_header *txhdr;
2557 	struct virtio_net_hdr *hdr;
2558 	struct mbuf *m;
2559 	int error;
2560 
2561 	m = *m_head;
2562 	M_ASSERTPKTHDR(m);
2563 
2564 	txhdr = uma_zalloc(vtnet_tx_header_zone, flags | M_ZERO);
2565 	if (txhdr == NULL) {
2566 		m_freem(m);
2567 		*m_head = NULL;
2568 		return (ENOMEM);
2569 	}
2570 
2571 	/*
2572 	 * Always use the non-mergeable header, regardless if mergable headers
2573 	 * were negotiated, because for transmit num_buffers is always zero.
2574 	 * The vtnet_hdr_size is used to enqueue the right header size segment.
2575 	 */
2576 	hdr = &txhdr->vth_uhdr.hdr;
2577 
2578 	if (m->m_flags & M_VLANTAG) {
2579 		m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
2580 		if ((*m_head = m) == NULL) {
2581 			error = ENOBUFS;
2582 			goto fail;
2583 		}
2584 		m->m_flags &= ~M_VLANTAG;
2585 	}
2586 
2587 	if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) {
2588 		m = vtnet_txq_offload(txq, m, hdr);
2589 		if ((*m_head = m) == NULL) {
2590 			error = ENOBUFS;
2591 			goto fail;
2592 		}
2593 	}
2594 
2595 	error = vtnet_txq_enqueue_buf(txq, m_head, txhdr);
2596 fail:
2597 	if (error)
2598 		uma_zfree(vtnet_tx_header_zone, txhdr);
2599 
2600 	return (error);
2601 }
2602 
2603 #ifdef VTNET_LEGACY_TX
2604 
2605 static void
2606 vtnet_start_locked(struct vtnet_txq *txq, struct ifnet *ifp)
2607 {
2608 	struct vtnet_softc *sc;
2609 	struct virtqueue *vq;
2610 	struct mbuf *m0;
2611 	int tries, enq;
2612 
2613 	sc = txq->vtntx_sc;
2614 	vq = txq->vtntx_vq;
2615 	tries = 0;
2616 
2617 	VTNET_TXQ_LOCK_ASSERT(txq);
2618 
2619 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2620 	    sc->vtnet_link_active == 0)
2621 		return;
2622 
2623 	vtnet_txq_eof(txq);
2624 
2625 again:
2626 	enq = 0;
2627 
2628 	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
2629 		if (virtqueue_full(vq))
2630 			break;
2631 
2632 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
2633 		if (m0 == NULL)
2634 			break;
2635 
2636 		if (vtnet_txq_encap(txq, &m0, M_NOWAIT) != 0) {
2637 			if (m0 != NULL)
2638 				IFQ_DRV_PREPEND(&ifp->if_snd, m0);
2639 			break;
2640 		}
2641 
2642 		enq++;
2643 		ETHER_BPF_MTAP(ifp, m0);
2644 	}
2645 
2646 	if (enq > 0 && vtnet_txq_notify(txq) != 0) {
2647 		if (tries++ < VTNET_NOTIFY_RETRIES)
2648 			goto again;
2649 
2650 		txq->vtntx_stats.vtxs_rescheduled++;
2651 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask);
2652 	}
2653 }
2654 
2655 static void
2656 vtnet_start(struct ifnet *ifp)
2657 {
2658 	struct vtnet_softc *sc;
2659 	struct vtnet_txq *txq;
2660 
2661 	sc = ifp->if_softc;
2662 	txq = &sc->vtnet_txqs[0];
2663 
2664 	VTNET_TXQ_LOCK(txq);
2665 	vtnet_start_locked(txq, ifp);
2666 	VTNET_TXQ_UNLOCK(txq);
2667 }
2668 
2669 #else /* !VTNET_LEGACY_TX */
2670 
2671 static int
2672 vtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m)
2673 {
2674 	struct vtnet_softc *sc;
2675 	struct virtqueue *vq;
2676 	struct buf_ring *br;
2677 	struct ifnet *ifp;
2678 	int enq, tries, error;
2679 
2680 	sc = txq->vtntx_sc;
2681 	vq = txq->vtntx_vq;
2682 	br = txq->vtntx_br;
2683 	ifp = sc->vtnet_ifp;
2684 	tries = 0;
2685 	error = 0;
2686 
2687 	VTNET_TXQ_LOCK_ASSERT(txq);
2688 
2689 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2690 	    sc->vtnet_link_active == 0) {
2691 		if (m != NULL)
2692 			error = drbr_enqueue(ifp, br, m);
2693 		return (error);
2694 	}
2695 
2696 	if (m != NULL) {
2697 		error = drbr_enqueue(ifp, br, m);
2698 		if (error)
2699 			return (error);
2700 	}
2701 
2702 	vtnet_txq_eof(txq);
2703 
2704 again:
2705 	enq = 0;
2706 
2707 	while ((m = drbr_peek(ifp, br)) != NULL) {
2708 		if (virtqueue_full(vq)) {
2709 			drbr_putback(ifp, br, m);
2710 			break;
2711 		}
2712 
2713 		if (vtnet_txq_encap(txq, &m, M_NOWAIT) != 0) {
2714 			if (m != NULL)
2715 				drbr_putback(ifp, br, m);
2716 			else
2717 				drbr_advance(ifp, br);
2718 			break;
2719 		}
2720 		drbr_advance(ifp, br);
2721 
2722 		enq++;
2723 		ETHER_BPF_MTAP(ifp, m);
2724 	}
2725 
2726 	if (enq > 0 && vtnet_txq_notify(txq) != 0) {
2727 		if (tries++ < VTNET_NOTIFY_RETRIES)
2728 			goto again;
2729 
2730 		txq->vtntx_stats.vtxs_rescheduled++;
2731 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask);
2732 	}
2733 
2734 	return (0);
2735 }
2736 
2737 static int
2738 vtnet_txq_mq_start(struct ifnet *ifp, struct mbuf *m)
2739 {
2740 	struct vtnet_softc *sc;
2741 	struct vtnet_txq *txq;
2742 	int i, npairs, error;
2743 
2744 	sc = ifp->if_softc;
2745 	npairs = sc->vtnet_act_vq_pairs;
2746 
2747 	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
2748 		i = m->m_pkthdr.flowid % npairs;
2749 	else
2750 		i = curcpu % npairs;
2751 
2752 	txq = &sc->vtnet_txqs[i];
2753 
2754 	if (VTNET_TXQ_TRYLOCK(txq) != 0) {
2755 		error = vtnet_txq_mq_start_locked(txq, m);
2756 		VTNET_TXQ_UNLOCK(txq);
2757 	} else {
2758 		error = drbr_enqueue(ifp, txq->vtntx_br, m);
2759 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask);
2760 	}
2761 
2762 	return (error);
2763 }
2764 
2765 static void
2766 vtnet_txq_tq_deferred(void *xtxq, int pending __unused)
2767 {
2768 	struct vtnet_softc *sc;
2769 	struct vtnet_txq *txq;
2770 
2771 	txq = xtxq;
2772 	sc = txq->vtntx_sc;
2773 
2774 	VTNET_TXQ_LOCK(txq);
2775 	if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br))
2776 		vtnet_txq_mq_start_locked(txq, NULL);
2777 	VTNET_TXQ_UNLOCK(txq);
2778 }
2779 
2780 #endif /* VTNET_LEGACY_TX */
2781 
2782 static void
2783 vtnet_txq_start(struct vtnet_txq *txq)
2784 {
2785 	struct vtnet_softc *sc;
2786 	struct ifnet *ifp;
2787 
2788 	sc = txq->vtntx_sc;
2789 	ifp = sc->vtnet_ifp;
2790 
2791 #ifdef VTNET_LEGACY_TX
2792 	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2793 		vtnet_start_locked(txq, ifp);
2794 #else
2795 	if (!drbr_empty(ifp, txq->vtntx_br))
2796 		vtnet_txq_mq_start_locked(txq, NULL);
2797 #endif
2798 }
2799 
2800 static void
2801 vtnet_txq_tq_intr(void *xtxq, int pending __unused)
2802 {
2803 	struct vtnet_softc *sc;
2804 	struct vtnet_txq *txq;
2805 	struct ifnet *ifp;
2806 
2807 	txq = xtxq;
2808 	sc = txq->vtntx_sc;
2809 	ifp = sc->vtnet_ifp;
2810 
2811 	VTNET_TXQ_LOCK(txq);
2812 
2813 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2814 		VTNET_TXQ_UNLOCK(txq);
2815 		return;
2816 	}
2817 
2818 	vtnet_txq_eof(txq);
2819 	vtnet_txq_start(txq);
2820 
2821 	VTNET_TXQ_UNLOCK(txq);
2822 }
2823 
2824 static int
2825 vtnet_txq_eof(struct vtnet_txq *txq)
2826 {
2827 	struct virtqueue *vq;
2828 	struct vtnet_tx_header *txhdr;
2829 	struct mbuf *m;
2830 	int deq;
2831 
2832 	vq = txq->vtntx_vq;
2833 	deq = 0;
2834 	VTNET_TXQ_LOCK_ASSERT(txq);
2835 
2836 	while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
2837 		m = txhdr->vth_mbuf;
2838 		deq++;
2839 
2840 		txq->vtntx_stats.vtxs_opackets++;
2841 		txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len;
2842 		if (m->m_flags & M_MCAST)
2843 			txq->vtntx_stats.vtxs_omcasts++;
2844 
2845 		m_freem(m);
2846 		uma_zfree(vtnet_tx_header_zone, txhdr);
2847 	}
2848 
2849 	if (virtqueue_empty(vq))
2850 		txq->vtntx_watchdog = 0;
2851 
2852 	return (deq);
2853 }
2854 
2855 static void
2856 vtnet_tx_vq_intr(void *xtxq)
2857 {
2858 	struct vtnet_softc *sc;
2859 	struct vtnet_txq *txq;
2860 	struct ifnet *ifp;
2861 
2862 	txq = xtxq;
2863 	sc = txq->vtntx_sc;
2864 	ifp = sc->vtnet_ifp;
2865 
2866 	if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) {
2867 		/*
2868 		 * Ignore this interrupt. Either this is a spurious interrupt
2869 		 * or multiqueue without per-VQ MSIX so every queue needs to
2870 		 * be polled (a brain dead configuration we could try harder
2871 		 * to avoid).
2872 		 */
2873 		vtnet_txq_disable_intr(txq);
2874 		return;
2875 	}
2876 
2877 #ifdef DEV_NETMAP
2878 	if (netmap_tx_irq(ifp, txq->vtntx_id) != NM_IRQ_PASS)
2879 		return;
2880 #endif /* DEV_NETMAP */
2881 
2882 	VTNET_TXQ_LOCK(txq);
2883 
2884 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2885 		VTNET_TXQ_UNLOCK(txq);
2886 		return;
2887 	}
2888 
2889 	vtnet_txq_eof(txq);
2890 	vtnet_txq_start(txq);
2891 
2892 	VTNET_TXQ_UNLOCK(txq);
2893 }
2894 
2895 static void
2896 vtnet_tx_start_all(struct vtnet_softc *sc)
2897 {
2898 	struct vtnet_txq *txq;
2899 	int i;
2900 
2901 	VTNET_CORE_LOCK_ASSERT(sc);
2902 
2903 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
2904 		txq = &sc->vtnet_txqs[i];
2905 
2906 		VTNET_TXQ_LOCK(txq);
2907 		vtnet_txq_start(txq);
2908 		VTNET_TXQ_UNLOCK(txq);
2909 	}
2910 }
2911 
2912 #ifndef VTNET_LEGACY_TX
2913 static void
2914 vtnet_qflush(struct ifnet *ifp)
2915 {
2916 	struct vtnet_softc *sc;
2917 	struct vtnet_txq *txq;
2918 	struct mbuf *m;
2919 	int i;
2920 
2921 	sc = ifp->if_softc;
2922 
2923 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
2924 		txq = &sc->vtnet_txqs[i];
2925 
2926 		VTNET_TXQ_LOCK(txq);
2927 		while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL)
2928 			m_freem(m);
2929 		VTNET_TXQ_UNLOCK(txq);
2930 	}
2931 
2932 	if_qflush(ifp);
2933 }
2934 #endif
2935 
2936 static int
2937 vtnet_watchdog(struct vtnet_txq *txq)
2938 {
2939 	struct ifnet *ifp;
2940 
2941 	ifp = txq->vtntx_sc->vtnet_ifp;
2942 
2943 	VTNET_TXQ_LOCK(txq);
2944 	if (txq->vtntx_watchdog == 1) {
2945 		/*
2946 		 * Only drain completed frames if the watchdog is about to
2947 		 * expire. If any frames were drained, there may be enough
2948 		 * free descriptors now available to transmit queued frames.
2949 		 * In that case, the timer will immediately be decremented
2950 		 * below, but the timeout is generous enough that should not
2951 		 * be a problem.
2952 		 */
2953 		if (vtnet_txq_eof(txq) != 0)
2954 			vtnet_txq_start(txq);
2955 	}
2956 
2957 	if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) {
2958 		VTNET_TXQ_UNLOCK(txq);
2959 		return (0);
2960 	}
2961 	VTNET_TXQ_UNLOCK(txq);
2962 
2963 	if_printf(ifp, "watchdog timeout on queue %d\n", txq->vtntx_id);
2964 	return (1);
2965 }
2966 
2967 static void
2968 vtnet_accum_stats(struct vtnet_softc *sc, struct vtnet_rxq_stats *rxacc,
2969     struct vtnet_txq_stats *txacc)
2970 {
2971 
2972 	bzero(rxacc, sizeof(struct vtnet_rxq_stats));
2973 	bzero(txacc, sizeof(struct vtnet_txq_stats));
2974 
2975 	for (int i = 0; i < sc->vtnet_max_vq_pairs; i++) {
2976 		struct vtnet_rxq_stats *rxst;
2977 		struct vtnet_txq_stats *txst;
2978 
2979 		rxst = &sc->vtnet_rxqs[i].vtnrx_stats;
2980 		rxacc->vrxs_ipackets += rxst->vrxs_ipackets;
2981 		rxacc->vrxs_ibytes += rxst->vrxs_ibytes;
2982 		rxacc->vrxs_iqdrops += rxst->vrxs_iqdrops;
2983 		rxacc->vrxs_csum += rxst->vrxs_csum;
2984 		rxacc->vrxs_csum_failed += rxst->vrxs_csum_failed;
2985 		rxacc->vrxs_rescheduled += rxst->vrxs_rescheduled;
2986 
2987 		txst = &sc->vtnet_txqs[i].vtntx_stats;
2988 		txacc->vtxs_opackets += txst->vtxs_opackets;
2989 		txacc->vtxs_obytes += txst->vtxs_obytes;
2990 		txacc->vtxs_csum += txst->vtxs_csum;
2991 		txacc->vtxs_tso += txst->vtxs_tso;
2992 		txacc->vtxs_rescheduled += txst->vtxs_rescheduled;
2993 	}
2994 }
2995 
2996 static uint64_t
2997 vtnet_get_counter(if_t ifp, ift_counter cnt)
2998 {
2999 	struct vtnet_softc *sc;
3000 	struct vtnet_rxq_stats rxaccum;
3001 	struct vtnet_txq_stats txaccum;
3002 
3003 	sc = if_getsoftc(ifp);
3004 	vtnet_accum_stats(sc, &rxaccum, &txaccum);
3005 
3006 	switch (cnt) {
3007 	case IFCOUNTER_IPACKETS:
3008 		return (rxaccum.vrxs_ipackets);
3009 	case IFCOUNTER_IQDROPS:
3010 		return (rxaccum.vrxs_iqdrops);
3011 	case IFCOUNTER_IERRORS:
3012 		return (rxaccum.vrxs_ierrors);
3013 	case IFCOUNTER_OPACKETS:
3014 		return (txaccum.vtxs_opackets);
3015 #ifndef VTNET_LEGACY_TX
3016 	case IFCOUNTER_OBYTES:
3017 		return (txaccum.vtxs_obytes);
3018 	case IFCOUNTER_OMCASTS:
3019 		return (txaccum.vtxs_omcasts);
3020 #endif
3021 	default:
3022 		return (if_get_counter_default(ifp, cnt));
3023 	}
3024 }
3025 
3026 static void
3027 vtnet_tick(void *xsc)
3028 {
3029 	struct vtnet_softc *sc;
3030 	struct ifnet *ifp;
3031 	int i, timedout;
3032 
3033 	sc = xsc;
3034 	ifp = sc->vtnet_ifp;
3035 	timedout = 0;
3036 
3037 	VTNET_CORE_LOCK_ASSERT(sc);
3038 
3039 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
3040 		timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]);
3041 
3042 	if (timedout != 0) {
3043 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3044 		vtnet_init_locked(sc, 0);
3045 	} else
3046 		callout_schedule(&sc->vtnet_tick_ch, hz);
3047 }
3048 
3049 static void
3050 vtnet_start_taskqueues(struct vtnet_softc *sc)
3051 {
3052 	device_t dev;
3053 	struct vtnet_rxq *rxq;
3054 	struct vtnet_txq *txq;
3055 	int i, error;
3056 
3057 	dev = sc->vtnet_dev;
3058 
3059 	/*
3060 	 * Errors here are very difficult to recover from - we cannot
3061 	 * easily fail because, if this is during boot, we will hang
3062 	 * when freeing any successfully started taskqueues because
3063 	 * the scheduler isn't up yet.
3064 	 *
3065 	 * Most drivers just ignore the return value - it only fails
3066 	 * with ENOMEM so an error is not likely.
3067 	 */
3068 	for (i = 0; i < sc->vtnet_req_vq_pairs; i++) {
3069 		rxq = &sc->vtnet_rxqs[i];
3070 		error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET,
3071 		    "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id);
3072 		if (error) {
3073 			device_printf(dev, "failed to start rx taskq %d\n",
3074 			    rxq->vtnrx_id);
3075 		}
3076 
3077 		txq = &sc->vtnet_txqs[i];
3078 		error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET,
3079 		    "%s txq %d", device_get_nameunit(dev), txq->vtntx_id);
3080 		if (error) {
3081 			device_printf(dev, "failed to start tx taskq %d\n",
3082 			    txq->vtntx_id);
3083 		}
3084 	}
3085 }
3086 
3087 static void
3088 vtnet_free_taskqueues(struct vtnet_softc *sc)
3089 {
3090 	struct vtnet_rxq *rxq;
3091 	struct vtnet_txq *txq;
3092 	int i;
3093 
3094 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
3095 		rxq = &sc->vtnet_rxqs[i];
3096 		if (rxq->vtnrx_tq != NULL) {
3097 			taskqueue_free(rxq->vtnrx_tq);
3098 			rxq->vtnrx_tq = NULL;
3099 		}
3100 
3101 		txq = &sc->vtnet_txqs[i];
3102 		if (txq->vtntx_tq != NULL) {
3103 			taskqueue_free(txq->vtntx_tq);
3104 			txq->vtntx_tq = NULL;
3105 		}
3106 	}
3107 }
3108 
3109 static void
3110 vtnet_drain_taskqueues(struct vtnet_softc *sc)
3111 {
3112 	struct vtnet_rxq *rxq;
3113 	struct vtnet_txq *txq;
3114 	int i;
3115 
3116 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
3117 		rxq = &sc->vtnet_rxqs[i];
3118 		if (rxq->vtnrx_tq != NULL)
3119 			taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
3120 
3121 		txq = &sc->vtnet_txqs[i];
3122 		if (txq->vtntx_tq != NULL) {
3123 			taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask);
3124 #ifndef VTNET_LEGACY_TX
3125 			taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask);
3126 #endif
3127 		}
3128 	}
3129 }
3130 
3131 static void
3132 vtnet_drain_rxtx_queues(struct vtnet_softc *sc)
3133 {
3134 	struct vtnet_rxq *rxq;
3135 	struct vtnet_txq *txq;
3136 	int i;
3137 
3138 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
3139 		rxq = &sc->vtnet_rxqs[i];
3140 		vtnet_rxq_free_mbufs(rxq);
3141 
3142 		txq = &sc->vtnet_txqs[i];
3143 		vtnet_txq_free_mbufs(txq);
3144 	}
3145 }
3146 
3147 static void
3148 vtnet_stop_rendezvous(struct vtnet_softc *sc)
3149 {
3150 	struct vtnet_rxq *rxq;
3151 	struct vtnet_txq *txq;
3152 	int i;
3153 
3154 	VTNET_CORE_LOCK_ASSERT(sc);
3155 
3156 	/*
3157 	 * Lock and unlock the per-queue mutex so we known the stop
3158 	 * state is visible. Doing only the active queues should be
3159 	 * sufficient, but it does not cost much extra to do all the
3160 	 * queues.
3161 	 */
3162 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
3163 		rxq = &sc->vtnet_rxqs[i];
3164 		VTNET_RXQ_LOCK(rxq);
3165 		VTNET_RXQ_UNLOCK(rxq);
3166 
3167 		txq = &sc->vtnet_txqs[i];
3168 		VTNET_TXQ_LOCK(txq);
3169 		VTNET_TXQ_UNLOCK(txq);
3170 	}
3171 }
3172 
3173 static void
3174 vtnet_stop(struct vtnet_softc *sc)
3175 {
3176 	device_t dev;
3177 	struct ifnet *ifp;
3178 
3179 	dev = sc->vtnet_dev;
3180 	ifp = sc->vtnet_ifp;
3181 
3182 	VTNET_CORE_LOCK_ASSERT(sc);
3183 
3184 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3185 	sc->vtnet_link_active = 0;
3186 	callout_stop(&sc->vtnet_tick_ch);
3187 
3188 	/* Only advisory. */
3189 	vtnet_disable_interrupts(sc);
3190 
3191 #ifdef DEV_NETMAP
3192 	/* Stop any pending txsync/rxsync and disable them. */
3193 	netmap_disable_all_rings(ifp);
3194 #endif /* DEV_NETMAP */
3195 
3196 	/*
3197 	 * Stop the host adapter. This resets it to the pre-initialized
3198 	 * state. It will not generate any interrupts until after it is
3199 	 * reinitialized.
3200 	 */
3201 	virtio_stop(dev);
3202 	vtnet_stop_rendezvous(sc);
3203 
3204 	vtnet_drain_rxtx_queues(sc);
3205 	sc->vtnet_act_vq_pairs = 1;
3206 }
3207 
3208 static int
3209 vtnet_virtio_reinit(struct vtnet_softc *sc)
3210 {
3211 	device_t dev;
3212 	struct ifnet *ifp;
3213 	uint64_t features;
3214 	int error;
3215 
3216 	dev = sc->vtnet_dev;
3217 	ifp = sc->vtnet_ifp;
3218 	features = sc->vtnet_negotiated_features;
3219 
3220 	/*
3221 	 * Re-negotiate with the host, removing any disabled receive
3222 	 * features. Transmit features are disabled only on our side
3223 	 * via if_capenable and if_hwassist.
3224 	 */
3225 
3226 	if ((ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) == 0)
3227 		features &= ~(VIRTIO_NET_F_GUEST_CSUM | VTNET_LRO_FEATURES);
3228 
3229 	if ((ifp->if_capenable & IFCAP_LRO) == 0)
3230 		features &= ~VTNET_LRO_FEATURES;
3231 
3232 	if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
3233 		features &= ~VIRTIO_NET_F_CTRL_VLAN;
3234 
3235 	error = virtio_reinit(dev, features);
3236 	if (error) {
3237 		device_printf(dev, "virtio reinit error %d\n", error);
3238 		return (error);
3239 	}
3240 
3241 	sc->vtnet_features = features;
3242 	virtio_reinit_complete(dev);
3243 
3244 	return (0);
3245 }
3246 
3247 static void
3248 vtnet_init_rx_filters(struct vtnet_softc *sc)
3249 {
3250 	struct ifnet *ifp;
3251 
3252 	ifp = sc->vtnet_ifp;
3253 
3254 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
3255 		vtnet_rx_filter(sc);
3256 		vtnet_rx_filter_mac(sc);
3257 	}
3258 
3259 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
3260 		vtnet_rx_filter_vlan(sc);
3261 }
3262 
3263 static int
3264 vtnet_init_rx_queues(struct vtnet_softc *sc)
3265 {
3266 	device_t dev;
3267 	struct ifnet *ifp;
3268 	struct vtnet_rxq *rxq;
3269 	int i, clustersz, error;
3270 
3271 	dev = sc->vtnet_dev;
3272 	ifp = sc->vtnet_ifp;
3273 
3274 	clustersz = vtnet_rx_cluster_size(sc, ifp->if_mtu);
3275 	sc->vtnet_rx_clustersz = clustersz;
3276 
3277 	if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) {
3278 		sc->vtnet_rx_nmbufs = howmany(sizeof(struct vtnet_rx_header) +
3279 		    VTNET_MAX_RX_SIZE, clustersz);
3280 		KASSERT(sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs,
3281 		    ("%s: too many rx mbufs %d for %d segments", __func__,
3282 		    sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs));
3283 	} else
3284 		sc->vtnet_rx_nmbufs = 1;
3285 
3286 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
3287 		rxq = &sc->vtnet_rxqs[i];
3288 
3289 		/* Hold the lock to satisfy asserts. */
3290 		VTNET_RXQ_LOCK(rxq);
3291 		error = vtnet_rxq_populate(rxq);
3292 		VTNET_RXQ_UNLOCK(rxq);
3293 
3294 		if (error) {
3295 			device_printf(dev, "cannot populate Rx queue %d\n", i);
3296 			return (error);
3297 		}
3298 	}
3299 
3300 	return (0);
3301 }
3302 
3303 static int
3304 vtnet_init_tx_queues(struct vtnet_softc *sc)
3305 {
3306 	struct vtnet_txq *txq;
3307 	int i;
3308 
3309 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
3310 		txq = &sc->vtnet_txqs[i];
3311 		txq->vtntx_watchdog = 0;
3312 		txq->vtntx_intr_threshold = vtnet_txq_intr_threshold(txq);
3313 #ifdef DEV_NETMAP
3314 		netmap_reset(NA(sc->vtnet_ifp), NR_TX, i, 0);
3315 #endif /* DEV_NETMAP */
3316 	}
3317 
3318 	return (0);
3319 }
3320 
3321 static int
3322 vtnet_init_rxtx_queues(struct vtnet_softc *sc)
3323 {
3324 	int error;
3325 
3326 	error = vtnet_init_rx_queues(sc);
3327 	if (error)
3328 		return (error);
3329 
3330 	error = vtnet_init_tx_queues(sc);
3331 	if (error)
3332 		return (error);
3333 
3334 	return (0);
3335 }
3336 
3337 static void
3338 vtnet_set_active_vq_pairs(struct vtnet_softc *sc)
3339 {
3340 	device_t dev;
3341 	int npairs;
3342 
3343 	dev = sc->vtnet_dev;
3344 
3345 	if ((sc->vtnet_flags & VTNET_FLAG_MQ) == 0) {
3346 		sc->vtnet_act_vq_pairs = 1;
3347 		return;
3348 	}
3349 
3350 	npairs = sc->vtnet_req_vq_pairs;
3351 
3352 	if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) {
3353 		device_printf(dev, "cannot set active queue pairs to %d, "
3354 		    "falling back to 1 queue pair\n", npairs);
3355 		npairs = 1;
3356 	}
3357 
3358 	sc->vtnet_act_vq_pairs = npairs;
3359 }
3360 
3361 static void
3362 vtnet_update_rx_offloads(struct vtnet_softc *sc)
3363 {
3364 	struct ifnet *ifp;
3365 	uint64_t features;
3366 	int error;
3367 
3368 	ifp = sc->vtnet_ifp;
3369 	features = sc->vtnet_features;
3370 
3371 	VTNET_CORE_LOCK_ASSERT(sc);
3372 
3373 	if (ifp->if_capabilities & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
3374 		if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
3375 			features |= VIRTIO_NET_F_GUEST_CSUM;
3376 		else
3377 			features &= ~VIRTIO_NET_F_GUEST_CSUM;
3378 	}
3379 
3380 	if (ifp->if_capabilities & IFCAP_LRO && !vtnet_software_lro(sc)) {
3381 		if (ifp->if_capenable & IFCAP_LRO)
3382 			features |= VTNET_LRO_FEATURES;
3383 		else
3384 			features &= ~VTNET_LRO_FEATURES;
3385 	}
3386 
3387 	error = vtnet_ctrl_guest_offloads(sc,
3388 	    features & (VIRTIO_NET_F_GUEST_CSUM | VIRTIO_NET_F_GUEST_TSO4 |
3389 		        VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN  |
3390 			VIRTIO_NET_F_GUEST_UFO));
3391 	if (error) {
3392 		device_printf(sc->vtnet_dev,
3393 		    "%s: cannot update Rx features\n", __func__);
3394 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3395 			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3396 			vtnet_init_locked(sc, 0);
3397 		}
3398 	} else
3399 		sc->vtnet_features = features;
3400 }
3401 
3402 static int
3403 vtnet_reinit(struct vtnet_softc *sc)
3404 {
3405 	struct ifnet *ifp;
3406 	int error;
3407 
3408 	ifp = sc->vtnet_ifp;
3409 
3410 	bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
3411 
3412 	error = vtnet_virtio_reinit(sc);
3413 	if (error)
3414 		return (error);
3415 
3416 	vtnet_set_macaddr(sc);
3417 	vtnet_set_active_vq_pairs(sc);
3418 
3419 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ)
3420 		vtnet_init_rx_filters(sc);
3421 
3422 	ifp->if_hwassist = 0;
3423 	if (ifp->if_capenable & IFCAP_TXCSUM)
3424 		ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
3425 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
3426 		ifp->if_hwassist |= VTNET_CSUM_OFFLOAD_IPV6;
3427 	if (ifp->if_capenable & IFCAP_TSO4)
3428 		ifp->if_hwassist |= CSUM_IP_TSO;
3429 	if (ifp->if_capenable & IFCAP_TSO6)
3430 		ifp->if_hwassist |= CSUM_IP6_TSO;
3431 
3432 	error = vtnet_init_rxtx_queues(sc);
3433 	if (error)
3434 		return (error);
3435 
3436 	return (0);
3437 }
3438 
3439 static void
3440 vtnet_init_locked(struct vtnet_softc *sc, int init_mode)
3441 {
3442 	struct ifnet *ifp;
3443 
3444 	ifp = sc->vtnet_ifp;
3445 
3446 	VTNET_CORE_LOCK_ASSERT(sc);
3447 
3448 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3449 		return;
3450 
3451 	vtnet_stop(sc);
3452 
3453 #ifdef DEV_NETMAP
3454 	/* Once stopped we can update the netmap flags, if necessary. */
3455 	switch (init_mode) {
3456 	case VTNET_INIT_NETMAP_ENTER:
3457 		nm_set_native_flags(NA(ifp));
3458 		break;
3459 	case VTNET_INIT_NETMAP_EXIT:
3460 		nm_clear_native_flags(NA(ifp));
3461 		break;
3462 	}
3463 #endif /* DEV_NETMAP */
3464 
3465 	if (vtnet_reinit(sc) != 0) {
3466 		vtnet_stop(sc);
3467 		return;
3468 	}
3469 
3470 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
3471 	vtnet_update_link_status(sc);
3472 	vtnet_enable_interrupts(sc);
3473 	callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
3474 
3475 #ifdef DEV_NETMAP
3476 	/* Re-enable txsync/rxsync. */
3477 	netmap_enable_all_rings(ifp);
3478 #endif /* DEV_NETMAP */
3479 }
3480 
3481 static void
3482 vtnet_init(void *xsc)
3483 {
3484 	struct vtnet_softc *sc;
3485 
3486 	sc = xsc;
3487 
3488 	VTNET_CORE_LOCK(sc);
3489 	vtnet_init_locked(sc, 0);
3490 	VTNET_CORE_UNLOCK(sc);
3491 }
3492 
3493 static void
3494 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
3495 {
3496 
3497 	/*
3498 	 * The control virtqueue is only polled and therefore it should
3499 	 * already be empty.
3500 	 */
3501 	KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
3502 	    ("%s: ctrl vq %p not empty", __func__, sc->vtnet_ctrl_vq));
3503 }
3504 
3505 static void
3506 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
3507     struct sglist *sg, int readable, int writable)
3508 {
3509 	struct virtqueue *vq;
3510 
3511 	vq = sc->vtnet_ctrl_vq;
3512 
3513 	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ);
3514 	VTNET_CORE_LOCK_ASSERT(sc);
3515 
3516 	if (!virtqueue_empty(vq))
3517 		return;
3518 
3519 	/*
3520 	 * Poll for the response, but the command is likely completed before
3521 	 * returning from the notify.
3522 	 */
3523 	if (virtqueue_enqueue(vq, cookie, sg, readable, writable) == 0)  {
3524 		virtqueue_notify(vq);
3525 		virtqueue_poll(vq, NULL);
3526 	}
3527 }
3528 
3529 static int
3530 vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr)
3531 {
3532 	struct sglist_seg segs[3];
3533 	struct sglist sg;
3534 	struct {
3535 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3536 		uint8_t pad1;
3537 		uint8_t addr[ETHER_ADDR_LEN] __aligned(8);
3538 		uint8_t pad2;
3539 		uint8_t ack;
3540 	} s;
3541 	int error;
3542 
3543 	error = 0;
3544 	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_MAC);
3545 
3546 	s.hdr.class = VIRTIO_NET_CTRL_MAC;
3547 	s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
3548 	bcopy(hwaddr, &s.addr[0], ETHER_ADDR_LEN);
3549 	s.ack = VIRTIO_NET_ERR;
3550 
3551 	sglist_init(&sg, nitems(segs), segs);
3552 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3553 	error |= sglist_append(&sg, &s.addr[0], ETHER_ADDR_LEN);
3554 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3555 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3556 
3557 	if (error == 0)
3558 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3559 
3560 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3561 }
3562 
3563 static int
3564 vtnet_ctrl_guest_offloads(struct vtnet_softc *sc, uint64_t offloads)
3565 {
3566 	struct sglist_seg segs[3];
3567 	struct sglist sg;
3568 	struct {
3569 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3570 		uint8_t pad1;
3571 		uint64_t offloads __aligned(8);
3572 		uint8_t pad2;
3573 		uint8_t ack;
3574 	} s;
3575 	int error;
3576 
3577 	error = 0;
3578 	MPASS(sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS);
3579 
3580 	s.hdr.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
3581 	s.hdr.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
3582 	s.offloads = vtnet_gtoh64(sc, offloads);
3583 	s.ack = VIRTIO_NET_ERR;
3584 
3585 	sglist_init(&sg, nitems(segs), segs);
3586 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3587 	error |= sglist_append(&sg, &s.offloads, sizeof(uint64_t));
3588 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3589 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3590 
3591 	if (error == 0)
3592 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3593 
3594 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3595 }
3596 
3597 static int
3598 vtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs)
3599 {
3600 	struct sglist_seg segs[3];
3601 	struct sglist sg;
3602 	struct {
3603 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3604 		uint8_t pad1;
3605 		struct virtio_net_ctrl_mq mq __aligned(2);
3606 		uint8_t pad2;
3607 		uint8_t ack;
3608 	} s;
3609 	int error;
3610 
3611 	error = 0;
3612 	MPASS(sc->vtnet_flags & VTNET_FLAG_MQ);
3613 
3614 	s.hdr.class = VIRTIO_NET_CTRL_MQ;
3615 	s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
3616 	s.mq.virtqueue_pairs = vtnet_gtoh16(sc, npairs);
3617 	s.ack = VIRTIO_NET_ERR;
3618 
3619 	sglist_init(&sg, nitems(segs), segs);
3620 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3621 	error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq));
3622 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3623 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3624 
3625 	if (error == 0)
3626 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3627 
3628 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3629 }
3630 
3631 static int
3632 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, uint8_t cmd, bool on)
3633 {
3634 	struct sglist_seg segs[3];
3635 	struct sglist sg;
3636 	struct {
3637 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3638 		uint8_t pad1;
3639 		uint8_t onoff;
3640 		uint8_t pad2;
3641 		uint8_t ack;
3642 	} s;
3643 	int error;
3644 
3645 	error = 0;
3646 	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX);
3647 
3648 	s.hdr.class = VIRTIO_NET_CTRL_RX;
3649 	s.hdr.cmd = cmd;
3650 	s.onoff = on;
3651 	s.ack = VIRTIO_NET_ERR;
3652 
3653 	sglist_init(&sg, nitems(segs), segs);
3654 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3655 	error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t));
3656 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3657 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3658 
3659 	if (error == 0)
3660 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3661 
3662 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3663 }
3664 
3665 static int
3666 vtnet_set_promisc(struct vtnet_softc *sc, bool on)
3667 {
3668 	return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
3669 }
3670 
3671 static int
3672 vtnet_set_allmulti(struct vtnet_softc *sc, bool on)
3673 {
3674 	return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
3675 }
3676 
3677 static void
3678 vtnet_rx_filter(struct vtnet_softc *sc)
3679 {
3680 	device_t dev;
3681 	struct ifnet *ifp;
3682 
3683 	dev = sc->vtnet_dev;
3684 	ifp = sc->vtnet_ifp;
3685 
3686 	VTNET_CORE_LOCK_ASSERT(sc);
3687 
3688 	if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0) {
3689 		device_printf(dev, "cannot %s promiscuous mode\n",
3690 		    ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
3691 	}
3692 
3693 	if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0) {
3694 		device_printf(dev, "cannot %s all-multicast mode\n",
3695 		    ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
3696 	}
3697 }
3698 
3699 static u_int
3700 vtnet_copy_ifaddr(void *arg, struct sockaddr_dl *sdl, u_int ucnt)
3701 {
3702 	struct vtnet_softc *sc = arg;
3703 
3704 	if (memcmp(LLADDR(sdl), sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0)
3705 		return (0);
3706 
3707 	if (ucnt < VTNET_MAX_MAC_ENTRIES)
3708 		bcopy(LLADDR(sdl),
3709 		    &sc->vtnet_mac_filter->vmf_unicast.macs[ucnt],
3710 		    ETHER_ADDR_LEN);
3711 
3712 	return (1);
3713 }
3714 
3715 static u_int
3716 vtnet_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int mcnt)
3717 {
3718 	struct vtnet_mac_filter *filter = arg;
3719 
3720 	if (mcnt < VTNET_MAX_MAC_ENTRIES)
3721 		bcopy(LLADDR(sdl), &filter->vmf_multicast.macs[mcnt],
3722 		    ETHER_ADDR_LEN);
3723 
3724 	return (1);
3725 }
3726 
3727 static void
3728 vtnet_rx_filter_mac(struct vtnet_softc *sc)
3729 {
3730 	struct virtio_net_ctrl_hdr hdr __aligned(2);
3731 	struct vtnet_mac_filter *filter;
3732 	struct sglist_seg segs[4];
3733 	struct sglist sg;
3734 	struct ifnet *ifp;
3735 	bool promisc, allmulti;
3736 	u_int ucnt, mcnt;
3737 	int error;
3738 	uint8_t ack;
3739 
3740 	ifp = sc->vtnet_ifp;
3741 	filter = sc->vtnet_mac_filter;
3742 	error = 0;
3743 
3744 	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX);
3745 	VTNET_CORE_LOCK_ASSERT(sc);
3746 
3747 	/* Unicast MAC addresses: */
3748 	ucnt = if_foreach_lladdr(ifp, vtnet_copy_ifaddr, sc);
3749 	promisc = (ucnt > VTNET_MAX_MAC_ENTRIES);
3750 
3751 	if (promisc) {
3752 		ucnt = 0;
3753 		if_printf(ifp, "more than %d MAC addresses assigned, "
3754 		    "falling back to promiscuous mode\n",
3755 		    VTNET_MAX_MAC_ENTRIES);
3756 	}
3757 
3758 	/* Multicast MAC addresses: */
3759 	mcnt = if_foreach_llmaddr(ifp, vtnet_copy_maddr, filter);
3760 	allmulti = (mcnt > VTNET_MAX_MAC_ENTRIES);
3761 
3762 	if (allmulti) {
3763 		mcnt = 0;
3764 		if_printf(ifp, "more than %d multicast MAC addresses "
3765 		    "assigned, falling back to all-multicast mode\n",
3766 		    VTNET_MAX_MAC_ENTRIES);
3767 	}
3768 
3769 	if (promisc && allmulti)
3770 		goto out;
3771 
3772 	filter->vmf_unicast.nentries = vtnet_gtoh32(sc, ucnt);
3773 	filter->vmf_multicast.nentries = vtnet_gtoh32(sc, mcnt);
3774 
3775 	hdr.class = VIRTIO_NET_CTRL_MAC;
3776 	hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
3777 	ack = VIRTIO_NET_ERR;
3778 
3779 	sglist_init(&sg, nitems(segs), segs);
3780 	error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
3781 	error |= sglist_append(&sg, &filter->vmf_unicast,
3782 	    sizeof(uint32_t) + ucnt * ETHER_ADDR_LEN);
3783 	error |= sglist_append(&sg, &filter->vmf_multicast,
3784 	    sizeof(uint32_t) + mcnt * ETHER_ADDR_LEN);
3785 	error |= sglist_append(&sg, &ack, sizeof(uint8_t));
3786 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3787 
3788 	if (error == 0)
3789 		vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
3790 	if (ack != VIRTIO_NET_OK)
3791 		if_printf(ifp, "error setting host MAC filter table\n");
3792 
3793 out:
3794 	if (promisc != 0 && vtnet_set_promisc(sc, true) != 0)
3795 		if_printf(ifp, "cannot enable promiscuous mode\n");
3796 	if (allmulti != 0 && vtnet_set_allmulti(sc, true) != 0)
3797 		if_printf(ifp, "cannot enable all-multicast mode\n");
3798 }
3799 
3800 static int
3801 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
3802 {
3803 	struct sglist_seg segs[3];
3804 	struct sglist sg;
3805 	struct {
3806 		struct virtio_net_ctrl_hdr hdr __aligned(2);
3807 		uint8_t pad1;
3808 		uint16_t tag __aligned(2);
3809 		uint8_t pad2;
3810 		uint8_t ack;
3811 	} s;
3812 	int error;
3813 
3814 	error = 0;
3815 	MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER);
3816 
3817 	s.hdr.class = VIRTIO_NET_CTRL_VLAN;
3818 	s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
3819 	s.tag = vtnet_gtoh16(sc, tag);
3820 	s.ack = VIRTIO_NET_ERR;
3821 
3822 	sglist_init(&sg, nitems(segs), segs);
3823 	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
3824 	error |= sglist_append(&sg, &s.tag, sizeof(uint16_t));
3825 	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
3826 	MPASS(error == 0 && sg.sg_nseg == nitems(segs));
3827 
3828 	if (error == 0)
3829 		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
3830 
3831 	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
3832 }
3833 
3834 static void
3835 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
3836 {
3837 	int i, bit;
3838 	uint32_t w;
3839 	uint16_t tag;
3840 
3841 	MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER);
3842 	VTNET_CORE_LOCK_ASSERT(sc);
3843 
3844 	/* Enable the filter for each configured VLAN. */
3845 	for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) {
3846 		w = sc->vtnet_vlan_filter[i];
3847 
3848 		while ((bit = ffs(w) - 1) != -1) {
3849 			w &= ~(1 << bit);
3850 			tag = sizeof(w) * CHAR_BIT * i + bit;
3851 
3852 			if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) {
3853 				device_printf(sc->vtnet_dev,
3854 				    "cannot enable VLAN %d filter\n", tag);
3855 			}
3856 		}
3857 	}
3858 }
3859 
3860 static void
3861 vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
3862 {
3863 	struct ifnet *ifp;
3864 	int idx, bit;
3865 
3866 	ifp = sc->vtnet_ifp;
3867 	idx = (tag >> 5) & 0x7F;
3868 	bit = tag & 0x1F;
3869 
3870 	if (tag == 0 || tag > 4095)
3871 		return;
3872 
3873 	VTNET_CORE_LOCK(sc);
3874 
3875 	if (add)
3876 		sc->vtnet_vlan_filter[idx] |= (1 << bit);
3877 	else
3878 		sc->vtnet_vlan_filter[idx] &= ~(1 << bit);
3879 
3880 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER &&
3881 	    ifp->if_drv_flags & IFF_DRV_RUNNING &&
3882 	    vtnet_exec_vlan_filter(sc, add, tag) != 0) {
3883 		device_printf(sc->vtnet_dev,
3884 		    "cannot %s VLAN %d %s the host filter table\n",
3885 		    add ? "add" : "remove", tag, add ? "to" : "from");
3886 	}
3887 
3888 	VTNET_CORE_UNLOCK(sc);
3889 }
3890 
3891 static void
3892 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3893 {
3894 
3895 	if (ifp->if_softc != arg)
3896 		return;
3897 
3898 	vtnet_update_vlan_filter(arg, 1, tag);
3899 }
3900 
3901 static void
3902 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3903 {
3904 
3905 	if (ifp->if_softc != arg)
3906 		return;
3907 
3908 	vtnet_update_vlan_filter(arg, 0, tag);
3909 }
3910 
3911 static void
3912 vtnet_update_speed_duplex(struct vtnet_softc *sc)
3913 {
3914 	struct ifnet *ifp;
3915 	uint32_t speed;
3916 
3917 	ifp = sc->vtnet_ifp;
3918 
3919 	if ((sc->vtnet_features & VIRTIO_NET_F_SPEED_DUPLEX) == 0)
3920 		return;
3921 
3922 	/* BMV: Ignore duplex. */
3923 	speed = virtio_read_dev_config_4(sc->vtnet_dev,
3924 	    offsetof(struct virtio_net_config, speed));
3925 	if (speed != UINT32_MAX)
3926 		ifp->if_baudrate = IF_Mbps(speed);
3927 }
3928 
3929 static int
3930 vtnet_is_link_up(struct vtnet_softc *sc)
3931 {
3932 	uint16_t status;
3933 
3934 	if ((sc->vtnet_features & VIRTIO_NET_F_STATUS) == 0)
3935 		return (1);
3936 
3937 	status = virtio_read_dev_config_2(sc->vtnet_dev,
3938 	    offsetof(struct virtio_net_config, status));
3939 
3940 	return ((status & VIRTIO_NET_S_LINK_UP) != 0);
3941 }
3942 
3943 static void
3944 vtnet_update_link_status(struct vtnet_softc *sc)
3945 {
3946 	struct ifnet *ifp;
3947 	int link;
3948 
3949 	ifp = sc->vtnet_ifp;
3950 	VTNET_CORE_LOCK_ASSERT(sc);
3951 	link = vtnet_is_link_up(sc);
3952 
3953 	/* Notify if the link status has changed. */
3954 	if (link != 0 && sc->vtnet_link_active == 0) {
3955 		vtnet_update_speed_duplex(sc);
3956 		sc->vtnet_link_active = 1;
3957 		if_link_state_change(ifp, LINK_STATE_UP);
3958 	} else if (link == 0 && sc->vtnet_link_active != 0) {
3959 		sc->vtnet_link_active = 0;
3960 		if_link_state_change(ifp, LINK_STATE_DOWN);
3961 	}
3962 }
3963 
3964 static int
3965 vtnet_ifmedia_upd(struct ifnet *ifp __unused)
3966 {
3967 	return (EOPNOTSUPP);
3968 }
3969 
3970 static void
3971 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3972 {
3973 	struct vtnet_softc *sc;
3974 
3975 	sc = ifp->if_softc;
3976 
3977 	ifmr->ifm_status = IFM_AVALID;
3978 	ifmr->ifm_active = IFM_ETHER;
3979 
3980 	VTNET_CORE_LOCK(sc);
3981 	if (vtnet_is_link_up(sc) != 0) {
3982 		ifmr->ifm_status |= IFM_ACTIVE;
3983 		ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
3984 	} else
3985 		ifmr->ifm_active |= IFM_NONE;
3986 	VTNET_CORE_UNLOCK(sc);
3987 }
3988 
3989 static void
3990 vtnet_get_macaddr(struct vtnet_softc *sc)
3991 {
3992 
3993 	if (sc->vtnet_flags & VTNET_FLAG_MAC) {
3994 		virtio_read_device_config_array(sc->vtnet_dev,
3995 		    offsetof(struct virtio_net_config, mac),
3996 		    &sc->vtnet_hwaddr[0], sizeof(uint8_t), ETHER_ADDR_LEN);
3997 	} else {
3998 		/* Generate a random locally administered unicast address. */
3999 		sc->vtnet_hwaddr[0] = 0xB2;
4000 		arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0);
4001 	}
4002 }
4003 
4004 static void
4005 vtnet_set_macaddr(struct vtnet_softc *sc)
4006 {
4007 	device_t dev;
4008 	int error;
4009 
4010 	dev = sc->vtnet_dev;
4011 
4012 	if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) {
4013 		error = vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr);
4014 		if (error)
4015 			device_printf(dev, "unable to set MAC address\n");
4016 		return;
4017 	}
4018 
4019 	/* MAC in config is read-only in modern VirtIO. */
4020 	if (!vtnet_modern(sc) && sc->vtnet_flags & VTNET_FLAG_MAC) {
4021 		for (int i = 0; i < ETHER_ADDR_LEN; i++) {
4022 			virtio_write_dev_config_1(dev,
4023 			    offsetof(struct virtio_net_config, mac) + i,
4024 			    sc->vtnet_hwaddr[i]);
4025 		}
4026 	}
4027 }
4028 
4029 static void
4030 vtnet_attached_set_macaddr(struct vtnet_softc *sc)
4031 {
4032 
4033 	/* Assign MAC address if it was generated. */
4034 	if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0)
4035 		vtnet_set_macaddr(sc);
4036 }
4037 
4038 static void
4039 vtnet_vlan_tag_remove(struct mbuf *m)
4040 {
4041 	struct ether_vlan_header *evh;
4042 
4043 	evh = mtod(m, struct ether_vlan_header *);
4044 	m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
4045 	m->m_flags |= M_VLANTAG;
4046 
4047 	/* Strip the 802.1Q header. */
4048 	bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
4049 	    ETHER_HDR_LEN - ETHER_TYPE_LEN);
4050 	m_adj(m, ETHER_VLAN_ENCAP_LEN);
4051 }
4052 
4053 static void
4054 vtnet_set_rx_process_limit(struct vtnet_softc *sc)
4055 {
4056 	int limit;
4057 
4058 	limit = vtnet_tunable_int(sc, "rx_process_limit",
4059 	    vtnet_rx_process_limit);
4060 	if (limit < 0)
4061 		limit = INT_MAX;
4062 	sc->vtnet_rx_process_limit = limit;
4063 }
4064 
4065 static void
4066 vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx,
4067     struct sysctl_oid_list *child, struct vtnet_rxq *rxq)
4068 {
4069 	struct sysctl_oid *node;
4070 	struct sysctl_oid_list *list;
4071 	struct vtnet_rxq_stats *stats;
4072 	char namebuf[16];
4073 
4074 	snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id);
4075 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
4076 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Receive Queue");
4077 	list = SYSCTL_CHILDREN(node);
4078 
4079 	stats = &rxq->vtnrx_stats;
4080 
4081 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD,
4082 	    &stats->vrxs_ipackets, "Receive packets");
4083 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD,
4084 	    &stats->vrxs_ibytes, "Receive bytes");
4085 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD,
4086 	    &stats->vrxs_iqdrops, "Receive drops");
4087 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD,
4088 	    &stats->vrxs_ierrors, "Receive errors");
4089 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
4090 	    &stats->vrxs_csum, "Receive checksum offloaded");
4091 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD,
4092 	    &stats->vrxs_csum_failed, "Receive checksum offload failed");
4093 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "host_lro", CTLFLAG_RD,
4094 	    &stats->vrxs_host_lro, "Receive host segmentation offloaded");
4095 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD,
4096 	    &stats->vrxs_rescheduled,
4097 	    "Receive interrupt handler rescheduled");
4098 }
4099 
4100 static void
4101 vtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx,
4102     struct sysctl_oid_list *child, struct vtnet_txq *txq)
4103 {
4104 	struct sysctl_oid *node;
4105 	struct sysctl_oid_list *list;
4106 	struct vtnet_txq_stats *stats;
4107 	char namebuf[16];
4108 
4109 	snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id);
4110 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
4111 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Transmit Queue");
4112 	list = SYSCTL_CHILDREN(node);
4113 
4114 	stats = &txq->vtntx_stats;
4115 
4116 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD,
4117 	    &stats->vtxs_opackets, "Transmit packets");
4118 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD,
4119 	    &stats->vtxs_obytes, "Transmit bytes");
4120 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD,
4121 	    &stats->vtxs_omcasts, "Transmit multicasts");
4122 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
4123 	    &stats->vtxs_csum, "Transmit checksum offloaded");
4124 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
4125 	    &stats->vtxs_tso, "Transmit TCP segmentation offloaded");
4126 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD,
4127 	    &stats->vtxs_rescheduled,
4128 	    "Transmit interrupt handler rescheduled");
4129 }
4130 
4131 static void
4132 vtnet_setup_queue_sysctl(struct vtnet_softc *sc)
4133 {
4134 	device_t dev;
4135 	struct sysctl_ctx_list *ctx;
4136 	struct sysctl_oid *tree;
4137 	struct sysctl_oid_list *child;
4138 	int i;
4139 
4140 	dev = sc->vtnet_dev;
4141 	ctx = device_get_sysctl_ctx(dev);
4142 	tree = device_get_sysctl_tree(dev);
4143 	child = SYSCTL_CHILDREN(tree);
4144 
4145 	for (i = 0; i < sc->vtnet_req_vq_pairs; i++) {
4146 		vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]);
4147 		vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]);
4148 	}
4149 }
4150 
4151 static void
4152 vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx,
4153     struct sysctl_oid_list *child, struct vtnet_softc *sc)
4154 {
4155 	struct vtnet_statistics *stats;
4156 	struct vtnet_rxq_stats rxaccum;
4157 	struct vtnet_txq_stats txaccum;
4158 
4159 	vtnet_accum_stats(sc, &rxaccum, &txaccum);
4160 
4161 	stats = &sc->vtnet_stats;
4162 	stats->rx_csum_offloaded = rxaccum.vrxs_csum;
4163 	stats->rx_csum_failed = rxaccum.vrxs_csum_failed;
4164 	stats->rx_task_rescheduled = rxaccum.vrxs_rescheduled;
4165 	stats->tx_csum_offloaded = txaccum.vtxs_csum;
4166 	stats->tx_tso_offloaded = txaccum.vtxs_tso;
4167 	stats->tx_task_rescheduled = txaccum.vtxs_rescheduled;
4168 
4169 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed",
4170 	    CTLFLAG_RD, &stats->mbuf_alloc_failed,
4171 	    "Mbuf cluster allocation failures");
4172 
4173 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large",
4174 	    CTLFLAG_RD, &stats->rx_frame_too_large,
4175 	    "Received frame larger than the mbuf chain");
4176 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
4177 	    CTLFLAG_RD, &stats->rx_enq_replacement_failed,
4178 	    "Enqueuing the replacement receive mbuf failed");
4179 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed",
4180 	    CTLFLAG_RD, &stats->rx_mergeable_failed,
4181 	    "Mergeable buffers receive failures");
4182 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
4183 	    CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
4184 	    "Received checksum offloaded buffer with unsupported "
4185 	    "Ethernet type");
4186 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
4187 	    CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
4188 	    "Received checksum offloaded buffer with incorrect IP protocol");
4189 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset",
4190 	    CTLFLAG_RD, &stats->rx_csum_bad_offset,
4191 	    "Received checksum offloaded buffer with incorrect offset");
4192 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_proto",
4193 	    CTLFLAG_RD, &stats->rx_csum_bad_proto,
4194 	    "Received checksum offloaded buffer with incorrect protocol");
4195 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed",
4196 	    CTLFLAG_RD, &stats->rx_csum_failed,
4197 	    "Received buffer checksum offload failed");
4198 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded",
4199 	    CTLFLAG_RD, &stats->rx_csum_offloaded,
4200 	    "Received buffer checksum offload succeeded");
4201 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled",
4202 	    CTLFLAG_RD, &stats->rx_task_rescheduled,
4203 	    "Times the receive interrupt task rescheduled itself");
4204 
4205 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_unknown_ethtype",
4206 	    CTLFLAG_RD, &stats->tx_csum_unknown_ethtype,
4207 	    "Aborted transmit of checksum offloaded buffer with unknown "
4208 	    "Ethernet type");
4209 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_proto_mismatch",
4210 	    CTLFLAG_RD, &stats->tx_csum_proto_mismatch,
4211 	    "Aborted transmit of checksum offloaded buffer because mismatched "
4212 	    "protocols");
4213 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp",
4214 	    CTLFLAG_RD, &stats->tx_tso_not_tcp,
4215 	    "Aborted transmit of TSO buffer with non TCP protocol");
4216 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_without_csum",
4217 	    CTLFLAG_RD, &stats->tx_tso_without_csum,
4218 	    "Aborted transmit of TSO buffer without TCP checksum offload");
4219 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged",
4220 	    CTLFLAG_RD, &stats->tx_defragged,
4221 	    "Transmit mbufs defragged");
4222 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed",
4223 	    CTLFLAG_RD, &stats->tx_defrag_failed,
4224 	    "Aborted transmit of buffer because defrag failed");
4225 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded",
4226 	    CTLFLAG_RD, &stats->tx_csum_offloaded,
4227 	    "Offloaded checksum of transmitted buffer");
4228 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded",
4229 	    CTLFLAG_RD, &stats->tx_tso_offloaded,
4230 	    "Segmentation offload of transmitted buffer");
4231 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled",
4232 	    CTLFLAG_RD, &stats->tx_task_rescheduled,
4233 	    "Times the transmit interrupt task rescheduled itself");
4234 }
4235 
4236 static void
4237 vtnet_setup_sysctl(struct vtnet_softc *sc)
4238 {
4239 	device_t dev;
4240 	struct sysctl_ctx_list *ctx;
4241 	struct sysctl_oid *tree;
4242 	struct sysctl_oid_list *child;
4243 
4244 	dev = sc->vtnet_dev;
4245 	ctx = device_get_sysctl_ctx(dev);
4246 	tree = device_get_sysctl_tree(dev);
4247 	child = SYSCTL_CHILDREN(tree);
4248 
4249 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs",
4250 	    CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0,
4251 	    "Number of maximum supported virtqueue pairs");
4252 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "req_vq_pairs",
4253 	    CTLFLAG_RD, &sc->vtnet_req_vq_pairs, 0,
4254 	    "Number of requested virtqueue pairs");
4255 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs",
4256 	    CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0,
4257 	    "Number of active virtqueue pairs");
4258 
4259 	vtnet_setup_stat_sysctl(ctx, child, sc);
4260 }
4261 
4262 static void
4263 vtnet_load_tunables(struct vtnet_softc *sc)
4264 {
4265 
4266 	sc->vtnet_lro_entry_count = vtnet_tunable_int(sc,
4267 	    "lro_entry_count", vtnet_lro_entry_count);
4268 	if (sc->vtnet_lro_entry_count < TCP_LRO_ENTRIES)
4269 		sc->vtnet_lro_entry_count = TCP_LRO_ENTRIES;
4270 
4271 	sc->vtnet_lro_mbufq_depth = vtnet_tunable_int(sc,
4272 	    "lro_mbufq_depth", vtnet_lro_mbufq_depth);
4273 }
4274 
4275 static int
4276 vtnet_rxq_enable_intr(struct vtnet_rxq *rxq)
4277 {
4278 
4279 	return (virtqueue_enable_intr(rxq->vtnrx_vq));
4280 }
4281 
4282 static void
4283 vtnet_rxq_disable_intr(struct vtnet_rxq *rxq)
4284 {
4285 
4286 	virtqueue_disable_intr(rxq->vtnrx_vq);
4287 }
4288 
4289 static int
4290 vtnet_txq_enable_intr(struct vtnet_txq *txq)
4291 {
4292 	struct virtqueue *vq;
4293 
4294 	vq = txq->vtntx_vq;
4295 
4296 	if (vtnet_txq_below_threshold(txq) != 0)
4297 		return (virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG));
4298 
4299 	/*
4300 	 * The free count is above our threshold. Keep the Tx interrupt
4301 	 * disabled until the queue is fuller.
4302 	 */
4303 	return (0);
4304 }
4305 
4306 static void
4307 vtnet_txq_disable_intr(struct vtnet_txq *txq)
4308 {
4309 
4310 	virtqueue_disable_intr(txq->vtntx_vq);
4311 }
4312 
4313 static void
4314 vtnet_enable_rx_interrupts(struct vtnet_softc *sc)
4315 {
4316 	struct vtnet_rxq *rxq;
4317 	int i;
4318 
4319 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
4320 		rxq = &sc->vtnet_rxqs[i];
4321 		if (vtnet_rxq_enable_intr(rxq) != 0)
4322 			taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
4323 	}
4324 }
4325 
4326 static void
4327 vtnet_enable_tx_interrupts(struct vtnet_softc *sc)
4328 {
4329 	int i;
4330 
4331 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
4332 		vtnet_txq_enable_intr(&sc->vtnet_txqs[i]);
4333 }
4334 
4335 static void
4336 vtnet_enable_interrupts(struct vtnet_softc *sc)
4337 {
4338 
4339 	vtnet_enable_rx_interrupts(sc);
4340 	vtnet_enable_tx_interrupts(sc);
4341 }
4342 
4343 static void
4344 vtnet_disable_rx_interrupts(struct vtnet_softc *sc)
4345 {
4346 	int i;
4347 
4348 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
4349 		vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]);
4350 }
4351 
4352 static void
4353 vtnet_disable_tx_interrupts(struct vtnet_softc *sc)
4354 {
4355 	int i;
4356 
4357 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
4358 		vtnet_txq_disable_intr(&sc->vtnet_txqs[i]);
4359 }
4360 
4361 static void
4362 vtnet_disable_interrupts(struct vtnet_softc *sc)
4363 {
4364 
4365 	vtnet_disable_rx_interrupts(sc);
4366 	vtnet_disable_tx_interrupts(sc);
4367 }
4368 
4369 static int
4370 vtnet_tunable_int(struct vtnet_softc *sc, const char *knob, int def)
4371 {
4372 	char path[64];
4373 
4374 	snprintf(path, sizeof(path),
4375 	    "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob);
4376 	TUNABLE_INT_FETCH(path, &def);
4377 
4378 	return (def);
4379 }
4380 
4381 #ifdef DEBUGNET
4382 static void
4383 vtnet_debugnet_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize)
4384 {
4385 	struct vtnet_softc *sc;
4386 
4387 	sc = if_getsoftc(ifp);
4388 
4389 	VTNET_CORE_LOCK(sc);
4390 	*nrxr = sc->vtnet_req_vq_pairs;
4391 	*ncl = DEBUGNET_MAX_IN_FLIGHT;
4392 	*clsize = sc->vtnet_rx_clustersz;
4393 	VTNET_CORE_UNLOCK(sc);
4394 }
4395 
4396 static void
4397 vtnet_debugnet_event(struct ifnet *ifp __unused, enum debugnet_ev event __unused)
4398 {
4399 }
4400 
4401 static int
4402 vtnet_debugnet_transmit(struct ifnet *ifp, struct mbuf *m)
4403 {
4404 	struct vtnet_softc *sc;
4405 	struct vtnet_txq *txq;
4406 	int error;
4407 
4408 	sc = if_getsoftc(ifp);
4409 	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
4410 	    IFF_DRV_RUNNING)
4411 		return (EBUSY);
4412 
4413 	txq = &sc->vtnet_txqs[0];
4414 	error = vtnet_txq_encap(txq, &m, M_NOWAIT | M_USE_RESERVE);
4415 	if (error == 0)
4416 		(void)vtnet_txq_notify(txq);
4417 	return (error);
4418 }
4419 
4420 static int
4421 vtnet_debugnet_poll(struct ifnet *ifp, int count)
4422 {
4423 	struct vtnet_softc *sc;
4424 	int i;
4425 
4426 	sc = if_getsoftc(ifp);
4427 	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
4428 	    IFF_DRV_RUNNING)
4429 		return (EBUSY);
4430 
4431 	(void)vtnet_txq_eof(&sc->vtnet_txqs[0]);
4432 	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
4433 		(void)vtnet_rxq_eof(&sc->vtnet_rxqs[i]);
4434 	return (0);
4435 }
4436 #endif /* DEBUGNET */
4437