xref: /freebsd/sys/dev/vmware/vmxnet3/if_vmx.c (revision c6db8143eda5c775467145ac73e8ebec47afdd8f)
1 /*-
2  * Copyright (c) 2013 Tsubai Masanari
3  * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
4  *
5  * Permission to use, copy, modify, and distribute this software for any
6  * purpose with or without fee is hereby granted, provided that the above
7  * copyright notice and this permission notice appear in all copies.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16  *
17  * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $
18  */
19 
20 /* Driver for VMware vmxnet3 virtual ethernet devices. */
21 
22 #include <sys/cdefs.h>
23 __FBSDID("$FreeBSD$");
24 
25 #include <sys/param.h>
26 #include <sys/systm.h>
27 #include <sys/eventhandler.h>
28 #include <sys/kernel.h>
29 #include <sys/endian.h>
30 #include <sys/sockio.h>
31 #include <sys/mbuf.h>
32 #include <sys/malloc.h>
33 #include <sys/module.h>
34 #include <sys/socket.h>
35 #include <sys/sysctl.h>
36 #include <sys/smp.h>
37 #include <sys/taskqueue.h>
38 #include <vm/vm.h>
39 #include <vm/pmap.h>
40 
41 #include <net/ethernet.h>
42 #include <net/if.h>
43 #include <net/if_var.h>
44 #include <net/if_arp.h>
45 #include <net/if_dl.h>
46 #include <net/if_types.h>
47 #include <net/if_media.h>
48 #include <net/if_vlan_var.h>
49 
50 #include <net/bpf.h>
51 
52 #include <netinet/in_systm.h>
53 #include <netinet/in.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip6.h>
56 #include <netinet6/ip6_var.h>
57 #include <netinet/udp.h>
58 #include <netinet/tcp.h>
59 
60 #include <machine/in_cksum.h>
61 
62 #include <machine/bus.h>
63 #include <machine/resource.h>
64 #include <sys/bus.h>
65 #include <sys/rman.h>
66 
67 #include <dev/pci/pcireg.h>
68 #include <dev/pci/pcivar.h>
69 
70 #include "if_vmxreg.h"
71 #include "if_vmxvar.h"
72 
73 #include "opt_inet.h"
74 #include "opt_inet6.h"
75 
76 #ifdef VMXNET3_FAILPOINTS
77 #include <sys/fail.h>
78 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0,
79     "vmxnet3 fail points");
80 #define VMXNET3_FP	_debug_fail_point_vmxnet3
81 #endif
82 
83 static int	vmxnet3_probe(device_t);
84 static int	vmxnet3_attach(device_t);
85 static int	vmxnet3_detach(device_t);
86 static int	vmxnet3_shutdown(device_t);
87 
88 static int	vmxnet3_alloc_resources(struct vmxnet3_softc *);
89 static void	vmxnet3_free_resources(struct vmxnet3_softc *);
90 static int	vmxnet3_check_version(struct vmxnet3_softc *);
91 static void	vmxnet3_initial_config(struct vmxnet3_softc *);
92 static void	vmxnet3_check_multiqueue(struct vmxnet3_softc *);
93 
94 static int	vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
95 static int	vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
96 static int	vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
97 static int	vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int,
98 		    struct vmxnet3_interrupt *);
99 static int	vmxnet3_alloc_intr_resources(struct vmxnet3_softc *);
100 static int	vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
101 static int	vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
102 static int	vmxnet3_setup_interrupts(struct vmxnet3_softc *);
103 static int	vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
104 
105 static void	vmxnet3_free_interrupt(struct vmxnet3_softc *,
106 		    struct vmxnet3_interrupt *);
107 static void	vmxnet3_free_interrupts(struct vmxnet3_softc *);
108 
109 #ifndef VMXNET3_LEGACY_TX
110 static int	vmxnet3_alloc_taskqueue(struct vmxnet3_softc *);
111 static void	vmxnet3_start_taskqueue(struct vmxnet3_softc *);
112 static void	vmxnet3_drain_taskqueue(struct vmxnet3_softc *);
113 static void	vmxnet3_free_taskqueue(struct vmxnet3_softc *);
114 #endif
115 
116 static int	vmxnet3_init_rxq(struct vmxnet3_softc *, int);
117 static int	vmxnet3_init_txq(struct vmxnet3_softc *, int);
118 static int	vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
119 static void	vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
120 static void	vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
121 static void	vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);
122 
123 static int	vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
124 static void	vmxnet3_free_shared_data(struct vmxnet3_softc *);
125 static int	vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
126 static void	vmxnet3_free_txq_data(struct vmxnet3_softc *);
127 static int	vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
128 static void	vmxnet3_free_rxq_data(struct vmxnet3_softc *);
129 static int	vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
130 static void	vmxnet3_free_queue_data(struct vmxnet3_softc *);
131 static int	vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
132 static void	vmxnet3_init_shared_data(struct vmxnet3_softc *);
133 static void	vmxnet3_reinit_interface(struct vmxnet3_softc *);
134 static void	vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
135 static void	vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
136 static int	vmxnet3_alloc_data(struct vmxnet3_softc *);
137 static void	vmxnet3_free_data(struct vmxnet3_softc *);
138 static int	vmxnet3_setup_interface(struct vmxnet3_softc *);
139 
140 static void	vmxnet3_evintr(struct vmxnet3_softc *);
141 static void	vmxnet3_txq_eof(struct vmxnet3_txqueue *);
142 static void	vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
143 static int	vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
144 static void	vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
145 		    struct vmxnet3_rxring *, int);
146 static void	vmxnet3_rxq_eof(struct vmxnet3_rxqueue *);
147 static void	vmxnet3_legacy_intr(void *);
148 static void	vmxnet3_txq_intr(void *);
149 static void	vmxnet3_rxq_intr(void *);
150 static void	vmxnet3_event_intr(void *);
151 
152 static void	vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
153 static void	vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
154 static void	vmxnet3_stop(struct vmxnet3_softc *);
155 
156 static void	vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
157 static int	vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
158 static int	vmxnet3_reinit_queues(struct vmxnet3_softc *);
159 static int	vmxnet3_enable_device(struct vmxnet3_softc *);
160 static void	vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
161 static int	vmxnet3_reinit(struct vmxnet3_softc *);
162 static void	vmxnet3_init_locked(struct vmxnet3_softc *);
163 static void	vmxnet3_init(void *);
164 
165 static int	vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *,
166 		    int *, int *, int *);
167 static int	vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **,
168 		    bus_dmamap_t, bus_dma_segment_t [], int *);
169 static void	vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
170 static int	vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
171 #ifdef VMXNET3_LEGACY_TX
172 static void	vmxnet3_start_locked(struct ifnet *);
173 static void	vmxnet3_start(struct ifnet *);
174 #else
175 static int	vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *,
176 		    struct mbuf *);
177 static int	vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *);
178 static void	vmxnet3_txq_tq_deferred(void *, int);
179 #endif
180 static void	vmxnet3_txq_start(struct vmxnet3_txqueue *);
181 static void	vmxnet3_tx_start_all(struct vmxnet3_softc *);
182 
183 static void	vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
184 		    uint16_t);
185 static void	vmxnet3_register_vlan(void *, struct ifnet *, uint16_t);
186 static void	vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t);
187 static void	vmxnet3_set_rxfilter(struct vmxnet3_softc *);
188 static int	vmxnet3_change_mtu(struct vmxnet3_softc *, int);
189 static int	vmxnet3_ioctl(struct ifnet *, u_long, caddr_t);
190 static uint64_t	vmxnet3_get_counter(struct ifnet *, ift_counter);
191 
192 #ifndef VMXNET3_LEGACY_TX
193 static void	vmxnet3_qflush(struct ifnet *);
194 #endif
195 
196 static int	vmxnet3_watchdog(struct vmxnet3_txqueue *);
197 static void	vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
198 static void	vmxnet3_tick(void *);
199 static void	vmxnet3_link_status(struct vmxnet3_softc *);
200 static void	vmxnet3_media_status(struct ifnet *, struct ifmediareq *);
201 static int	vmxnet3_media_change(struct ifnet *);
202 static void	vmxnet3_set_lladdr(struct vmxnet3_softc *);
203 static void	vmxnet3_get_lladdr(struct vmxnet3_softc *);
204 
205 static void	vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *,
206 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
207 static void	vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *,
208 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
209 static void	vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *,
210 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
211 static void	vmxnet3_setup_sysctl(struct vmxnet3_softc *);
212 
213 static void	vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t,
214 		    uint32_t);
215 static uint32_t	vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t);
216 static void	vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t,
217 		    uint32_t);
218 static void	vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t);
219 static uint32_t	vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t);
220 
221 static void	vmxnet3_enable_intr(struct vmxnet3_softc *, int);
222 static void	vmxnet3_disable_intr(struct vmxnet3_softc *, int);
223 static void	vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
224 static void	vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
225 
226 static int	vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t,
227 		    bus_size_t, struct vmxnet3_dma_alloc *);
228 static void	vmxnet3_dma_free(struct vmxnet3_softc *,
229 		    struct vmxnet3_dma_alloc *);
230 static int	vmxnet3_tunable_int(struct vmxnet3_softc *,
231 		    const char *, int);
232 
233 typedef enum {
234 	VMXNET3_BARRIER_RD,
235 	VMXNET3_BARRIER_WR,
236 	VMXNET3_BARRIER_RDWR,
237 } vmxnet3_barrier_t;
238 
239 static void	vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
240 
241 /* Tunables. */
242 static int vmxnet3_mq_disable = 0;
243 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable);
244 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES;
245 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue);
246 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES;
247 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue);
248 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC;
249 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc);
250 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC;
251 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc);
252 
253 static device_method_t vmxnet3_methods[] = {
254 	/* Device interface. */
255 	DEVMETHOD(device_probe,		vmxnet3_probe),
256 	DEVMETHOD(device_attach,	vmxnet3_attach),
257 	DEVMETHOD(device_detach,	vmxnet3_detach),
258 	DEVMETHOD(device_shutdown,	vmxnet3_shutdown),
259 
260 	DEVMETHOD_END
261 };
262 
263 static driver_t vmxnet3_driver = {
264 	"vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc)
265 };
266 
267 static devclass_t vmxnet3_devclass;
268 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0);
269 
270 MODULE_DEPEND(vmx, pci, 1, 1, 1);
271 MODULE_DEPEND(vmx, ether, 1, 1, 1);
272 
273 #define VMXNET3_VMWARE_VENDOR_ID	0x15AD
274 #define VMXNET3_VMWARE_DEVICE_ID	0x07B0
275 
276 static int
277 vmxnet3_probe(device_t dev)
278 {
279 
280 	if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID &&
281 	    pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) {
282 		device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter");
283 		return (BUS_PROBE_DEFAULT);
284 	}
285 
286 	return (ENXIO);
287 }
288 
289 static int
290 vmxnet3_attach(device_t dev)
291 {
292 	struct vmxnet3_softc *sc;
293 	int error;
294 
295 	sc = device_get_softc(dev);
296 	sc->vmx_dev = dev;
297 
298 	pci_enable_busmaster(dev);
299 
300 	VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev));
301 	callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0);
302 
303 	vmxnet3_initial_config(sc);
304 
305 	error = vmxnet3_alloc_resources(sc);
306 	if (error)
307 		goto fail;
308 
309 	error = vmxnet3_check_version(sc);
310 	if (error)
311 		goto fail;
312 
313 	error = vmxnet3_alloc_rxtx_queues(sc);
314 	if (error)
315 		goto fail;
316 
317 #ifndef VMXNET3_LEGACY_TX
318 	error = vmxnet3_alloc_taskqueue(sc);
319 	if (error)
320 		goto fail;
321 #endif
322 
323 	error = vmxnet3_alloc_interrupts(sc);
324 	if (error)
325 		goto fail;
326 
327 	vmxnet3_check_multiqueue(sc);
328 
329 	error = vmxnet3_alloc_data(sc);
330 	if (error)
331 		goto fail;
332 
333 	error = vmxnet3_setup_interface(sc);
334 	if (error)
335 		goto fail;
336 
337 	error = vmxnet3_setup_interrupts(sc);
338 	if (error) {
339 		ether_ifdetach(sc->vmx_ifp);
340 		device_printf(dev, "could not set up interrupt\n");
341 		goto fail;
342 	}
343 
344 	vmxnet3_setup_sysctl(sc);
345 #ifndef VMXNET3_LEGACY_TX
346 	vmxnet3_start_taskqueue(sc);
347 #endif
348 
349 fail:
350 	if (error)
351 		vmxnet3_detach(dev);
352 
353 	return (error);
354 }
355 
356 static int
357 vmxnet3_detach(device_t dev)
358 {
359 	struct vmxnet3_softc *sc;
360 	struct ifnet *ifp;
361 
362 	sc = device_get_softc(dev);
363 	ifp = sc->vmx_ifp;
364 
365 	if (device_is_attached(dev)) {
366 		VMXNET3_CORE_LOCK(sc);
367 		vmxnet3_stop(sc);
368 		VMXNET3_CORE_UNLOCK(sc);
369 
370 		callout_drain(&sc->vmx_tick);
371 #ifndef VMXNET3_LEGACY_TX
372 		vmxnet3_drain_taskqueue(sc);
373 #endif
374 
375 		ether_ifdetach(ifp);
376 	}
377 
378 	if (sc->vmx_vlan_attach != NULL) {
379 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach);
380 		sc->vmx_vlan_attach = NULL;
381 	}
382 	if (sc->vmx_vlan_detach != NULL) {
383 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach);
384 		sc->vmx_vlan_detach = NULL;
385 	}
386 
387 #ifndef VMXNET3_LEGACY_TX
388 	vmxnet3_free_taskqueue(sc);
389 #endif
390 	vmxnet3_free_interrupts(sc);
391 
392 	if (ifp != NULL) {
393 		if_free(ifp);
394 		sc->vmx_ifp = NULL;
395 	}
396 
397 	ifmedia_removeall(&sc->vmx_media);
398 
399 	vmxnet3_free_data(sc);
400 	vmxnet3_free_resources(sc);
401 	vmxnet3_free_rxtx_queues(sc);
402 
403 	VMXNET3_CORE_LOCK_DESTROY(sc);
404 
405 	return (0);
406 }
407 
408 static int
409 vmxnet3_shutdown(device_t dev)
410 {
411 
412 	return (0);
413 }
414 
415 static int
416 vmxnet3_alloc_resources(struct vmxnet3_softc *sc)
417 {
418 	device_t dev;
419 	int rid;
420 
421 	dev = sc->vmx_dev;
422 
423 	rid = PCIR_BAR(0);
424 	sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
425 	    RF_ACTIVE);
426 	if (sc->vmx_res0 == NULL) {
427 		device_printf(dev,
428 		    "could not map BAR0 memory\n");
429 		return (ENXIO);
430 	}
431 
432 	sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0);
433 	sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0);
434 
435 	rid = PCIR_BAR(1);
436 	sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
437 	    RF_ACTIVE);
438 	if (sc->vmx_res1 == NULL) {
439 		device_printf(dev,
440 		    "could not map BAR1 memory\n");
441 		return (ENXIO);
442 	}
443 
444 	sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1);
445 	sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1);
446 
447 	if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
448 		rid = PCIR_BAR(2);
449 		sc->vmx_msix_res = bus_alloc_resource_any(dev,
450 		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
451 	}
452 
453 	if (sc->vmx_msix_res == NULL)
454 		sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
455 
456 	return (0);
457 }
458 
459 static void
460 vmxnet3_free_resources(struct vmxnet3_softc *sc)
461 {
462 	device_t dev;
463 	int rid;
464 
465 	dev = sc->vmx_dev;
466 
467 	if (sc->vmx_res0 != NULL) {
468 		rid = PCIR_BAR(0);
469 		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0);
470 		sc->vmx_res0 = NULL;
471 	}
472 
473 	if (sc->vmx_res1 != NULL) {
474 		rid = PCIR_BAR(1);
475 		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1);
476 		sc->vmx_res1 = NULL;
477 	}
478 
479 	if (sc->vmx_msix_res != NULL) {
480 		rid = PCIR_BAR(2);
481 		bus_release_resource(dev, SYS_RES_MEMORY, rid,
482 		    sc->vmx_msix_res);
483 		sc->vmx_msix_res = NULL;
484 	}
485 }
486 
487 static int
488 vmxnet3_check_version(struct vmxnet3_softc *sc)
489 {
490 	device_t dev;
491 	uint32_t version;
492 
493 	dev = sc->vmx_dev;
494 
495 	version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
496 	if ((version & 0x01) == 0) {
497 		device_printf(dev, "unsupported hardware version %#x\n",
498 		    version);
499 		return (ENOTSUP);
500 	}
501 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
502 
503 	version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
504 	if ((version & 0x01) == 0) {
505 		device_printf(dev, "unsupported UPT version %#x\n", version);
506 		return (ENOTSUP);
507 	}
508 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
509 
510 	return (0);
511 }
512 
513 static void
514 vmxnet3_initial_config(struct vmxnet3_softc *sc)
515 {
516 	int nqueue, ndesc;
517 
518 	nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue);
519 	if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1)
520 		nqueue = VMXNET3_DEF_TX_QUEUES;
521 	if (nqueue > mp_ncpus)
522 		nqueue = mp_ncpus;
523 	sc->vmx_max_ntxqueues = nqueue;
524 
525 	nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue);
526 	if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1)
527 		nqueue = VMXNET3_DEF_RX_QUEUES;
528 	if (nqueue > mp_ncpus)
529 		nqueue = mp_ncpus;
530 	sc->vmx_max_nrxqueues = nqueue;
531 
532 	if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) {
533 		sc->vmx_max_nrxqueues = 1;
534 		sc->vmx_max_ntxqueues = 1;
535 	}
536 
537 	ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc);
538 	if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC)
539 		ndesc = VMXNET3_DEF_TX_NDESC;
540 	if (ndesc & VMXNET3_MASK_TX_NDESC)
541 		ndesc &= ~VMXNET3_MASK_TX_NDESC;
542 	sc->vmx_ntxdescs = ndesc;
543 
544 	ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc);
545 	if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC)
546 		ndesc = VMXNET3_DEF_RX_NDESC;
547 	if (ndesc & VMXNET3_MASK_RX_NDESC)
548 		ndesc &= ~VMXNET3_MASK_RX_NDESC;
549 	sc->vmx_nrxdescs = ndesc;
550 	sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
551 }
552 
553 static void
554 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
555 {
556 
557 	if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
558 		goto out;
559 
560 	/* BMV: Just use the maximum configured for now. */
561 	sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
562 	sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;
563 
564 	if (sc->vmx_nrxqueues > 1)
565 		sc->vmx_flags |= VMXNET3_FLAG_RSS;
566 
567 	return;
568 
569 out:
570 	sc->vmx_ntxqueues = 1;
571 	sc->vmx_nrxqueues = 1;
572 }
573 
574 static int
575 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
576 {
577 	device_t dev;
578 	int nmsix, cnt, required;
579 
580 	dev = sc->vmx_dev;
581 
582 	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
583 		return (1);
584 
585 	/* Allocate an additional vector for the events interrupt. */
586 	required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1;
587 
588 	nmsix = pci_msix_count(dev);
589 	if (nmsix < required)
590 		return (1);
591 
592 	cnt = required;
593 	if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
594 		sc->vmx_nintrs = required;
595 		return (0);
596 	} else
597 		pci_release_msi(dev);
598 
599 	/* BMV TODO Fallback to sharing MSIX vectors if possible. */
600 
601 	return (1);
602 }
603 
604 static int
605 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
606 {
607 	device_t dev;
608 	int nmsi, cnt, required;
609 
610 	dev = sc->vmx_dev;
611 	required = 1;
612 
613 	nmsi = pci_msi_count(dev);
614 	if (nmsi < required)
615 		return (1);
616 
617 	cnt = required;
618 	if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) {
619 		sc->vmx_nintrs = 1;
620 		return (0);
621 	} else
622 		pci_release_msi(dev);
623 
624 	return (1);
625 }
626 
627 static int
628 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
629 {
630 
631 	sc->vmx_nintrs = 1;
632 	return (0);
633 }
634 
635 static int
636 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags,
637     struct vmxnet3_interrupt *intr)
638 {
639 	struct resource *irq;
640 
641 	irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags);
642 	if (irq == NULL)
643 		return (ENXIO);
644 
645 	intr->vmxi_irq = irq;
646 	intr->vmxi_rid = rid;
647 
648 	return (0);
649 }
650 
651 static int
652 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc)
653 {
654 	int i, rid, flags, error;
655 
656 	rid = 0;
657 	flags = RF_ACTIVE;
658 
659 	if (sc->vmx_intr_type == VMXNET3_IT_LEGACY)
660 		flags |= RF_SHAREABLE;
661 	else
662 		rid = 1;
663 
664 	for (i = 0; i < sc->vmx_nintrs; i++, rid++) {
665 		error = vmxnet3_alloc_interrupt(sc, rid, flags,
666 		    &sc->vmx_intrs[i]);
667 		if (error)
668 			return (error);
669 	}
670 
671 	return (0);
672 }
673 
674 static int
675 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
676 {
677 	device_t dev;
678 	struct vmxnet3_txqueue *txq;
679 	struct vmxnet3_rxqueue *rxq;
680 	struct vmxnet3_interrupt *intr;
681 	enum intr_type type;
682 	int i, error;
683 
684 	dev = sc->vmx_dev;
685 	intr = &sc->vmx_intrs[0];
686 	type = INTR_TYPE_NET | INTR_MPSAFE;
687 
688 	for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) {
689 		txq = &sc->vmx_txq[i];
690 		error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
691 		     vmxnet3_txq_intr, txq, &intr->vmxi_handler);
692 		if (error)
693 			return (error);
694 		bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
695 		    "tq%d", i);
696 		txq->vxtxq_intr_idx = intr->vmxi_rid - 1;
697 	}
698 
699 	for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) {
700 		rxq = &sc->vmx_rxq[i];
701 		error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
702 		    vmxnet3_rxq_intr, rxq, &intr->vmxi_handler);
703 		if (error)
704 			return (error);
705 		bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
706 		    "rq%d", i);
707 		rxq->vxrxq_intr_idx = intr->vmxi_rid - 1;
708 	}
709 
710 	error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
711 	    vmxnet3_event_intr, sc, &intr->vmxi_handler);
712 	if (error)
713 		return (error);
714 	bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event");
715 	sc->vmx_event_intr_idx = intr->vmxi_rid - 1;
716 
717 	return (0);
718 }
719 
720 static int
721 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
722 {
723 	struct vmxnet3_interrupt *intr;
724 	int i, error;
725 
726 	intr = &sc->vmx_intrs[0];
727 	error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq,
728 	    INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc,
729 	    &intr->vmxi_handler);
730 
731 	for (i = 0; i < sc->vmx_ntxqueues; i++)
732 		sc->vmx_txq[i].vxtxq_intr_idx = 0;
733 	for (i = 0; i < sc->vmx_nrxqueues; i++)
734 		sc->vmx_rxq[i].vxrxq_intr_idx = 0;
735 	sc->vmx_event_intr_idx = 0;
736 
737 	return (error);
738 }
739 
740 static void
741 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
742 {
743 	struct vmxnet3_txqueue *txq;
744 	struct vmxnet3_txq_shared *txs;
745 	struct vmxnet3_rxqueue *rxq;
746 	struct vmxnet3_rxq_shared *rxs;
747 	int i;
748 
749 	sc->vmx_ds->evintr = sc->vmx_event_intr_idx;
750 
751 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
752 		txq = &sc->vmx_txq[i];
753 		txs = txq->vxtxq_ts;
754 		txs->intr_idx = txq->vxtxq_intr_idx;
755 	}
756 
757 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
758 		rxq = &sc->vmx_rxq[i];
759 		rxs = rxq->vxrxq_rs;
760 		rxs->intr_idx = rxq->vxrxq_intr_idx;
761 	}
762 }
763 
764 static int
765 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
766 {
767 	int error;
768 
769 	error = vmxnet3_alloc_intr_resources(sc);
770 	if (error)
771 		return (error);
772 
773 	switch (sc->vmx_intr_type) {
774 	case VMXNET3_IT_MSIX:
775 		error = vmxnet3_setup_msix_interrupts(sc);
776 		break;
777 	case VMXNET3_IT_MSI:
778 	case VMXNET3_IT_LEGACY:
779 		error = vmxnet3_setup_legacy_interrupt(sc);
780 		break;
781 	default:
782 		panic("%s: invalid interrupt type %d", __func__,
783 		    sc->vmx_intr_type);
784 	}
785 
786 	if (error == 0)
787 		vmxnet3_set_interrupt_idx(sc);
788 
789 	return (error);
790 }
791 
792 static int
793 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
794 {
795 	device_t dev;
796 	uint32_t config;
797 	int error;
798 
799 	dev = sc->vmx_dev;
800 	config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
801 
802 	sc->vmx_intr_type = config & 0x03;
803 	sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
804 
805 	switch (sc->vmx_intr_type) {
806 	case VMXNET3_IT_AUTO:
807 		sc->vmx_intr_type = VMXNET3_IT_MSIX;
808 		/* FALLTHROUGH */
809 	case VMXNET3_IT_MSIX:
810 		error = vmxnet3_alloc_msix_interrupts(sc);
811 		if (error == 0)
812 			break;
813 		sc->vmx_intr_type = VMXNET3_IT_MSI;
814 		/* FALLTHROUGH */
815 	case VMXNET3_IT_MSI:
816 		error = vmxnet3_alloc_msi_interrupts(sc);
817 		if (error == 0)
818 			break;
819 		sc->vmx_intr_type = VMXNET3_IT_LEGACY;
820 		/* FALLTHROUGH */
821 	case VMXNET3_IT_LEGACY:
822 		error = vmxnet3_alloc_legacy_interrupts(sc);
823 		if (error == 0)
824 			break;
825 		/* FALLTHROUGH */
826 	default:
827 		sc->vmx_intr_type = -1;
828 		device_printf(dev, "cannot allocate any interrupt resources\n");
829 		return (ENXIO);
830 	}
831 
832 	return (error);
833 }
834 
835 static void
836 vmxnet3_free_interrupt(struct vmxnet3_softc *sc,
837     struct vmxnet3_interrupt *intr)
838 {
839 	device_t dev;
840 
841 	dev = sc->vmx_dev;
842 
843 	if (intr->vmxi_handler != NULL) {
844 		bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler);
845 		intr->vmxi_handler = NULL;
846 	}
847 
848 	if (intr->vmxi_irq != NULL) {
849 		bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid,
850 		    intr->vmxi_irq);
851 		intr->vmxi_irq = NULL;
852 		intr->vmxi_rid = -1;
853 	}
854 }
855 
856 static void
857 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
858 {
859 	int i;
860 
861 	for (i = 0; i < sc->vmx_nintrs; i++)
862 		vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]);
863 
864 	if (sc->vmx_intr_type == VMXNET3_IT_MSI ||
865 	    sc->vmx_intr_type == VMXNET3_IT_MSIX)
866 		pci_release_msi(sc->vmx_dev);
867 }
868 
869 #ifndef VMXNET3_LEGACY_TX
870 static int
871 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc)
872 {
873 	device_t dev;
874 
875 	dev = sc->vmx_dev;
876 
877 	sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT,
878 	    taskqueue_thread_enqueue, &sc->vmx_tq);
879 	if (sc->vmx_tq == NULL)
880 		return (ENOMEM);
881 
882 	return (0);
883 }
884 
885 static void
886 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc)
887 {
888 	device_t dev;
889 	int nthreads, error;
890 
891 	dev = sc->vmx_dev;
892 
893 	/*
894 	 * The taskqueue is typically not frequently used, so a dedicated
895 	 * thread for each queue is unnecessary.
896 	 */
897 	nthreads = MAX(1, sc->vmx_ntxqueues / 2);
898 
899 	/*
900 	 * Most drivers just ignore the return value - it only fails
901 	 * with ENOMEM so an error is not likely. It is hard for us
902 	 * to recover from an error here.
903 	 */
904 	error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET,
905 	    "%s taskq", device_get_nameunit(dev));
906 	if (error)
907 		device_printf(dev, "failed to start taskqueue: %d", error);
908 }
909 
910 static void
911 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc)
912 {
913 	struct vmxnet3_txqueue *txq;
914 	int i;
915 
916 	if (sc->vmx_tq != NULL) {
917 		for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
918 			txq = &sc->vmx_txq[i];
919 			taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask);
920 		}
921 	}
922 }
923 
924 static void
925 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc)
926 {
927 	if (sc->vmx_tq != NULL) {
928 		taskqueue_free(sc->vmx_tq);
929 		sc->vmx_tq = NULL;
930 	}
931 }
932 #endif
933 
934 static int
935 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
936 {
937 	struct vmxnet3_rxqueue *rxq;
938 	struct vmxnet3_rxring *rxr;
939 	int i;
940 
941 	rxq = &sc->vmx_rxq[q];
942 
943 	snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
944 	    device_get_nameunit(sc->vmx_dev), q);
945 	mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF);
946 
947 	rxq->vxrxq_sc = sc;
948 	rxq->vxrxq_id = q;
949 
950 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
951 		rxr = &rxq->vxrxq_cmd_ring[i];
952 		rxr->vxrxr_rid = i;
953 		rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
954 		rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc *
955 		    sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO);
956 		if (rxr->vxrxr_rxbuf == NULL)
957 			return (ENOMEM);
958 
959 		rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
960 	}
961 
962 	return (0);
963 }
964 
965 static int
966 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
967 {
968 	struct vmxnet3_txqueue *txq;
969 	struct vmxnet3_txring *txr;
970 
971 	txq = &sc->vmx_txq[q];
972 	txr = &txq->vxtxq_cmd_ring;
973 
974 	snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
975 	    device_get_nameunit(sc->vmx_dev), q);
976 	mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF);
977 
978 	txq->vxtxq_sc = sc;
979 	txq->vxtxq_id = q;
980 
981 	txr->vxtxr_ndesc = sc->vmx_ntxdescs;
982 	txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc *
983 	    sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO);
984 	if (txr->vxtxr_txbuf == NULL)
985 		return (ENOMEM);
986 
987 	txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
988 
989 #ifndef VMXNET3_LEGACY_TX
990 	TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq);
991 
992 	txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF,
993 	    M_NOWAIT, &txq->vxtxq_mtx);
994 	if (txq->vxtxq_br == NULL)
995 		return (ENOMEM);
996 #endif
997 
998 	return (0);
999 }
1000 
1001 static int
1002 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
1003 {
1004 	int i, error;
1005 
1006 	/*
1007 	 * Only attempt to create multiple queues if MSIX is available. MSIX is
1008 	 * disabled by default because its apparently broken for devices passed
1009 	 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable
1010 	 * must be set to zero for MSIX. This check prevents us from allocating
1011 	 * queue structures that we will not use.
1012 	 */
1013 	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
1014 		sc->vmx_max_nrxqueues = 1;
1015 		sc->vmx_max_ntxqueues = 1;
1016 	}
1017 
1018 	sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) *
1019 	    sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
1020 	sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) *
1021 	    sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
1022 	if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL)
1023 		return (ENOMEM);
1024 
1025 	for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
1026 		error = vmxnet3_init_rxq(sc, i);
1027 		if (error)
1028 			return (error);
1029 	}
1030 
1031 	for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
1032 		error = vmxnet3_init_txq(sc, i);
1033 		if (error)
1034 			return (error);
1035 	}
1036 
1037 	return (0);
1038 }
1039 
1040 static void
1041 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
1042 {
1043 	struct vmxnet3_rxring *rxr;
1044 	int i;
1045 
1046 	rxq->vxrxq_sc = NULL;
1047 	rxq->vxrxq_id = -1;
1048 
1049 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1050 		rxr = &rxq->vxrxq_cmd_ring[i];
1051 
1052 		if (rxr->vxrxr_rxbuf != NULL) {
1053 			free(rxr->vxrxr_rxbuf, M_DEVBUF);
1054 			rxr->vxrxr_rxbuf = NULL;
1055 		}
1056 	}
1057 
1058 	if (mtx_initialized(&rxq->vxrxq_mtx) != 0)
1059 		mtx_destroy(&rxq->vxrxq_mtx);
1060 }
1061 
1062 static void
1063 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
1064 {
1065 	struct vmxnet3_txring *txr;
1066 
1067 	txr = &txq->vxtxq_cmd_ring;
1068 
1069 	txq->vxtxq_sc = NULL;
1070 	txq->vxtxq_id = -1;
1071 
1072 #ifndef VMXNET3_LEGACY_TX
1073 	if (txq->vxtxq_br != NULL) {
1074 		buf_ring_free(txq->vxtxq_br, M_DEVBUF);
1075 		txq->vxtxq_br = NULL;
1076 	}
1077 #endif
1078 
1079 	if (txr->vxtxr_txbuf != NULL) {
1080 		free(txr->vxtxr_txbuf, M_DEVBUF);
1081 		txr->vxtxr_txbuf = NULL;
1082 	}
1083 
1084 	if (mtx_initialized(&txq->vxtxq_mtx) != 0)
1085 		mtx_destroy(&txq->vxtxq_mtx);
1086 }
1087 
1088 static void
1089 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
1090 {
1091 	int i;
1092 
1093 	if (sc->vmx_rxq != NULL) {
1094 		for (i = 0; i < sc->vmx_max_nrxqueues; i++)
1095 			vmxnet3_destroy_rxq(&sc->vmx_rxq[i]);
1096 		free(sc->vmx_rxq, M_DEVBUF);
1097 		sc->vmx_rxq = NULL;
1098 	}
1099 
1100 	if (sc->vmx_txq != NULL) {
1101 		for (i = 0; i < sc->vmx_max_ntxqueues; i++)
1102 			vmxnet3_destroy_txq(&sc->vmx_txq[i]);
1103 		free(sc->vmx_txq, M_DEVBUF);
1104 		sc->vmx_txq = NULL;
1105 	}
1106 }
1107 
1108 static int
1109 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
1110 {
1111 	device_t dev;
1112 	uint8_t *kva;
1113 	size_t size;
1114 	int i, error;
1115 
1116 	dev = sc->vmx_dev;
1117 
1118 	size = sizeof(struct vmxnet3_driver_shared);
1119 	error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
1120 	if (error) {
1121 		device_printf(dev, "cannot alloc shared memory\n");
1122 		return (error);
1123 	}
1124 	sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;
1125 
1126 	size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
1127 	    sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
1128 	error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
1129 	if (error) {
1130 		device_printf(dev, "cannot alloc queue shared memory\n");
1131 		return (error);
1132 	}
1133 	sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
1134 	kva = sc->vmx_qs;
1135 
1136 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
1137 		sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
1138 		kva += sizeof(struct vmxnet3_txq_shared);
1139 	}
1140 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
1141 		sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
1142 		kva += sizeof(struct vmxnet3_rxq_shared);
1143 	}
1144 
1145 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1146 		size = sizeof(struct vmxnet3_rss_shared);
1147 		error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
1148 		if (error) {
1149 			device_printf(dev, "cannot alloc rss shared memory\n");
1150 			return (error);
1151 		}
1152 		sc->vmx_rss =
1153 		    (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
1154 	}
1155 
1156 	return (0);
1157 }
1158 
1159 static void
1160 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
1161 {
1162 
1163 	if (sc->vmx_rss != NULL) {
1164 		vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
1165 		sc->vmx_rss = NULL;
1166 	}
1167 
1168 	if (sc->vmx_qs != NULL) {
1169 		vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
1170 		sc->vmx_qs = NULL;
1171 	}
1172 
1173 	if (sc->vmx_ds != NULL) {
1174 		vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
1175 		sc->vmx_ds = NULL;
1176 	}
1177 }
1178 
1179 static int
1180 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
1181 {
1182 	device_t dev;
1183 	struct vmxnet3_txqueue *txq;
1184 	struct vmxnet3_txring *txr;
1185 	struct vmxnet3_comp_ring *txc;
1186 	size_t descsz, compsz;
1187 	int i, q, error;
1188 
1189 	dev = sc->vmx_dev;
1190 
1191 	for (q = 0; q < sc->vmx_ntxqueues; q++) {
1192 		txq = &sc->vmx_txq[q];
1193 		txr = &txq->vxtxq_cmd_ring;
1194 		txc = &txq->vxtxq_comp_ring;
1195 
1196 		descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
1197 		compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);
1198 
1199 		error = bus_dma_tag_create(bus_get_dma_tag(dev),
1200 		    1, 0,			/* alignment, boundary */
1201 		    BUS_SPACE_MAXADDR,		/* lowaddr */
1202 		    BUS_SPACE_MAXADDR,		/* highaddr */
1203 		    NULL, NULL,			/* filter, filterarg */
1204 		    VMXNET3_TX_MAXSIZE,		/* maxsize */
1205 		    VMXNET3_TX_MAXSEGS,		/* nsegments */
1206 		    VMXNET3_TX_MAXSEGSIZE,	/* maxsegsize */
1207 		    0,				/* flags */
1208 		    NULL, NULL,			/* lockfunc, lockarg */
1209 		    &txr->vxtxr_txtag);
1210 		if (error) {
1211 			device_printf(dev,
1212 			    "unable to create Tx buffer tag for queue %d\n", q);
1213 			return (error);
1214 		}
1215 
1216 		error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
1217 		if (error) {
1218 			device_printf(dev, "cannot alloc Tx descriptors for "
1219 			    "queue %d error %d\n", q, error);
1220 			return (error);
1221 		}
1222 		txr->vxtxr_txd =
1223 		    (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;
1224 
1225 		error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
1226 		if (error) {
1227 			device_printf(dev, "cannot alloc Tx comp descriptors "
1228 			   "for queue %d error %d\n", q, error);
1229 			return (error);
1230 		}
1231 		txc->vxcr_u.txcd =
1232 		    (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;
1233 
1234 		for (i = 0; i < txr->vxtxr_ndesc; i++) {
1235 			error = bus_dmamap_create(txr->vxtxr_txtag, 0,
1236 			    &txr->vxtxr_txbuf[i].vtxb_dmamap);
1237 			if (error) {
1238 				device_printf(dev, "unable to create Tx buf "
1239 				    "dmamap for queue %d idx %d\n", q, i);
1240 				return (error);
1241 			}
1242 		}
1243 	}
1244 
1245 	return (0);
1246 }
1247 
1248 static void
1249 vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
1250 {
1251 	device_t dev;
1252 	struct vmxnet3_txqueue *txq;
1253 	struct vmxnet3_txring *txr;
1254 	struct vmxnet3_comp_ring *txc;
1255 	struct vmxnet3_txbuf *txb;
1256 	int i, q;
1257 
1258 	dev = sc->vmx_dev;
1259 
1260 	for (q = 0; q < sc->vmx_ntxqueues; q++) {
1261 		txq = &sc->vmx_txq[q];
1262 		txr = &txq->vxtxq_cmd_ring;
1263 		txc = &txq->vxtxq_comp_ring;
1264 
1265 		for (i = 0; i < txr->vxtxr_ndesc; i++) {
1266 			txb = &txr->vxtxr_txbuf[i];
1267 			if (txb->vtxb_dmamap != NULL) {
1268 				bus_dmamap_destroy(txr->vxtxr_txtag,
1269 				    txb->vtxb_dmamap);
1270 				txb->vtxb_dmamap = NULL;
1271 			}
1272 		}
1273 
1274 		if (txc->vxcr_u.txcd != NULL) {
1275 			vmxnet3_dma_free(sc, &txc->vxcr_dma);
1276 			txc->vxcr_u.txcd = NULL;
1277 		}
1278 
1279 		if (txr->vxtxr_txd != NULL) {
1280 			vmxnet3_dma_free(sc, &txr->vxtxr_dma);
1281 			txr->vxtxr_txd = NULL;
1282 		}
1283 
1284 		if (txr->vxtxr_txtag != NULL) {
1285 			bus_dma_tag_destroy(txr->vxtxr_txtag);
1286 			txr->vxtxr_txtag = NULL;
1287 		}
1288 	}
1289 }
1290 
1291 static int
1292 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
1293 {
1294 	device_t dev;
1295 	struct vmxnet3_rxqueue *rxq;
1296 	struct vmxnet3_rxring *rxr;
1297 	struct vmxnet3_comp_ring *rxc;
1298 	int descsz, compsz;
1299 	int i, j, q, error;
1300 
1301 	dev = sc->vmx_dev;
1302 
1303 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
1304 		rxq = &sc->vmx_rxq[q];
1305 		rxc = &rxq->vxrxq_comp_ring;
1306 		compsz = 0;
1307 
1308 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1309 			rxr = &rxq->vxrxq_cmd_ring[i];
1310 
1311 			descsz = rxr->vxrxr_ndesc *
1312 			    sizeof(struct vmxnet3_rxdesc);
1313 			compsz += rxr->vxrxr_ndesc *
1314 			    sizeof(struct vmxnet3_rxcompdesc);
1315 
1316 			error = bus_dma_tag_create(bus_get_dma_tag(dev),
1317 			    1, 0,		/* alignment, boundary */
1318 			    BUS_SPACE_MAXADDR,	/* lowaddr */
1319 			    BUS_SPACE_MAXADDR,	/* highaddr */
1320 			    NULL, NULL,		/* filter, filterarg */
1321 			    MJUMPAGESIZE,	/* maxsize */
1322 			    1,			/* nsegments */
1323 			    MJUMPAGESIZE,	/* maxsegsize */
1324 			    0,			/* flags */
1325 			    NULL, NULL,		/* lockfunc, lockarg */
1326 			    &rxr->vxrxr_rxtag);
1327 			if (error) {
1328 				device_printf(dev,
1329 				    "unable to create Rx buffer tag for "
1330 				    "queue %d\n", q);
1331 				return (error);
1332 			}
1333 
1334 			error = vmxnet3_dma_malloc(sc, descsz, 512,
1335 			    &rxr->vxrxr_dma);
1336 			if (error) {
1337 				device_printf(dev, "cannot allocate Rx "
1338 				    "descriptors for queue %d/%d error %d\n",
1339 				    i, q, error);
1340 				return (error);
1341 			}
1342 			rxr->vxrxr_rxd =
1343 			    (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
1344 		}
1345 
1346 		error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
1347 		if (error) {
1348 			device_printf(dev, "cannot alloc Rx comp descriptors "
1349 			    "for queue %d error %d\n", q, error);
1350 			return (error);
1351 		}
1352 		rxc->vxcr_u.rxcd =
1353 		    (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;
1354 
1355 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1356 			rxr = &rxq->vxrxq_cmd_ring[i];
1357 
1358 			error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1359 			    &rxr->vxrxr_spare_dmap);
1360 			if (error) {
1361 				device_printf(dev, "unable to create spare "
1362 				    "dmamap for queue %d/%d error %d\n",
1363 				    q, i, error);
1364 				return (error);
1365 			}
1366 
1367 			for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1368 				error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1369 				    &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
1370 				if (error) {
1371 					device_printf(dev, "unable to create "
1372 					    "dmamap for queue %d/%d slot %d "
1373 					    "error %d\n",
1374 					    q, i, j, error);
1375 					return (error);
1376 				}
1377 			}
1378 		}
1379 	}
1380 
1381 	return (0);
1382 }
1383 
1384 static void
1385 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
1386 {
1387 	device_t dev;
1388 	struct vmxnet3_rxqueue *rxq;
1389 	struct vmxnet3_rxring *rxr;
1390 	struct vmxnet3_comp_ring *rxc;
1391 	struct vmxnet3_rxbuf *rxb;
1392 	int i, j, q;
1393 
1394 	dev = sc->vmx_dev;
1395 
1396 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
1397 		rxq = &sc->vmx_rxq[q];
1398 		rxc = &rxq->vxrxq_comp_ring;
1399 
1400 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1401 			rxr = &rxq->vxrxq_cmd_ring[i];
1402 
1403 			if (rxr->vxrxr_spare_dmap != NULL) {
1404 				bus_dmamap_destroy(rxr->vxrxr_rxtag,
1405 				    rxr->vxrxr_spare_dmap);
1406 				rxr->vxrxr_spare_dmap = NULL;
1407 			}
1408 
1409 			for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1410 				rxb = &rxr->vxrxr_rxbuf[j];
1411 				if (rxb->vrxb_dmamap != NULL) {
1412 					bus_dmamap_destroy(rxr->vxrxr_rxtag,
1413 					    rxb->vrxb_dmamap);
1414 					rxb->vrxb_dmamap = NULL;
1415 				}
1416 			}
1417 		}
1418 
1419 		if (rxc->vxcr_u.rxcd != NULL) {
1420 			vmxnet3_dma_free(sc, &rxc->vxcr_dma);
1421 			rxc->vxcr_u.rxcd = NULL;
1422 		}
1423 
1424 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1425 			rxr = &rxq->vxrxq_cmd_ring[i];
1426 
1427 			if (rxr->vxrxr_rxd != NULL) {
1428 				vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
1429 				rxr->vxrxr_rxd = NULL;
1430 			}
1431 
1432 			if (rxr->vxrxr_rxtag != NULL) {
1433 				bus_dma_tag_destroy(rxr->vxrxr_rxtag);
1434 				rxr->vxrxr_rxtag = NULL;
1435 			}
1436 		}
1437 	}
1438 }
1439 
1440 static int
1441 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
1442 {
1443 	int error;
1444 
1445 	error = vmxnet3_alloc_txq_data(sc);
1446 	if (error)
1447 		return (error);
1448 
1449 	error = vmxnet3_alloc_rxq_data(sc);
1450 	if (error)
1451 		return (error);
1452 
1453 	return (0);
1454 }
1455 
1456 static void
1457 vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
1458 {
1459 
1460 	if (sc->vmx_rxq != NULL)
1461 		vmxnet3_free_rxq_data(sc);
1462 
1463 	if (sc->vmx_txq != NULL)
1464 		vmxnet3_free_txq_data(sc);
1465 }
1466 
1467 static int
1468 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1469 {
1470 	int error;
1471 
1472 	error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
1473 	    32, &sc->vmx_mcast_dma);
1474 	if (error)
1475 		device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1476 	else
1477 		sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;
1478 
1479 	return (error);
1480 }
1481 
1482 static void
1483 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1484 {
1485 
1486 	if (sc->vmx_mcast != NULL) {
1487 		vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
1488 		sc->vmx_mcast = NULL;
1489 	}
1490 }
1491 
1492 static void
1493 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1494 {
1495 	struct vmxnet3_driver_shared *ds;
1496 	struct vmxnet3_txqueue *txq;
1497 	struct vmxnet3_txq_shared *txs;
1498 	struct vmxnet3_rxqueue *rxq;
1499 	struct vmxnet3_rxq_shared *rxs;
1500 	int i;
1501 
1502 	ds = sc->vmx_ds;
1503 
1504 	/*
1505 	 * Initialize fields of the shared data that remains the same across
1506 	 * reinits. Note the shared data is zero'd when allocated.
1507 	 */
1508 
1509 	ds->magic = VMXNET3_REV1_MAGIC;
1510 
1511 	/* DriverInfo */
1512 	ds->version = VMXNET3_DRIVER_VERSION;
1513 	ds->guest = VMXNET3_GOS_FREEBSD |
1514 #ifdef __LP64__
1515 	    VMXNET3_GOS_64BIT;
1516 #else
1517 	    VMXNET3_GOS_32BIT;
1518 #endif
1519 	ds->vmxnet3_revision = 1;
1520 	ds->upt_version = 1;
1521 
1522 	/* Misc. conf */
1523 	ds->driver_data = vtophys(sc);
1524 	ds->driver_data_len = sizeof(struct vmxnet3_softc);
1525 	ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
1526 	ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
1527 	ds->nrxsg_max = sc->vmx_max_rxsegs;
1528 
1529 	/* RSS conf */
1530 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1531 		ds->rss.version = 1;
1532 		ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
1533 		ds->rss.len = sc->vmx_rss_dma.dma_size;
1534 	}
1535 
1536 	/* Interrupt control. */
1537 	ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1538 	ds->nintr = sc->vmx_nintrs;
1539 	ds->evintr = sc->vmx_event_intr_idx;
1540 	ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1541 
1542 	for (i = 0; i < sc->vmx_nintrs; i++)
1543 		ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1544 
1545 	/* Receive filter. */
1546 	ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
1547 	ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;
1548 
1549 	/* Tx queues */
1550 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
1551 		txq = &sc->vmx_txq[i];
1552 		txs = txq->vxtxq_ts;
1553 
1554 		txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
1555 		txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1556 		txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
1557 		txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1558 		txs->driver_data = vtophys(txq);
1559 		txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1560 	}
1561 
1562 	/* Rx queues */
1563 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
1564 		rxq = &sc->vmx_rxq[i];
1565 		rxs = rxq->vxrxq_rs;
1566 
1567 		rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
1568 		rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1569 		rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
1570 		rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1571 		rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
1572 		rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1573 		rxs->driver_data = vtophys(rxq);
1574 		rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1575 	}
1576 }
1577 
1578 static void
1579 vmxnet3_reinit_interface(struct vmxnet3_softc *sc)
1580 {
1581 	struct ifnet *ifp;
1582 
1583 	ifp = sc->vmx_ifp;
1584 
1585 	/* Use the current MAC address. */
1586 	bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN);
1587 	vmxnet3_set_lladdr(sc);
1588 
1589 	ifp->if_hwassist = 0;
1590 	if (ifp->if_capenable & IFCAP_TXCSUM)
1591 		ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD;
1592 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1593 		ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6;
1594 	if (ifp->if_capenable & IFCAP_TSO4)
1595 		ifp->if_hwassist |= CSUM_IP_TSO;
1596 	if (ifp->if_capenable & IFCAP_TSO6)
1597 		ifp->if_hwassist |= CSUM_IP6_TSO;
1598 }
1599 
1600 static void
1601 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
1602 {
1603 	/*
1604 	 * Use the same key as the Linux driver until FreeBSD can do
1605 	 * RSS (presumably Toeplitz) in software.
1606 	 */
1607 	static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
1608 	    0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
1609 	    0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
1610 	    0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
1611 	    0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
1612 	    0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
1613 	};
1614 
1615 	struct vmxnet3_driver_shared *ds;
1616 	struct vmxnet3_rss_shared *rss;
1617 	int i;
1618 
1619 	ds = sc->vmx_ds;
1620 	rss = sc->vmx_rss;
1621 
1622 	rss->hash_type =
1623 	    UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
1624 	    UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
1625 	rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
1626 	rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
1627 	rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
1628 	memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
1629 
1630 	for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
1631 		rss->ind_table[i] = i % sc->vmx_nrxqueues;
1632 }
1633 
1634 static void
1635 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1636 {
1637 	struct ifnet *ifp;
1638 	struct vmxnet3_driver_shared *ds;
1639 
1640 	ifp = sc->vmx_ifp;
1641 	ds = sc->vmx_ds;
1642 
1643 	ds->mtu = ifp->if_mtu;
1644 	ds->ntxqueue = sc->vmx_ntxqueues;
1645 	ds->nrxqueue = sc->vmx_nrxqueues;
1646 
1647 	ds->upt_features = 0;
1648 	if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
1649 		ds->upt_features |= UPT1_F_CSUM;
1650 	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1651 		ds->upt_features |= UPT1_F_VLAN;
1652 	if (ifp->if_capenable & IFCAP_LRO)
1653 		ds->upt_features |= UPT1_F_LRO;
1654 
1655 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1656 		ds->upt_features |= UPT1_F_RSS;
1657 		vmxnet3_reinit_rss_shared_data(sc);
1658 	}
1659 
1660 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
1661 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1662 	    (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
1663 }
1664 
1665 static int
1666 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1667 {
1668 	int error;
1669 
1670 	error = vmxnet3_alloc_shared_data(sc);
1671 	if (error)
1672 		return (error);
1673 
1674 	error = vmxnet3_alloc_queue_data(sc);
1675 	if (error)
1676 		return (error);
1677 
1678 	error = vmxnet3_alloc_mcast_table(sc);
1679 	if (error)
1680 		return (error);
1681 
1682 	vmxnet3_init_shared_data(sc);
1683 
1684 	return (0);
1685 }
1686 
1687 static void
1688 vmxnet3_free_data(struct vmxnet3_softc *sc)
1689 {
1690 
1691 	vmxnet3_free_mcast_table(sc);
1692 	vmxnet3_free_queue_data(sc);
1693 	vmxnet3_free_shared_data(sc);
1694 }
1695 
1696 static int
1697 vmxnet3_setup_interface(struct vmxnet3_softc *sc)
1698 {
1699 	device_t dev;
1700 	struct ifnet *ifp;
1701 
1702 	dev = sc->vmx_dev;
1703 
1704 	ifp = sc->vmx_ifp = if_alloc(IFT_ETHER);
1705 	if (ifp == NULL) {
1706 		device_printf(dev, "cannot allocate ifnet structure\n");
1707 		return (ENOSPC);
1708 	}
1709 
1710 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1711 #if __FreeBSD_version < 1000025
1712 	ifp->if_baudrate = 1000000000;
1713 #elif __FreeBSD_version < 1100011
1714 	if_initbaudrate(ifp, IF_Gbps(10));
1715 #else
1716 	ifp->if_baudrate = IF_Gbps(10);
1717 #endif
1718 	ifp->if_softc = sc;
1719 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1720 	ifp->if_init = vmxnet3_init;
1721 	ifp->if_ioctl = vmxnet3_ioctl;
1722 	ifp->if_get_counter = vmxnet3_get_counter;
1723 	ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1724 	ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS;
1725 	ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE;
1726 
1727 #ifdef VMXNET3_LEGACY_TX
1728 	ifp->if_start = vmxnet3_start;
1729 	ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1;
1730 	IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1);
1731 	IFQ_SET_READY(&ifp->if_snd);
1732 #else
1733 	ifp->if_transmit = vmxnet3_txq_mq_start;
1734 	ifp->if_qflush = vmxnet3_qflush;
1735 #endif
1736 
1737 	vmxnet3_get_lladdr(sc);
1738 	ether_ifattach(ifp, sc->vmx_lladdr);
1739 
1740 	ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM;
1741 	ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6;
1742 	ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
1743 	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
1744 	    IFCAP_VLAN_HWCSUM;
1745 	ifp->if_capenable = ifp->if_capabilities;
1746 
1747 	/* These capabilities are not enabled by default. */
1748 	ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER;
1749 
1750 	sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
1751 	    vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
1752 	sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config,
1753 	    vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
1754 
1755 	ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change,
1756 	    vmxnet3_media_status);
1757 	ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1758 	ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO);
1759 
1760 	return (0);
1761 }
1762 
1763 static void
1764 vmxnet3_evintr(struct vmxnet3_softc *sc)
1765 {
1766 	device_t dev;
1767 	struct ifnet *ifp;
1768 	struct vmxnet3_txq_shared *ts;
1769 	struct vmxnet3_rxq_shared *rs;
1770 	uint32_t event;
1771 	int reset;
1772 
1773 	dev = sc->vmx_dev;
1774 	ifp = sc->vmx_ifp;
1775 	reset = 0;
1776 
1777 	VMXNET3_CORE_LOCK(sc);
1778 
1779 	/* Clear events. */
1780 	event = sc->vmx_ds->event;
1781 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
1782 
1783 	if (event & VMXNET3_EVENT_LINK) {
1784 		vmxnet3_link_status(sc);
1785 		if (sc->vmx_link_active != 0)
1786 			vmxnet3_tx_start_all(sc);
1787 	}
1788 
1789 	if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
1790 		reset = 1;
1791 		vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
1792 		ts = sc->vmx_txq[0].vxtxq_ts;
1793 		if (ts->stopped != 0)
1794 			device_printf(dev, "Tx queue error %#x\n", ts->error);
1795 		rs = sc->vmx_rxq[0].vxrxq_rs;
1796 		if (rs->stopped != 0)
1797 			device_printf(dev, "Rx queue error %#x\n", rs->error);
1798 		device_printf(dev, "Rx/Tx queue error event ... resetting\n");
1799 	}
1800 
1801 	if (event & VMXNET3_EVENT_DIC)
1802 		device_printf(dev, "device implementation change event\n");
1803 	if (event & VMXNET3_EVENT_DEBUG)
1804 		device_printf(dev, "debug event\n");
1805 
1806 	if (reset != 0) {
1807 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1808 		vmxnet3_init_locked(sc);
1809 	}
1810 
1811 	VMXNET3_CORE_UNLOCK(sc);
1812 }
1813 
1814 static void
1815 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq)
1816 {
1817 	struct vmxnet3_softc *sc;
1818 	struct ifnet *ifp;
1819 	struct vmxnet3_txring *txr;
1820 	struct vmxnet3_comp_ring *txc;
1821 	struct vmxnet3_txcompdesc *txcd;
1822 	struct vmxnet3_txbuf *txb;
1823 	struct mbuf *m;
1824 	u_int sop;
1825 
1826 	sc = txq->vxtxq_sc;
1827 	ifp = sc->vmx_ifp;
1828 	txr = &txq->vxtxq_cmd_ring;
1829 	txc = &txq->vxtxq_comp_ring;
1830 
1831 	VMXNET3_TXQ_LOCK_ASSERT(txq);
1832 
1833 	for (;;) {
1834 		txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
1835 		if (txcd->gen != txc->vxcr_gen)
1836 			break;
1837 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1838 
1839 		if (++txc->vxcr_next == txc->vxcr_ndesc) {
1840 			txc->vxcr_next = 0;
1841 			txc->vxcr_gen ^= 1;
1842 		}
1843 
1844 		sop = txr->vxtxr_next;
1845 		txb = &txr->vxtxr_txbuf[sop];
1846 
1847 		if ((m = txb->vtxb_m) != NULL) {
1848 			bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
1849 			    BUS_DMASYNC_POSTWRITE);
1850 			bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
1851 
1852 			txq->vxtxq_stats.vmtxs_opackets++;
1853 			txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len;
1854 			if (m->m_flags & M_MCAST)
1855 				txq->vxtxq_stats.vmtxs_omcasts++;
1856 
1857 			m_freem(m);
1858 			txb->vtxb_m = NULL;
1859 		}
1860 
1861 		txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
1862 	}
1863 
1864 	if (txr->vxtxr_head == txr->vxtxr_next)
1865 		txq->vxtxq_watchdog = 0;
1866 }
1867 
1868 static int
1869 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr)
1870 {
1871 	struct ifnet *ifp;
1872 	struct mbuf *m;
1873 	struct vmxnet3_rxdesc *rxd;
1874 	struct vmxnet3_rxbuf *rxb;
1875 	bus_dma_tag_t tag;
1876 	bus_dmamap_t dmap;
1877 	bus_dma_segment_t segs[1];
1878 	int idx, clsize, btype, flags, nsegs, error;
1879 
1880 	ifp = sc->vmx_ifp;
1881 	tag = rxr->vxrxr_rxtag;
1882 	dmap = rxr->vxrxr_spare_dmap;
1883 	idx = rxr->vxrxr_fill;
1884 	rxd = &rxr->vxrxr_rxd[idx];
1885 	rxb = &rxr->vxrxr_rxbuf[idx];
1886 
1887 #ifdef VMXNET3_FAILPOINTS
1888 	KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS);
1889 	if (rxr->vxrxr_rid != 0)
1890 		KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS);
1891 #endif
1892 
1893 	if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) {
1894 		flags = M_PKTHDR;
1895 		clsize = MCLBYTES;
1896 		btype = VMXNET3_BTYPE_HEAD;
1897 	} else {
1898 #if __FreeBSD_version < 902001
1899 		/*
1900 		 * These mbufs will never be used for the start of a frame.
1901 		 * Roughly prior to branching releng/9.2, the load_mbuf_sg()
1902 		 * required the mbuf to always be a packet header. Avoid
1903 		 * unnecessary mbuf initialization in newer versions where
1904 		 * that is not the case.
1905 		 */
1906 		flags = M_PKTHDR;
1907 #else
1908 		flags = 0;
1909 #endif
1910 		clsize = MJUMPAGESIZE;
1911 		btype = VMXNET3_BTYPE_BODY;
1912 	}
1913 
1914 	m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize);
1915 	if (m == NULL) {
1916 		sc->vmx_stats.vmst_mgetcl_failed++;
1917 		return (ENOBUFS);
1918 	}
1919 
1920 	if (btype == VMXNET3_BTYPE_HEAD) {
1921 		m->m_len = m->m_pkthdr.len = clsize;
1922 		m_adj(m, ETHER_ALIGN);
1923 	} else
1924 		m->m_len = clsize;
1925 
1926 	error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs,
1927 	    BUS_DMA_NOWAIT);
1928 	if (error) {
1929 		m_freem(m);
1930 		sc->vmx_stats.vmst_mbuf_load_failed++;
1931 		return (error);
1932 	}
1933 	KASSERT(nsegs == 1,
1934 	    ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
1935 #if __FreeBSD_version < 902001
1936 	if (btype == VMXNET3_BTYPE_BODY)
1937 		m->m_flags &= ~M_PKTHDR;
1938 #endif
1939 
1940 	if (rxb->vrxb_m != NULL) {
1941 		bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD);
1942 		bus_dmamap_unload(tag, rxb->vrxb_dmamap);
1943 	}
1944 
1945 	rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
1946 	rxb->vrxb_dmamap = dmap;
1947 	rxb->vrxb_m = m;
1948 
1949 	rxd->addr = segs[0].ds_addr;
1950 	rxd->len = segs[0].ds_len;
1951 	rxd->btype = btype;
1952 	rxd->gen = rxr->vxrxr_gen;
1953 
1954 	vmxnet3_rxr_increment_fill(rxr);
1955 	return (0);
1956 }
1957 
1958 static void
1959 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
1960     struct vmxnet3_rxring *rxr, int idx)
1961 {
1962 	struct vmxnet3_rxdesc *rxd;
1963 
1964 	rxd = &rxr->vxrxr_rxd[idx];
1965 	rxd->gen = rxr->vxrxr_gen;
1966 	vmxnet3_rxr_increment_fill(rxr);
1967 }
1968 
1969 static void
1970 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
1971 {
1972 	struct vmxnet3_softc *sc;
1973 	struct vmxnet3_rxring *rxr;
1974 	struct vmxnet3_comp_ring *rxc;
1975 	struct vmxnet3_rxcompdesc *rxcd;
1976 	int idx, eof;
1977 
1978 	sc = rxq->vxrxq_sc;
1979 	rxc = &rxq->vxrxq_comp_ring;
1980 
1981 	do {
1982 		rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
1983 		if (rxcd->gen != rxc->vxcr_gen)
1984 			break;		/* Not expected. */
1985 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1986 
1987 		if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
1988 			rxc->vxcr_next = 0;
1989 			rxc->vxcr_gen ^= 1;
1990 		}
1991 
1992 		idx = rxcd->rxd_idx;
1993 		eof = rxcd->eop;
1994 		if (rxcd->qid < sc->vmx_nrxqueues)
1995 			rxr = &rxq->vxrxq_cmd_ring[0];
1996 		else
1997 			rxr = &rxq->vxrxq_cmd_ring[1];
1998 		vmxnet3_rxq_eof_discard(rxq, rxr, idx);
1999 	} while (!eof);
2000 }
2001 
2002 static void
2003 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2004 {
2005 
2006 	if (rxcd->ipv4) {
2007 		m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2008 		if (rxcd->ipcsum_ok)
2009 			m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2010 	}
2011 
2012 	if (!rxcd->fragment) {
2013 		if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) {
2014 			m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2015 			    CSUM_PSEUDO_HDR;
2016 			m->m_pkthdr.csum_data = 0xFFFF;
2017 		}
2018 	}
2019 }
2020 
2021 static void
2022 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
2023     struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2024 {
2025 	struct vmxnet3_softc *sc;
2026 	struct ifnet *ifp;
2027 
2028 	sc = rxq->vxrxq_sc;
2029 	ifp = sc->vmx_ifp;
2030 
2031 	if (rxcd->error) {
2032 		rxq->vxrxq_stats.vmrxs_ierrors++;
2033 		m_freem(m);
2034 		return;
2035 	}
2036 
2037 #ifdef notyet
2038 	switch (rxcd->rss_type) {
2039 	case VMXNET3_RCD_RSS_TYPE_IPV4:
2040 		m->m_pkthdr.flowid = rxcd->rss_hash;
2041 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4);
2042 		break;
2043 	case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
2044 		m->m_pkthdr.flowid = rxcd->rss_hash;
2045 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4);
2046 		break;
2047 	case VMXNET3_RCD_RSS_TYPE_IPV6:
2048 		m->m_pkthdr.flowid = rxcd->rss_hash;
2049 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6);
2050 		break;
2051 	case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
2052 		m->m_pkthdr.flowid = rxcd->rss_hash;
2053 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6);
2054 		break;
2055 	default: /* VMXNET3_RCD_RSS_TYPE_NONE */
2056 		m->m_pkthdr.flowid = rxq->vxrxq_id;
2057 		M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2058 		break;
2059 	}
2060 #else
2061 	m->m_pkthdr.flowid = rxq->vxrxq_id;
2062 	m->m_flags |= M_FLOWID;
2063 #endif
2064 
2065 	if (!rxcd->no_csum)
2066 		vmxnet3_rx_csum(rxcd, m);
2067 	if (rxcd->vlan) {
2068 		m->m_flags |= M_VLANTAG;
2069 		m->m_pkthdr.ether_vtag = rxcd->vtag;
2070 	}
2071 
2072 	rxq->vxrxq_stats.vmrxs_ipackets++;
2073 	rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len;
2074 
2075 	VMXNET3_RXQ_UNLOCK(rxq);
2076 	(*ifp->if_input)(ifp, m);
2077 	VMXNET3_RXQ_LOCK(rxq);
2078 }
2079 
2080 static void
2081 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq)
2082 {
2083 	struct vmxnet3_softc *sc;
2084 	struct ifnet *ifp;
2085 	struct vmxnet3_rxring *rxr;
2086 	struct vmxnet3_comp_ring *rxc;
2087 	struct vmxnet3_rxdesc *rxd;
2088 	struct vmxnet3_rxcompdesc *rxcd;
2089 	struct mbuf *m, *m_head, *m_tail;
2090 	int idx, length;
2091 
2092 	sc = rxq->vxrxq_sc;
2093 	ifp = sc->vmx_ifp;
2094 	rxc = &rxq->vxrxq_comp_ring;
2095 
2096 	VMXNET3_RXQ_LOCK_ASSERT(rxq);
2097 
2098 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2099 		return;
2100 
2101 	m_head = rxq->vxrxq_mhead;
2102 	rxq->vxrxq_mhead = NULL;
2103 	m_tail = rxq->vxrxq_mtail;
2104 	rxq->vxrxq_mtail = NULL;
2105 	MPASS(m_head == NULL || m_tail != NULL);
2106 
2107 	for (;;) {
2108 		rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2109 		if (rxcd->gen != rxc->vxcr_gen) {
2110 			rxq->vxrxq_mhead = m_head;
2111 			rxq->vxrxq_mtail = m_tail;
2112 			break;
2113 		}
2114 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2115 
2116 		if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2117 			rxc->vxcr_next = 0;
2118 			rxc->vxcr_gen ^= 1;
2119 		}
2120 
2121 		idx = rxcd->rxd_idx;
2122 		length = rxcd->len;
2123 		if (rxcd->qid < sc->vmx_nrxqueues)
2124 			rxr = &rxq->vxrxq_cmd_ring[0];
2125 		else
2126 			rxr = &rxq->vxrxq_cmd_ring[1];
2127 		rxd = &rxr->vxrxr_rxd[idx];
2128 
2129 		m = rxr->vxrxr_rxbuf[idx].vrxb_m;
2130 		KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf",
2131 		    __func__, rxcd->qid, idx));
2132 
2133 		/*
2134 		 * The host may skip descriptors. We detect this when this
2135 		 * descriptor does not match the previous fill index. Catch
2136 		 * up with the host now.
2137 		 */
2138 		if (__predict_false(rxr->vxrxr_fill != idx)) {
2139 			while (rxr->vxrxr_fill != idx) {
2140 				rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
2141 				    rxr->vxrxr_gen;
2142 				vmxnet3_rxr_increment_fill(rxr);
2143 			}
2144 		}
2145 
2146 		if (rxcd->sop) {
2147 			KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD,
2148 			    ("%s: start of frame w/o head buffer", __func__));
2149 			KASSERT(rxr == &rxq->vxrxq_cmd_ring[0],
2150 			    ("%s: start of frame not in ring 0", __func__));
2151 			KASSERT((idx % sc->vmx_rx_max_chain) == 0,
2152 			    ("%s: start of frame at unexcepted index %d (%d)",
2153 			     __func__, idx, sc->vmx_rx_max_chain));
2154 			KASSERT(m_head == NULL,
2155 			    ("%s: duplicate start of frame?", __func__));
2156 
2157 			if (length == 0) {
2158 				/* Just ignore this descriptor. */
2159 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2160 				goto nextp;
2161 			}
2162 
2163 			if (vmxnet3_newbuf(sc, rxr) != 0) {
2164 				rxq->vxrxq_stats.vmrxs_iqdrops++;
2165 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2166 				if (!rxcd->eop)
2167 					vmxnet3_rxq_discard_chain(rxq);
2168 				goto nextp;
2169 			}
2170 
2171 			m->m_pkthdr.rcvif = ifp;
2172 			m->m_pkthdr.len = m->m_len = length;
2173 			m->m_pkthdr.csum_flags = 0;
2174 			m_head = m_tail = m;
2175 
2176 		} else {
2177 			KASSERT(rxd->btype == VMXNET3_BTYPE_BODY,
2178 			    ("%s: non start of frame w/o body buffer", __func__));
2179 			KASSERT(m_head != NULL,
2180 			    ("%s: frame not started?", __func__));
2181 
2182 			if (vmxnet3_newbuf(sc, rxr) != 0) {
2183 				rxq->vxrxq_stats.vmrxs_iqdrops++;
2184 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2185 				if (!rxcd->eop)
2186 					vmxnet3_rxq_discard_chain(rxq);
2187 				m_freem(m_head);
2188 				m_head = m_tail = NULL;
2189 				goto nextp;
2190 			}
2191 
2192 			m->m_len = length;
2193 			m_head->m_pkthdr.len += length;
2194 			m_tail->m_next = m;
2195 			m_tail = m;
2196 		}
2197 
2198 		if (rxcd->eop) {
2199 			vmxnet3_rxq_input(rxq, rxcd, m_head);
2200 			m_head = m_tail = NULL;
2201 
2202 			/* Must recheck after dropping the Rx lock. */
2203 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2204 				break;
2205 		}
2206 
2207 nextp:
2208 		if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
2209 			int qid = rxcd->qid;
2210 			bus_size_t r;
2211 
2212 			idx = (idx + 1) % rxr->vxrxr_ndesc;
2213 			if (qid >= sc->vmx_nrxqueues) {
2214 				qid -= sc->vmx_nrxqueues;
2215 				r = VMXNET3_BAR0_RXH2(qid);
2216 			} else
2217 				r = VMXNET3_BAR0_RXH1(qid);
2218 			vmxnet3_write_bar0(sc, r, idx);
2219 		}
2220 	}
2221 }
2222 
2223 static void
2224 vmxnet3_legacy_intr(void *xsc)
2225 {
2226 	struct vmxnet3_softc *sc;
2227 	struct vmxnet3_rxqueue *rxq;
2228 	struct vmxnet3_txqueue *txq;
2229 
2230 	sc = xsc;
2231 	rxq = &sc->vmx_rxq[0];
2232 	txq = &sc->vmx_txq[0];
2233 
2234 	if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
2235 		if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
2236 			return;
2237 	}
2238 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2239 		vmxnet3_disable_all_intrs(sc);
2240 
2241 	if (sc->vmx_ds->event != 0)
2242 		vmxnet3_evintr(sc);
2243 
2244 	VMXNET3_RXQ_LOCK(rxq);
2245 	vmxnet3_rxq_eof(rxq);
2246 	VMXNET3_RXQ_UNLOCK(rxq);
2247 
2248 	VMXNET3_TXQ_LOCK(txq);
2249 	vmxnet3_txq_eof(txq);
2250 	vmxnet3_txq_start(txq);
2251 	VMXNET3_TXQ_UNLOCK(txq);
2252 
2253 	vmxnet3_enable_all_intrs(sc);
2254 }
2255 
2256 static void
2257 vmxnet3_txq_intr(void *xtxq)
2258 {
2259 	struct vmxnet3_softc *sc;
2260 	struct vmxnet3_txqueue *txq;
2261 
2262 	txq = xtxq;
2263 	sc = txq->vxtxq_sc;
2264 
2265 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2266 		vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx);
2267 
2268 	VMXNET3_TXQ_LOCK(txq);
2269 	vmxnet3_txq_eof(txq);
2270 	vmxnet3_txq_start(txq);
2271 	VMXNET3_TXQ_UNLOCK(txq);
2272 
2273 	vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx);
2274 }
2275 
2276 static void
2277 vmxnet3_rxq_intr(void *xrxq)
2278 {
2279 	struct vmxnet3_softc *sc;
2280 	struct vmxnet3_rxqueue *rxq;
2281 
2282 	rxq = xrxq;
2283 	sc = rxq->vxrxq_sc;
2284 
2285 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2286 		vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
2287 
2288 	VMXNET3_RXQ_LOCK(rxq);
2289 	vmxnet3_rxq_eof(rxq);
2290 	VMXNET3_RXQ_UNLOCK(rxq);
2291 
2292 	vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx);
2293 }
2294 
2295 static void
2296 vmxnet3_event_intr(void *xsc)
2297 {
2298 	struct vmxnet3_softc *sc;
2299 
2300 	sc = xsc;
2301 
2302 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2303 		vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
2304 
2305 	if (sc->vmx_ds->event != 0)
2306 		vmxnet3_evintr(sc);
2307 
2308 	vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2309 }
2310 
2311 static void
2312 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2313 {
2314 	struct vmxnet3_txring *txr;
2315 	struct vmxnet3_txbuf *txb;
2316 	int i;
2317 
2318 	txr = &txq->vxtxq_cmd_ring;
2319 
2320 	for (i = 0; i < txr->vxtxr_ndesc; i++) {
2321 		txb = &txr->vxtxr_txbuf[i];
2322 
2323 		if (txb->vtxb_m == NULL)
2324 			continue;
2325 
2326 		bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
2327 		    BUS_DMASYNC_POSTWRITE);
2328 		bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
2329 		m_freem(txb->vtxb_m);
2330 		txb->vtxb_m = NULL;
2331 	}
2332 }
2333 
2334 static void
2335 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2336 {
2337 	struct vmxnet3_rxring *rxr;
2338 	struct vmxnet3_rxbuf *rxb;
2339 	int i, j;
2340 
2341 	if (rxq->vxrxq_mhead != NULL) {
2342 		m_freem(rxq->vxrxq_mhead);
2343 		rxq->vxrxq_mhead = NULL;
2344 		rxq->vxrxq_mtail = NULL;
2345 	}
2346 
2347 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
2348 		rxr = &rxq->vxrxq_cmd_ring[i];
2349 
2350 		for (j = 0; j < rxr->vxrxr_ndesc; j++) {
2351 			rxb = &rxr->vxrxr_rxbuf[j];
2352 
2353 			if (rxb->vrxb_m == NULL)
2354 				continue;
2355 
2356 			bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap,
2357 			    BUS_DMASYNC_POSTREAD);
2358 			bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap);
2359 			m_freem(rxb->vrxb_m);
2360 			rxb->vrxb_m = NULL;
2361 		}
2362 	}
2363 }
2364 
2365 static void
2366 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
2367 {
2368 	struct vmxnet3_rxqueue *rxq;
2369 	struct vmxnet3_txqueue *txq;
2370 	int i;
2371 
2372 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
2373 		rxq = &sc->vmx_rxq[i];
2374 		VMXNET3_RXQ_LOCK(rxq);
2375 		VMXNET3_RXQ_UNLOCK(rxq);
2376 	}
2377 
2378 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
2379 		txq = &sc->vmx_txq[i];
2380 		VMXNET3_TXQ_LOCK(txq);
2381 		VMXNET3_TXQ_UNLOCK(txq);
2382 	}
2383 }
2384 
2385 static void
2386 vmxnet3_stop(struct vmxnet3_softc *sc)
2387 {
2388 	struct ifnet *ifp;
2389 	int q;
2390 
2391 	ifp = sc->vmx_ifp;
2392 	VMXNET3_CORE_LOCK_ASSERT(sc);
2393 
2394 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2395 	sc->vmx_link_active = 0;
2396 	callout_stop(&sc->vmx_tick);
2397 
2398 	/* Disable interrupts. */
2399 	vmxnet3_disable_all_intrs(sc);
2400 	vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
2401 
2402 	vmxnet3_stop_rendezvous(sc);
2403 
2404 	for (q = 0; q < sc->vmx_ntxqueues; q++)
2405 		vmxnet3_txstop(sc, &sc->vmx_txq[q]);
2406 	for (q = 0; q < sc->vmx_nrxqueues; q++)
2407 		vmxnet3_rxstop(sc, &sc->vmx_rxq[q]);
2408 
2409 	vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
2410 }
2411 
2412 static void
2413 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2414 {
2415 	struct vmxnet3_txring *txr;
2416 	struct vmxnet3_comp_ring *txc;
2417 
2418 	txr = &txq->vxtxq_cmd_ring;
2419 	txr->vxtxr_head = 0;
2420 	txr->vxtxr_next = 0;
2421 	txr->vxtxr_gen = VMXNET3_INIT_GEN;
2422 	bzero(txr->vxtxr_txd,
2423 	    txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));
2424 
2425 	txc = &txq->vxtxq_comp_ring;
2426 	txc->vxcr_next = 0;
2427 	txc->vxcr_gen = VMXNET3_INIT_GEN;
2428 	bzero(txc->vxcr_u.txcd,
2429 	    txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
2430 }
2431 
2432 static int
2433 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2434 {
2435 	struct ifnet *ifp;
2436 	struct vmxnet3_rxring *rxr;
2437 	struct vmxnet3_comp_ring *rxc;
2438 	int i, populate, idx, frame_size, error;
2439 
2440 	ifp = sc->vmx_ifp;
2441 	frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) +
2442 	    ifp->if_mtu;
2443 
2444 	/*
2445 	 * If the MTU causes us to exceed what a regular sized cluster can
2446 	 * handle, we allocate a second MJUMPAGESIZE cluster after it in
2447 	 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters.
2448 	 *
2449 	 * Keep rx_max_chain a divisor of the maximum Rx ring size to make
2450 	 * our life easier. We do not support changing the ring size after
2451 	 * the attach.
2452 	 */
2453 	if (frame_size <= MCLBYTES)
2454 		sc->vmx_rx_max_chain = 1;
2455 	else
2456 		sc->vmx_rx_max_chain = 2;
2457 
2458 	/*
2459 	 * Only populate ring 1 if the configuration will take advantage
2460 	 * of it. That is either when LRO is enabled or the frame size
2461 	 * exceeds what ring 0 can contain.
2462 	 */
2463 	if ((ifp->if_capenable & IFCAP_LRO) == 0 &&
2464 	    frame_size <= MCLBYTES + MJUMPAGESIZE)
2465 		populate = 1;
2466 	else
2467 		populate = VMXNET3_RXRINGS_PERQ;
2468 
2469 	for (i = 0; i < populate; i++) {
2470 		rxr = &rxq->vxrxq_cmd_ring[i];
2471 		rxr->vxrxr_fill = 0;
2472 		rxr->vxrxr_gen = VMXNET3_INIT_GEN;
2473 		bzero(rxr->vxrxr_rxd,
2474 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2475 
2476 		for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
2477 			error = vmxnet3_newbuf(sc, rxr);
2478 			if (error)
2479 				return (error);
2480 		}
2481 	}
2482 
2483 	for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
2484 		rxr = &rxq->vxrxq_cmd_ring[i];
2485 		rxr->vxrxr_fill = 0;
2486 		rxr->vxrxr_gen = 0;
2487 		bzero(rxr->vxrxr_rxd,
2488 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2489 	}
2490 
2491 	rxc = &rxq->vxrxq_comp_ring;
2492 	rxc->vxcr_next = 0;
2493 	rxc->vxcr_gen = VMXNET3_INIT_GEN;
2494 	bzero(rxc->vxcr_u.rxcd,
2495 	    rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));
2496 
2497 	return (0);
2498 }
2499 
2500 static int
2501 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
2502 {
2503 	device_t dev;
2504 	int q, error;
2505 
2506 	dev = sc->vmx_dev;
2507 
2508 	for (q = 0; q < sc->vmx_ntxqueues; q++)
2509 		vmxnet3_txinit(sc, &sc->vmx_txq[q]);
2510 
2511 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
2512 		error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
2513 		if (error) {
2514 			device_printf(dev, "cannot populate Rx queue %d\n", q);
2515 			return (error);
2516 		}
2517 	}
2518 
2519 	return (0);
2520 }
2521 
2522 static int
2523 vmxnet3_enable_device(struct vmxnet3_softc *sc)
2524 {
2525 	int q;
2526 
2527 	if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
2528 		device_printf(sc->vmx_dev, "device enable command failed!\n");
2529 		return (1);
2530 	}
2531 
2532 	/* Reset the Rx queue heads. */
2533 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
2534 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
2535 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
2536 	}
2537 
2538 	return (0);
2539 }
2540 
2541 static void
2542 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
2543 {
2544 	struct ifnet *ifp;
2545 
2546 	ifp = sc->vmx_ifp;
2547 
2548 	vmxnet3_set_rxfilter(sc);
2549 
2550 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2551 		bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter,
2552 		    sizeof(sc->vmx_ds->vlan_filter));
2553 	else
2554 		bzero(sc->vmx_ds->vlan_filter,
2555 		    sizeof(sc->vmx_ds->vlan_filter));
2556 	vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
2557 }
2558 
2559 static int
2560 vmxnet3_reinit(struct vmxnet3_softc *sc)
2561 {
2562 
2563 	vmxnet3_reinit_interface(sc);
2564 	vmxnet3_reinit_shared_data(sc);
2565 
2566 	if (vmxnet3_reinit_queues(sc) != 0)
2567 		return (ENXIO);
2568 
2569 	if (vmxnet3_enable_device(sc) != 0)
2570 		return (ENXIO);
2571 
2572 	vmxnet3_reinit_rxfilters(sc);
2573 
2574 	return (0);
2575 }
2576 
2577 static void
2578 vmxnet3_init_locked(struct vmxnet3_softc *sc)
2579 {
2580 	struct ifnet *ifp;
2581 
2582 	ifp = sc->vmx_ifp;
2583 
2584 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2585 		return;
2586 
2587 	vmxnet3_stop(sc);
2588 
2589 	if (vmxnet3_reinit(sc) != 0) {
2590 		vmxnet3_stop(sc);
2591 		return;
2592 	}
2593 
2594 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
2595 	vmxnet3_link_status(sc);
2596 
2597 	vmxnet3_enable_all_intrs(sc);
2598 	callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
2599 }
2600 
2601 static void
2602 vmxnet3_init(void *xsc)
2603 {
2604 	struct vmxnet3_softc *sc;
2605 
2606 	sc = xsc;
2607 
2608 	VMXNET3_CORE_LOCK(sc);
2609 	vmxnet3_init_locked(sc);
2610 	VMXNET3_CORE_UNLOCK(sc);
2611 }
2612 
2613 /*
2614  * BMV: Much of this can go away once we finally have offsets in
2615  * the mbuf packet header. Bug andre@.
2616  */
2617 static int
2618 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
2619     int *etype, int *proto, int *start)
2620 {
2621 	struct ether_vlan_header *evh;
2622 	int offset;
2623 #if defined(INET)
2624 	struct ip *ip = NULL;
2625 	struct ip iphdr;
2626 #endif
2627 #if defined(INET6)
2628 	struct ip6_hdr *ip6 = NULL;
2629 	struct ip6_hdr ip6hdr;
2630 #endif
2631 
2632 	evh = mtod(m, struct ether_vlan_header *);
2633 	if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2634 		/* BMV: We should handle nested VLAN tags too. */
2635 		*etype = ntohs(evh->evl_proto);
2636 		offset = sizeof(struct ether_vlan_header);
2637 	} else {
2638 		*etype = ntohs(evh->evl_encap_proto);
2639 		offset = sizeof(struct ether_header);
2640 	}
2641 
2642 	switch (*etype) {
2643 #if defined(INET)
2644 	case ETHERTYPE_IP:
2645 		if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
2646 			m_copydata(m, offset, sizeof(struct ip),
2647 			    (caddr_t) &iphdr);
2648 			ip = &iphdr;
2649 		} else
2650 			ip = mtodo(m, offset);
2651 		*proto = ip->ip_p;
2652 		*start = offset + (ip->ip_hl << 2);
2653 		break;
2654 #endif
2655 #if defined(INET6)
2656 	case ETHERTYPE_IPV6:
2657 		if (__predict_false(m->m_len <
2658 		    offset + sizeof(struct ip6_hdr))) {
2659 			m_copydata(m, offset, sizeof(struct ip6_hdr),
2660 			    (caddr_t) &ip6hdr);
2661 			ip6 = &ip6hdr;
2662 		} else
2663 			ip6 = mtodo(m, offset);
2664 		*proto = -1;
2665 		*start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
2666 		/* Assert the network stack sent us a valid packet. */
2667 		KASSERT(*start > offset,
2668 		    ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
2669 		    *start, offset, *proto));
2670 		break;
2671 #endif
2672 	default:
2673 		return (EINVAL);
2674 	}
2675 
2676 	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2677 		struct tcphdr *tcp, tcphdr;
2678 		uint16_t sum;
2679 
2680 		if (__predict_false(*proto != IPPROTO_TCP)) {
2681 			/* Likely failed to correctly parse the mbuf. */
2682 			return (EINVAL);
2683 		}
2684 
2685 		txq->vxtxq_stats.vmtxs_tso++;
2686 
2687 		switch (*etype) {
2688 #if defined(INET)
2689 		case ETHERTYPE_IP:
2690 			sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2691 			    htons(IPPROTO_TCP));
2692 			break;
2693 #endif
2694 #if defined(INET6)
2695 		case ETHERTYPE_IPV6:
2696 			sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
2697 			break;
2698 #endif
2699 		default:
2700 			sum = 0;
2701 			break;
2702 		}
2703 
2704 		if (m->m_len < *start + sizeof(struct tcphdr)) {
2705 			m_copyback(m, *start + offsetof(struct tcphdr, th_sum),
2706 			    sizeof(uint16_t), (caddr_t) &sum);
2707 			m_copydata(m, *start, sizeof(struct tcphdr),
2708 			    (caddr_t) &tcphdr);
2709 			tcp = &tcphdr;
2710 		} else {
2711 			tcp = mtodo(m, *start);
2712 			tcp->th_sum = sum;
2713 		}
2714 
2715 		/*
2716 		 * For TSO, the size of the protocol header is also
2717 		 * included in the descriptor header size.
2718 		 */
2719 		*start += (tcp->th_off << 2);
2720 	} else
2721 		txq->vxtxq_stats.vmtxs_csum++;
2722 
2723 	return (0);
2724 }
2725 
2726 static int
2727 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
2728     bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs)
2729 {
2730 	struct vmxnet3_txring *txr;
2731 	struct mbuf *m;
2732 	bus_dma_tag_t tag;
2733 	int error;
2734 
2735 	txr = &txq->vxtxq_cmd_ring;
2736 	m = *m0;
2737 	tag = txr->vxtxr_txtag;
2738 
2739 	error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
2740 	if (error == 0 || error != EFBIG)
2741 		return (error);
2742 
2743 	m = m_defrag(m, M_NOWAIT);
2744 	if (m != NULL) {
2745 		*m0 = m;
2746 		error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
2747 	} else
2748 		error = ENOBUFS;
2749 
2750 	if (error) {
2751 		m_freem(*m0);
2752 		*m0 = NULL;
2753 		txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++;
2754 	} else
2755 		txq->vxtxq_sc->vmx_stats.vmst_defragged++;
2756 
2757 	return (error);
2758 }
2759 
2760 static void
2761 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
2762 {
2763 	struct vmxnet3_txring *txr;
2764 
2765 	txr = &txq->vxtxq_cmd_ring;
2766 	bus_dmamap_unload(txr->vxtxr_txtag, dmap);
2767 }
2768 
2769 static int
2770 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
2771 {
2772 	struct vmxnet3_softc *sc;
2773 	struct vmxnet3_txring *txr;
2774 	struct vmxnet3_txdesc *txd, *sop;
2775 	struct mbuf *m;
2776 	bus_dmamap_t dmap;
2777 	bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS];
2778 	int i, gen, nsegs, etype, proto, start, error;
2779 
2780 	sc = txq->vxtxq_sc;
2781 	start = 0;
2782 	txd = NULL;
2783 	txr = &txq->vxtxq_cmd_ring;
2784 	dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
2785 
2786 	error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs);
2787 	if (error)
2788 		return (error);
2789 
2790 	m = *m0;
2791 	M_ASSERTPKTHDR(m);
2792 	KASSERT(nsegs <= VMXNET3_TX_MAXSEGS,
2793 	    ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
2794 
2795 	if (VMXNET3_TXRING_AVAIL(txr) < nsegs) {
2796 		txq->vxtxq_stats.vmtxs_full++;
2797 		vmxnet3_txq_unload_mbuf(txq, dmap);
2798 		return (ENOSPC);
2799 	} else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
2800 		error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start);
2801 		if (error) {
2802 			txq->vxtxq_stats.vmtxs_offload_failed++;
2803 			vmxnet3_txq_unload_mbuf(txq, dmap);
2804 			m_freem(m);
2805 			*m0 = NULL;
2806 			return (error);
2807 		}
2808 	}
2809 
2810 	txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m;
2811 	sop = &txr->vxtxr_txd[txr->vxtxr_head];
2812 	gen = txr->vxtxr_gen ^ 1;	/* Owned by cpu (yet) */
2813 
2814 	for (i = 0; i < nsegs; i++) {
2815 		txd = &txr->vxtxr_txd[txr->vxtxr_head];
2816 
2817 		txd->addr = segs[i].ds_addr;
2818 		txd->len = segs[i].ds_len;
2819 		txd->gen = gen;
2820 		txd->dtype = 0;
2821 		txd->offload_mode = VMXNET3_OM_NONE;
2822 		txd->offload_pos = 0;
2823 		txd->hlen = 0;
2824 		txd->eop = 0;
2825 		txd->compreq = 0;
2826 		txd->vtag_mode = 0;
2827 		txd->vtag = 0;
2828 
2829 		if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
2830 			txr->vxtxr_head = 0;
2831 			txr->vxtxr_gen ^= 1;
2832 		}
2833 		gen = txr->vxtxr_gen;
2834 	}
2835 	txd->eop = 1;
2836 	txd->compreq = 1;
2837 
2838 	if (m->m_flags & M_VLANTAG) {
2839 		sop->vtag_mode = 1;
2840 		sop->vtag = m->m_pkthdr.ether_vtag;
2841 	}
2842 
2843 	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2844 		sop->offload_mode = VMXNET3_OM_TSO;
2845 		sop->hlen = start;
2846 		sop->offload_pos = m->m_pkthdr.tso_segsz;
2847 	} else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
2848 	    VMXNET3_CSUM_OFFLOAD_IPV6)) {
2849 		sop->offload_mode = VMXNET3_OM_CSUM;
2850 		sop->hlen = start;
2851 		sop->offload_pos = start + m->m_pkthdr.csum_data;
2852 	}
2853 
2854 	/* Finally, change the ownership. */
2855 	vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
2856 	sop->gen ^= 1;
2857 
2858 	txq->vxtxq_ts->npending += nsegs;
2859 	if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
2860 		txq->vxtxq_ts->npending = 0;
2861 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
2862 		    txr->vxtxr_head);
2863 	}
2864 
2865 	return (0);
2866 }
2867 
2868 #ifdef VMXNET3_LEGACY_TX
2869 
2870 static void
2871 vmxnet3_start_locked(struct ifnet *ifp)
2872 {
2873 	struct vmxnet3_softc *sc;
2874 	struct vmxnet3_txqueue *txq;
2875 	struct vmxnet3_txring *txr;
2876 	struct mbuf *m_head;
2877 	int tx, avail;
2878 
2879 	sc = ifp->if_softc;
2880 	txq = &sc->vmx_txq[0];
2881 	txr = &txq->vxtxq_cmd_ring;
2882 	tx = 0;
2883 
2884 	VMXNET3_TXQ_LOCK_ASSERT(txq);
2885 
2886 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2887 	    sc->vmx_link_active == 0)
2888 		return;
2889 
2890 	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
2891 		if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2)
2892 			break;
2893 
2894 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
2895 		if (m_head == NULL)
2896 			break;
2897 
2898 		/* Assume worse case if this mbuf is the head of a chain. */
2899 		if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
2900 			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2901 			break;
2902 		}
2903 
2904 		if (vmxnet3_txq_encap(txq, &m_head) != 0) {
2905 			if (m_head != NULL)
2906 				IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2907 			break;
2908 		}
2909 
2910 		tx++;
2911 		ETHER_BPF_MTAP(ifp, m_head);
2912 	}
2913 
2914 	if (tx > 0)
2915 		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
2916 }
2917 
2918 static void
2919 vmxnet3_start(struct ifnet *ifp)
2920 {
2921 	struct vmxnet3_softc *sc;
2922 	struct vmxnet3_txqueue *txq;
2923 
2924 	sc = ifp->if_softc;
2925 	txq = &sc->vmx_txq[0];
2926 
2927 	VMXNET3_TXQ_LOCK(txq);
2928 	vmxnet3_start_locked(ifp);
2929 	VMXNET3_TXQ_UNLOCK(txq);
2930 }
2931 
2932 #else /* !VMXNET3_LEGACY_TX */
2933 
2934 static int
2935 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m)
2936 {
2937 	struct vmxnet3_softc *sc;
2938 	struct vmxnet3_txring *txr;
2939 	struct buf_ring *br;
2940 	struct ifnet *ifp;
2941 	int tx, avail, error;
2942 
2943 	sc = txq->vxtxq_sc;
2944 	br = txq->vxtxq_br;
2945 	ifp = sc->vmx_ifp;
2946 	txr = &txq->vxtxq_cmd_ring;
2947 	tx = 0;
2948 	error = 0;
2949 
2950 	VMXNET3_TXQ_LOCK_ASSERT(txq);
2951 
2952 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2953 	    sc->vmx_link_active == 0) {
2954 		if (m != NULL)
2955 			error = drbr_enqueue(ifp, br, m);
2956 		return (error);
2957 	}
2958 
2959 	if (m != NULL) {
2960 		error = drbr_enqueue(ifp, br, m);
2961 		if (error)
2962 			return (error);
2963 	}
2964 
2965 	while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) {
2966 		m = drbr_peek(ifp, br);
2967 		if (m == NULL)
2968 			break;
2969 
2970 		/* Assume worse case if this mbuf is the head of a chain. */
2971 		if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
2972 			drbr_putback(ifp, br, m);
2973 			break;
2974 		}
2975 
2976 		if (vmxnet3_txq_encap(txq, &m) != 0) {
2977 			if (m != NULL)
2978 				drbr_putback(ifp, br, m);
2979 			else
2980 				drbr_advance(ifp, br);
2981 			break;
2982 		}
2983 		drbr_advance(ifp, br);
2984 
2985 		tx++;
2986 		ETHER_BPF_MTAP(ifp, m);
2987 	}
2988 
2989 	if (tx > 0)
2990 		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
2991 
2992 	return (0);
2993 }
2994 
2995 static int
2996 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m)
2997 {
2998 	struct vmxnet3_softc *sc;
2999 	struct vmxnet3_txqueue *txq;
3000 	int i, ntxq, error;
3001 
3002 	sc = ifp->if_softc;
3003 	ntxq = sc->vmx_ntxqueues;
3004 
3005 	if (m->m_flags & M_FLOWID)
3006 		i = m->m_pkthdr.flowid % ntxq;
3007 	else
3008 		i = curcpu % ntxq;
3009 
3010 	txq = &sc->vmx_txq[i];
3011 
3012 	if (VMXNET3_TXQ_TRYLOCK(txq) != 0) {
3013 		error = vmxnet3_txq_mq_start_locked(txq, m);
3014 		VMXNET3_TXQ_UNLOCK(txq);
3015 	} else {
3016 		error = drbr_enqueue(ifp, txq->vxtxq_br, m);
3017 		taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask);
3018 	}
3019 
3020 	return (error);
3021 }
3022 
3023 static void
3024 vmxnet3_txq_tq_deferred(void *xtxq, int pending)
3025 {
3026 	struct vmxnet3_softc *sc;
3027 	struct vmxnet3_txqueue *txq;
3028 
3029 	txq = xtxq;
3030 	sc = txq->vxtxq_sc;
3031 
3032 	VMXNET3_TXQ_LOCK(txq);
3033 	if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br))
3034 		vmxnet3_txq_mq_start_locked(txq, NULL);
3035 	VMXNET3_TXQ_UNLOCK(txq);
3036 }
3037 
3038 #endif /* VMXNET3_LEGACY_TX */
3039 
3040 static void
3041 vmxnet3_txq_start(struct vmxnet3_txqueue *txq)
3042 {
3043 	struct vmxnet3_softc *sc;
3044 	struct ifnet *ifp;
3045 
3046 	sc = txq->vxtxq_sc;
3047 	ifp = sc->vmx_ifp;
3048 
3049 #ifdef VMXNET3_LEGACY_TX
3050 	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3051 		vmxnet3_start_locked(ifp);
3052 #else
3053 	if (!drbr_empty(ifp, txq->vxtxq_br))
3054 		vmxnet3_txq_mq_start_locked(txq, NULL);
3055 #endif
3056 }
3057 
3058 static void
3059 vmxnet3_tx_start_all(struct vmxnet3_softc *sc)
3060 {
3061 	struct vmxnet3_txqueue *txq;
3062 	int i;
3063 
3064 	VMXNET3_CORE_LOCK_ASSERT(sc);
3065 
3066 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3067 		txq = &sc->vmx_txq[i];
3068 
3069 		VMXNET3_TXQ_LOCK(txq);
3070 		vmxnet3_txq_start(txq);
3071 		VMXNET3_TXQ_UNLOCK(txq);
3072 	}
3073 }
3074 
3075 static void
3076 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
3077 {
3078 	struct ifnet *ifp;
3079 	int idx, bit;
3080 
3081 	ifp = sc->vmx_ifp;
3082 	idx = (tag >> 5) & 0x7F;
3083 	bit = tag & 0x1F;
3084 
3085 	if (tag == 0 || tag > 4095)
3086 		return;
3087 
3088 	VMXNET3_CORE_LOCK(sc);
3089 
3090 	/* Update our private VLAN bitvector. */
3091 	if (add)
3092 		sc->vmx_vlan_filter[idx] |= (1 << bit);
3093 	else
3094 		sc->vmx_vlan_filter[idx] &= ~(1 << bit);
3095 
3096 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
3097 		if (add)
3098 			sc->vmx_ds->vlan_filter[idx] |= (1 << bit);
3099 		else
3100 			sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit);
3101 		vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
3102 	}
3103 
3104 	VMXNET3_CORE_UNLOCK(sc);
3105 }
3106 
3107 static void
3108 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3109 {
3110 
3111 	if (ifp->if_softc == arg)
3112 		vmxnet3_update_vlan_filter(arg, 1, tag);
3113 }
3114 
3115 static void
3116 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3117 {
3118 
3119 	if (ifp->if_softc == arg)
3120 		vmxnet3_update_vlan_filter(arg, 0, tag);
3121 }
3122 
3123 static void
3124 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
3125 {
3126 	struct ifnet *ifp;
3127 	struct vmxnet3_driver_shared *ds;
3128 	struct ifmultiaddr *ifma;
3129 	u_int mode;
3130 
3131 	ifp = sc->vmx_ifp;
3132 	ds = sc->vmx_ds;
3133 
3134 	mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST;
3135 	if (ifp->if_flags & IFF_PROMISC)
3136 		mode |= VMXNET3_RXMODE_PROMISC;
3137 	if (ifp->if_flags & IFF_ALLMULTI)
3138 		mode |= VMXNET3_RXMODE_ALLMULTI;
3139 	else {
3140 		int cnt = 0, overflow = 0;
3141 
3142 		if_maddr_rlock(ifp);
3143 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3144 			if (ifma->ifma_addr->sa_family != AF_LINK)
3145 				continue;
3146 			else if (cnt == VMXNET3_MULTICAST_MAX) {
3147 				overflow = 1;
3148 				break;
3149 			}
3150 
3151 			bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
3152 			   &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN);
3153 			cnt++;
3154 		}
3155 		if_maddr_runlock(ifp);
3156 
3157 		if (overflow != 0) {
3158 			cnt = 0;
3159 			mode |= VMXNET3_RXMODE_ALLMULTI;
3160 		} else if (cnt > 0)
3161 			mode |= VMXNET3_RXMODE_MCAST;
3162 		ds->mcast_tablelen = cnt * ETHER_ADDR_LEN;
3163 	}
3164 
3165 	ds->rxmode = mode;
3166 
3167 	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
3168 	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
3169 }
3170 
3171 static int
3172 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu)
3173 {
3174 	struct ifnet *ifp;
3175 
3176 	ifp = sc->vmx_ifp;
3177 
3178 	if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU)
3179 		return (EINVAL);
3180 
3181 	ifp->if_mtu = mtu;
3182 
3183 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3184 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3185 		vmxnet3_init_locked(sc);
3186 	}
3187 
3188 	return (0);
3189 }
3190 
3191 static int
3192 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
3193 {
3194 	struct vmxnet3_softc *sc;
3195 	struct ifreq *ifr;
3196 	int reinit, mask, error;
3197 
3198 	sc = ifp->if_softc;
3199 	ifr = (struct ifreq *) data;
3200 	error = 0;
3201 
3202 	switch (cmd) {
3203 	case SIOCSIFMTU:
3204 		if (ifp->if_mtu != ifr->ifr_mtu) {
3205 			VMXNET3_CORE_LOCK(sc);
3206 			error = vmxnet3_change_mtu(sc, ifr->ifr_mtu);
3207 			VMXNET3_CORE_UNLOCK(sc);
3208 		}
3209 		break;
3210 
3211 	case SIOCSIFFLAGS:
3212 		VMXNET3_CORE_LOCK(sc);
3213 		if (ifp->if_flags & IFF_UP) {
3214 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3215 				if ((ifp->if_flags ^ sc->vmx_if_flags) &
3216 				    (IFF_PROMISC | IFF_ALLMULTI)) {
3217 					vmxnet3_set_rxfilter(sc);
3218 				}
3219 			} else
3220 				vmxnet3_init_locked(sc);
3221 		} else {
3222 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3223 				vmxnet3_stop(sc);
3224 		}
3225 		sc->vmx_if_flags = ifp->if_flags;
3226 		VMXNET3_CORE_UNLOCK(sc);
3227 		break;
3228 
3229 	case SIOCADDMULTI:
3230 	case SIOCDELMULTI:
3231 		VMXNET3_CORE_LOCK(sc);
3232 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3233 			vmxnet3_set_rxfilter(sc);
3234 		VMXNET3_CORE_UNLOCK(sc);
3235 		break;
3236 
3237 	case SIOCSIFMEDIA:
3238 	case SIOCGIFMEDIA:
3239 		error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd);
3240 		break;
3241 
3242 	case SIOCSIFCAP:
3243 		VMXNET3_CORE_LOCK(sc);
3244 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3245 
3246 		if (mask & IFCAP_TXCSUM)
3247 			ifp->if_capenable ^= IFCAP_TXCSUM;
3248 		if (mask & IFCAP_TXCSUM_IPV6)
3249 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3250 		if (mask & IFCAP_TSO4)
3251 			ifp->if_capenable ^= IFCAP_TSO4;
3252 		if (mask & IFCAP_TSO6)
3253 			ifp->if_capenable ^= IFCAP_TSO6;
3254 
3255 		if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO |
3256 		    IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) {
3257 			/* Changing these features requires us to reinit. */
3258 			reinit = 1;
3259 
3260 			if (mask & IFCAP_RXCSUM)
3261 				ifp->if_capenable ^= IFCAP_RXCSUM;
3262 			if (mask & IFCAP_RXCSUM_IPV6)
3263 				ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3264 			if (mask & IFCAP_LRO)
3265 				ifp->if_capenable ^= IFCAP_LRO;
3266 			if (mask & IFCAP_VLAN_HWTAGGING)
3267 				ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3268 			if (mask & IFCAP_VLAN_HWFILTER)
3269 				ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3270 		} else
3271 			reinit = 0;
3272 
3273 		if (mask & IFCAP_VLAN_HWTSO)
3274 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3275 
3276 		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3277 			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3278 			vmxnet3_init_locked(sc);
3279 		}
3280 
3281 		VMXNET3_CORE_UNLOCK(sc);
3282 		VLAN_CAPABILITIES(ifp);
3283 		break;
3284 
3285 	default:
3286 		error = ether_ioctl(ifp, cmd, data);
3287 		break;
3288 	}
3289 
3290 	VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc);
3291 
3292 	return (error);
3293 }
3294 
3295 #ifndef VMXNET3_LEGACY_TX
3296 static void
3297 vmxnet3_qflush(struct ifnet *ifp)
3298 {
3299 	struct vmxnet3_softc *sc;
3300 	struct vmxnet3_txqueue *txq;
3301 	struct mbuf *m;
3302 	int i;
3303 
3304 	sc = ifp->if_softc;
3305 
3306 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3307 		txq = &sc->vmx_txq[i];
3308 
3309 		VMXNET3_TXQ_LOCK(txq);
3310 		while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL)
3311 			m_freem(m);
3312 		VMXNET3_TXQ_UNLOCK(txq);
3313 	}
3314 
3315 	if_qflush(ifp);
3316 }
3317 #endif
3318 
3319 static int
3320 vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
3321 {
3322 	struct vmxnet3_softc *sc;
3323 
3324 	sc = txq->vxtxq_sc;
3325 
3326 	VMXNET3_TXQ_LOCK(txq);
3327 	if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
3328 		VMXNET3_TXQ_UNLOCK(txq);
3329 		return (0);
3330 	}
3331 	VMXNET3_TXQ_UNLOCK(txq);
3332 
3333 	if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n",
3334 	    txq->vxtxq_id);
3335 	return (1);
3336 }
3337 
3338 static void
3339 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
3340 {
3341 
3342 	vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
3343 }
3344 
3345 static uint64_t
3346 vmxnet3_get_counter(struct ifnet *ifp, ift_counter cnt)
3347 {
3348 	struct vmxnet3_softc *sc;
3349 	uint64_t rv;
3350 
3351 	sc = if_getsoftc(ifp);
3352 	rv = 0;
3353 
3354 	/*
3355 	 * With the exception of if_ierrors, these ifnet statistics are
3356 	 * only updated in the driver, so just set them to our accumulated
3357 	 * values. if_ierrors is updated in ether_input() for malformed
3358 	 * frames that we should have already discarded.
3359 	 */
3360 	switch (cnt) {
3361 	case IFCOUNTER_IPACKETS:
3362 		for (int i = 0; i < sc->vmx_nrxqueues; i++)
3363 			rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ipackets;
3364 		return (rv);
3365 	case IFCOUNTER_IQDROPS:
3366 		for (int i = 0; i < sc->vmx_nrxqueues; i++)
3367 			rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_iqdrops;
3368 		return (rv);
3369 	case IFCOUNTER_IERRORS:
3370 		for (int i = 0; i < sc->vmx_nrxqueues; i++)
3371 			rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ierrors;
3372 		return (rv);
3373 	case IFCOUNTER_OPACKETS:
3374 		for (int i = 0; i < sc->vmx_ntxqueues; i++)
3375 			rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_opackets;
3376 		return (rv);
3377 #ifndef VMXNET3_LEGACY_TX
3378 	case IFCOUNTER_OBYTES:
3379 		for (int i = 0; i < sc->vmx_ntxqueues; i++)
3380 			rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_obytes;
3381 		return (rv);
3382 	case IFCOUNTER_OMCASTS:
3383 		for (int i = 0; i < sc->vmx_ntxqueues; i++)
3384 			rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_omcasts;
3385 		return (rv);
3386 #endif
3387 	default:
3388 		return (if_get_counter_default(ifp, cnt));
3389 	}
3390 }
3391 
3392 static void
3393 vmxnet3_tick(void *xsc)
3394 {
3395 	struct vmxnet3_softc *sc;
3396 	struct ifnet *ifp;
3397 	int i, timedout;
3398 
3399 	sc = xsc;
3400 	ifp = sc->vmx_ifp;
3401 	timedout = 0;
3402 
3403 	VMXNET3_CORE_LOCK_ASSERT(sc);
3404 
3405 	vmxnet3_refresh_host_stats(sc);
3406 
3407 	for (i = 0; i < sc->vmx_ntxqueues; i++)
3408 		timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]);
3409 
3410 	if (timedout != 0) {
3411 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3412 		vmxnet3_init_locked(sc);
3413 	} else
3414 		callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
3415 }
3416 
3417 static int
3418 vmxnet3_link_is_up(struct vmxnet3_softc *sc)
3419 {
3420 	uint32_t status;
3421 
3422 	/* Also update the link speed while here. */
3423 	status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
3424 	sc->vmx_link_speed = status >> 16;
3425 	return !!(status & 0x1);
3426 }
3427 
3428 static void
3429 vmxnet3_link_status(struct vmxnet3_softc *sc)
3430 {
3431 	struct ifnet *ifp;
3432 	int link;
3433 
3434 	ifp = sc->vmx_ifp;
3435 	link = vmxnet3_link_is_up(sc);
3436 
3437 	if (link != 0 && sc->vmx_link_active == 0) {
3438 		sc->vmx_link_active = 1;
3439 		if_link_state_change(ifp, LINK_STATE_UP);
3440 	} else if (link == 0 && sc->vmx_link_active != 0) {
3441 		sc->vmx_link_active = 0;
3442 		if_link_state_change(ifp, LINK_STATE_DOWN);
3443 	}
3444 }
3445 
3446 static void
3447 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
3448 {
3449 	struct vmxnet3_softc *sc;
3450 
3451 	sc = ifp->if_softc;
3452 
3453 	ifmr->ifm_active = IFM_ETHER | IFM_AUTO;
3454 	ifmr->ifm_status = IFM_AVALID;
3455 
3456 	VMXNET3_CORE_LOCK(sc);
3457 	if (vmxnet3_link_is_up(sc) != 0)
3458 		ifmr->ifm_status |= IFM_ACTIVE;
3459 	else
3460 		ifmr->ifm_status |= IFM_NONE;
3461 	VMXNET3_CORE_UNLOCK(sc);
3462 }
3463 
3464 static int
3465 vmxnet3_media_change(struct ifnet *ifp)
3466 {
3467 
3468 	/* Ignore. */
3469 	return (0);
3470 }
3471 
3472 static void
3473 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
3474 {
3475 	uint32_t ml, mh;
3476 
3477 	ml  = sc->vmx_lladdr[0];
3478 	ml |= sc->vmx_lladdr[1] << 8;
3479 	ml |= sc->vmx_lladdr[2] << 16;
3480 	ml |= sc->vmx_lladdr[3] << 24;
3481 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
3482 
3483 	mh  = sc->vmx_lladdr[4];
3484 	mh |= sc->vmx_lladdr[5] << 8;
3485 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
3486 }
3487 
3488 static void
3489 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
3490 {
3491 	uint32_t ml, mh;
3492 
3493 	ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
3494 	mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
3495 
3496 	sc->vmx_lladdr[0] = ml;
3497 	sc->vmx_lladdr[1] = ml >> 8;
3498 	sc->vmx_lladdr[2] = ml >> 16;
3499 	sc->vmx_lladdr[3] = ml >> 24;
3500 	sc->vmx_lladdr[4] = mh;
3501 	sc->vmx_lladdr[5] = mh >> 8;
3502 }
3503 
3504 static void
3505 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq,
3506     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3507 {
3508 	struct sysctl_oid *node, *txsnode;
3509 	struct sysctl_oid_list *list, *txslist;
3510 	struct vmxnet3_txq_stats *stats;
3511 	struct UPT1_TxStats *txstats;
3512 	char namebuf[16];
3513 
3514 	stats = &txq->vxtxq_stats;
3515 	txstats = &txq->vxtxq_ts->stats;
3516 
3517 	snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id);
3518 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3519 	    NULL, "Transmit Queue");
3520 	txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node);
3521 
3522 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD,
3523 	    &stats->vmtxs_opackets, "Transmit packets");
3524 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD,
3525 	    &stats->vmtxs_obytes, "Transmit bytes");
3526 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD,
3527 	    &stats->vmtxs_omcasts, "Transmit multicasts");
3528 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
3529 	    &stats->vmtxs_csum, "Transmit checksum offloaded");
3530 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
3531 	    &stats->vmtxs_tso, "Transmit TCP segmentation offloaded");
3532 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD,
3533 	    &stats->vmtxs_full, "Transmit ring full");
3534 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD,
3535 	    &stats->vmtxs_offload_failed, "Transmit checksum offload failed");
3536 
3537 	/*
3538 	 * Add statistics reported by the host. These are updated once
3539 	 * per second.
3540 	 */
3541 	txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3542 	    NULL, "Host Statistics");
3543 	txslist = SYSCTL_CHILDREN(txsnode);
3544 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD,
3545 	    &txstats->TSO_packets, "TSO packets");
3546 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD,
3547 	    &txstats->TSO_bytes, "TSO bytes");
3548 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3549 	    &txstats->ucast_packets, "Unicast packets");
3550 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3551 	    &txstats->ucast_bytes, "Unicast bytes");
3552 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3553 	    &txstats->mcast_packets, "Multicast packets");
3554 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3555 	    &txstats->mcast_bytes, "Multicast bytes");
3556 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD,
3557 	    &txstats->error, "Errors");
3558 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD,
3559 	    &txstats->discard, "Discards");
3560 }
3561 
3562 static void
3563 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq,
3564     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3565 {
3566 	struct sysctl_oid *node, *rxsnode;
3567 	struct sysctl_oid_list *list, *rxslist;
3568 	struct vmxnet3_rxq_stats *stats;
3569 	struct UPT1_RxStats *rxstats;
3570 	char namebuf[16];
3571 
3572 	stats = &rxq->vxrxq_stats;
3573 	rxstats = &rxq->vxrxq_rs->stats;
3574 
3575 	snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id);
3576 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3577 	    NULL, "Receive Queue");
3578 	rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node);
3579 
3580 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD,
3581 	    &stats->vmrxs_ipackets, "Receive packets");
3582 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD,
3583 	    &stats->vmrxs_ibytes, "Receive bytes");
3584 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD,
3585 	    &stats->vmrxs_iqdrops, "Receive drops");
3586 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD,
3587 	    &stats->vmrxs_ierrors, "Receive errors");
3588 
3589 	/*
3590 	 * Add statistics reported by the host. These are updated once
3591 	 * per second.
3592 	 */
3593 	rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3594 	    NULL, "Host Statistics");
3595 	rxslist = SYSCTL_CHILDREN(rxsnode);
3596 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD,
3597 	    &rxstats->LRO_packets, "LRO packets");
3598 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD,
3599 	    &rxstats->LRO_bytes, "LRO bytes");
3600 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3601 	    &rxstats->ucast_packets, "Unicast packets");
3602 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3603 	    &rxstats->ucast_bytes, "Unicast bytes");
3604 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3605 	    &rxstats->mcast_packets, "Multicast packets");
3606 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3607 	    &rxstats->mcast_bytes, "Multicast bytes");
3608 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD,
3609 	    &rxstats->bcast_packets, "Broadcast packets");
3610 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD,
3611 	    &rxstats->bcast_bytes, "Broadcast bytes");
3612 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD,
3613 	    &rxstats->nobuffer, "No buffer");
3614 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD,
3615 	    &rxstats->error, "Errors");
3616 }
3617 
3618 static void
3619 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc,
3620     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3621 {
3622 	struct sysctl_oid *node;
3623 	struct sysctl_oid_list *list;
3624 	int i;
3625 
3626 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3627 		struct vmxnet3_txqueue *txq = &sc->vmx_txq[i];
3628 
3629 		node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO,
3630 		    "debug", CTLFLAG_RD, NULL, "");
3631 		list = SYSCTL_CHILDREN(node);
3632 
3633 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD,
3634 		    &txq->vxtxq_cmd_ring.vxtxr_head, 0, "");
3635 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD,
3636 		    &txq->vxtxq_cmd_ring.vxtxr_next, 0, "");
3637 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD,
3638 		    &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, "");
3639 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD,
3640 		    &txq->vxtxq_cmd_ring.vxtxr_gen, 0, "");
3641 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3642 		    &txq->vxtxq_comp_ring.vxcr_next, 0, "");
3643 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3644 		    &txq->vxtxq_comp_ring.vxcr_ndesc, 0,"");
3645 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3646 		    &txq->vxtxq_comp_ring.vxcr_gen, 0, "");
3647 	}
3648 
3649 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
3650 		struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i];
3651 
3652 		node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
3653 		    "debug", CTLFLAG_RD, NULL, "");
3654 		list = SYSCTL_CHILDREN(node);
3655 
3656 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD,
3657 		    &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, "");
3658 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
3659 		    &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
3660 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
3661 		    &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
3662 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD,
3663 		    &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, "");
3664 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
3665 		    &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
3666 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
3667 		    &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
3668 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3669 		    &rxq->vxrxq_comp_ring.vxcr_next, 0, "");
3670 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3671 		    &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
3672 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3673 		    &rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
3674 	}
3675 }
3676 
3677 static void
3678 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
3679     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3680 {
3681 	int i;
3682 
3683 	for (i = 0; i < sc->vmx_ntxqueues; i++)
3684 		vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child);
3685 	for (i = 0; i < sc->vmx_nrxqueues; i++)
3686 		vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child);
3687 
3688 	vmxnet3_setup_debug_sysctl(sc, ctx, child);
3689 }
3690 
3691 static void
3692 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
3693 {
3694 	device_t dev;
3695 	struct vmxnet3_statistics *stats;
3696 	struct sysctl_ctx_list *ctx;
3697 	struct sysctl_oid *tree;
3698 	struct sysctl_oid_list *child;
3699 
3700 	dev = sc->vmx_dev;
3701 	ctx = device_get_sysctl_ctx(dev);
3702 	tree = device_get_sysctl_tree(dev);
3703 	child = SYSCTL_CHILDREN(tree);
3704 
3705 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD,
3706 	    &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues");
3707 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD,
3708 	    &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues");
3709 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD,
3710 	    &sc->vmx_ntxqueues, 0, "Number of Tx queues");
3711 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD,
3712 	    &sc->vmx_nrxqueues, 0, "Number of Rx queues");
3713 
3714 	stats = &sc->vmx_stats;
3715 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD,
3716 	    &stats->vmst_defragged, 0, "Tx mbuf chains defragged");
3717 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD,
3718 	    &stats->vmst_defrag_failed, 0,
3719 	    "Tx mbuf dropped because defrag failed");
3720 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD,
3721 	    &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed");
3722 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD,
3723 	    &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed");
3724 
3725 	vmxnet3_setup_queue_sysctl(sc, ctx, child);
3726 }
3727 
3728 static void
3729 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3730 {
3731 
3732 	bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
3733 }
3734 
3735 static uint32_t
3736 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
3737 {
3738 
3739 	return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
3740 }
3741 
3742 static void
3743 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3744 {
3745 
3746 	bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
3747 }
3748 
3749 static void
3750 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3751 {
3752 
3753 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
3754 }
3755 
3756 static uint32_t
3757 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3758 {
3759 
3760 	vmxnet3_write_cmd(sc, cmd);
3761 	bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0,
3762 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
3763 	return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
3764 }
3765 
3766 static void
3767 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
3768 {
3769 
3770 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
3771 }
3772 
3773 static void
3774 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
3775 {
3776 
3777 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
3778 }
3779 
3780 static void
3781 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
3782 {
3783 	int i;
3784 
3785 	sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
3786 	for (i = 0; i < sc->vmx_nintrs; i++)
3787 		vmxnet3_enable_intr(sc, i);
3788 }
3789 
3790 static void
3791 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
3792 {
3793 	int i;
3794 
3795 	sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
3796 	for (i = 0; i < sc->vmx_nintrs; i++)
3797 		vmxnet3_disable_intr(sc, i);
3798 }
3799 
3800 static void
3801 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3802 {
3803 	bus_addr_t *baddr = arg;
3804 
3805 	if (error == 0)
3806 		*baddr = segs->ds_addr;
3807 }
3808 
3809 static int
3810 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
3811     struct vmxnet3_dma_alloc *dma)
3812 {
3813 	device_t dev;
3814 	int error;
3815 
3816 	dev = sc->vmx_dev;
3817 	bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3818 
3819 	error = bus_dma_tag_create(bus_get_dma_tag(dev),
3820 	    align, 0,		/* alignment, bounds */
3821 	    BUS_SPACE_MAXADDR,	/* lowaddr */
3822 	    BUS_SPACE_MAXADDR,	/* highaddr */
3823 	    NULL, NULL,		/* filter, filterarg */
3824 	    size,		/* maxsize */
3825 	    1,			/* nsegments */
3826 	    size,		/* maxsegsize */
3827 	    BUS_DMA_ALLOCNOW,	/* flags */
3828 	    NULL,		/* lockfunc */
3829 	    NULL,		/* lockfuncarg */
3830 	    &dma->dma_tag);
3831 	if (error) {
3832 		device_printf(dev, "bus_dma_tag_create failed: %d\n", error);
3833 		goto fail;
3834 	}
3835 
3836 	error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
3837 	    BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
3838 	if (error) {
3839 		device_printf(dev, "bus_dmamem_alloc failed: %d\n", error);
3840 		goto fail;
3841 	}
3842 
3843 	error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3844 	    size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
3845 	if (error) {
3846 		device_printf(dev, "bus_dmamap_load failed: %d\n", error);
3847 		goto fail;
3848 	}
3849 
3850 	dma->dma_size = size;
3851 
3852 fail:
3853 	if (error)
3854 		vmxnet3_dma_free(sc, dma);
3855 
3856 	return (error);
3857 }
3858 
3859 static void
3860 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
3861 {
3862 
3863 	if (dma->dma_tag != NULL) {
3864 		if (dma->dma_paddr != 0) {
3865 			bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3866 			    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3867 			bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3868 		}
3869 
3870 		if (dma->dma_vaddr != NULL) {
3871 			bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
3872 			    dma->dma_map);
3873 		}
3874 
3875 		bus_dma_tag_destroy(dma->dma_tag);
3876 	}
3877 	bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3878 }
3879 
3880 static int
3881 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def)
3882 {
3883 	char path[64];
3884 
3885 	snprintf(path, sizeof(path),
3886 	    "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob);
3887 	TUNABLE_INT_FETCH(path, &def);
3888 
3889 	return (def);
3890 }
3891 
3892 /*
3893  * Since this is a purely paravirtualized device, we do not have
3894  * to worry about DMA coherency. But at times, we must make sure
3895  * both the compiler and CPU do not reorder memory operations.
3896  */
3897 static inline void
3898 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
3899 {
3900 
3901 	switch (type) {
3902 	case VMXNET3_BARRIER_RD:
3903 		rmb();
3904 		break;
3905 	case VMXNET3_BARRIER_WR:
3906 		wmb();
3907 		break;
3908 	case VMXNET3_BARRIER_RDWR:
3909 		mb();
3910 		break;
3911 	default:
3912 		panic("%s: bad barrier type %d", __func__, type);
3913 	}
3914 }
3915