xref: /freebsd/sys/dev/vmware/vmxnet3/if_vmx.c (revision 5608fd23c27fa1e8ee595d7b678cbfd35d657fbe)
1 /*-
2  * Copyright (c) 2013 Tsubai Masanari
3  * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
4  *
5  * Permission to use, copy, modify, and distribute this software for any
6  * purpose with or without fee is hereby granted, provided that the above
7  * copyright notice and this permission notice appear in all copies.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16  *
17  * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $
18  */
19 
20 /* Driver for VMware vmxnet3 virtual ethernet devices. */
21 
22 #include <sys/cdefs.h>
23 __FBSDID("$FreeBSD$");
24 
25 #include <sys/param.h>
26 #include <sys/systm.h>
27 #include <sys/eventhandler.h>
28 #include <sys/kernel.h>
29 #include <sys/endian.h>
30 #include <sys/sockio.h>
31 #include <sys/mbuf.h>
32 #include <sys/malloc.h>
33 #include <sys/module.h>
34 #include <sys/socket.h>
35 #include <sys/sysctl.h>
36 #include <sys/smp.h>
37 #include <sys/taskqueue.h>
38 #include <vm/vm.h>
39 #include <vm/pmap.h>
40 
41 #include <net/ethernet.h>
42 #include <net/if.h>
43 #include <net/if_var.h>
44 #include <net/if_arp.h>
45 #include <net/if_dl.h>
46 #include <net/if_types.h>
47 #include <net/if_media.h>
48 #include <net/if_vlan_var.h>
49 
50 #include <net/bpf.h>
51 
52 #include <netinet/in_systm.h>
53 #include <netinet/in.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip6.h>
56 #include <netinet6/ip6_var.h>
57 #include <netinet/udp.h>
58 #include <netinet/tcp.h>
59 
60 #include <machine/in_cksum.h>
61 
62 #include <machine/bus.h>
63 #include <machine/resource.h>
64 #include <sys/bus.h>
65 #include <sys/rman.h>
66 
67 #include <dev/pci/pcireg.h>
68 #include <dev/pci/pcivar.h>
69 
70 #include "if_vmxreg.h"
71 #include "if_vmxvar.h"
72 
73 #include "opt_inet.h"
74 #include "opt_inet6.h"
75 
76 #ifdef VMXNET3_FAILPOINTS
77 #include <sys/fail.h>
78 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0,
79     "vmxnet3 fail points");
80 #define VMXNET3_FP	_debug_fail_point_vmxnet3
81 #endif
82 
83 static int	vmxnet3_probe(device_t);
84 static int	vmxnet3_attach(device_t);
85 static int	vmxnet3_detach(device_t);
86 static int	vmxnet3_shutdown(device_t);
87 
88 static int	vmxnet3_alloc_resources(struct vmxnet3_softc *);
89 static void	vmxnet3_free_resources(struct vmxnet3_softc *);
90 static int	vmxnet3_check_version(struct vmxnet3_softc *);
91 static void	vmxnet3_initial_config(struct vmxnet3_softc *);
92 static void	vmxnet3_check_multiqueue(struct vmxnet3_softc *);
93 
94 static int	vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
95 static int	vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
96 static int	vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
97 static int	vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int,
98 		    struct vmxnet3_interrupt *);
99 static int	vmxnet3_alloc_intr_resources(struct vmxnet3_softc *);
100 static int	vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
101 static int	vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
102 static int	vmxnet3_setup_interrupts(struct vmxnet3_softc *);
103 static int	vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
104 
105 static void	vmxnet3_free_interrupt(struct vmxnet3_softc *,
106 		    struct vmxnet3_interrupt *);
107 static void	vmxnet3_free_interrupts(struct vmxnet3_softc *);
108 
109 #ifndef VMXNET3_LEGACY_TX
110 static int	vmxnet3_alloc_taskqueue(struct vmxnet3_softc *);
111 static void	vmxnet3_start_taskqueue(struct vmxnet3_softc *);
112 static void	vmxnet3_drain_taskqueue(struct vmxnet3_softc *);
113 static void	vmxnet3_free_taskqueue(struct vmxnet3_softc *);
114 #endif
115 
116 static int	vmxnet3_init_rxq(struct vmxnet3_softc *, int);
117 static int	vmxnet3_init_txq(struct vmxnet3_softc *, int);
118 static int	vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
119 static void	vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
120 static void	vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
121 static void	vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);
122 
123 static int	vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
124 static void	vmxnet3_free_shared_data(struct vmxnet3_softc *);
125 static int	vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
126 static void	vmxnet3_free_txq_data(struct vmxnet3_softc *);
127 static int	vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
128 static void	vmxnet3_free_rxq_data(struct vmxnet3_softc *);
129 static int	vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
130 static void	vmxnet3_free_queue_data(struct vmxnet3_softc *);
131 static int	vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
132 static void	vmxnet3_init_shared_data(struct vmxnet3_softc *);
133 static void	vmxnet3_reinit_interface(struct vmxnet3_softc *);
134 static void	vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
135 static void	vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
136 static int	vmxnet3_alloc_data(struct vmxnet3_softc *);
137 static void	vmxnet3_free_data(struct vmxnet3_softc *);
138 static int	vmxnet3_setup_interface(struct vmxnet3_softc *);
139 
140 static void	vmxnet3_evintr(struct vmxnet3_softc *);
141 static void	vmxnet3_txq_eof(struct vmxnet3_txqueue *);
142 static void	vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
143 static int	vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
144 static void	vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
145 		    struct vmxnet3_rxring *, int);
146 static void	vmxnet3_rxq_eof(struct vmxnet3_rxqueue *);
147 static void	vmxnet3_legacy_intr(void *);
148 static void	vmxnet3_txq_intr(void *);
149 static void	vmxnet3_rxq_intr(void *);
150 static void	vmxnet3_event_intr(void *);
151 
152 static void	vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
153 static void	vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
154 static void	vmxnet3_stop(struct vmxnet3_softc *);
155 
156 static void	vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
157 static int	vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
158 static int	vmxnet3_reinit_queues(struct vmxnet3_softc *);
159 static int	vmxnet3_enable_device(struct vmxnet3_softc *);
160 static void	vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
161 static int	vmxnet3_reinit(struct vmxnet3_softc *);
162 static void	vmxnet3_init_locked(struct vmxnet3_softc *);
163 static void	vmxnet3_init(void *);
164 
165 static int	vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *,
166 		    int *, int *, int *);
167 static int	vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **,
168 		    bus_dmamap_t, bus_dma_segment_t [], int *);
169 static void	vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
170 static int	vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
171 #ifdef VMXNET3_LEGACY_TX
172 static void	vmxnet3_start_locked(struct ifnet *);
173 static void	vmxnet3_start(struct ifnet *);
174 #else
175 static int	vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *,
176 		    struct mbuf *);
177 static int	vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *);
178 static void	vmxnet3_txq_tq_deferred(void *, int);
179 #endif
180 static void	vmxnet3_txq_start(struct vmxnet3_txqueue *);
181 static void	vmxnet3_tx_start_all(struct vmxnet3_softc *);
182 
183 static void	vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
184 		    uint16_t);
185 static void	vmxnet3_register_vlan(void *, struct ifnet *, uint16_t);
186 static void	vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t);
187 static void	vmxnet3_set_rxfilter(struct vmxnet3_softc *);
188 static int	vmxnet3_change_mtu(struct vmxnet3_softc *, int);
189 static int	vmxnet3_ioctl(struct ifnet *, u_long, caddr_t);
190 
191 #ifndef VMXNET3_LEGACY_TX
192 static void	vmxnet3_qflush(struct ifnet *);
193 #endif
194 
195 static int	vmxnet3_watchdog(struct vmxnet3_txqueue *);
196 static void	vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
197 static void	vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *,
198 		    struct vmxnet3_txq_stats *);
199 static void	vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *,
200 		    struct vmxnet3_rxq_stats *);
201 static void	vmxnet3_tick(void *);
202 static void	vmxnet3_link_status(struct vmxnet3_softc *);
203 static void	vmxnet3_media_status(struct ifnet *, struct ifmediareq *);
204 static int	vmxnet3_media_change(struct ifnet *);
205 static void	vmxnet3_set_lladdr(struct vmxnet3_softc *);
206 static void	vmxnet3_get_lladdr(struct vmxnet3_softc *);
207 
208 static void	vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *,
209 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
210 static void	vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *,
211 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
212 static void	vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *,
213 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
214 static void	vmxnet3_setup_sysctl(struct vmxnet3_softc *);
215 
216 static void	vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t,
217 		    uint32_t);
218 static uint32_t	vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t);
219 static void	vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t,
220 		    uint32_t);
221 static void	vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t);
222 static uint32_t	vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t);
223 
224 static void	vmxnet3_enable_intr(struct vmxnet3_softc *, int);
225 static void	vmxnet3_disable_intr(struct vmxnet3_softc *, int);
226 static void	vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
227 static void	vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
228 
229 static int	vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t,
230 		    bus_size_t, struct vmxnet3_dma_alloc *);
231 static void	vmxnet3_dma_free(struct vmxnet3_softc *,
232 		    struct vmxnet3_dma_alloc *);
233 static int	vmxnet3_tunable_int(struct vmxnet3_softc *,
234 		    const char *, int);
235 
236 typedef enum {
237 	VMXNET3_BARRIER_RD,
238 	VMXNET3_BARRIER_WR,
239 	VMXNET3_BARRIER_RDWR,
240 } vmxnet3_barrier_t;
241 
242 static void	vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
243 
244 /* Tunables. */
245 static int vmxnet3_mq_disable = 0;
246 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable);
247 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES;
248 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue);
249 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES;
250 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue);
251 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC;
252 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc);
253 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC;
254 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc);
255 
256 static device_method_t vmxnet3_methods[] = {
257 	/* Device interface. */
258 	DEVMETHOD(device_probe,		vmxnet3_probe),
259 	DEVMETHOD(device_attach,	vmxnet3_attach),
260 	DEVMETHOD(device_detach,	vmxnet3_detach),
261 	DEVMETHOD(device_shutdown,	vmxnet3_shutdown),
262 
263 	DEVMETHOD_END
264 };
265 
266 static driver_t vmxnet3_driver = {
267 	"vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc)
268 };
269 
270 static devclass_t vmxnet3_devclass;
271 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0);
272 
273 MODULE_DEPEND(vmx, pci, 1, 1, 1);
274 MODULE_DEPEND(vmx, ether, 1, 1, 1);
275 
276 #define VMXNET3_VMWARE_VENDOR_ID	0x15AD
277 #define VMXNET3_VMWARE_DEVICE_ID	0x07B0
278 
279 static int
280 vmxnet3_probe(device_t dev)
281 {
282 
283 	if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID &&
284 	    pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) {
285 		device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter");
286 		return (BUS_PROBE_DEFAULT);
287 	}
288 
289 	return (ENXIO);
290 }
291 
292 static int
293 vmxnet3_attach(device_t dev)
294 {
295 	struct vmxnet3_softc *sc;
296 	int error;
297 
298 	sc = device_get_softc(dev);
299 	sc->vmx_dev = dev;
300 
301 	pci_enable_busmaster(dev);
302 
303 	VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev));
304 	callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0);
305 
306 	vmxnet3_initial_config(sc);
307 
308 	error = vmxnet3_alloc_resources(sc);
309 	if (error)
310 		goto fail;
311 
312 	error = vmxnet3_check_version(sc);
313 	if (error)
314 		goto fail;
315 
316 	error = vmxnet3_alloc_rxtx_queues(sc);
317 	if (error)
318 		goto fail;
319 
320 #ifndef VMXNET3_LEGACY_TX
321 	error = vmxnet3_alloc_taskqueue(sc);
322 	if (error)
323 		goto fail;
324 #endif
325 
326 	error = vmxnet3_alloc_interrupts(sc);
327 	if (error)
328 		goto fail;
329 
330 	vmxnet3_check_multiqueue(sc);
331 
332 	error = vmxnet3_alloc_data(sc);
333 	if (error)
334 		goto fail;
335 
336 	error = vmxnet3_setup_interface(sc);
337 	if (error)
338 		goto fail;
339 
340 	error = vmxnet3_setup_interrupts(sc);
341 	if (error) {
342 		ether_ifdetach(sc->vmx_ifp);
343 		device_printf(dev, "could not set up interrupt\n");
344 		goto fail;
345 	}
346 
347 	vmxnet3_setup_sysctl(sc);
348 #ifndef VMXNET3_LEGACY_TX
349 	vmxnet3_start_taskqueue(sc);
350 #endif
351 
352 fail:
353 	if (error)
354 		vmxnet3_detach(dev);
355 
356 	return (error);
357 }
358 
359 static int
360 vmxnet3_detach(device_t dev)
361 {
362 	struct vmxnet3_softc *sc;
363 	struct ifnet *ifp;
364 
365 	sc = device_get_softc(dev);
366 	ifp = sc->vmx_ifp;
367 
368 	if (device_is_attached(dev)) {
369 		VMXNET3_CORE_LOCK(sc);
370 		vmxnet3_stop(sc);
371 		VMXNET3_CORE_UNLOCK(sc);
372 
373 		callout_drain(&sc->vmx_tick);
374 #ifndef VMXNET3_LEGACY_TX
375 		vmxnet3_drain_taskqueue(sc);
376 #endif
377 
378 		ether_ifdetach(ifp);
379 	}
380 
381 	if (sc->vmx_vlan_attach != NULL) {
382 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach);
383 		sc->vmx_vlan_attach = NULL;
384 	}
385 	if (sc->vmx_vlan_detach != NULL) {
386 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach);
387 		sc->vmx_vlan_detach = NULL;
388 	}
389 
390 #ifndef VMXNET3_LEGACY_TX
391 	vmxnet3_free_taskqueue(sc);
392 #endif
393 	vmxnet3_free_interrupts(sc);
394 
395 	if (ifp != NULL) {
396 		if_free(ifp);
397 		sc->vmx_ifp = NULL;
398 	}
399 
400 	ifmedia_removeall(&sc->vmx_media);
401 
402 	vmxnet3_free_data(sc);
403 	vmxnet3_free_resources(sc);
404 	vmxnet3_free_rxtx_queues(sc);
405 
406 	VMXNET3_CORE_LOCK_DESTROY(sc);
407 
408 	return (0);
409 }
410 
411 static int
412 vmxnet3_shutdown(device_t dev)
413 {
414 
415 	return (0);
416 }
417 
418 static int
419 vmxnet3_alloc_resources(struct vmxnet3_softc *sc)
420 {
421 	device_t dev;
422 	int rid;
423 
424 	dev = sc->vmx_dev;
425 
426 	rid = PCIR_BAR(0);
427 	sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
428 	    RF_ACTIVE);
429 	if (sc->vmx_res0 == NULL) {
430 		device_printf(dev,
431 		    "could not map BAR0 memory\n");
432 		return (ENXIO);
433 	}
434 
435 	sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0);
436 	sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0);
437 
438 	rid = PCIR_BAR(1);
439 	sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
440 	    RF_ACTIVE);
441 	if (sc->vmx_res1 == NULL) {
442 		device_printf(dev,
443 		    "could not map BAR1 memory\n");
444 		return (ENXIO);
445 	}
446 
447 	sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1);
448 	sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1);
449 
450 	if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
451 		rid = PCIR_BAR(2);
452 		sc->vmx_msix_res = bus_alloc_resource_any(dev,
453 		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
454 	}
455 
456 	if (sc->vmx_msix_res == NULL)
457 		sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
458 
459 	return (0);
460 }
461 
462 static void
463 vmxnet3_free_resources(struct vmxnet3_softc *sc)
464 {
465 	device_t dev;
466 	int rid;
467 
468 	dev = sc->vmx_dev;
469 
470 	if (sc->vmx_res0 != NULL) {
471 		rid = PCIR_BAR(0);
472 		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0);
473 		sc->vmx_res0 = NULL;
474 	}
475 
476 	if (sc->vmx_res1 != NULL) {
477 		rid = PCIR_BAR(1);
478 		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1);
479 		sc->vmx_res1 = NULL;
480 	}
481 
482 	if (sc->vmx_msix_res != NULL) {
483 		rid = PCIR_BAR(2);
484 		bus_release_resource(dev, SYS_RES_MEMORY, rid,
485 		    sc->vmx_msix_res);
486 		sc->vmx_msix_res = NULL;
487 	}
488 }
489 
490 static int
491 vmxnet3_check_version(struct vmxnet3_softc *sc)
492 {
493 	device_t dev;
494 	uint32_t version;
495 
496 	dev = sc->vmx_dev;
497 
498 	version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
499 	if ((version & 0x01) == 0) {
500 		device_printf(dev, "unsupported hardware version %#x\n",
501 		    version);
502 		return (ENOTSUP);
503 	}
504 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
505 
506 	version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
507 	if ((version & 0x01) == 0) {
508 		device_printf(dev, "unsupported UPT version %#x\n", version);
509 		return (ENOTSUP);
510 	}
511 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
512 
513 	return (0);
514 }
515 
516 static void
517 vmxnet3_initial_config(struct vmxnet3_softc *sc)
518 {
519 	int nqueue, ndesc;
520 
521 	nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue);
522 	if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1)
523 		nqueue = VMXNET3_DEF_TX_QUEUES;
524 	if (nqueue > mp_ncpus)
525 		nqueue = mp_ncpus;
526 	sc->vmx_max_ntxqueues = nqueue;
527 
528 	nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue);
529 	if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1)
530 		nqueue = VMXNET3_DEF_RX_QUEUES;
531 	if (nqueue > mp_ncpus)
532 		nqueue = mp_ncpus;
533 	sc->vmx_max_nrxqueues = nqueue;
534 
535 	if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) {
536 		sc->vmx_max_nrxqueues = 1;
537 		sc->vmx_max_ntxqueues = 1;
538 	}
539 
540 	ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc);
541 	if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC)
542 		ndesc = VMXNET3_DEF_TX_NDESC;
543 	if (ndesc & VMXNET3_MASK_TX_NDESC)
544 		ndesc &= ~VMXNET3_MASK_TX_NDESC;
545 	sc->vmx_ntxdescs = ndesc;
546 
547 	ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc);
548 	if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC)
549 		ndesc = VMXNET3_DEF_RX_NDESC;
550 	if (ndesc & VMXNET3_MASK_RX_NDESC)
551 		ndesc &= ~VMXNET3_MASK_RX_NDESC;
552 	sc->vmx_nrxdescs = ndesc;
553 	sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
554 }
555 
556 static void
557 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
558 {
559 
560 	if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
561 		goto out;
562 
563 	/* BMV: Just use the maximum configured for now. */
564 	sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
565 	sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;
566 
567 	if (sc->vmx_nrxqueues > 1)
568 		sc->vmx_flags |= VMXNET3_FLAG_RSS;
569 
570 	return;
571 
572 out:
573 	sc->vmx_ntxqueues = 1;
574 	sc->vmx_nrxqueues = 1;
575 }
576 
577 static int
578 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
579 {
580 	device_t dev;
581 	int nmsix, cnt, required;
582 
583 	dev = sc->vmx_dev;
584 
585 	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
586 		return (1);
587 
588 	/* Allocate an additional vector for the events interrupt. */
589 	required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1;
590 
591 	nmsix = pci_msix_count(dev);
592 	if (nmsix < required)
593 		return (1);
594 
595 	cnt = required;
596 	if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
597 		sc->vmx_nintrs = required;
598 		return (0);
599 	} else
600 		pci_release_msi(dev);
601 
602 	/* BMV TODO Fallback to sharing MSIX vectors if possible. */
603 
604 	return (1);
605 }
606 
607 static int
608 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
609 {
610 	device_t dev;
611 	int nmsi, cnt, required;
612 
613 	dev = sc->vmx_dev;
614 	required = 1;
615 
616 	nmsi = pci_msi_count(dev);
617 	if (nmsi < required)
618 		return (1);
619 
620 	cnt = required;
621 	if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) {
622 		sc->vmx_nintrs = 1;
623 		return (0);
624 	} else
625 		pci_release_msi(dev);
626 
627 	return (1);
628 }
629 
630 static int
631 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
632 {
633 
634 	sc->vmx_nintrs = 1;
635 	return (0);
636 }
637 
638 static int
639 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags,
640     struct vmxnet3_interrupt *intr)
641 {
642 	struct resource *irq;
643 
644 	irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags);
645 	if (irq == NULL)
646 		return (ENXIO);
647 
648 	intr->vmxi_irq = irq;
649 	intr->vmxi_rid = rid;
650 
651 	return (0);
652 }
653 
654 static int
655 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc)
656 {
657 	int i, rid, flags, error;
658 
659 	rid = 0;
660 	flags = RF_ACTIVE;
661 
662 	if (sc->vmx_intr_type == VMXNET3_IT_LEGACY)
663 		flags |= RF_SHAREABLE;
664 	else
665 		rid = 1;
666 
667 	for (i = 0; i < sc->vmx_nintrs; i++, rid++) {
668 		error = vmxnet3_alloc_interrupt(sc, rid, flags,
669 		    &sc->vmx_intrs[i]);
670 		if (error)
671 			return (error);
672 	}
673 
674 	return (0);
675 }
676 
677 static int
678 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
679 {
680 	device_t dev;
681 	struct vmxnet3_txqueue *txq;
682 	struct vmxnet3_rxqueue *rxq;
683 	struct vmxnet3_interrupt *intr;
684 	enum intr_type type;
685 	int i, error;
686 
687 	dev = sc->vmx_dev;
688 	intr = &sc->vmx_intrs[0];
689 	type = INTR_TYPE_NET | INTR_MPSAFE;
690 
691 	for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) {
692 		txq = &sc->vmx_txq[i];
693 		error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
694 		     vmxnet3_txq_intr, txq, &intr->vmxi_handler);
695 		if (error)
696 			return (error);
697 		bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
698 		    "tq%d", i);
699 		txq->vxtxq_intr_idx = intr->vmxi_rid - 1;
700 	}
701 
702 	for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) {
703 		rxq = &sc->vmx_rxq[i];
704 		error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
705 		    vmxnet3_rxq_intr, rxq, &intr->vmxi_handler);
706 		if (error)
707 			return (error);
708 		bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
709 		    "rq%d", i);
710 		rxq->vxrxq_intr_idx = intr->vmxi_rid - 1;
711 	}
712 
713 	error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
714 	    vmxnet3_event_intr, sc, &intr->vmxi_handler);
715 	if (error)
716 		return (error);
717 	bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event");
718 	sc->vmx_event_intr_idx = intr->vmxi_rid - 1;
719 
720 	return (0);
721 }
722 
723 static int
724 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
725 {
726 	struct vmxnet3_interrupt *intr;
727 	int i, error;
728 
729 	intr = &sc->vmx_intrs[0];
730 	error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq,
731 	    INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc,
732 	    &intr->vmxi_handler);
733 
734 	for (i = 0; i < sc->vmx_ntxqueues; i++)
735 		sc->vmx_txq[i].vxtxq_intr_idx = 0;
736 	for (i = 0; i < sc->vmx_nrxqueues; i++)
737 		sc->vmx_rxq[i].vxrxq_intr_idx = 0;
738 	sc->vmx_event_intr_idx = 0;
739 
740 	return (error);
741 }
742 
743 static void
744 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
745 {
746 	struct vmxnet3_txqueue *txq;
747 	struct vmxnet3_txq_shared *txs;
748 	struct vmxnet3_rxqueue *rxq;
749 	struct vmxnet3_rxq_shared *rxs;
750 	int i;
751 
752 	sc->vmx_ds->evintr = sc->vmx_event_intr_idx;
753 
754 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
755 		txq = &sc->vmx_txq[i];
756 		txs = txq->vxtxq_ts;
757 		txs->intr_idx = txq->vxtxq_intr_idx;
758 	}
759 
760 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
761 		rxq = &sc->vmx_rxq[i];
762 		rxs = rxq->vxrxq_rs;
763 		rxs->intr_idx = rxq->vxrxq_intr_idx;
764 	}
765 }
766 
767 static int
768 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
769 {
770 	int error;
771 
772 	error = vmxnet3_alloc_intr_resources(sc);
773 	if (error)
774 		return (error);
775 
776 	switch (sc->vmx_intr_type) {
777 	case VMXNET3_IT_MSIX:
778 		error = vmxnet3_setup_msix_interrupts(sc);
779 		break;
780 	case VMXNET3_IT_MSI:
781 	case VMXNET3_IT_LEGACY:
782 		error = vmxnet3_setup_legacy_interrupt(sc);
783 		break;
784 	default:
785 		panic("%s: invalid interrupt type %d", __func__,
786 		    sc->vmx_intr_type);
787 	}
788 
789 	if (error == 0)
790 		vmxnet3_set_interrupt_idx(sc);
791 
792 	return (error);
793 }
794 
795 static int
796 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
797 {
798 	device_t dev;
799 	uint32_t config;
800 	int error;
801 
802 	dev = sc->vmx_dev;
803 	config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
804 
805 	sc->vmx_intr_type = config & 0x03;
806 	sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
807 
808 	switch (sc->vmx_intr_type) {
809 	case VMXNET3_IT_AUTO:
810 		sc->vmx_intr_type = VMXNET3_IT_MSIX;
811 		/* FALLTHROUGH */
812 	case VMXNET3_IT_MSIX:
813 		error = vmxnet3_alloc_msix_interrupts(sc);
814 		if (error == 0)
815 			break;
816 		sc->vmx_intr_type = VMXNET3_IT_MSI;
817 		/* FALLTHROUGH */
818 	case VMXNET3_IT_MSI:
819 		error = vmxnet3_alloc_msi_interrupts(sc);
820 		if (error == 0)
821 			break;
822 		sc->vmx_intr_type = VMXNET3_IT_LEGACY;
823 		/* FALLTHROUGH */
824 	case VMXNET3_IT_LEGACY:
825 		error = vmxnet3_alloc_legacy_interrupts(sc);
826 		if (error == 0)
827 			break;
828 		/* FALLTHROUGH */
829 	default:
830 		sc->vmx_intr_type = -1;
831 		device_printf(dev, "cannot allocate any interrupt resources\n");
832 		return (ENXIO);
833 	}
834 
835 	return (error);
836 }
837 
838 static void
839 vmxnet3_free_interrupt(struct vmxnet3_softc *sc,
840     struct vmxnet3_interrupt *intr)
841 {
842 	device_t dev;
843 
844 	dev = sc->vmx_dev;
845 
846 	if (intr->vmxi_handler != NULL) {
847 		bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler);
848 		intr->vmxi_handler = NULL;
849 	}
850 
851 	if (intr->vmxi_irq != NULL) {
852 		bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid,
853 		    intr->vmxi_irq);
854 		intr->vmxi_irq = NULL;
855 		intr->vmxi_rid = -1;
856 	}
857 }
858 
859 static void
860 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
861 {
862 	int i;
863 
864 	for (i = 0; i < sc->vmx_nintrs; i++)
865 		vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]);
866 
867 	if (sc->vmx_intr_type == VMXNET3_IT_MSI ||
868 	    sc->vmx_intr_type == VMXNET3_IT_MSIX)
869 		pci_release_msi(sc->vmx_dev);
870 }
871 
872 #ifndef VMXNET3_LEGACY_TX
873 static int
874 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc)
875 {
876 	device_t dev;
877 
878 	dev = sc->vmx_dev;
879 
880 	sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT,
881 	    taskqueue_thread_enqueue, &sc->vmx_tq);
882 	if (sc->vmx_tq == NULL)
883 		return (ENOMEM);
884 
885 	return (0);
886 }
887 
888 static void
889 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc)
890 {
891 	device_t dev;
892 	int nthreads, error;
893 
894 	dev = sc->vmx_dev;
895 
896 	/*
897 	 * The taskqueue is typically not frequently used, so a dedicated
898 	 * thread for each queue is unnecessary.
899 	 */
900 	nthreads = MAX(1, sc->vmx_ntxqueues / 2);
901 
902 	/*
903 	 * Most drivers just ignore the return value - it only fails
904 	 * with ENOMEM so an error is not likely. It is hard for us
905 	 * to recover from an error here.
906 	 */
907 	error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET,
908 	    "%s taskq", device_get_nameunit(dev));
909 	if (error)
910 		device_printf(dev, "failed to start taskqueue: %d", error);
911 }
912 
913 static void
914 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc)
915 {
916 	struct vmxnet3_txqueue *txq;
917 	int i;
918 
919 	if (sc->vmx_tq != NULL) {
920 		for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
921 			txq = &sc->vmx_txq[i];
922 			taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask);
923 		}
924 	}
925 }
926 
927 static void
928 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc)
929 {
930 	if (sc->vmx_tq != NULL) {
931 		taskqueue_free(sc->vmx_tq);
932 		sc->vmx_tq = NULL;
933 	}
934 }
935 #endif
936 
937 static int
938 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
939 {
940 	struct vmxnet3_rxqueue *rxq;
941 	struct vmxnet3_rxring *rxr;
942 	int i;
943 
944 	rxq = &sc->vmx_rxq[q];
945 
946 	snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
947 	    device_get_nameunit(sc->vmx_dev), q);
948 	mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF);
949 
950 	rxq->vxrxq_sc = sc;
951 	rxq->vxrxq_id = q;
952 
953 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
954 		rxr = &rxq->vxrxq_cmd_ring[i];
955 		rxr->vxrxr_rid = i;
956 		rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
957 		rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc *
958 		    sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO);
959 		if (rxr->vxrxr_rxbuf == NULL)
960 			return (ENOMEM);
961 
962 		rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
963 	}
964 
965 	return (0);
966 }
967 
968 static int
969 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
970 {
971 	struct vmxnet3_txqueue *txq;
972 	struct vmxnet3_txring *txr;
973 
974 	txq = &sc->vmx_txq[q];
975 	txr = &txq->vxtxq_cmd_ring;
976 
977 	snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
978 	    device_get_nameunit(sc->vmx_dev), q);
979 	mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF);
980 
981 	txq->vxtxq_sc = sc;
982 	txq->vxtxq_id = q;
983 
984 	txr->vxtxr_ndesc = sc->vmx_ntxdescs;
985 	txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc *
986 	    sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO);
987 	if (txr->vxtxr_txbuf == NULL)
988 		return (ENOMEM);
989 
990 	txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
991 
992 #ifndef VMXNET3_LEGACY_TX
993 	TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq);
994 
995 	txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF,
996 	    M_NOWAIT, &txq->vxtxq_mtx);
997 	if (txq->vxtxq_br == NULL)
998 		return (ENOMEM);
999 #endif
1000 
1001 	return (0);
1002 }
1003 
1004 static int
1005 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
1006 {
1007 	int i, error;
1008 
1009 	/*
1010 	 * Only attempt to create multiple queues if MSIX is available. MSIX is
1011 	 * disabled by default because its apparently broken for devices passed
1012 	 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable
1013 	 * must be set to zero for MSIX. This check prevents us from allocating
1014 	 * queue structures that we will not use.
1015 	 */
1016 	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
1017 		sc->vmx_max_nrxqueues = 1;
1018 		sc->vmx_max_ntxqueues = 1;
1019 	}
1020 
1021 	sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) *
1022 	    sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
1023 	sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) *
1024 	    sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
1025 	if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL)
1026 		return (ENOMEM);
1027 
1028 	for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
1029 		error = vmxnet3_init_rxq(sc, i);
1030 		if (error)
1031 			return (error);
1032 	}
1033 
1034 	for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
1035 		error = vmxnet3_init_txq(sc, i);
1036 		if (error)
1037 			return (error);
1038 	}
1039 
1040 	return (0);
1041 }
1042 
1043 static void
1044 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
1045 {
1046 	struct vmxnet3_rxring *rxr;
1047 	int i;
1048 
1049 	rxq->vxrxq_sc = NULL;
1050 	rxq->vxrxq_id = -1;
1051 
1052 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1053 		rxr = &rxq->vxrxq_cmd_ring[i];
1054 
1055 		if (rxr->vxrxr_rxbuf != NULL) {
1056 			free(rxr->vxrxr_rxbuf, M_DEVBUF);
1057 			rxr->vxrxr_rxbuf = NULL;
1058 		}
1059 	}
1060 
1061 	if (mtx_initialized(&rxq->vxrxq_mtx) != 0)
1062 		mtx_destroy(&rxq->vxrxq_mtx);
1063 }
1064 
1065 static void
1066 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
1067 {
1068 	struct vmxnet3_txring *txr;
1069 
1070 	txr = &txq->vxtxq_cmd_ring;
1071 
1072 	txq->vxtxq_sc = NULL;
1073 	txq->vxtxq_id = -1;
1074 
1075 #ifndef VMXNET3_LEGACY_TX
1076 	if (txq->vxtxq_br != NULL) {
1077 		buf_ring_free(txq->vxtxq_br, M_DEVBUF);
1078 		txq->vxtxq_br = NULL;
1079 	}
1080 #endif
1081 
1082 	if (txr->vxtxr_txbuf != NULL) {
1083 		free(txr->vxtxr_txbuf, M_DEVBUF);
1084 		txr->vxtxr_txbuf = NULL;
1085 	}
1086 
1087 	if (mtx_initialized(&txq->vxtxq_mtx) != 0)
1088 		mtx_destroy(&txq->vxtxq_mtx);
1089 }
1090 
1091 static void
1092 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
1093 {
1094 	int i;
1095 
1096 	if (sc->vmx_rxq != NULL) {
1097 		for (i = 0; i < sc->vmx_max_nrxqueues; i++)
1098 			vmxnet3_destroy_rxq(&sc->vmx_rxq[i]);
1099 		free(sc->vmx_rxq, M_DEVBUF);
1100 		sc->vmx_rxq = NULL;
1101 	}
1102 
1103 	if (sc->vmx_txq != NULL) {
1104 		for (i = 0; i < sc->vmx_max_ntxqueues; i++)
1105 			vmxnet3_destroy_txq(&sc->vmx_txq[i]);
1106 		free(sc->vmx_txq, M_DEVBUF);
1107 		sc->vmx_txq = NULL;
1108 	}
1109 }
1110 
1111 static int
1112 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
1113 {
1114 	device_t dev;
1115 	uint8_t *kva;
1116 	size_t size;
1117 	int i, error;
1118 
1119 	dev = sc->vmx_dev;
1120 
1121 	size = sizeof(struct vmxnet3_driver_shared);
1122 	error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
1123 	if (error) {
1124 		device_printf(dev, "cannot alloc shared memory\n");
1125 		return (error);
1126 	}
1127 	sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;
1128 
1129 	size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
1130 	    sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
1131 	error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
1132 	if (error) {
1133 		device_printf(dev, "cannot alloc queue shared memory\n");
1134 		return (error);
1135 	}
1136 	sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
1137 	kva = sc->vmx_qs;
1138 
1139 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
1140 		sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
1141 		kva += sizeof(struct vmxnet3_txq_shared);
1142 	}
1143 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
1144 		sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
1145 		kva += sizeof(struct vmxnet3_rxq_shared);
1146 	}
1147 
1148 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1149 		size = sizeof(struct vmxnet3_rss_shared);
1150 		error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
1151 		if (error) {
1152 			device_printf(dev, "cannot alloc rss shared memory\n");
1153 			return (error);
1154 		}
1155 		sc->vmx_rss =
1156 		    (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
1157 	}
1158 
1159 	return (0);
1160 }
1161 
1162 static void
1163 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
1164 {
1165 
1166 	if (sc->vmx_rss != NULL) {
1167 		vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
1168 		sc->vmx_rss = NULL;
1169 	}
1170 
1171 	if (sc->vmx_qs != NULL) {
1172 		vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
1173 		sc->vmx_qs = NULL;
1174 	}
1175 
1176 	if (sc->vmx_ds != NULL) {
1177 		vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
1178 		sc->vmx_ds = NULL;
1179 	}
1180 }
1181 
1182 static int
1183 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
1184 {
1185 	device_t dev;
1186 	struct vmxnet3_txqueue *txq;
1187 	struct vmxnet3_txring *txr;
1188 	struct vmxnet3_comp_ring *txc;
1189 	size_t descsz, compsz;
1190 	int i, q, error;
1191 
1192 	dev = sc->vmx_dev;
1193 
1194 	for (q = 0; q < sc->vmx_ntxqueues; q++) {
1195 		txq = &sc->vmx_txq[q];
1196 		txr = &txq->vxtxq_cmd_ring;
1197 		txc = &txq->vxtxq_comp_ring;
1198 
1199 		descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
1200 		compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);
1201 
1202 		error = bus_dma_tag_create(bus_get_dma_tag(dev),
1203 		    1, 0,			/* alignment, boundary */
1204 		    BUS_SPACE_MAXADDR,		/* lowaddr */
1205 		    BUS_SPACE_MAXADDR,		/* highaddr */
1206 		    NULL, NULL,			/* filter, filterarg */
1207 		    VMXNET3_TX_MAXSIZE,		/* maxsize */
1208 		    VMXNET3_TX_MAXSEGS,		/* nsegments */
1209 		    VMXNET3_TX_MAXSEGSIZE,	/* maxsegsize */
1210 		    0,				/* flags */
1211 		    NULL, NULL,			/* lockfunc, lockarg */
1212 		    &txr->vxtxr_txtag);
1213 		if (error) {
1214 			device_printf(dev,
1215 			    "unable to create Tx buffer tag for queue %d\n", q);
1216 			return (error);
1217 		}
1218 
1219 		error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
1220 		if (error) {
1221 			device_printf(dev, "cannot alloc Tx descriptors for "
1222 			    "queue %d error %d\n", q, error);
1223 			return (error);
1224 		}
1225 		txr->vxtxr_txd =
1226 		    (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;
1227 
1228 		error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
1229 		if (error) {
1230 			device_printf(dev, "cannot alloc Tx comp descriptors "
1231 			   "for queue %d error %d\n", q, error);
1232 			return (error);
1233 		}
1234 		txc->vxcr_u.txcd =
1235 		    (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;
1236 
1237 		for (i = 0; i < txr->vxtxr_ndesc; i++) {
1238 			error = bus_dmamap_create(txr->vxtxr_txtag, 0,
1239 			    &txr->vxtxr_txbuf[i].vtxb_dmamap);
1240 			if (error) {
1241 				device_printf(dev, "unable to create Tx buf "
1242 				    "dmamap for queue %d idx %d\n", q, i);
1243 				return (error);
1244 			}
1245 		}
1246 	}
1247 
1248 	return (0);
1249 }
1250 
1251 static void
1252 vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
1253 {
1254 	device_t dev;
1255 	struct vmxnet3_txqueue *txq;
1256 	struct vmxnet3_txring *txr;
1257 	struct vmxnet3_comp_ring *txc;
1258 	struct vmxnet3_txbuf *txb;
1259 	int i, q;
1260 
1261 	dev = sc->vmx_dev;
1262 
1263 	for (q = 0; q < sc->vmx_ntxqueues; q++) {
1264 		txq = &sc->vmx_txq[q];
1265 		txr = &txq->vxtxq_cmd_ring;
1266 		txc = &txq->vxtxq_comp_ring;
1267 
1268 		for (i = 0; i < txr->vxtxr_ndesc; i++) {
1269 			txb = &txr->vxtxr_txbuf[i];
1270 			if (txb->vtxb_dmamap != NULL) {
1271 				bus_dmamap_destroy(txr->vxtxr_txtag,
1272 				    txb->vtxb_dmamap);
1273 				txb->vtxb_dmamap = NULL;
1274 			}
1275 		}
1276 
1277 		if (txc->vxcr_u.txcd != NULL) {
1278 			vmxnet3_dma_free(sc, &txc->vxcr_dma);
1279 			txc->vxcr_u.txcd = NULL;
1280 		}
1281 
1282 		if (txr->vxtxr_txd != NULL) {
1283 			vmxnet3_dma_free(sc, &txr->vxtxr_dma);
1284 			txr->vxtxr_txd = NULL;
1285 		}
1286 
1287 		if (txr->vxtxr_txtag != NULL) {
1288 			bus_dma_tag_destroy(txr->vxtxr_txtag);
1289 			txr->vxtxr_txtag = NULL;
1290 		}
1291 	}
1292 }
1293 
1294 static int
1295 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
1296 {
1297 	device_t dev;
1298 	struct vmxnet3_rxqueue *rxq;
1299 	struct vmxnet3_rxring *rxr;
1300 	struct vmxnet3_comp_ring *rxc;
1301 	int descsz, compsz;
1302 	int i, j, q, error;
1303 
1304 	dev = sc->vmx_dev;
1305 
1306 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
1307 		rxq = &sc->vmx_rxq[q];
1308 		rxc = &rxq->vxrxq_comp_ring;
1309 		compsz = 0;
1310 
1311 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1312 			rxr = &rxq->vxrxq_cmd_ring[i];
1313 
1314 			descsz = rxr->vxrxr_ndesc *
1315 			    sizeof(struct vmxnet3_rxdesc);
1316 			compsz += rxr->vxrxr_ndesc *
1317 			    sizeof(struct vmxnet3_rxcompdesc);
1318 
1319 			error = bus_dma_tag_create(bus_get_dma_tag(dev),
1320 			    1, 0,		/* alignment, boundary */
1321 			    BUS_SPACE_MAXADDR,	/* lowaddr */
1322 			    BUS_SPACE_MAXADDR,	/* highaddr */
1323 			    NULL, NULL,		/* filter, filterarg */
1324 			    MJUMPAGESIZE,	/* maxsize */
1325 			    1,			/* nsegments */
1326 			    MJUMPAGESIZE,	/* maxsegsize */
1327 			    0,			/* flags */
1328 			    NULL, NULL,		/* lockfunc, lockarg */
1329 			    &rxr->vxrxr_rxtag);
1330 			if (error) {
1331 				device_printf(dev,
1332 				    "unable to create Rx buffer tag for "
1333 				    "queue %d\n", q);
1334 				return (error);
1335 			}
1336 
1337 			error = vmxnet3_dma_malloc(sc, descsz, 512,
1338 			    &rxr->vxrxr_dma);
1339 			if (error) {
1340 				device_printf(dev, "cannot allocate Rx "
1341 				    "descriptors for queue %d/%d error %d\n",
1342 				    i, q, error);
1343 				return (error);
1344 			}
1345 			rxr->vxrxr_rxd =
1346 			    (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
1347 		}
1348 
1349 		error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
1350 		if (error) {
1351 			device_printf(dev, "cannot alloc Rx comp descriptors "
1352 			    "for queue %d error %d\n", q, error);
1353 			return (error);
1354 		}
1355 		rxc->vxcr_u.rxcd =
1356 		    (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;
1357 
1358 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1359 			rxr = &rxq->vxrxq_cmd_ring[i];
1360 
1361 			error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1362 			    &rxr->vxrxr_spare_dmap);
1363 			if (error) {
1364 				device_printf(dev, "unable to create spare "
1365 				    "dmamap for queue %d/%d error %d\n",
1366 				    q, i, error);
1367 				return (error);
1368 			}
1369 
1370 			for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1371 				error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1372 				    &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
1373 				if (error) {
1374 					device_printf(dev, "unable to create "
1375 					    "dmamap for queue %d/%d slot %d "
1376 					    "error %d\n",
1377 					    q, i, j, error);
1378 					return (error);
1379 				}
1380 			}
1381 		}
1382 	}
1383 
1384 	return (0);
1385 }
1386 
1387 static void
1388 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
1389 {
1390 	device_t dev;
1391 	struct vmxnet3_rxqueue *rxq;
1392 	struct vmxnet3_rxring *rxr;
1393 	struct vmxnet3_comp_ring *rxc;
1394 	struct vmxnet3_rxbuf *rxb;
1395 	int i, j, q;
1396 
1397 	dev = sc->vmx_dev;
1398 
1399 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
1400 		rxq = &sc->vmx_rxq[q];
1401 		rxc = &rxq->vxrxq_comp_ring;
1402 
1403 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1404 			rxr = &rxq->vxrxq_cmd_ring[i];
1405 
1406 			if (rxr->vxrxr_spare_dmap != NULL) {
1407 				bus_dmamap_destroy(rxr->vxrxr_rxtag,
1408 				    rxr->vxrxr_spare_dmap);
1409 				rxr->vxrxr_spare_dmap = NULL;
1410 			}
1411 
1412 			for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1413 				rxb = &rxr->vxrxr_rxbuf[j];
1414 				if (rxb->vrxb_dmamap != NULL) {
1415 					bus_dmamap_destroy(rxr->vxrxr_rxtag,
1416 					    rxb->vrxb_dmamap);
1417 					rxb->vrxb_dmamap = NULL;
1418 				}
1419 			}
1420 		}
1421 
1422 		if (rxc->vxcr_u.rxcd != NULL) {
1423 			vmxnet3_dma_free(sc, &rxc->vxcr_dma);
1424 			rxc->vxcr_u.rxcd = NULL;
1425 		}
1426 
1427 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1428 			rxr = &rxq->vxrxq_cmd_ring[i];
1429 
1430 			if (rxr->vxrxr_rxd != NULL) {
1431 				vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
1432 				rxr->vxrxr_rxd = NULL;
1433 			}
1434 
1435 			if (rxr->vxrxr_rxtag != NULL) {
1436 				bus_dma_tag_destroy(rxr->vxrxr_rxtag);
1437 				rxr->vxrxr_rxtag = NULL;
1438 			}
1439 		}
1440 	}
1441 }
1442 
1443 static int
1444 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
1445 {
1446 	int error;
1447 
1448 	error = vmxnet3_alloc_txq_data(sc);
1449 	if (error)
1450 		return (error);
1451 
1452 	error = vmxnet3_alloc_rxq_data(sc);
1453 	if (error)
1454 		return (error);
1455 
1456 	return (0);
1457 }
1458 
1459 static void
1460 vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
1461 {
1462 
1463 	if (sc->vmx_rxq != NULL)
1464 		vmxnet3_free_rxq_data(sc);
1465 
1466 	if (sc->vmx_txq != NULL)
1467 		vmxnet3_free_txq_data(sc);
1468 }
1469 
1470 static int
1471 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1472 {
1473 	int error;
1474 
1475 	error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
1476 	    32, &sc->vmx_mcast_dma);
1477 	if (error)
1478 		device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1479 	else
1480 		sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;
1481 
1482 	return (error);
1483 }
1484 
1485 static void
1486 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1487 {
1488 
1489 	if (sc->vmx_mcast != NULL) {
1490 		vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
1491 		sc->vmx_mcast = NULL;
1492 	}
1493 }
1494 
1495 static void
1496 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1497 {
1498 	struct vmxnet3_driver_shared *ds;
1499 	struct vmxnet3_txqueue *txq;
1500 	struct vmxnet3_txq_shared *txs;
1501 	struct vmxnet3_rxqueue *rxq;
1502 	struct vmxnet3_rxq_shared *rxs;
1503 	int i;
1504 
1505 	ds = sc->vmx_ds;
1506 
1507 	/*
1508 	 * Initialize fields of the shared data that remains the same across
1509 	 * reinits. Note the shared data is zero'd when allocated.
1510 	 */
1511 
1512 	ds->magic = VMXNET3_REV1_MAGIC;
1513 
1514 	/* DriverInfo */
1515 	ds->version = VMXNET3_DRIVER_VERSION;
1516 	ds->guest = VMXNET3_GOS_FREEBSD |
1517 #ifdef __LP64__
1518 	    VMXNET3_GOS_64BIT;
1519 #else
1520 	    VMXNET3_GOS_32BIT;
1521 #endif
1522 	ds->vmxnet3_revision = 1;
1523 	ds->upt_version = 1;
1524 
1525 	/* Misc. conf */
1526 	ds->driver_data = vtophys(sc);
1527 	ds->driver_data_len = sizeof(struct vmxnet3_softc);
1528 	ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
1529 	ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
1530 	ds->nrxsg_max = sc->vmx_max_rxsegs;
1531 
1532 	/* RSS conf */
1533 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1534 		ds->rss.version = 1;
1535 		ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
1536 		ds->rss.len = sc->vmx_rss_dma.dma_size;
1537 	}
1538 
1539 	/* Interrupt control. */
1540 	ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1541 	ds->nintr = sc->vmx_nintrs;
1542 	ds->evintr = sc->vmx_event_intr_idx;
1543 	ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1544 
1545 	for (i = 0; i < sc->vmx_nintrs; i++)
1546 		ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1547 
1548 	/* Receive filter. */
1549 	ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
1550 	ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;
1551 
1552 	/* Tx queues */
1553 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
1554 		txq = &sc->vmx_txq[i];
1555 		txs = txq->vxtxq_ts;
1556 
1557 		txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
1558 		txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1559 		txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
1560 		txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1561 		txs->driver_data = vtophys(txq);
1562 		txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1563 	}
1564 
1565 	/* Rx queues */
1566 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
1567 		rxq = &sc->vmx_rxq[i];
1568 		rxs = rxq->vxrxq_rs;
1569 
1570 		rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
1571 		rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1572 		rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
1573 		rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1574 		rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
1575 		rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1576 		rxs->driver_data = vtophys(rxq);
1577 		rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1578 	}
1579 }
1580 
1581 static void
1582 vmxnet3_reinit_interface(struct vmxnet3_softc *sc)
1583 {
1584 	struct ifnet *ifp;
1585 
1586 	ifp = sc->vmx_ifp;
1587 
1588 	/* Use the current MAC address. */
1589 	bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN);
1590 	vmxnet3_set_lladdr(sc);
1591 
1592 	ifp->if_hwassist = 0;
1593 	if (ifp->if_capenable & IFCAP_TXCSUM)
1594 		ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD;
1595 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1596 		ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6;
1597 	if (ifp->if_capenable & IFCAP_TSO4)
1598 		ifp->if_hwassist |= CSUM_IP_TSO;
1599 	if (ifp->if_capenable & IFCAP_TSO6)
1600 		ifp->if_hwassist |= CSUM_IP6_TSO;
1601 }
1602 
1603 static void
1604 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
1605 {
1606 	/*
1607 	 * Use the same key as the Linux driver until FreeBSD can do
1608 	 * RSS (presumably Toeplitz) in software.
1609 	 */
1610 	static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
1611 	    0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
1612 	    0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
1613 	    0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
1614 	    0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
1615 	    0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
1616 	};
1617 
1618 	struct vmxnet3_driver_shared *ds;
1619 	struct vmxnet3_rss_shared *rss;
1620 	int i;
1621 
1622 	ds = sc->vmx_ds;
1623 	rss = sc->vmx_rss;
1624 
1625 	rss->hash_type =
1626 	    UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
1627 	    UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
1628 	rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
1629 	rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
1630 	rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
1631 	memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
1632 
1633 	for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
1634 		rss->ind_table[i] = i % sc->vmx_nrxqueues;
1635 }
1636 
1637 static void
1638 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1639 {
1640 	struct ifnet *ifp;
1641 	struct vmxnet3_driver_shared *ds;
1642 
1643 	ifp = sc->vmx_ifp;
1644 	ds = sc->vmx_ds;
1645 
1646 	ds->mtu = ifp->if_mtu;
1647 	ds->ntxqueue = sc->vmx_ntxqueues;
1648 	ds->nrxqueue = sc->vmx_nrxqueues;
1649 
1650 	ds->upt_features = 0;
1651 	if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
1652 		ds->upt_features |= UPT1_F_CSUM;
1653 	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1654 		ds->upt_features |= UPT1_F_VLAN;
1655 	if (ifp->if_capenable & IFCAP_LRO)
1656 		ds->upt_features |= UPT1_F_LRO;
1657 
1658 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1659 		ds->upt_features |= UPT1_F_RSS;
1660 		vmxnet3_reinit_rss_shared_data(sc);
1661 	}
1662 
1663 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
1664 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1665 	    (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
1666 }
1667 
1668 static int
1669 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1670 {
1671 	int error;
1672 
1673 	error = vmxnet3_alloc_shared_data(sc);
1674 	if (error)
1675 		return (error);
1676 
1677 	error = vmxnet3_alloc_queue_data(sc);
1678 	if (error)
1679 		return (error);
1680 
1681 	error = vmxnet3_alloc_mcast_table(sc);
1682 	if (error)
1683 		return (error);
1684 
1685 	vmxnet3_init_shared_data(sc);
1686 
1687 	return (0);
1688 }
1689 
1690 static void
1691 vmxnet3_free_data(struct vmxnet3_softc *sc)
1692 {
1693 
1694 	vmxnet3_free_mcast_table(sc);
1695 	vmxnet3_free_queue_data(sc);
1696 	vmxnet3_free_shared_data(sc);
1697 }
1698 
1699 static int
1700 vmxnet3_setup_interface(struct vmxnet3_softc *sc)
1701 {
1702 	device_t dev;
1703 	struct ifnet *ifp;
1704 
1705 	dev = sc->vmx_dev;
1706 
1707 	ifp = sc->vmx_ifp = if_alloc(IFT_ETHER);
1708 	if (ifp == NULL) {
1709 		device_printf(dev, "cannot allocate ifnet structure\n");
1710 		return (ENOSPC);
1711 	}
1712 
1713 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1714 #if __FreeBSD_version < 1000025
1715 	ifp->if_baudrate = 1000000000;
1716 #elif __FreeBSD_version < 1100011
1717 	if_initbaudrate(ifp, IF_Gbps(10));
1718 #else
1719 	ifp->if_baudrate = IF_Gbps(10);
1720 #endif
1721 	ifp->if_softc = sc;
1722 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1723 	ifp->if_init = vmxnet3_init;
1724 	ifp->if_ioctl = vmxnet3_ioctl;
1725 	ifp->if_hw_tsomax = VMXNET3_TSO_MAXSIZE;
1726 
1727 #ifdef VMXNET3_LEGACY_TX
1728 	ifp->if_start = vmxnet3_start;
1729 	ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1;
1730 	IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1);
1731 	IFQ_SET_READY(&ifp->if_snd);
1732 #else
1733 	ifp->if_transmit = vmxnet3_txq_mq_start;
1734 	ifp->if_qflush = vmxnet3_qflush;
1735 #endif
1736 
1737 	vmxnet3_get_lladdr(sc);
1738 	ether_ifattach(ifp, sc->vmx_lladdr);
1739 
1740 	ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM;
1741 	ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6;
1742 	ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
1743 	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
1744 	    IFCAP_VLAN_HWCSUM;
1745 	ifp->if_capenable = ifp->if_capabilities;
1746 
1747 	/* These capabilities are not enabled by default. */
1748 	ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER;
1749 
1750 	sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
1751 	    vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
1752 	sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config,
1753 	    vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
1754 
1755 	ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change,
1756 	    vmxnet3_media_status);
1757 	ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1758 	ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO);
1759 
1760 	return (0);
1761 }
1762 
1763 static void
1764 vmxnet3_evintr(struct vmxnet3_softc *sc)
1765 {
1766 	device_t dev;
1767 	struct ifnet *ifp;
1768 	struct vmxnet3_txq_shared *ts;
1769 	struct vmxnet3_rxq_shared *rs;
1770 	uint32_t event;
1771 	int reset;
1772 
1773 	dev = sc->vmx_dev;
1774 	ifp = sc->vmx_ifp;
1775 	reset = 0;
1776 
1777 	VMXNET3_CORE_LOCK(sc);
1778 
1779 	/* Clear events. */
1780 	event = sc->vmx_ds->event;
1781 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
1782 
1783 	if (event & VMXNET3_EVENT_LINK) {
1784 		vmxnet3_link_status(sc);
1785 		if (sc->vmx_link_active != 0)
1786 			vmxnet3_tx_start_all(sc);
1787 	}
1788 
1789 	if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
1790 		reset = 1;
1791 		vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
1792 		ts = sc->vmx_txq[0].vxtxq_ts;
1793 		if (ts->stopped != 0)
1794 			device_printf(dev, "Tx queue error %#x\n", ts->error);
1795 		rs = sc->vmx_rxq[0].vxrxq_rs;
1796 		if (rs->stopped != 0)
1797 			device_printf(dev, "Rx queue error %#x\n", rs->error);
1798 		device_printf(dev, "Rx/Tx queue error event ... resetting\n");
1799 	}
1800 
1801 	if (event & VMXNET3_EVENT_DIC)
1802 		device_printf(dev, "device implementation change event\n");
1803 	if (event & VMXNET3_EVENT_DEBUG)
1804 		device_printf(dev, "debug event\n");
1805 
1806 	if (reset != 0) {
1807 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1808 		vmxnet3_init_locked(sc);
1809 	}
1810 
1811 	VMXNET3_CORE_UNLOCK(sc);
1812 }
1813 
1814 static void
1815 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq)
1816 {
1817 	struct vmxnet3_softc *sc;
1818 	struct ifnet *ifp;
1819 	struct vmxnet3_txring *txr;
1820 	struct vmxnet3_comp_ring *txc;
1821 	struct vmxnet3_txcompdesc *txcd;
1822 	struct vmxnet3_txbuf *txb;
1823 	struct mbuf *m;
1824 	u_int sop;
1825 
1826 	sc = txq->vxtxq_sc;
1827 	ifp = sc->vmx_ifp;
1828 	txr = &txq->vxtxq_cmd_ring;
1829 	txc = &txq->vxtxq_comp_ring;
1830 
1831 	VMXNET3_TXQ_LOCK_ASSERT(txq);
1832 
1833 	for (;;) {
1834 		txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
1835 		if (txcd->gen != txc->vxcr_gen)
1836 			break;
1837 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1838 
1839 		if (++txc->vxcr_next == txc->vxcr_ndesc) {
1840 			txc->vxcr_next = 0;
1841 			txc->vxcr_gen ^= 1;
1842 		}
1843 
1844 		sop = txr->vxtxr_next;
1845 		txb = &txr->vxtxr_txbuf[sop];
1846 
1847 		if ((m = txb->vtxb_m) != NULL) {
1848 			bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
1849 			    BUS_DMASYNC_POSTWRITE);
1850 			bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
1851 
1852 			txq->vxtxq_stats.vmtxs_opackets++;
1853 			txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len;
1854 			if (m->m_flags & M_MCAST)
1855 				txq->vxtxq_stats.vmtxs_omcasts++;
1856 
1857 			m_freem(m);
1858 			txb->vtxb_m = NULL;
1859 		}
1860 
1861 		txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
1862 	}
1863 
1864 	if (txr->vxtxr_head == txr->vxtxr_next)
1865 		txq->vxtxq_watchdog = 0;
1866 }
1867 
1868 static int
1869 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr)
1870 {
1871 	struct ifnet *ifp;
1872 	struct mbuf *m;
1873 	struct vmxnet3_rxdesc *rxd;
1874 	struct vmxnet3_rxbuf *rxb;
1875 	bus_dma_tag_t tag;
1876 	bus_dmamap_t dmap;
1877 	bus_dma_segment_t segs[1];
1878 	int idx, clsize, btype, flags, nsegs, error;
1879 
1880 	ifp = sc->vmx_ifp;
1881 	tag = rxr->vxrxr_rxtag;
1882 	dmap = rxr->vxrxr_spare_dmap;
1883 	idx = rxr->vxrxr_fill;
1884 	rxd = &rxr->vxrxr_rxd[idx];
1885 	rxb = &rxr->vxrxr_rxbuf[idx];
1886 
1887 #ifdef VMXNET3_FAILPOINTS
1888 	KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS);
1889 	if (rxr->vxrxr_rid != 0)
1890 		KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS);
1891 #endif
1892 
1893 	if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) {
1894 		flags = M_PKTHDR;
1895 		clsize = MCLBYTES;
1896 		btype = VMXNET3_BTYPE_HEAD;
1897 	} else {
1898 #if __FreeBSD_version < 902001
1899 		/*
1900 		 * These mbufs will never be used for the start of a frame.
1901 		 * Roughly prior to branching releng/9.2, the load_mbuf_sg()
1902 		 * required the mbuf to always be a packet header. Avoid
1903 		 * unnecessary mbuf initialization in newer versions where
1904 		 * that is not the case.
1905 		 */
1906 		flags = M_PKTHDR;
1907 #else
1908 		flags = 0;
1909 #endif
1910 		clsize = MJUMPAGESIZE;
1911 		btype = VMXNET3_BTYPE_BODY;
1912 	}
1913 
1914 	m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize);
1915 	if (m == NULL) {
1916 		sc->vmx_stats.vmst_mgetcl_failed++;
1917 		return (ENOBUFS);
1918 	}
1919 
1920 	if (btype == VMXNET3_BTYPE_HEAD) {
1921 		m->m_len = m->m_pkthdr.len = clsize;
1922 		m_adj(m, ETHER_ALIGN);
1923 	} else
1924 		m->m_len = clsize;
1925 
1926 	error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs,
1927 	    BUS_DMA_NOWAIT);
1928 	if (error) {
1929 		m_freem(m);
1930 		sc->vmx_stats.vmst_mbuf_load_failed++;
1931 		return (error);
1932 	}
1933 	KASSERT(nsegs == 1,
1934 	    ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
1935 #if __FreeBSD_version < 902001
1936 	if (btype == VMXNET3_BTYPE_BODY)
1937 		m->m_flags &= ~M_PKTHDR;
1938 #endif
1939 
1940 	if (rxb->vrxb_m != NULL) {
1941 		bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD);
1942 		bus_dmamap_unload(tag, rxb->vrxb_dmamap);
1943 	}
1944 
1945 	rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
1946 	rxb->vrxb_dmamap = dmap;
1947 	rxb->vrxb_m = m;
1948 
1949 	rxd->addr = segs[0].ds_addr;
1950 	rxd->len = segs[0].ds_len;
1951 	rxd->btype = btype;
1952 	rxd->gen = rxr->vxrxr_gen;
1953 
1954 	vmxnet3_rxr_increment_fill(rxr);
1955 	return (0);
1956 }
1957 
1958 static void
1959 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
1960     struct vmxnet3_rxring *rxr, int idx)
1961 {
1962 	struct vmxnet3_rxdesc *rxd;
1963 
1964 	rxd = &rxr->vxrxr_rxd[idx];
1965 	rxd->gen = rxr->vxrxr_gen;
1966 	vmxnet3_rxr_increment_fill(rxr);
1967 }
1968 
1969 static void
1970 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
1971 {
1972 	struct vmxnet3_softc *sc;
1973 	struct vmxnet3_rxring *rxr;
1974 	struct vmxnet3_comp_ring *rxc;
1975 	struct vmxnet3_rxcompdesc *rxcd;
1976 	int idx, eof;
1977 
1978 	sc = rxq->vxrxq_sc;
1979 	rxc = &rxq->vxrxq_comp_ring;
1980 
1981 	do {
1982 		rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
1983 		if (rxcd->gen != rxc->vxcr_gen)
1984 			break;		/* Not expected. */
1985 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1986 
1987 		if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
1988 			rxc->vxcr_next = 0;
1989 			rxc->vxcr_gen ^= 1;
1990 		}
1991 
1992 		idx = rxcd->rxd_idx;
1993 		eof = rxcd->eop;
1994 		if (rxcd->qid < sc->vmx_nrxqueues)
1995 			rxr = &rxq->vxrxq_cmd_ring[0];
1996 		else
1997 			rxr = &rxq->vxrxq_cmd_ring[1];
1998 		vmxnet3_rxq_eof_discard(rxq, rxr, idx);
1999 	} while (!eof);
2000 }
2001 
2002 static void
2003 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2004 {
2005 
2006 	if (rxcd->ipv4) {
2007 		m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2008 		if (rxcd->ipcsum_ok)
2009 			m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2010 	}
2011 
2012 	if (!rxcd->fragment) {
2013 		if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) {
2014 			m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2015 			    CSUM_PSEUDO_HDR;
2016 			m->m_pkthdr.csum_data = 0xFFFF;
2017 		}
2018 	}
2019 }
2020 
2021 static void
2022 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
2023     struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2024 {
2025 	struct vmxnet3_softc *sc;
2026 	struct ifnet *ifp;
2027 
2028 	sc = rxq->vxrxq_sc;
2029 	ifp = sc->vmx_ifp;
2030 
2031 	if (rxcd->error) {
2032 		rxq->vxrxq_stats.vmrxs_ierrors++;
2033 		m_freem(m);
2034 		return;
2035 	}
2036 
2037 #ifdef notyet
2038 	switch (rxcd->rss_type) {
2039 	case VMXNET3_RCD_RSS_TYPE_IPV4:
2040 		m->m_pkthdr.flowid = rxcd->rss_hash;
2041 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4);
2042 		break;
2043 	case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
2044 		m->m_pkthdr.flowid = rxcd->rss_hash;
2045 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4);
2046 		break;
2047 	case VMXNET3_RCD_RSS_TYPE_IPV6:
2048 		m->m_pkthdr.flowid = rxcd->rss_hash;
2049 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6);
2050 		break;
2051 	case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
2052 		m->m_pkthdr.flowid = rxcd->rss_hash;
2053 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6);
2054 		break;
2055 	default: /* VMXNET3_RCD_RSS_TYPE_NONE */
2056 		m->m_pkthdr.flowid = rxq->vxrxq_id;
2057 		M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2058 		break;
2059 	}
2060 #else
2061 	m->m_pkthdr.flowid = rxq->vxrxq_id;
2062 	m->m_flags |= M_FLOWID;
2063 #endif
2064 
2065 	if (!rxcd->no_csum)
2066 		vmxnet3_rx_csum(rxcd, m);
2067 	if (rxcd->vlan) {
2068 		m->m_flags |= M_VLANTAG;
2069 		m->m_pkthdr.ether_vtag = rxcd->vtag;
2070 	}
2071 
2072 	rxq->vxrxq_stats.vmrxs_ipackets++;
2073 	rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len;
2074 
2075 	VMXNET3_RXQ_UNLOCK(rxq);
2076 	(*ifp->if_input)(ifp, m);
2077 	VMXNET3_RXQ_LOCK(rxq);
2078 }
2079 
2080 static void
2081 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq)
2082 {
2083 	struct vmxnet3_softc *sc;
2084 	struct ifnet *ifp;
2085 	struct vmxnet3_rxring *rxr;
2086 	struct vmxnet3_comp_ring *rxc;
2087 	struct vmxnet3_rxdesc *rxd;
2088 	struct vmxnet3_rxcompdesc *rxcd;
2089 	struct mbuf *m, *m_head, *m_tail;
2090 	int idx, length;
2091 
2092 	sc = rxq->vxrxq_sc;
2093 	ifp = sc->vmx_ifp;
2094 	rxc = &rxq->vxrxq_comp_ring;
2095 
2096 	VMXNET3_RXQ_LOCK_ASSERT(rxq);
2097 
2098 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2099 		return;
2100 
2101 	m_head = rxq->vxrxq_mhead;
2102 	rxq->vxrxq_mhead = NULL;
2103 	m_tail = rxq->vxrxq_mtail;
2104 	rxq->vxrxq_mtail = NULL;
2105 	MPASS(m_head == NULL || m_tail != NULL);
2106 
2107 	for (;;) {
2108 		rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2109 		if (rxcd->gen != rxc->vxcr_gen) {
2110 			rxq->vxrxq_mhead = m_head;
2111 			rxq->vxrxq_mtail = m_tail;
2112 			break;
2113 		}
2114 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2115 
2116 		if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2117 			rxc->vxcr_next = 0;
2118 			rxc->vxcr_gen ^= 1;
2119 		}
2120 
2121 		idx = rxcd->rxd_idx;
2122 		length = rxcd->len;
2123 		if (rxcd->qid < sc->vmx_nrxqueues)
2124 			rxr = &rxq->vxrxq_cmd_ring[0];
2125 		else
2126 			rxr = &rxq->vxrxq_cmd_ring[1];
2127 		rxd = &rxr->vxrxr_rxd[idx];
2128 
2129 		m = rxr->vxrxr_rxbuf[idx].vrxb_m;
2130 		KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf",
2131 		    __func__, rxcd->qid, idx));
2132 
2133 		/*
2134 		 * The host may skip descriptors. We detect this when this
2135 		 * descriptor does not match the previous fill index. Catch
2136 		 * up with the host now.
2137 		 */
2138 		if (__predict_false(rxr->vxrxr_fill != idx)) {
2139 			while (rxr->vxrxr_fill != idx) {
2140 				rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
2141 				    rxr->vxrxr_gen;
2142 				vmxnet3_rxr_increment_fill(rxr);
2143 			}
2144 		}
2145 
2146 		if (rxcd->sop) {
2147 			KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD,
2148 			    ("%s: start of frame w/o head buffer", __func__));
2149 			KASSERT(rxr == &rxq->vxrxq_cmd_ring[0],
2150 			    ("%s: start of frame not in ring 0", __func__));
2151 			KASSERT((idx % sc->vmx_rx_max_chain) == 0,
2152 			    ("%s: start of frame at unexcepted index %d (%d)",
2153 			     __func__, idx, sc->vmx_rx_max_chain));
2154 			KASSERT(m_head == NULL,
2155 			    ("%s: duplicate start of frame?", __func__));
2156 
2157 			if (length == 0) {
2158 				/* Just ignore this descriptor. */
2159 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2160 				goto nextp;
2161 			}
2162 
2163 			if (vmxnet3_newbuf(sc, rxr) != 0) {
2164 				rxq->vxrxq_stats.vmrxs_iqdrops++;
2165 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2166 				if (!rxcd->eop)
2167 					vmxnet3_rxq_discard_chain(rxq);
2168 				goto nextp;
2169 			}
2170 
2171 			m->m_pkthdr.rcvif = ifp;
2172 			m->m_pkthdr.len = m->m_len = length;
2173 			m->m_pkthdr.csum_flags = 0;
2174 			m_head = m_tail = m;
2175 
2176 		} else {
2177 			KASSERT(rxd->btype == VMXNET3_BTYPE_BODY,
2178 			    ("%s: non start of frame w/o body buffer", __func__));
2179 			KASSERT(m_head != NULL,
2180 			    ("%s: frame not started?", __func__));
2181 
2182 			if (vmxnet3_newbuf(sc, rxr) != 0) {
2183 				rxq->vxrxq_stats.vmrxs_iqdrops++;
2184 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2185 				if (!rxcd->eop)
2186 					vmxnet3_rxq_discard_chain(rxq);
2187 				m_freem(m_head);
2188 				m_head = m_tail = NULL;
2189 				goto nextp;
2190 			}
2191 
2192 			m->m_len = length;
2193 			m_head->m_pkthdr.len += length;
2194 			m_tail->m_next = m;
2195 			m_tail = m;
2196 		}
2197 
2198 		if (rxcd->eop) {
2199 			vmxnet3_rxq_input(rxq, rxcd, m_head);
2200 			m_head = m_tail = NULL;
2201 
2202 			/* Must recheck after dropping the Rx lock. */
2203 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2204 				break;
2205 		}
2206 
2207 nextp:
2208 		if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
2209 			int qid = rxcd->qid;
2210 			bus_size_t r;
2211 
2212 			idx = (idx + 1) % rxr->vxrxr_ndesc;
2213 			if (qid >= sc->vmx_nrxqueues) {
2214 				qid -= sc->vmx_nrxqueues;
2215 				r = VMXNET3_BAR0_RXH2(qid);
2216 			} else
2217 				r = VMXNET3_BAR0_RXH1(qid);
2218 			vmxnet3_write_bar0(sc, r, idx);
2219 		}
2220 	}
2221 }
2222 
2223 static void
2224 vmxnet3_legacy_intr(void *xsc)
2225 {
2226 	struct vmxnet3_softc *sc;
2227 	struct vmxnet3_rxqueue *rxq;
2228 	struct vmxnet3_txqueue *txq;
2229 
2230 	sc = xsc;
2231 	rxq = &sc->vmx_rxq[0];
2232 	txq = &sc->vmx_txq[0];
2233 
2234 	if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
2235 		if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
2236 			return;
2237 	}
2238 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2239 		vmxnet3_disable_all_intrs(sc);
2240 
2241 	if (sc->vmx_ds->event != 0)
2242 		vmxnet3_evintr(sc);
2243 
2244 	VMXNET3_RXQ_LOCK(rxq);
2245 	vmxnet3_rxq_eof(rxq);
2246 	VMXNET3_RXQ_UNLOCK(rxq);
2247 
2248 	VMXNET3_TXQ_LOCK(txq);
2249 	vmxnet3_txq_eof(txq);
2250 	vmxnet3_txq_start(txq);
2251 	VMXNET3_TXQ_UNLOCK(txq);
2252 
2253 	vmxnet3_enable_all_intrs(sc);
2254 }
2255 
2256 static void
2257 vmxnet3_txq_intr(void *xtxq)
2258 {
2259 	struct vmxnet3_softc *sc;
2260 	struct vmxnet3_txqueue *txq;
2261 
2262 	txq = xtxq;
2263 	sc = txq->vxtxq_sc;
2264 
2265 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2266 		vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx);
2267 
2268 	VMXNET3_TXQ_LOCK(txq);
2269 	vmxnet3_txq_eof(txq);
2270 	vmxnet3_txq_start(txq);
2271 	VMXNET3_TXQ_UNLOCK(txq);
2272 
2273 	vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx);
2274 }
2275 
2276 static void
2277 vmxnet3_rxq_intr(void *xrxq)
2278 {
2279 	struct vmxnet3_softc *sc;
2280 	struct vmxnet3_rxqueue *rxq;
2281 
2282 	rxq = xrxq;
2283 	sc = rxq->vxrxq_sc;
2284 
2285 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2286 		vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
2287 
2288 	VMXNET3_RXQ_LOCK(rxq);
2289 	vmxnet3_rxq_eof(rxq);
2290 	VMXNET3_RXQ_UNLOCK(rxq);
2291 
2292 	vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx);
2293 }
2294 
2295 static void
2296 vmxnet3_event_intr(void *xsc)
2297 {
2298 	struct vmxnet3_softc *sc;
2299 
2300 	sc = xsc;
2301 
2302 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2303 		vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
2304 
2305 	if (sc->vmx_ds->event != 0)
2306 		vmxnet3_evintr(sc);
2307 
2308 	vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2309 }
2310 
2311 static void
2312 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2313 {
2314 	struct vmxnet3_txring *txr;
2315 	struct vmxnet3_txbuf *txb;
2316 	int i;
2317 
2318 	txr = &txq->vxtxq_cmd_ring;
2319 
2320 	for (i = 0; i < txr->vxtxr_ndesc; i++) {
2321 		txb = &txr->vxtxr_txbuf[i];
2322 
2323 		if (txb->vtxb_m == NULL)
2324 			continue;
2325 
2326 		bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
2327 		    BUS_DMASYNC_POSTWRITE);
2328 		bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
2329 		m_freem(txb->vtxb_m);
2330 		txb->vtxb_m = NULL;
2331 	}
2332 }
2333 
2334 static void
2335 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2336 {
2337 	struct vmxnet3_rxring *rxr;
2338 	struct vmxnet3_rxbuf *rxb;
2339 	int i, j;
2340 
2341 	if (rxq->vxrxq_mhead != NULL) {
2342 		m_freem(rxq->vxrxq_mhead);
2343 		rxq->vxrxq_mhead = NULL;
2344 		rxq->vxrxq_mtail = NULL;
2345 	}
2346 
2347 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
2348 		rxr = &rxq->vxrxq_cmd_ring[i];
2349 
2350 		for (j = 0; j < rxr->vxrxr_ndesc; j++) {
2351 			rxb = &rxr->vxrxr_rxbuf[j];
2352 
2353 			if (rxb->vrxb_m == NULL)
2354 				continue;
2355 
2356 			bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap,
2357 			    BUS_DMASYNC_POSTREAD);
2358 			bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap);
2359 			m_freem(rxb->vrxb_m);
2360 			rxb->vrxb_m = NULL;
2361 		}
2362 	}
2363 }
2364 
2365 static void
2366 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
2367 {
2368 	struct vmxnet3_rxqueue *rxq;
2369 	struct vmxnet3_txqueue *txq;
2370 	int i;
2371 
2372 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
2373 		rxq = &sc->vmx_rxq[i];
2374 		VMXNET3_RXQ_LOCK(rxq);
2375 		VMXNET3_RXQ_UNLOCK(rxq);
2376 	}
2377 
2378 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
2379 		txq = &sc->vmx_txq[i];
2380 		VMXNET3_TXQ_LOCK(txq);
2381 		VMXNET3_TXQ_UNLOCK(txq);
2382 	}
2383 }
2384 
2385 static void
2386 vmxnet3_stop(struct vmxnet3_softc *sc)
2387 {
2388 	struct ifnet *ifp;
2389 	int q;
2390 
2391 	ifp = sc->vmx_ifp;
2392 	VMXNET3_CORE_LOCK_ASSERT(sc);
2393 
2394 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2395 	sc->vmx_link_active = 0;
2396 	callout_stop(&sc->vmx_tick);
2397 
2398 	/* Disable interrupts. */
2399 	vmxnet3_disable_all_intrs(sc);
2400 	vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
2401 
2402 	vmxnet3_stop_rendezvous(sc);
2403 
2404 	for (q = 0; q < sc->vmx_ntxqueues; q++)
2405 		vmxnet3_txstop(sc, &sc->vmx_txq[q]);
2406 	for (q = 0; q < sc->vmx_nrxqueues; q++)
2407 		vmxnet3_rxstop(sc, &sc->vmx_rxq[q]);
2408 
2409 	vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
2410 }
2411 
2412 static void
2413 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2414 {
2415 	struct vmxnet3_txring *txr;
2416 	struct vmxnet3_comp_ring *txc;
2417 
2418 	txr = &txq->vxtxq_cmd_ring;
2419 	txr->vxtxr_head = 0;
2420 	txr->vxtxr_next = 0;
2421 	txr->vxtxr_gen = VMXNET3_INIT_GEN;
2422 	bzero(txr->vxtxr_txd,
2423 	    txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));
2424 
2425 	txc = &txq->vxtxq_comp_ring;
2426 	txc->vxcr_next = 0;
2427 	txc->vxcr_gen = VMXNET3_INIT_GEN;
2428 	bzero(txc->vxcr_u.txcd,
2429 	    txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
2430 }
2431 
2432 static int
2433 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2434 {
2435 	struct ifnet *ifp;
2436 	struct vmxnet3_rxring *rxr;
2437 	struct vmxnet3_comp_ring *rxc;
2438 	int i, populate, idx, frame_size, error;
2439 
2440 	ifp = sc->vmx_ifp;
2441 	frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) +
2442 	    ifp->if_mtu;
2443 
2444 	/*
2445 	 * If the MTU causes us to exceed what a regular sized cluster can
2446 	 * handle, we allocate a second MJUMPAGESIZE cluster after it in
2447 	 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters.
2448 	 *
2449 	 * Keep rx_max_chain a divisor of the maximum Rx ring size to make
2450 	 * our life easier. We do not support changing the ring size after
2451 	 * the attach.
2452 	 */
2453 	if (frame_size <= MCLBYTES)
2454 		sc->vmx_rx_max_chain = 1;
2455 	else
2456 		sc->vmx_rx_max_chain = 2;
2457 
2458 	/*
2459 	 * Only populate ring 1 if the configuration will take advantage
2460 	 * of it. That is either when LRO is enabled or the frame size
2461 	 * exceeds what ring 0 can contain.
2462 	 */
2463 	if ((ifp->if_capenable & IFCAP_LRO) == 0 &&
2464 	    frame_size <= MCLBYTES + MJUMPAGESIZE)
2465 		populate = 1;
2466 	else
2467 		populate = VMXNET3_RXRINGS_PERQ;
2468 
2469 	for (i = 0; i < populate; i++) {
2470 		rxr = &rxq->vxrxq_cmd_ring[i];
2471 		rxr->vxrxr_fill = 0;
2472 		rxr->vxrxr_gen = VMXNET3_INIT_GEN;
2473 		bzero(rxr->vxrxr_rxd,
2474 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2475 
2476 		for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
2477 			error = vmxnet3_newbuf(sc, rxr);
2478 			if (error)
2479 				return (error);
2480 		}
2481 	}
2482 
2483 	for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
2484 		rxr = &rxq->vxrxq_cmd_ring[i];
2485 		rxr->vxrxr_fill = 0;
2486 		rxr->vxrxr_gen = 0;
2487 		bzero(rxr->vxrxr_rxd,
2488 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2489 	}
2490 
2491 	rxc = &rxq->vxrxq_comp_ring;
2492 	rxc->vxcr_next = 0;
2493 	rxc->vxcr_gen = VMXNET3_INIT_GEN;
2494 	bzero(rxc->vxcr_u.rxcd,
2495 	    rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));
2496 
2497 	return (0);
2498 }
2499 
2500 static int
2501 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
2502 {
2503 	device_t dev;
2504 	int q, error;
2505 
2506 	dev = sc->vmx_dev;
2507 
2508 	for (q = 0; q < sc->vmx_ntxqueues; q++)
2509 		vmxnet3_txinit(sc, &sc->vmx_txq[q]);
2510 
2511 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
2512 		error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
2513 		if (error) {
2514 			device_printf(dev, "cannot populate Rx queue %d\n", q);
2515 			return (error);
2516 		}
2517 	}
2518 
2519 	return (0);
2520 }
2521 
2522 static int
2523 vmxnet3_enable_device(struct vmxnet3_softc *sc)
2524 {
2525 	int q;
2526 
2527 	if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
2528 		device_printf(sc->vmx_dev, "device enable command failed!\n");
2529 		return (1);
2530 	}
2531 
2532 	/* Reset the Rx queue heads. */
2533 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
2534 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
2535 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
2536 	}
2537 
2538 	return (0);
2539 }
2540 
2541 static void
2542 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
2543 {
2544 	struct ifnet *ifp;
2545 
2546 	ifp = sc->vmx_ifp;
2547 
2548 	vmxnet3_set_rxfilter(sc);
2549 
2550 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2551 		bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter,
2552 		    sizeof(sc->vmx_ds->vlan_filter));
2553 	else
2554 		bzero(sc->vmx_ds->vlan_filter,
2555 		    sizeof(sc->vmx_ds->vlan_filter));
2556 	vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
2557 }
2558 
2559 static int
2560 vmxnet3_reinit(struct vmxnet3_softc *sc)
2561 {
2562 
2563 	vmxnet3_reinit_interface(sc);
2564 	vmxnet3_reinit_shared_data(sc);
2565 
2566 	if (vmxnet3_reinit_queues(sc) != 0)
2567 		return (ENXIO);
2568 
2569 	if (vmxnet3_enable_device(sc) != 0)
2570 		return (ENXIO);
2571 
2572 	vmxnet3_reinit_rxfilters(sc);
2573 
2574 	return (0);
2575 }
2576 
2577 static void
2578 vmxnet3_init_locked(struct vmxnet3_softc *sc)
2579 {
2580 	struct ifnet *ifp;
2581 
2582 	ifp = sc->vmx_ifp;
2583 
2584 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2585 		return;
2586 
2587 	vmxnet3_stop(sc);
2588 
2589 	if (vmxnet3_reinit(sc) != 0) {
2590 		vmxnet3_stop(sc);
2591 		return;
2592 	}
2593 
2594 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
2595 	vmxnet3_link_status(sc);
2596 
2597 	vmxnet3_enable_all_intrs(sc);
2598 	callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
2599 }
2600 
2601 static void
2602 vmxnet3_init(void *xsc)
2603 {
2604 	struct vmxnet3_softc *sc;
2605 
2606 	sc = xsc;
2607 
2608 	VMXNET3_CORE_LOCK(sc);
2609 	vmxnet3_init_locked(sc);
2610 	VMXNET3_CORE_UNLOCK(sc);
2611 }
2612 
2613 /*
2614  * BMV: Much of this can go away once we finally have offsets in
2615  * the mbuf packet header. Bug andre@.
2616  */
2617 static int
2618 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
2619     int *etype, int *proto, int *start)
2620 {
2621 	struct ether_vlan_header *evh;
2622 	int offset;
2623 #if defined(INET)
2624 	struct ip *ip = NULL;
2625 	struct ip iphdr;
2626 #endif
2627 #if defined(INET6)
2628 	struct ip6_hdr *ip6 = NULL;
2629 	struct ip6_hdr ip6hdr;
2630 #endif
2631 
2632 	evh = mtod(m, struct ether_vlan_header *);
2633 	if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2634 		/* BMV: We should handle nested VLAN tags too. */
2635 		*etype = ntohs(evh->evl_proto);
2636 		offset = sizeof(struct ether_vlan_header);
2637 	} else {
2638 		*etype = ntohs(evh->evl_encap_proto);
2639 		offset = sizeof(struct ether_header);
2640 	}
2641 
2642 	switch (*etype) {
2643 #if defined(INET)
2644 	case ETHERTYPE_IP:
2645 		if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
2646 			m_copydata(m, offset, sizeof(struct ip),
2647 			    (caddr_t) &iphdr);
2648 			ip = &iphdr;
2649 		} else
2650 			ip = mtodo(m, offset);
2651 		*proto = ip->ip_p;
2652 		*start = offset + (ip->ip_hl << 2);
2653 		break;
2654 #endif
2655 #if defined(INET6)
2656 	case ETHERTYPE_IPV6:
2657 		if (__predict_false(m->m_len <
2658 		    offset + sizeof(struct ip6_hdr))) {
2659 			m_copydata(m, offset, sizeof(struct ip6_hdr),
2660 			    (caddr_t) &ip6hdr);
2661 			ip6 = &ip6hdr;
2662 		} else
2663 			ip6 = mtodo(m, offset);
2664 		*proto = -1;
2665 		*start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
2666 		/* Assert the network stack sent us a valid packet. */
2667 		KASSERT(*start > offset,
2668 		    ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
2669 		    *start, offset, *proto));
2670 		break;
2671 #endif
2672 	default:
2673 		return (EINVAL);
2674 	}
2675 
2676 	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2677 		struct tcphdr *tcp, tcphdr;
2678 		uint16_t sum;
2679 
2680 		if (__predict_false(*proto != IPPROTO_TCP)) {
2681 			/* Likely failed to correctly parse the mbuf. */
2682 			return (EINVAL);
2683 		}
2684 
2685 		txq->vxtxq_stats.vmtxs_tso++;
2686 
2687 		switch (*etype) {
2688 #if defined(INET)
2689 		case ETHERTYPE_IP:
2690 			sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2691 			    htons(IPPROTO_TCP));
2692 			break;
2693 #endif
2694 #if defined(INET6)
2695 		case ETHERTYPE_IPV6:
2696 			sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
2697 			break;
2698 #endif
2699 		default:
2700 			sum = 0;
2701 			break;
2702 		}
2703 
2704 		if (m->m_len < *start + sizeof(struct tcphdr)) {
2705 			m_copyback(m, *start + offsetof(struct tcphdr, th_sum),
2706 			    sizeof(uint16_t), (caddr_t) &sum);
2707 			m_copydata(m, *start, sizeof(struct tcphdr),
2708 			    (caddr_t) &tcphdr);
2709 			tcp = &tcphdr;
2710 		} else {
2711 			tcp = mtodo(m, *start);
2712 			tcp->th_sum = sum;
2713 		}
2714 
2715 		/*
2716 		 * For TSO, the size of the protocol header is also
2717 		 * included in the descriptor header size.
2718 		 */
2719 		*start += (tcp->th_off << 2);
2720 	} else
2721 		txq->vxtxq_stats.vmtxs_csum++;
2722 
2723 	return (0);
2724 }
2725 
2726 static int
2727 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
2728     bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs)
2729 {
2730 	struct vmxnet3_txring *txr;
2731 	struct mbuf *m;
2732 	bus_dma_tag_t tag;
2733 	int error;
2734 
2735 	txr = &txq->vxtxq_cmd_ring;
2736 	m = *m0;
2737 	tag = txr->vxtxr_txtag;
2738 
2739 	error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
2740 	if (error == 0 || error != EFBIG)
2741 		return (error);
2742 
2743 	m = m_defrag(m, M_NOWAIT);
2744 	if (m != NULL) {
2745 		*m0 = m;
2746 		error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
2747 	} else
2748 		error = ENOBUFS;
2749 
2750 	if (error) {
2751 		m_freem(*m0);
2752 		*m0 = NULL;
2753 		txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++;
2754 	} else
2755 		txq->vxtxq_sc->vmx_stats.vmst_defragged++;
2756 
2757 	return (error);
2758 }
2759 
2760 static void
2761 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
2762 {
2763 	struct vmxnet3_txring *txr;
2764 
2765 	txr = &txq->vxtxq_cmd_ring;
2766 	bus_dmamap_unload(txr->vxtxr_txtag, dmap);
2767 }
2768 
2769 static int
2770 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
2771 {
2772 	struct vmxnet3_softc *sc;
2773 	struct vmxnet3_txring *txr;
2774 	struct vmxnet3_txdesc *txd, *sop;
2775 	struct mbuf *m;
2776 	bus_dmamap_t dmap;
2777 	bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS];
2778 	int i, gen, nsegs, etype, proto, start, error;
2779 
2780 	sc = txq->vxtxq_sc;
2781 	start = 0;
2782 	txd = NULL;
2783 	txr = &txq->vxtxq_cmd_ring;
2784 	dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
2785 
2786 	error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs);
2787 	if (error)
2788 		return (error);
2789 
2790 	m = *m0;
2791 	M_ASSERTPKTHDR(m);
2792 	KASSERT(nsegs <= VMXNET3_TX_MAXSEGS,
2793 	    ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
2794 
2795 	if (VMXNET3_TXRING_AVAIL(txr) < nsegs) {
2796 		txq->vxtxq_stats.vmtxs_full++;
2797 		vmxnet3_txq_unload_mbuf(txq, dmap);
2798 		return (ENOSPC);
2799 	} else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
2800 		error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start);
2801 		if (error) {
2802 			txq->vxtxq_stats.vmtxs_offload_failed++;
2803 			vmxnet3_txq_unload_mbuf(txq, dmap);
2804 			m_freem(m);
2805 			*m0 = NULL;
2806 			return (error);
2807 		}
2808 	}
2809 
2810 	txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m;
2811 	sop = &txr->vxtxr_txd[txr->vxtxr_head];
2812 	gen = txr->vxtxr_gen ^ 1;	/* Owned by cpu (yet) */
2813 
2814 	for (i = 0; i < nsegs; i++) {
2815 		txd = &txr->vxtxr_txd[txr->vxtxr_head];
2816 
2817 		txd->addr = segs[i].ds_addr;
2818 		txd->len = segs[i].ds_len;
2819 		txd->gen = gen;
2820 		txd->dtype = 0;
2821 		txd->offload_mode = VMXNET3_OM_NONE;
2822 		txd->offload_pos = 0;
2823 		txd->hlen = 0;
2824 		txd->eop = 0;
2825 		txd->compreq = 0;
2826 		txd->vtag_mode = 0;
2827 		txd->vtag = 0;
2828 
2829 		if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
2830 			txr->vxtxr_head = 0;
2831 			txr->vxtxr_gen ^= 1;
2832 		}
2833 		gen = txr->vxtxr_gen;
2834 	}
2835 	txd->eop = 1;
2836 	txd->compreq = 1;
2837 
2838 	if (m->m_flags & M_VLANTAG) {
2839 		sop->vtag_mode = 1;
2840 		sop->vtag = m->m_pkthdr.ether_vtag;
2841 	}
2842 
2843 	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2844 		sop->offload_mode = VMXNET3_OM_TSO;
2845 		sop->hlen = start;
2846 		sop->offload_pos = m->m_pkthdr.tso_segsz;
2847 	} else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
2848 	    VMXNET3_CSUM_OFFLOAD_IPV6)) {
2849 		sop->offload_mode = VMXNET3_OM_CSUM;
2850 		sop->hlen = start;
2851 		sop->offload_pos = start + m->m_pkthdr.csum_data;
2852 	}
2853 
2854 	/* Finally, change the ownership. */
2855 	vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
2856 	sop->gen ^= 1;
2857 
2858 	txq->vxtxq_ts->npending += nsegs;
2859 	if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
2860 		txq->vxtxq_ts->npending = 0;
2861 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
2862 		    txr->vxtxr_head);
2863 	}
2864 
2865 	return (0);
2866 }
2867 
2868 #ifdef VMXNET3_LEGACY_TX
2869 
2870 static void
2871 vmxnet3_start_locked(struct ifnet *ifp)
2872 {
2873 	struct vmxnet3_softc *sc;
2874 	struct vmxnet3_txqueue *txq;
2875 	struct vmxnet3_txring *txr;
2876 	struct mbuf *m_head;
2877 	int tx, avail;
2878 
2879 	sc = ifp->if_softc;
2880 	txq = &sc->vmx_txq[0];
2881 	txr = &txq->vxtxq_cmd_ring;
2882 	tx = 0;
2883 
2884 	VMXNET3_TXQ_LOCK_ASSERT(txq);
2885 
2886 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2887 	    sc->vmx_link_active == 0)
2888 		return;
2889 
2890 	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
2891 		if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2)
2892 			break;
2893 
2894 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
2895 		if (m_head == NULL)
2896 			break;
2897 
2898 		/* Assume worse case if this mbuf is the head of a chain. */
2899 		if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
2900 			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2901 			break;
2902 		}
2903 
2904 		if (vmxnet3_txq_encap(txq, &m_head) != 0) {
2905 			if (m_head != NULL)
2906 				IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2907 			break;
2908 		}
2909 
2910 		tx++;
2911 		ETHER_BPF_MTAP(ifp, m_head);
2912 	}
2913 
2914 	if (tx > 0)
2915 		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
2916 }
2917 
2918 static void
2919 vmxnet3_start(struct ifnet *ifp)
2920 {
2921 	struct vmxnet3_softc *sc;
2922 	struct vmxnet3_txqueue *txq;
2923 
2924 	sc = ifp->if_softc;
2925 	txq = &sc->vmx_txq[0];
2926 
2927 	VMXNET3_TXQ_LOCK(txq);
2928 	vmxnet3_start_locked(ifp);
2929 	VMXNET3_TXQ_UNLOCK(txq);
2930 }
2931 
2932 #else /* !VMXNET3_LEGACY_TX */
2933 
2934 static int
2935 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m)
2936 {
2937 	struct vmxnet3_softc *sc;
2938 	struct vmxnet3_txring *txr;
2939 	struct buf_ring *br;
2940 	struct ifnet *ifp;
2941 	int tx, avail, error;
2942 
2943 	sc = txq->vxtxq_sc;
2944 	br = txq->vxtxq_br;
2945 	ifp = sc->vmx_ifp;
2946 	txr = &txq->vxtxq_cmd_ring;
2947 	tx = 0;
2948 	error = 0;
2949 
2950 	VMXNET3_TXQ_LOCK_ASSERT(txq);
2951 
2952 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2953 	    sc->vmx_link_active == 0) {
2954 		if (m != NULL)
2955 			error = drbr_enqueue(ifp, br, m);
2956 		return (error);
2957 	}
2958 
2959 	if (m != NULL) {
2960 		error = drbr_enqueue(ifp, br, m);
2961 		if (error)
2962 			return (error);
2963 	}
2964 
2965 	while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) {
2966 		m = drbr_peek(ifp, br);
2967 		if (m == NULL)
2968 			break;
2969 
2970 		/* Assume worse case if this mbuf is the head of a chain. */
2971 		if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
2972 			drbr_putback(ifp, br, m);
2973 			break;
2974 		}
2975 
2976 		if (vmxnet3_txq_encap(txq, &m) != 0) {
2977 			if (m != NULL)
2978 				drbr_putback(ifp, br, m);
2979 			else
2980 				drbr_advance(ifp, br);
2981 			break;
2982 		}
2983 		drbr_advance(ifp, br);
2984 
2985 		tx++;
2986 		ETHER_BPF_MTAP(ifp, m);
2987 	}
2988 
2989 	if (tx > 0)
2990 		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
2991 
2992 	return (0);
2993 }
2994 
2995 static int
2996 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m)
2997 {
2998 	struct vmxnet3_softc *sc;
2999 	struct vmxnet3_txqueue *txq;
3000 	int i, ntxq, error;
3001 
3002 	sc = ifp->if_softc;
3003 	ntxq = sc->vmx_ntxqueues;
3004 
3005 	if (m->m_flags & M_FLOWID)
3006 		i = m->m_pkthdr.flowid % ntxq;
3007 	else
3008 		i = curcpu % ntxq;
3009 
3010 	txq = &sc->vmx_txq[i];
3011 
3012 	if (VMXNET3_TXQ_TRYLOCK(txq) != 0) {
3013 		error = vmxnet3_txq_mq_start_locked(txq, m);
3014 		VMXNET3_TXQ_UNLOCK(txq);
3015 	} else {
3016 		error = drbr_enqueue(ifp, txq->vxtxq_br, m);
3017 		taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask);
3018 	}
3019 
3020 	return (error);
3021 }
3022 
3023 static void
3024 vmxnet3_txq_tq_deferred(void *xtxq, int pending)
3025 {
3026 	struct vmxnet3_softc *sc;
3027 	struct vmxnet3_txqueue *txq;
3028 
3029 	txq = xtxq;
3030 	sc = txq->vxtxq_sc;
3031 
3032 	VMXNET3_TXQ_LOCK(txq);
3033 	if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br))
3034 		vmxnet3_txq_mq_start_locked(txq, NULL);
3035 	VMXNET3_TXQ_UNLOCK(txq);
3036 }
3037 
3038 #endif /* VMXNET3_LEGACY_TX */
3039 
3040 static void
3041 vmxnet3_txq_start(struct vmxnet3_txqueue *txq)
3042 {
3043 	struct vmxnet3_softc *sc;
3044 	struct ifnet *ifp;
3045 
3046 	sc = txq->vxtxq_sc;
3047 	ifp = sc->vmx_ifp;
3048 
3049 #ifdef VMXNET3_LEGACY_TX
3050 	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3051 		vmxnet3_start_locked(ifp);
3052 #else
3053 	if (!drbr_empty(ifp, txq->vxtxq_br))
3054 		vmxnet3_txq_mq_start_locked(txq, NULL);
3055 #endif
3056 }
3057 
3058 static void
3059 vmxnet3_tx_start_all(struct vmxnet3_softc *sc)
3060 {
3061 	struct vmxnet3_txqueue *txq;
3062 	int i;
3063 
3064 	VMXNET3_CORE_LOCK_ASSERT(sc);
3065 
3066 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3067 		txq = &sc->vmx_txq[i];
3068 
3069 		VMXNET3_TXQ_LOCK(txq);
3070 		vmxnet3_txq_start(txq);
3071 		VMXNET3_TXQ_UNLOCK(txq);
3072 	}
3073 }
3074 
3075 static void
3076 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
3077 {
3078 	struct ifnet *ifp;
3079 	int idx, bit;
3080 
3081 	ifp = sc->vmx_ifp;
3082 	idx = (tag >> 5) & 0x7F;
3083 	bit = tag & 0x1F;
3084 
3085 	if (tag == 0 || tag > 4095)
3086 		return;
3087 
3088 	VMXNET3_CORE_LOCK(sc);
3089 
3090 	/* Update our private VLAN bitvector. */
3091 	if (add)
3092 		sc->vmx_vlan_filter[idx] |= (1 << bit);
3093 	else
3094 		sc->vmx_vlan_filter[idx] &= ~(1 << bit);
3095 
3096 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
3097 		if (add)
3098 			sc->vmx_ds->vlan_filter[idx] |= (1 << bit);
3099 		else
3100 			sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit);
3101 		vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
3102 	}
3103 
3104 	VMXNET3_CORE_UNLOCK(sc);
3105 }
3106 
3107 static void
3108 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3109 {
3110 
3111 	if (ifp->if_softc == arg)
3112 		vmxnet3_update_vlan_filter(arg, 1, tag);
3113 }
3114 
3115 static void
3116 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3117 {
3118 
3119 	if (ifp->if_softc == arg)
3120 		vmxnet3_update_vlan_filter(arg, 0, tag);
3121 }
3122 
3123 static void
3124 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
3125 {
3126 	struct ifnet *ifp;
3127 	struct vmxnet3_driver_shared *ds;
3128 	struct ifmultiaddr *ifma;
3129 	u_int mode;
3130 
3131 	ifp = sc->vmx_ifp;
3132 	ds = sc->vmx_ds;
3133 
3134 	mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST;
3135 	if (ifp->if_flags & IFF_PROMISC)
3136 		mode |= VMXNET3_RXMODE_PROMISC;
3137 	if (ifp->if_flags & IFF_ALLMULTI)
3138 		mode |= VMXNET3_RXMODE_ALLMULTI;
3139 	else {
3140 		int cnt = 0, overflow = 0;
3141 
3142 		if_maddr_rlock(ifp);
3143 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3144 			if (ifma->ifma_addr->sa_family != AF_LINK)
3145 				continue;
3146 			else if (cnt == VMXNET3_MULTICAST_MAX) {
3147 				overflow = 1;
3148 				break;
3149 			}
3150 
3151 			bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
3152 			   &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN);
3153 			cnt++;
3154 		}
3155 		if_maddr_runlock(ifp);
3156 
3157 		if (overflow != 0) {
3158 			cnt = 0;
3159 			mode |= VMXNET3_RXMODE_ALLMULTI;
3160 		} else if (cnt > 0)
3161 			mode |= VMXNET3_RXMODE_MCAST;
3162 		ds->mcast_tablelen = cnt * ETHER_ADDR_LEN;
3163 	}
3164 
3165 	ds->rxmode = mode;
3166 
3167 	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
3168 	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
3169 }
3170 
3171 static int
3172 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu)
3173 {
3174 	struct ifnet *ifp;
3175 
3176 	ifp = sc->vmx_ifp;
3177 
3178 	if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU)
3179 		return (EINVAL);
3180 
3181 	ifp->if_mtu = mtu;
3182 
3183 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3184 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3185 		vmxnet3_init_locked(sc);
3186 	}
3187 
3188 	return (0);
3189 }
3190 
3191 static int
3192 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
3193 {
3194 	struct vmxnet3_softc *sc;
3195 	struct ifreq *ifr;
3196 	int reinit, mask, error;
3197 
3198 	sc = ifp->if_softc;
3199 	ifr = (struct ifreq *) data;
3200 	error = 0;
3201 
3202 	switch (cmd) {
3203 	case SIOCSIFMTU:
3204 		if (ifp->if_mtu != ifr->ifr_mtu) {
3205 			VMXNET3_CORE_LOCK(sc);
3206 			error = vmxnet3_change_mtu(sc, ifr->ifr_mtu);
3207 			VMXNET3_CORE_UNLOCK(sc);
3208 		}
3209 		break;
3210 
3211 	case SIOCSIFFLAGS:
3212 		VMXNET3_CORE_LOCK(sc);
3213 		if (ifp->if_flags & IFF_UP) {
3214 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3215 				if ((ifp->if_flags ^ sc->vmx_if_flags) &
3216 				    (IFF_PROMISC | IFF_ALLMULTI)) {
3217 					vmxnet3_set_rxfilter(sc);
3218 				}
3219 			} else
3220 				vmxnet3_init_locked(sc);
3221 		} else {
3222 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3223 				vmxnet3_stop(sc);
3224 		}
3225 		sc->vmx_if_flags = ifp->if_flags;
3226 		VMXNET3_CORE_UNLOCK(sc);
3227 		break;
3228 
3229 	case SIOCADDMULTI:
3230 	case SIOCDELMULTI:
3231 		VMXNET3_CORE_LOCK(sc);
3232 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3233 			vmxnet3_set_rxfilter(sc);
3234 		VMXNET3_CORE_UNLOCK(sc);
3235 		break;
3236 
3237 	case SIOCSIFMEDIA:
3238 	case SIOCGIFMEDIA:
3239 		error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd);
3240 		break;
3241 
3242 	case SIOCSIFCAP:
3243 		VMXNET3_CORE_LOCK(sc);
3244 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3245 
3246 		if (mask & IFCAP_TXCSUM)
3247 			ifp->if_capenable ^= IFCAP_TXCSUM;
3248 		if (mask & IFCAP_TXCSUM_IPV6)
3249 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3250 		if (mask & IFCAP_TSO4)
3251 			ifp->if_capenable ^= IFCAP_TSO4;
3252 		if (mask & IFCAP_TSO6)
3253 			ifp->if_capenable ^= IFCAP_TSO6;
3254 
3255 		if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO |
3256 		    IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) {
3257 			/* Changing these features requires us to reinit. */
3258 			reinit = 1;
3259 
3260 			if (mask & IFCAP_RXCSUM)
3261 				ifp->if_capenable ^= IFCAP_RXCSUM;
3262 			if (mask & IFCAP_RXCSUM_IPV6)
3263 				ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3264 			if (mask & IFCAP_LRO)
3265 				ifp->if_capenable ^= IFCAP_LRO;
3266 			if (mask & IFCAP_VLAN_HWTAGGING)
3267 				ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3268 			if (mask & IFCAP_VLAN_HWFILTER)
3269 				ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3270 		} else
3271 			reinit = 0;
3272 
3273 		if (mask & IFCAP_VLAN_HWTSO)
3274 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3275 
3276 		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3277 			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3278 			vmxnet3_init_locked(sc);
3279 		}
3280 
3281 		VMXNET3_CORE_UNLOCK(sc);
3282 		VLAN_CAPABILITIES(ifp);
3283 		break;
3284 
3285 	default:
3286 		error = ether_ioctl(ifp, cmd, data);
3287 		break;
3288 	}
3289 
3290 	VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc);
3291 
3292 	return (error);
3293 }
3294 
3295 #ifndef VMXNET3_LEGACY_TX
3296 static void
3297 vmxnet3_qflush(struct ifnet *ifp)
3298 {
3299 	struct vmxnet3_softc *sc;
3300 	struct vmxnet3_txqueue *txq;
3301 	struct mbuf *m;
3302 	int i;
3303 
3304 	sc = ifp->if_softc;
3305 
3306 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3307 		txq = &sc->vmx_txq[i];
3308 
3309 		VMXNET3_TXQ_LOCK(txq);
3310 		while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL)
3311 			m_freem(m);
3312 		VMXNET3_TXQ_UNLOCK(txq);
3313 	}
3314 
3315 	if_qflush(ifp);
3316 }
3317 #endif
3318 
3319 static int
3320 vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
3321 {
3322 	struct vmxnet3_softc *sc;
3323 
3324 	sc = txq->vxtxq_sc;
3325 
3326 	VMXNET3_TXQ_LOCK(txq);
3327 	if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
3328 		VMXNET3_TXQ_UNLOCK(txq);
3329 		return (0);
3330 	}
3331 	VMXNET3_TXQ_UNLOCK(txq);
3332 
3333 	if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n",
3334 	    txq->vxtxq_id);
3335 	return (1);
3336 }
3337 
3338 static void
3339 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
3340 {
3341 
3342 	vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
3343 }
3344 
3345 static void
3346 vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *txq,
3347     struct vmxnet3_txq_stats *accum)
3348 {
3349 	struct vmxnet3_txq_stats *st;
3350 
3351 	st = &txq->vxtxq_stats;
3352 
3353 	accum->vmtxs_opackets += st->vmtxs_opackets;
3354 	accum->vmtxs_obytes += st->vmtxs_obytes;
3355 	accum->vmtxs_omcasts += st->vmtxs_omcasts;
3356 	accum->vmtxs_csum += st->vmtxs_csum;
3357 	accum->vmtxs_tso += st->vmtxs_tso;
3358 	accum->vmtxs_full += st->vmtxs_full;
3359 	accum->vmtxs_offload_failed += st->vmtxs_offload_failed;
3360 }
3361 
3362 static void
3363 vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *rxq,
3364     struct vmxnet3_rxq_stats *accum)
3365 {
3366 	struct vmxnet3_rxq_stats *st;
3367 
3368 	st = &rxq->vxrxq_stats;
3369 
3370 	accum->vmrxs_ipackets += st->vmrxs_ipackets;
3371 	accum->vmrxs_ibytes += st->vmrxs_ibytes;
3372 	accum->vmrxs_iqdrops += st->vmrxs_iqdrops;
3373 	accum->vmrxs_ierrors += st->vmrxs_ierrors;
3374 }
3375 
3376 static void
3377 vmxnet3_accumulate_stats(struct vmxnet3_softc *sc)
3378 {
3379 	struct ifnet *ifp;
3380 	struct vmxnet3_statistics *st;
3381 	struct vmxnet3_txq_stats txaccum;
3382 	struct vmxnet3_rxq_stats rxaccum;
3383 	int i;
3384 
3385 	ifp = sc->vmx_ifp;
3386 	st = &sc->vmx_stats;
3387 
3388 	bzero(&txaccum, sizeof(struct vmxnet3_txq_stats));
3389 	bzero(&rxaccum, sizeof(struct vmxnet3_rxq_stats));
3390 
3391 	for (i = 0; i < sc->vmx_ntxqueues; i++)
3392 		vmxnet3_txq_accum_stats(&sc->vmx_txq[i], &txaccum);
3393 	for (i = 0; i < sc->vmx_nrxqueues; i++)
3394 		vmxnet3_rxq_accum_stats(&sc->vmx_rxq[i], &rxaccum);
3395 
3396 	/*
3397 	 * With the exception of if_ierrors, these ifnet statistics are
3398 	 * only updated in the driver, so just set them to our accumulated
3399 	 * values. if_ierrors is updated in ether_input() for malformed
3400 	 * frames that we should have already discarded.
3401 	 */
3402 	ifp->if_ipackets = rxaccum.vmrxs_ipackets;
3403 	ifp->if_iqdrops = rxaccum.vmrxs_iqdrops;
3404 	ifp->if_ierrors = rxaccum.vmrxs_ierrors;
3405 	ifp->if_opackets = txaccum.vmtxs_opackets;
3406 #ifndef VMXNET3_LEGACY_TX
3407 	ifp->if_obytes = txaccum.vmtxs_obytes;
3408 	ifp->if_omcasts = txaccum.vmtxs_omcasts;
3409 #endif
3410 }
3411 
3412 static void
3413 vmxnet3_tick(void *xsc)
3414 {
3415 	struct vmxnet3_softc *sc;
3416 	struct ifnet *ifp;
3417 	int i, timedout;
3418 
3419 	sc = xsc;
3420 	ifp = sc->vmx_ifp;
3421 	timedout = 0;
3422 
3423 	VMXNET3_CORE_LOCK_ASSERT(sc);
3424 
3425 	vmxnet3_accumulate_stats(sc);
3426 	vmxnet3_refresh_host_stats(sc);
3427 
3428 	for (i = 0; i < sc->vmx_ntxqueues; i++)
3429 		timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]);
3430 
3431 	if (timedout != 0) {
3432 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3433 		vmxnet3_init_locked(sc);
3434 	} else
3435 		callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
3436 }
3437 
3438 static int
3439 vmxnet3_link_is_up(struct vmxnet3_softc *sc)
3440 {
3441 	uint32_t status;
3442 
3443 	/* Also update the link speed while here. */
3444 	status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
3445 	sc->vmx_link_speed = status >> 16;
3446 	return !!(status & 0x1);
3447 }
3448 
3449 static void
3450 vmxnet3_link_status(struct vmxnet3_softc *sc)
3451 {
3452 	struct ifnet *ifp;
3453 	int link;
3454 
3455 	ifp = sc->vmx_ifp;
3456 	link = vmxnet3_link_is_up(sc);
3457 
3458 	if (link != 0 && sc->vmx_link_active == 0) {
3459 		sc->vmx_link_active = 1;
3460 		if_link_state_change(ifp, LINK_STATE_UP);
3461 	} else if (link == 0 && sc->vmx_link_active != 0) {
3462 		sc->vmx_link_active = 0;
3463 		if_link_state_change(ifp, LINK_STATE_DOWN);
3464 	}
3465 }
3466 
3467 static void
3468 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
3469 {
3470 	struct vmxnet3_softc *sc;
3471 
3472 	sc = ifp->if_softc;
3473 
3474 	ifmr->ifm_active = IFM_ETHER | IFM_AUTO;
3475 	ifmr->ifm_status = IFM_AVALID;
3476 
3477 	VMXNET3_CORE_LOCK(sc);
3478 	if (vmxnet3_link_is_up(sc) != 0)
3479 		ifmr->ifm_status |= IFM_ACTIVE;
3480 	else
3481 		ifmr->ifm_status |= IFM_NONE;
3482 	VMXNET3_CORE_UNLOCK(sc);
3483 }
3484 
3485 static int
3486 vmxnet3_media_change(struct ifnet *ifp)
3487 {
3488 
3489 	/* Ignore. */
3490 	return (0);
3491 }
3492 
3493 static void
3494 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
3495 {
3496 	uint32_t ml, mh;
3497 
3498 	ml  = sc->vmx_lladdr[0];
3499 	ml |= sc->vmx_lladdr[1] << 8;
3500 	ml |= sc->vmx_lladdr[2] << 16;
3501 	ml |= sc->vmx_lladdr[3] << 24;
3502 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
3503 
3504 	mh  = sc->vmx_lladdr[4];
3505 	mh |= sc->vmx_lladdr[5] << 8;
3506 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
3507 }
3508 
3509 static void
3510 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
3511 {
3512 	uint32_t ml, mh;
3513 
3514 	ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
3515 	mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
3516 
3517 	sc->vmx_lladdr[0] = ml;
3518 	sc->vmx_lladdr[1] = ml >> 8;
3519 	sc->vmx_lladdr[2] = ml >> 16;
3520 	sc->vmx_lladdr[3] = ml >> 24;
3521 	sc->vmx_lladdr[4] = mh;
3522 	sc->vmx_lladdr[5] = mh >> 8;
3523 }
3524 
3525 static void
3526 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq,
3527     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3528 {
3529 	struct sysctl_oid *node, *txsnode;
3530 	struct sysctl_oid_list *list, *txslist;
3531 	struct vmxnet3_txq_stats *stats;
3532 	struct UPT1_TxStats *txstats;
3533 	char namebuf[16];
3534 
3535 	stats = &txq->vxtxq_stats;
3536 	txstats = &txq->vxtxq_ts->stats;
3537 
3538 	snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id);
3539 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3540 	    NULL, "Transmit Queue");
3541 	txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node);
3542 
3543 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD,
3544 	    &stats->vmtxs_opackets, "Transmit packets");
3545 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD,
3546 	    &stats->vmtxs_obytes, "Transmit bytes");
3547 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD,
3548 	    &stats->vmtxs_omcasts, "Transmit multicasts");
3549 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
3550 	    &stats->vmtxs_csum, "Transmit checksum offloaded");
3551 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
3552 	    &stats->vmtxs_tso, "Transmit TCP segmentation offloaded");
3553 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD,
3554 	    &stats->vmtxs_full, "Transmit ring full");
3555 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD,
3556 	    &stats->vmtxs_offload_failed, "Transmit checksum offload failed");
3557 
3558 	/*
3559 	 * Add statistics reported by the host. These are updated once
3560 	 * per second.
3561 	 */
3562 	txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3563 	    NULL, "Host Statistics");
3564 	txslist = SYSCTL_CHILDREN(txsnode);
3565 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD,
3566 	    &txstats->TSO_packets, "TSO packets");
3567 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD,
3568 	    &txstats->TSO_bytes, "TSO bytes");
3569 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3570 	    &txstats->ucast_packets, "Unicast packets");
3571 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3572 	    &txstats->ucast_bytes, "Unicast bytes");
3573 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3574 	    &txstats->mcast_packets, "Multicast packets");
3575 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3576 	    &txstats->mcast_bytes, "Multicast bytes");
3577 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD,
3578 	    &txstats->error, "Errors");
3579 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD,
3580 	    &txstats->discard, "Discards");
3581 }
3582 
3583 static void
3584 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq,
3585     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3586 {
3587 	struct sysctl_oid *node, *rxsnode;
3588 	struct sysctl_oid_list *list, *rxslist;
3589 	struct vmxnet3_rxq_stats *stats;
3590 	struct UPT1_RxStats *rxstats;
3591 	char namebuf[16];
3592 
3593 	stats = &rxq->vxrxq_stats;
3594 	rxstats = &rxq->vxrxq_rs->stats;
3595 
3596 	snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id);
3597 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3598 	    NULL, "Receive Queue");
3599 	rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node);
3600 
3601 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD,
3602 	    &stats->vmrxs_ipackets, "Receive packets");
3603 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD,
3604 	    &stats->vmrxs_ibytes, "Receive bytes");
3605 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD,
3606 	    &stats->vmrxs_iqdrops, "Receive drops");
3607 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD,
3608 	    &stats->vmrxs_ierrors, "Receive errors");
3609 
3610 	/*
3611 	 * Add statistics reported by the host. These are updated once
3612 	 * per second.
3613 	 */
3614 	rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3615 	    NULL, "Host Statistics");
3616 	rxslist = SYSCTL_CHILDREN(rxsnode);
3617 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD,
3618 	    &rxstats->LRO_packets, "LRO packets");
3619 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD,
3620 	    &rxstats->LRO_bytes, "LRO bytes");
3621 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3622 	    &rxstats->ucast_packets, "Unicast packets");
3623 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3624 	    &rxstats->ucast_bytes, "Unicast bytes");
3625 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3626 	    &rxstats->mcast_packets, "Multicast packets");
3627 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3628 	    &rxstats->mcast_bytes, "Multicast bytes");
3629 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD,
3630 	    &rxstats->bcast_packets, "Broadcast packets");
3631 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD,
3632 	    &rxstats->bcast_bytes, "Broadcast bytes");
3633 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD,
3634 	    &rxstats->nobuffer, "No buffer");
3635 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD,
3636 	    &rxstats->error, "Errors");
3637 }
3638 
3639 static void
3640 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc,
3641     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3642 {
3643 	struct sysctl_oid *node;
3644 	struct sysctl_oid_list *list;
3645 	int i;
3646 
3647 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3648 		struct vmxnet3_txqueue *txq = &sc->vmx_txq[i];
3649 
3650 		node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO,
3651 		    "debug", CTLFLAG_RD, NULL, "");
3652 		list = SYSCTL_CHILDREN(node);
3653 
3654 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD,
3655 		    &txq->vxtxq_cmd_ring.vxtxr_head, 0, "");
3656 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD,
3657 		    &txq->vxtxq_cmd_ring.vxtxr_next, 0, "");
3658 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD,
3659 		    &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, "");
3660 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD,
3661 		    &txq->vxtxq_cmd_ring.vxtxr_gen, 0, "");
3662 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3663 		    &txq->vxtxq_comp_ring.vxcr_next, 0, "");
3664 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3665 		    &txq->vxtxq_comp_ring.vxcr_ndesc, 0,"");
3666 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3667 		    &txq->vxtxq_comp_ring.vxcr_gen, 0, "");
3668 	}
3669 
3670 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
3671 		struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i];
3672 
3673 		node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
3674 		    "debug", CTLFLAG_RD, NULL, "");
3675 		list = SYSCTL_CHILDREN(node);
3676 
3677 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD,
3678 		    &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, "");
3679 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
3680 		    &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
3681 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
3682 		    &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
3683 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD,
3684 		    &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, "");
3685 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
3686 		    &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
3687 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
3688 		    &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
3689 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3690 		    &rxq->vxrxq_comp_ring.vxcr_next, 0, "");
3691 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3692 		    &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
3693 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3694 		    &rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
3695 	}
3696 }
3697 
3698 static void
3699 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
3700     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3701 {
3702 	int i;
3703 
3704 	for (i = 0; i < sc->vmx_ntxqueues; i++)
3705 		vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child);
3706 	for (i = 0; i < sc->vmx_nrxqueues; i++)
3707 		vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child);
3708 
3709 	vmxnet3_setup_debug_sysctl(sc, ctx, child);
3710 }
3711 
3712 static void
3713 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
3714 {
3715 	device_t dev;
3716 	struct vmxnet3_statistics *stats;
3717 	struct sysctl_ctx_list *ctx;
3718 	struct sysctl_oid *tree;
3719 	struct sysctl_oid_list *child;
3720 
3721 	dev = sc->vmx_dev;
3722 	ctx = device_get_sysctl_ctx(dev);
3723 	tree = device_get_sysctl_tree(dev);
3724 	child = SYSCTL_CHILDREN(tree);
3725 
3726 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD,
3727 	    &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues");
3728 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD,
3729 	    &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues");
3730 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD,
3731 	    &sc->vmx_ntxqueues, 0, "Number of Tx queues");
3732 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD,
3733 	    &sc->vmx_nrxqueues, 0, "Number of Rx queues");
3734 
3735 	stats = &sc->vmx_stats;
3736 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD,
3737 	    &stats->vmst_defragged, 0, "Tx mbuf chains defragged");
3738 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD,
3739 	    &stats->vmst_defrag_failed, 0,
3740 	    "Tx mbuf dropped because defrag failed");
3741 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD,
3742 	    &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed");
3743 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD,
3744 	    &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed");
3745 
3746 	vmxnet3_setup_queue_sysctl(sc, ctx, child);
3747 }
3748 
3749 static void
3750 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3751 {
3752 
3753 	bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
3754 }
3755 
3756 static uint32_t
3757 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
3758 {
3759 
3760 	return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
3761 }
3762 
3763 static void
3764 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3765 {
3766 
3767 	bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
3768 }
3769 
3770 static void
3771 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3772 {
3773 
3774 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
3775 }
3776 
3777 static uint32_t
3778 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3779 {
3780 
3781 	vmxnet3_write_cmd(sc, cmd);
3782 	bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0,
3783 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
3784 	return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
3785 }
3786 
3787 static void
3788 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
3789 {
3790 
3791 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
3792 }
3793 
3794 static void
3795 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
3796 {
3797 
3798 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
3799 }
3800 
3801 static void
3802 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
3803 {
3804 	int i;
3805 
3806 	sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
3807 	for (i = 0; i < sc->vmx_nintrs; i++)
3808 		vmxnet3_enable_intr(sc, i);
3809 }
3810 
3811 static void
3812 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
3813 {
3814 	int i;
3815 
3816 	sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
3817 	for (i = 0; i < sc->vmx_nintrs; i++)
3818 		vmxnet3_disable_intr(sc, i);
3819 }
3820 
3821 static void
3822 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3823 {
3824 	bus_addr_t *baddr = arg;
3825 
3826 	if (error == 0)
3827 		*baddr = segs->ds_addr;
3828 }
3829 
3830 static int
3831 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
3832     struct vmxnet3_dma_alloc *dma)
3833 {
3834 	device_t dev;
3835 	int error;
3836 
3837 	dev = sc->vmx_dev;
3838 	bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3839 
3840 	error = bus_dma_tag_create(bus_get_dma_tag(dev),
3841 	    align, 0,		/* alignment, bounds */
3842 	    BUS_SPACE_MAXADDR,	/* lowaddr */
3843 	    BUS_SPACE_MAXADDR,	/* highaddr */
3844 	    NULL, NULL,		/* filter, filterarg */
3845 	    size,		/* maxsize */
3846 	    1,			/* nsegments */
3847 	    size,		/* maxsegsize */
3848 	    BUS_DMA_ALLOCNOW,	/* flags */
3849 	    NULL,		/* lockfunc */
3850 	    NULL,		/* lockfuncarg */
3851 	    &dma->dma_tag);
3852 	if (error) {
3853 		device_printf(dev, "bus_dma_tag_create failed: %d\n", error);
3854 		goto fail;
3855 	}
3856 
3857 	error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
3858 	    BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
3859 	if (error) {
3860 		device_printf(dev, "bus_dmamem_alloc failed: %d\n", error);
3861 		goto fail;
3862 	}
3863 
3864 	error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3865 	    size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
3866 	if (error) {
3867 		device_printf(dev, "bus_dmamap_load failed: %d\n", error);
3868 		goto fail;
3869 	}
3870 
3871 	dma->dma_size = size;
3872 
3873 fail:
3874 	if (error)
3875 		vmxnet3_dma_free(sc, dma);
3876 
3877 	return (error);
3878 }
3879 
3880 static void
3881 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
3882 {
3883 
3884 	if (dma->dma_tag != NULL) {
3885 		if (dma->dma_paddr != 0) {
3886 			bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3887 			    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3888 			bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3889 		}
3890 
3891 		if (dma->dma_vaddr != NULL) {
3892 			bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
3893 			    dma->dma_map);
3894 		}
3895 
3896 		bus_dma_tag_destroy(dma->dma_tag);
3897 	}
3898 	bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3899 }
3900 
3901 static int
3902 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def)
3903 {
3904 	char path[64];
3905 
3906 	snprintf(path, sizeof(path),
3907 	    "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob);
3908 	TUNABLE_INT_FETCH(path, &def);
3909 
3910 	return (def);
3911 }
3912 
3913 /*
3914  * Since this is a purely paravirtualized device, we do not have
3915  * to worry about DMA coherency. But at times, we must make sure
3916  * both the compiler and CPU do not reorder memory operations.
3917  */
3918 static inline void
3919 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
3920 {
3921 
3922 	switch (type) {
3923 	case VMXNET3_BARRIER_RD:
3924 		rmb();
3925 		break;
3926 	case VMXNET3_BARRIER_WR:
3927 		wmb();
3928 		break;
3929 	case VMXNET3_BARRIER_RDWR:
3930 		mb();
3931 		break;
3932 	default:
3933 		panic("%s: bad barrier type %d", __func__, type);
3934 	}
3935 }
3936