xref: /freebsd/sys/dev/vmware/vmxnet3/if_vmx.c (revision a521f2116473fbd8c09db395518f060a27d02334)
1 /*-
2  * Copyright (c) 2013 Tsubai Masanari
3  * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
4  * Copyright (c) 2018 Patrick Kelsey
5  *
6  * Permission to use, copy, modify, and distribute this software for any
7  * purpose with or without fee is hereby granted, provided that the above
8  * copyright notice and this permission notice appear in all copies.
9  *
10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17  *
18  * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $
19  */
20 
21 /* Driver for VMware vmxnet3 virtual ethernet devices. */
22 
23 #include <sys/cdefs.h>
24 __FBSDID("$FreeBSD$");
25 
26 #include "opt_rss.h"
27 
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/kernel.h>
31 #include <sys/endian.h>
32 #include <sys/sockio.h>
33 #include <sys/mbuf.h>
34 #include <sys/malloc.h>
35 #include <sys/module.h>
36 #include <sys/socket.h>
37 #include <sys/sysctl.h>
38 #include <sys/smp.h>
39 #include <vm/vm.h>
40 #include <vm/pmap.h>
41 
42 #include <net/ethernet.h>
43 #include <net/if.h>
44 #include <net/if_var.h>
45 #include <net/if_arp.h>
46 #include <net/if_dl.h>
47 #include <net/if_types.h>
48 #include <net/if_media.h>
49 #include <net/if_vlan_var.h>
50 #include <net/iflib.h>
51 #ifdef RSS
52 #include <net/rss_config.h>
53 #endif
54 
55 #include <netinet/in_systm.h>
56 #include <netinet/in.h>
57 #include <netinet/ip.h>
58 #include <netinet/ip6.h>
59 #include <netinet6/ip6_var.h>
60 #include <netinet/udp.h>
61 #include <netinet/tcp.h>
62 
63 #include <machine/bus.h>
64 #include <machine/resource.h>
65 #include <sys/bus.h>
66 #include <sys/rman.h>
67 
68 #include <dev/pci/pcireg.h>
69 #include <dev/pci/pcivar.h>
70 
71 #include "ifdi_if.h"
72 
73 #include "if_vmxreg.h"
74 #include "if_vmxvar.h"
75 
76 #include "opt_inet.h"
77 #include "opt_inet6.h"
78 
79 #define VMXNET3_VMWARE_VENDOR_ID	0x15AD
80 #define VMXNET3_VMWARE_DEVICE_ID	0x07B0
81 
82 static pci_vendor_info_t vmxnet3_vendor_info_array[] =
83 {
84 	PVID(VMXNET3_VMWARE_VENDOR_ID, VMXNET3_VMWARE_DEVICE_ID, "VMware VMXNET3 Ethernet Adapter"),
85 	/* required last entry */
86 	PVID_END
87 };
88 
89 static void	*vmxnet3_register(device_t);
90 static int	vmxnet3_attach_pre(if_ctx_t);
91 static int	vmxnet3_msix_intr_assign(if_ctx_t, int);
92 static void	vmxnet3_free_irqs(struct vmxnet3_softc *);
93 static int	vmxnet3_attach_post(if_ctx_t);
94 static int	vmxnet3_detach(if_ctx_t);
95 static int	vmxnet3_shutdown(if_ctx_t);
96 static int	vmxnet3_suspend(if_ctx_t);
97 static int	vmxnet3_resume(if_ctx_t);
98 
99 static int	vmxnet3_alloc_resources(struct vmxnet3_softc *);
100 static void	vmxnet3_free_resources(struct vmxnet3_softc *);
101 static int	vmxnet3_check_version(struct vmxnet3_softc *);
102 static void	vmxnet3_set_interrupt_idx(struct vmxnet3_softc *);
103 
104 static int	vmxnet3_queues_shared_alloc(struct vmxnet3_softc *);
105 static void	vmxnet3_init_txq(struct vmxnet3_softc *, int);
106 static int	vmxnet3_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
107 static void	vmxnet3_init_rxq(struct vmxnet3_softc *, int, int);
108 static int	vmxnet3_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
109 static void	vmxnet3_queues_free(if_ctx_t);
110 
111 static int	vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
112 static void	vmxnet3_free_shared_data(struct vmxnet3_softc *);
113 static int	vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
114 static void	vmxnet3_free_mcast_table(struct vmxnet3_softc *);
115 static void	vmxnet3_init_shared_data(struct vmxnet3_softc *);
116 static void	vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
117 static void	vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
118 static int	vmxnet3_alloc_data(struct vmxnet3_softc *);
119 static void	vmxnet3_free_data(struct vmxnet3_softc *);
120 
121 static void	vmxnet3_evintr(struct vmxnet3_softc *);
122 static int	vmxnet3_isc_txd_encap(void *, if_pkt_info_t);
123 static void	vmxnet3_isc_txd_flush(void *, uint16_t, qidx_t);
124 static int	vmxnet3_isc_txd_credits_update(void *, uint16_t, bool);
125 static int	vmxnet3_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
126 static int	vmxnet3_isc_rxd_pkt_get(void *, if_rxd_info_t);
127 static void	vmxnet3_isc_rxd_refill(void *, if_rxd_update_t);
128 static void	vmxnet3_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t);
129 static int	vmxnet3_legacy_intr(void *);
130 static int	vmxnet3_rxq_intr(void *);
131 static int	vmxnet3_event_intr(void *);
132 
133 static void	vmxnet3_stop(if_ctx_t);
134 
135 static void	vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
136 static void	vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
137 static void	vmxnet3_reinit_queues(struct vmxnet3_softc *);
138 static int	vmxnet3_enable_device(struct vmxnet3_softc *);
139 static void	vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
140 static void	vmxnet3_init(if_ctx_t);
141 static void	vmxnet3_multi_set(if_ctx_t);
142 static int	vmxnet3_mtu_set(if_ctx_t, uint32_t);
143 static void	vmxnet3_media_status(if_ctx_t, struct ifmediareq *);
144 static int	vmxnet3_media_change(if_ctx_t);
145 static int	vmxnet3_promisc_set(if_ctx_t, int);
146 static uint64_t	vmxnet3_get_counter(if_ctx_t, ift_counter);
147 static void	vmxnet3_update_admin_status(if_ctx_t);
148 static void	vmxnet3_txq_timer(if_ctx_t, uint16_t);
149 
150 static void	vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
151 		    uint16_t);
152 static void	vmxnet3_vlan_register(if_ctx_t, uint16_t);
153 static void	vmxnet3_vlan_unregister(if_ctx_t, uint16_t);
154 static void	vmxnet3_set_rxfilter(struct vmxnet3_softc *, int);
155 
156 static void	vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
157 static int	vmxnet3_link_is_up(struct vmxnet3_softc *);
158 static void	vmxnet3_link_status(struct vmxnet3_softc *);
159 static void	vmxnet3_set_lladdr(struct vmxnet3_softc *);
160 static void	vmxnet3_get_lladdr(struct vmxnet3_softc *);
161 
162 static void	vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *,
163 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
164 static void	vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *,
165 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
166 static void	vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *,
167 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
168 static void	vmxnet3_setup_sysctl(struct vmxnet3_softc *);
169 
170 static void	vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t,
171 		    uint32_t);
172 static uint32_t	vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t);
173 static void	vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t,
174 		    uint32_t);
175 static void	vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t);
176 static uint32_t	vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t);
177 
178 static int	vmxnet3_tx_queue_intr_enable(if_ctx_t, uint16_t);
179 static int	vmxnet3_rx_queue_intr_enable(if_ctx_t, uint16_t);
180 static void	vmxnet3_link_intr_enable(if_ctx_t);
181 static void	vmxnet3_enable_intr(struct vmxnet3_softc *, int);
182 static void	vmxnet3_disable_intr(struct vmxnet3_softc *, int);
183 static void	vmxnet3_intr_enable_all(if_ctx_t);
184 static void	vmxnet3_intr_disable_all(if_ctx_t);
185 
186 typedef enum {
187 	VMXNET3_BARRIER_RD,
188 	VMXNET3_BARRIER_WR,
189 	VMXNET3_BARRIER_RDWR,
190 } vmxnet3_barrier_t;
191 
192 static void	vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
193 
194 static device_method_t vmxnet3_methods[] = {
195 	/* Device interface */
196 	DEVMETHOD(device_register, vmxnet3_register),
197 	DEVMETHOD(device_probe, iflib_device_probe),
198 	DEVMETHOD(device_attach, iflib_device_attach),
199 	DEVMETHOD(device_detach, iflib_device_detach),
200 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
201 	DEVMETHOD(device_suspend, iflib_device_suspend),
202 	DEVMETHOD(device_resume, iflib_device_resume),
203 	DEVMETHOD_END
204 };
205 
206 static driver_t vmxnet3_driver = {
207 	"vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc)
208 };
209 
210 static devclass_t vmxnet3_devclass;
211 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0);
212 IFLIB_PNP_INFO(pci, vmx, vmxnet3_vendor_info_array);
213 MODULE_VERSION(vmx, 2);
214 
215 MODULE_DEPEND(vmx, pci, 1, 1, 1);
216 MODULE_DEPEND(vmx, ether, 1, 1, 1);
217 MODULE_DEPEND(vmx, iflib, 1, 1, 1);
218 
219 static device_method_t vmxnet3_iflib_methods[] = {
220 	DEVMETHOD(ifdi_tx_queues_alloc, vmxnet3_tx_queues_alloc),
221 	DEVMETHOD(ifdi_rx_queues_alloc, vmxnet3_rx_queues_alloc),
222 	DEVMETHOD(ifdi_queues_free, vmxnet3_queues_free),
223 
224 	DEVMETHOD(ifdi_attach_pre, vmxnet3_attach_pre),
225 	DEVMETHOD(ifdi_attach_post, vmxnet3_attach_post),
226 	DEVMETHOD(ifdi_detach, vmxnet3_detach),
227 
228 	DEVMETHOD(ifdi_init, vmxnet3_init),
229 	DEVMETHOD(ifdi_stop, vmxnet3_stop),
230 	DEVMETHOD(ifdi_multi_set, vmxnet3_multi_set),
231 	DEVMETHOD(ifdi_mtu_set, vmxnet3_mtu_set),
232 	DEVMETHOD(ifdi_media_status, vmxnet3_media_status),
233 	DEVMETHOD(ifdi_media_change, vmxnet3_media_change),
234 	DEVMETHOD(ifdi_promisc_set, vmxnet3_promisc_set),
235 	DEVMETHOD(ifdi_get_counter, vmxnet3_get_counter),
236 	DEVMETHOD(ifdi_update_admin_status, vmxnet3_update_admin_status),
237 	DEVMETHOD(ifdi_timer, vmxnet3_txq_timer),
238 
239 	DEVMETHOD(ifdi_tx_queue_intr_enable, vmxnet3_tx_queue_intr_enable),
240 	DEVMETHOD(ifdi_rx_queue_intr_enable, vmxnet3_rx_queue_intr_enable),
241 	DEVMETHOD(ifdi_link_intr_enable, vmxnet3_link_intr_enable),
242 	DEVMETHOD(ifdi_intr_enable, vmxnet3_intr_enable_all),
243 	DEVMETHOD(ifdi_intr_disable, vmxnet3_intr_disable_all),
244 	DEVMETHOD(ifdi_msix_intr_assign, vmxnet3_msix_intr_assign),
245 
246 	DEVMETHOD(ifdi_vlan_register, vmxnet3_vlan_register),
247 	DEVMETHOD(ifdi_vlan_unregister, vmxnet3_vlan_unregister),
248 
249 	DEVMETHOD(ifdi_shutdown, vmxnet3_shutdown),
250 	DEVMETHOD(ifdi_suspend, vmxnet3_suspend),
251 	DEVMETHOD(ifdi_resume, vmxnet3_resume),
252 
253 	DEVMETHOD_END
254 };
255 
256 static driver_t vmxnet3_iflib_driver = {
257 	"vmx", vmxnet3_iflib_methods, sizeof(struct vmxnet3_softc)
258 };
259 
260 struct if_txrx vmxnet3_txrx = {
261 	.ift_txd_encap = vmxnet3_isc_txd_encap,
262 	.ift_txd_flush = vmxnet3_isc_txd_flush,
263 	.ift_txd_credits_update = vmxnet3_isc_txd_credits_update,
264 	.ift_rxd_available = vmxnet3_isc_rxd_available,
265 	.ift_rxd_pkt_get = vmxnet3_isc_rxd_pkt_get,
266 	.ift_rxd_refill = vmxnet3_isc_rxd_refill,
267 	.ift_rxd_flush = vmxnet3_isc_rxd_flush,
268 	.ift_legacy_intr = vmxnet3_legacy_intr
269 };
270 
271 static struct if_shared_ctx vmxnet3_sctx_init = {
272 	.isc_magic = IFLIB_MAGIC,
273 	.isc_q_align = 512,
274 
275 	.isc_tx_maxsize = VMXNET3_TX_MAXSIZE,
276 	.isc_tx_maxsegsize = VMXNET3_TX_MAXSEGSIZE,
277 	.isc_tso_maxsize = VMXNET3_TSO_MAXSIZE + sizeof(struct ether_vlan_header),
278 	.isc_tso_maxsegsize = VMXNET3_TX_MAXSEGSIZE,
279 
280 	/*
281 	 * These values are used to configure the busdma tag used for
282 	 * receive descriptors.  Each receive descriptor only points to one
283 	 * buffer.
284 	 */
285 	.isc_rx_maxsize = VMXNET3_RX_MAXSEGSIZE, /* One buf per descriptor */
286 	.isc_rx_nsegments = 1,  /* One mapping per descriptor */
287 	.isc_rx_maxsegsize = VMXNET3_RX_MAXSEGSIZE,
288 
289 	.isc_admin_intrcnt = 1,
290 	.isc_vendor_info = vmxnet3_vendor_info_array,
291 	.isc_driver_version = "2",
292 	.isc_driver = &vmxnet3_iflib_driver,
293 	.isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_SINGLE_IRQ_RX_ONLY,
294 
295 	/*
296 	 * Number of receive queues per receive queue set, with associated
297 	 * descriptor settings for each.
298 	 */
299 	.isc_nrxqs = 3,
300 	.isc_nfl = 2, /* one free list for each receive command queue */
301 	.isc_nrxd_min = {VMXNET3_MIN_RX_NDESC, VMXNET3_MIN_RX_NDESC, VMXNET3_MIN_RX_NDESC},
302 	.isc_nrxd_max = {VMXNET3_MAX_RX_NDESC, VMXNET3_MAX_RX_NDESC, VMXNET3_MAX_RX_NDESC},
303 	.isc_nrxd_default = {VMXNET3_DEF_RX_NDESC, VMXNET3_DEF_RX_NDESC, VMXNET3_DEF_RX_NDESC},
304 
305 	/*
306 	 * Number of transmit queues per transmit queue set, with associated
307 	 * descriptor settings for each.
308 	 */
309 	.isc_ntxqs = 2,
310 	.isc_ntxd_min = {VMXNET3_MIN_TX_NDESC, VMXNET3_MIN_TX_NDESC},
311 	.isc_ntxd_max = {VMXNET3_MAX_TX_NDESC, VMXNET3_MAX_TX_NDESC},
312 	.isc_ntxd_default = {VMXNET3_DEF_TX_NDESC, VMXNET3_DEF_TX_NDESC},
313 };
314 
315 static void *
316 vmxnet3_register(device_t dev)
317 {
318 	return (&vmxnet3_sctx_init);
319 }
320 
321 static int
322 trunc_powerof2(int val)
323 {
324 
325 	return (1U << (fls(val) - 1));
326 }
327 
328 static int
329 vmxnet3_attach_pre(if_ctx_t ctx)
330 {
331 	device_t dev;
332 	if_softc_ctx_t scctx;
333 	struct vmxnet3_softc *sc;
334 	uint32_t intr_config;
335 	int error;
336 
337 	dev = iflib_get_dev(ctx);
338 	sc = iflib_get_softc(ctx);
339 	sc->vmx_dev = dev;
340 	sc->vmx_ctx = ctx;
341 	sc->vmx_sctx = iflib_get_sctx(ctx);
342 	sc->vmx_scctx = iflib_get_softc_ctx(ctx);
343 	sc->vmx_ifp = iflib_get_ifp(ctx);
344 	sc->vmx_media = iflib_get_media(ctx);
345 	scctx = sc->vmx_scctx;
346 
347 	scctx->isc_tx_nsegments = VMXNET3_TX_MAXSEGS;
348 	scctx->isc_tx_tso_segments_max = VMXNET3_TX_MAXSEGS;
349 	/* isc_tx_tso_size_max doesn't include possible vlan header */
350 	scctx->isc_tx_tso_size_max = VMXNET3_TSO_MAXSIZE;
351 	scctx->isc_tx_tso_segsize_max = VMXNET3_TX_MAXSEGSIZE;
352 	scctx->isc_txrx = &vmxnet3_txrx;
353 
354 	/* If 0, the iflib tunable was not set, so set to the default */
355 	if (scctx->isc_nrxqsets == 0)
356 		scctx->isc_nrxqsets = VMXNET3_DEF_RX_QUEUES;
357 	scctx->isc_nrxqsets = trunc_powerof2(scctx->isc_nrxqsets);
358 	scctx->isc_nrxqsets_max = min(VMXNET3_MAX_RX_QUEUES, mp_ncpus);
359 	scctx->isc_nrxqsets_max = trunc_powerof2(scctx->isc_nrxqsets_max);
360 
361 	/* If 0, the iflib tunable was not set, so set to the default */
362 	if (scctx->isc_ntxqsets == 0)
363 		scctx->isc_ntxqsets = VMXNET3_DEF_TX_QUEUES;
364 	scctx->isc_ntxqsets = trunc_powerof2(scctx->isc_ntxqsets);
365 	scctx->isc_ntxqsets_max = min(VMXNET3_MAX_TX_QUEUES, mp_ncpus);
366 	scctx->isc_ntxqsets_max = trunc_powerof2(scctx->isc_ntxqsets_max);
367 
368 	/*
369 	 * Enforce that the transmit completion queue descriptor count is
370 	 * the same as the transmit command queue descriptor count.
371 	 */
372 	scctx->isc_ntxd[0] = scctx->isc_ntxd[1];
373 	scctx->isc_txqsizes[0] =
374 	    sizeof(struct vmxnet3_txcompdesc) * scctx->isc_ntxd[0];
375 	scctx->isc_txqsizes[1] =
376 	    sizeof(struct vmxnet3_txdesc) * scctx->isc_ntxd[1];
377 
378 	/*
379 	 * Enforce that the receive completion queue descriptor count is the
380 	 * sum of the receive command queue descriptor counts, and that the
381 	 * second receive command queue descriptor count is the same as the
382 	 * first one.
383 	 */
384 	scctx->isc_nrxd[2] = scctx->isc_nrxd[1];
385 	scctx->isc_nrxd[0] = scctx->isc_nrxd[1] + scctx->isc_nrxd[2];
386 	scctx->isc_rxqsizes[0] =
387 	    sizeof(struct vmxnet3_rxcompdesc) * scctx->isc_nrxd[0];
388 	scctx->isc_rxqsizes[1] =
389 	    sizeof(struct vmxnet3_rxdesc) * scctx->isc_nrxd[1];
390 	scctx->isc_rxqsizes[2] =
391 	    sizeof(struct vmxnet3_rxdesc) * scctx->isc_nrxd[2];
392 
393 	/*
394 	 * Initialize the max frame size and descriptor queue buffer
395 	 * sizes.
396 	 */
397 	vmxnet3_mtu_set(ctx, if_getmtu(sc->vmx_ifp));
398 
399 	scctx->isc_rss_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
400 
401 	/* Map PCI BARs */
402 	error = vmxnet3_alloc_resources(sc);
403 	if (error)
404 		goto fail;
405 
406 	/* Check device versions */
407 	error = vmxnet3_check_version(sc);
408 	if (error)
409 		goto fail;
410 
411 	/*
412 	 * The interrupt mode can be set in the hypervisor configuration via
413 	 * the parameter ethernet<N>.intrMode.
414 	 */
415 	intr_config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
416 	sc->vmx_intr_mask_mode = (intr_config >> 2) & 0x03;
417 
418 	/*
419 	 * Configure the softc context to attempt to configure the interrupt
420 	 * mode now indicated by intr_config.  iflib will follow the usual
421 	 * fallback path MSI-X -> MSI -> LEGACY, starting at the configured
422 	 * starting mode.
423 	 */
424 	switch (intr_config & 0x03) {
425 	case VMXNET3_IT_AUTO:
426 	case VMXNET3_IT_MSIX:
427 		scctx->isc_msix_bar = pci_msix_table_bar(dev);
428 		break;
429 	case VMXNET3_IT_MSI:
430 		scctx->isc_msix_bar = -1;
431 		scctx->isc_disable_msix = 1;
432 		break;
433 	case VMXNET3_IT_LEGACY:
434 		scctx->isc_msix_bar = 0;
435 		break;
436 	}
437 
438 	scctx->isc_tx_csum_flags = VMXNET3_CSUM_ALL_OFFLOAD;
439 	scctx->isc_capabilities = scctx->isc_capenable =
440 	    IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 |
441 	    IFCAP_TSO4 | IFCAP_TSO6 |
442 	    IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 |
443 	    IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
444 	    IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO |
445 	    IFCAP_JUMBO_MTU;
446 
447 	/* These capabilities are not enabled by default. */
448 	scctx->isc_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER;
449 
450 	vmxnet3_get_lladdr(sc);
451 	iflib_set_mac(ctx, sc->vmx_lladdr);
452 
453 	return (0);
454 fail:
455 	/*
456 	 * We must completely clean up anything allocated above as iflib
457 	 * will not invoke any other driver entry points as a result of this
458 	 * failure.
459 	 */
460 	vmxnet3_free_resources(sc);
461 
462 	return (error);
463 }
464 
465 static int
466 vmxnet3_msix_intr_assign(if_ctx_t ctx, int msix)
467 {
468 	struct vmxnet3_softc *sc;
469 	if_softc_ctx_t scctx;
470 	struct vmxnet3_rxqueue *rxq;
471 	int error;
472 	int i;
473 	char irq_name[16];
474 
475 	sc = iflib_get_softc(ctx);
476 	scctx = sc->vmx_scctx;
477 
478 	for (i = 0; i < scctx->isc_nrxqsets; i++) {
479 		snprintf(irq_name, sizeof(irq_name), "rxq%d", i);
480 
481 		rxq = &sc->vmx_rxq[i];
482 		error = iflib_irq_alloc_generic(ctx, &rxq->vxrxq_irq, i + 1,
483 		    IFLIB_INTR_RX, vmxnet3_rxq_intr, rxq, i, irq_name);
484 		if (error) {
485 			device_printf(iflib_get_dev(ctx),
486 			    "Failed to register rxq %d interrupt handler\n", i);
487 			return (error);
488 		}
489 	}
490 
491 	for (i = 0; i < scctx->isc_ntxqsets; i++) {
492 		snprintf(irq_name, sizeof(irq_name), "txq%d", i);
493 
494 		/*
495 		 * Don't provide the corresponding rxq irq for reference -
496 		 * we want the transmit task to be attached to a task queue
497 		 * that is different from the one used by the corresponding
498 		 * rxq irq.  That is because the TX doorbell writes are very
499 		 * expensive as virtualized MMIO operations, so we want to
500 		 * be able to defer them to another core when possible so
501 		 * that they don't steal receive processing cycles during
502 		 * stack turnarounds like TCP ACK generation.  The other
503 		 * piece to this approach is enabling the iflib abdicate
504 		 * option (currently via an interface-specific
505 		 * tunable/sysctl).
506 		 */
507 		iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_TX, NULL, i,
508 		    irq_name);
509 	}
510 
511 	error = iflib_irq_alloc_generic(ctx, &sc->vmx_event_intr_irq,
512 	    scctx->isc_nrxqsets + 1, IFLIB_INTR_ADMIN, vmxnet3_event_intr, sc, 0,
513 	    "event");
514 	if (error) {
515 		device_printf(iflib_get_dev(ctx),
516 		    "Failed to register event interrupt handler\n");
517 		return (error);
518 	}
519 
520 	return (0);
521 }
522 
523 static void
524 vmxnet3_free_irqs(struct vmxnet3_softc *sc)
525 {
526 	if_softc_ctx_t scctx;
527 	struct vmxnet3_rxqueue *rxq;
528 	int i;
529 
530 	scctx = sc->vmx_scctx;
531 
532 	for (i = 0; i < scctx->isc_nrxqsets; i++) {
533 		rxq = &sc->vmx_rxq[i];
534 		iflib_irq_free(sc->vmx_ctx, &rxq->vxrxq_irq);
535 	}
536 
537 	iflib_irq_free(sc->vmx_ctx, &sc->vmx_event_intr_irq);
538 }
539 
540 static int
541 vmxnet3_attach_post(if_ctx_t ctx)
542 {
543 	device_t dev;
544 	if_softc_ctx_t scctx;
545 	struct vmxnet3_softc *sc;
546 	int error;
547 
548 	dev = iflib_get_dev(ctx);
549 	scctx = iflib_get_softc_ctx(ctx);
550 	sc = iflib_get_softc(ctx);
551 
552 	if (scctx->isc_nrxqsets > 1)
553 		sc->vmx_flags |= VMXNET3_FLAG_RSS;
554 
555 	error = vmxnet3_alloc_data(sc);
556 	if (error)
557 		goto fail;
558 
559 	vmxnet3_set_interrupt_idx(sc);
560 	vmxnet3_setup_sysctl(sc);
561 
562 	ifmedia_add(sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
563 	ifmedia_set(sc->vmx_media, IFM_ETHER | IFM_AUTO);
564 
565 fail:
566 	return (error);
567 }
568 
569 static int
570 vmxnet3_detach(if_ctx_t ctx)
571 {
572 	struct vmxnet3_softc *sc;
573 
574 	sc = iflib_get_softc(ctx);
575 
576 	vmxnet3_free_irqs(sc);
577 	vmxnet3_free_data(sc);
578 	vmxnet3_free_resources(sc);
579 
580 	return (0);
581 }
582 
583 static int
584 vmxnet3_shutdown(if_ctx_t ctx)
585 {
586 
587 	return (0);
588 }
589 
590 static int
591 vmxnet3_suspend(if_ctx_t ctx)
592 {
593 
594 	return (0);
595 }
596 
597 static int
598 vmxnet3_resume(if_ctx_t ctx)
599 {
600 
601 	return (0);
602 }
603 
604 static int
605 vmxnet3_alloc_resources(struct vmxnet3_softc *sc)
606 {
607 	device_t dev;
608 	int rid;
609 
610 	dev = sc->vmx_dev;
611 
612 	rid = PCIR_BAR(0);
613 	sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
614 	    RF_ACTIVE);
615 	if (sc->vmx_res0 == NULL) {
616 		device_printf(dev,
617 		    "could not map BAR0 memory\n");
618 		return (ENXIO);
619 	}
620 
621 	sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0);
622 	sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0);
623 
624 	rid = PCIR_BAR(1);
625 	sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
626 	    RF_ACTIVE);
627 	if (sc->vmx_res1 == NULL) {
628 		device_printf(dev,
629 		    "could not map BAR1 memory\n");
630 		return (ENXIO);
631 	}
632 
633 	sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1);
634 	sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1);
635 
636 	return (0);
637 }
638 
639 static void
640 vmxnet3_free_resources(struct vmxnet3_softc *sc)
641 {
642 	device_t dev;
643 
644 	dev = sc->vmx_dev;
645 
646 	if (sc->vmx_res0 != NULL) {
647 		bus_release_resource(dev, SYS_RES_MEMORY,
648 		    rman_get_rid(sc->vmx_res0), sc->vmx_res0);
649 		sc->vmx_res0 = NULL;
650 	}
651 
652 	if (sc->vmx_res1 != NULL) {
653 		bus_release_resource(dev, SYS_RES_MEMORY,
654 		    rman_get_rid(sc->vmx_res1), sc->vmx_res1);
655 		sc->vmx_res1 = NULL;
656 	}
657 }
658 
659 static int
660 vmxnet3_check_version(struct vmxnet3_softc *sc)
661 {
662 	device_t dev;
663 	uint32_t version;
664 
665 	dev = sc->vmx_dev;
666 
667 	version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
668 	if ((version & 0x01) == 0) {
669 		device_printf(dev, "unsupported hardware version %#x\n",
670 		    version);
671 		return (ENOTSUP);
672 	}
673 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
674 
675 	version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
676 	if ((version & 0x01) == 0) {
677 		device_printf(dev, "unsupported UPT version %#x\n", version);
678 		return (ENOTSUP);
679 	}
680 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
681 
682 	return (0);
683 }
684 
685 static void
686 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
687 {
688 	if_softc_ctx_t scctx;
689 	struct vmxnet3_txqueue *txq;
690 	struct vmxnet3_txq_shared *txs;
691 	struct vmxnet3_rxqueue *rxq;
692 	struct vmxnet3_rxq_shared *rxs;
693 	int intr_idx;
694 	int i;
695 
696 	scctx = sc->vmx_scctx;
697 
698 	/*
699 	 * There is always one interrupt per receive queue, assigned
700 	 * starting with the first interrupt.  When there is only one
701 	 * interrupt available, the event interrupt shares the receive queue
702 	 * interrupt, otherwise it uses the interrupt following the last
703 	 * receive queue interrupt.  Transmit queues are not assigned
704 	 * interrupts, so they are given indexes beyond the indexes that
705 	 * correspond to the real interrupts.
706 	 */
707 
708 	/* The event interrupt is always the last vector. */
709 	sc->vmx_event_intr_idx = scctx->isc_vectors - 1;
710 
711 	intr_idx = 0;
712 	for (i = 0; i < scctx->isc_nrxqsets; i++, intr_idx++) {
713 		rxq = &sc->vmx_rxq[i];
714 		rxs = rxq->vxrxq_rs;
715 		rxq->vxrxq_intr_idx = intr_idx;
716 		rxs->intr_idx = rxq->vxrxq_intr_idx;
717 	}
718 
719 	/*
720 	 * Assign the tx queues interrupt indexes above what we are actually
721 	 * using.  These interrupts will never be enabled.
722 	 */
723 	intr_idx = scctx->isc_vectors;
724 	for (i = 0; i < scctx->isc_ntxqsets; i++, intr_idx++) {
725 		txq = &sc->vmx_txq[i];
726 		txs = txq->vxtxq_ts;
727 		txq->vxtxq_intr_idx = intr_idx;
728 		txs->intr_idx = txq->vxtxq_intr_idx;
729 	}
730 }
731 
732 static int
733 vmxnet3_queues_shared_alloc(struct vmxnet3_softc *sc)
734 {
735 	if_softc_ctx_t scctx;
736 	int size;
737 	int error;
738 
739 	scctx = sc->vmx_scctx;
740 
741 	/*
742 	 * The txq and rxq shared data areas must be allocated contiguously
743 	 * as vmxnet3_driver_shared contains only a single address member
744 	 * for the shared queue data area.
745 	 */
746 	size = scctx->isc_ntxqsets * sizeof(struct vmxnet3_txq_shared) +
747 	    scctx->isc_nrxqsets * sizeof(struct vmxnet3_rxq_shared);
748 	error = iflib_dma_alloc_align(sc->vmx_ctx, size, 128, &sc->vmx_qs_dma, 0);
749 	if (error) {
750 		device_printf(sc->vmx_dev, "cannot alloc queue shared memory\n");
751 		return (error);
752 	}
753 
754 	return (0);
755 }
756 
757 static void
758 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
759 {
760 	struct vmxnet3_txqueue *txq;
761 	struct vmxnet3_comp_ring *txc;
762 	struct vmxnet3_txring *txr;
763 	if_softc_ctx_t scctx;
764 
765 	txq = &sc->vmx_txq[q];
766 	txc = &txq->vxtxq_comp_ring;
767 	txr = &txq->vxtxq_cmd_ring;
768 	scctx = sc->vmx_scctx;
769 
770 	snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
771 	    device_get_nameunit(sc->vmx_dev), q);
772 
773 	txq->vxtxq_sc = sc;
774 	txq->vxtxq_id = q;
775 	txc->vxcr_ndesc = scctx->isc_ntxd[0];
776 	txr->vxtxr_ndesc = scctx->isc_ntxd[1];
777 }
778 
779 static int
780 vmxnet3_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
781     int ntxqs, int ntxqsets)
782 {
783 	struct vmxnet3_softc *sc;
784 	int q;
785 	int error;
786 	caddr_t kva;
787 
788 	sc = iflib_get_softc(ctx);
789 
790 	/* Allocate the array of transmit queues */
791 	sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) *
792 	    ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
793 	if (sc->vmx_txq == NULL)
794 		return (ENOMEM);
795 
796 	/* Initialize driver state for each transmit queue */
797 	for (q = 0; q < ntxqsets; q++)
798 		vmxnet3_init_txq(sc, q);
799 
800 	/*
801 	 * Allocate queue state that is shared with the device.  This check
802 	 * and call is performed in both vmxnet3_tx_queues_alloc() and
803 	 * vmxnet3_rx_queues_alloc() so that we don't have to care which
804 	 * order iflib invokes those routines in.
805 	 */
806 	if (sc->vmx_qs_dma.idi_size == 0) {
807 		error = vmxnet3_queues_shared_alloc(sc);
808 		if (error)
809 			return (error);
810 	}
811 
812 	kva = sc->vmx_qs_dma.idi_vaddr;
813 	for (q = 0; q < ntxqsets; q++) {
814 		sc->vmx_txq[q].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
815 		kva += sizeof(struct vmxnet3_txq_shared);
816 	}
817 
818 	/* Record descriptor ring vaddrs and paddrs */
819 	for (q = 0; q < ntxqsets; q++) {
820 		struct vmxnet3_txqueue *txq;
821 		struct vmxnet3_txring *txr;
822 		struct vmxnet3_comp_ring *txc;
823 
824 		txq = &sc->vmx_txq[q];
825 		txc = &txq->vxtxq_comp_ring;
826 		txr = &txq->vxtxq_cmd_ring;
827 
828 		/* Completion ring */
829 		txc->vxcr_u.txcd =
830 		    (struct vmxnet3_txcompdesc *) vaddrs[q * ntxqs + 0];
831 		txc->vxcr_paddr = paddrs[q * ntxqs + 0];
832 
833 		/* Command ring */
834 		txr->vxtxr_txd =
835 		    (struct vmxnet3_txdesc *) vaddrs[q * ntxqs + 1];
836 		txr->vxtxr_paddr = paddrs[q * ntxqs + 1];
837 	}
838 
839 	return (0);
840 }
841 
842 static void
843 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q, int nrxqs)
844 {
845 	struct vmxnet3_rxqueue *rxq;
846 	struct vmxnet3_comp_ring *rxc;
847 	struct vmxnet3_rxring *rxr;
848 	if_softc_ctx_t scctx;
849 	int i;
850 
851 	rxq = &sc->vmx_rxq[q];
852 	rxc = &rxq->vxrxq_comp_ring;
853 	scctx = sc->vmx_scctx;
854 
855 	snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
856 	    device_get_nameunit(sc->vmx_dev), q);
857 
858 	rxq->vxrxq_sc = sc;
859 	rxq->vxrxq_id = q;
860 
861 	/*
862 	 * First rxq is the completion queue, so there are nrxqs - 1 command
863 	 * rings starting at iflib queue id 1.
864 	 */
865 	rxc->vxcr_ndesc = scctx->isc_nrxd[0];
866 	for (i = 0; i < nrxqs - 1; i++) {
867 		rxr = &rxq->vxrxq_cmd_ring[i];
868 		rxr->vxrxr_ndesc = scctx->isc_nrxd[i + 1];
869 	}
870 }
871 
872 static int
873 vmxnet3_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
874     int nrxqs, int nrxqsets)
875 {
876 	struct vmxnet3_softc *sc;
877 	if_softc_ctx_t scctx;
878 	int q;
879 	int i;
880 	int error;
881 	caddr_t kva;
882 
883 	sc = iflib_get_softc(ctx);
884 	scctx = sc->vmx_scctx;
885 
886 	/* Allocate the array of receive queues */
887 	sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) *
888 	    nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
889 	if (sc->vmx_rxq == NULL)
890 		return (ENOMEM);
891 
892 	/* Initialize driver state for each receive queue */
893 	for (q = 0; q < nrxqsets; q++)
894 		vmxnet3_init_rxq(sc, q, nrxqs);
895 
896 	/*
897 	 * Allocate queue state that is shared with the device.  This check
898 	 * and call is performed in both vmxnet3_tx_queues_alloc() and
899 	 * vmxnet3_rx_queues_alloc() so that we don't have to care which
900 	 * order iflib invokes those routines in.
901 	 */
902 	if (sc->vmx_qs_dma.idi_size == 0) {
903 		error = vmxnet3_queues_shared_alloc(sc);
904 		if (error)
905 			return (error);
906 	}
907 
908 	kva = sc->vmx_qs_dma.idi_vaddr +
909 	    scctx->isc_ntxqsets * sizeof(struct vmxnet3_txq_shared);
910 	for (q = 0; q < nrxqsets; q++) {
911 		sc->vmx_rxq[q].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
912 		kva += sizeof(struct vmxnet3_rxq_shared);
913 	}
914 
915 	/* Record descriptor ring vaddrs and paddrs */
916 	for (q = 0; q < nrxqsets; q++) {
917 		struct vmxnet3_rxqueue *rxq;
918 		struct vmxnet3_rxring *rxr;
919 		struct vmxnet3_comp_ring *rxc;
920 
921 		rxq = &sc->vmx_rxq[q];
922 		rxc = &rxq->vxrxq_comp_ring;
923 
924 		/* Completion ring */
925 		rxc->vxcr_u.rxcd =
926 		    (struct vmxnet3_rxcompdesc *) vaddrs[q * nrxqs + 0];
927 		rxc->vxcr_paddr = paddrs[q * nrxqs + 0];
928 
929 		/* Command ring(s) */
930 		for (i = 0; i < nrxqs - 1; i++) {
931 			rxr = &rxq->vxrxq_cmd_ring[i];
932 
933 			rxr->vxrxr_rxd =
934 			    (struct vmxnet3_rxdesc *) vaddrs[q * nrxqs + 1 + i];
935 			rxr->vxrxr_paddr = paddrs[q * nrxqs + 1 + i];
936 		}
937 	}
938 
939 	return (0);
940 }
941 
942 static void
943 vmxnet3_queues_free(if_ctx_t ctx)
944 {
945 	struct vmxnet3_softc *sc;
946 
947 	sc = iflib_get_softc(ctx);
948 
949 	/* Free queue state area that is shared with the device */
950 	if (sc->vmx_qs_dma.idi_size != 0) {
951 		iflib_dma_free(&sc->vmx_qs_dma);
952 		sc->vmx_qs_dma.idi_size = 0;
953 	}
954 
955 	/* Free array of receive queues */
956 	if (sc->vmx_rxq != NULL) {
957 		free(sc->vmx_rxq, M_DEVBUF);
958 		sc->vmx_rxq = NULL;
959 	}
960 
961 	/* Free array of transmit queues */
962 	if (sc->vmx_txq != NULL) {
963 		free(sc->vmx_txq, M_DEVBUF);
964 		sc->vmx_txq = NULL;
965 	}
966 }
967 
968 static int
969 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
970 {
971 	device_t dev;
972 	size_t size;
973 	int error;
974 
975 	dev = sc->vmx_dev;
976 
977 	/* Top level state structure shared with the device */
978 	size = sizeof(struct vmxnet3_driver_shared);
979 	error = iflib_dma_alloc_align(sc->vmx_ctx, size, 1, &sc->vmx_ds_dma, 0);
980 	if (error) {
981 		device_printf(dev, "cannot alloc shared memory\n");
982 		return (error);
983 	}
984 	sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.idi_vaddr;
985 
986 	/* RSS table state shared with the device */
987 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
988 		size = sizeof(struct vmxnet3_rss_shared);
989 		error = iflib_dma_alloc_align(sc->vmx_ctx, size, 128,
990 		    &sc->vmx_rss_dma, 0);
991 		if (error) {
992 			device_printf(dev, "cannot alloc rss shared memory\n");
993 			return (error);
994 		}
995 		sc->vmx_rss =
996 		    (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.idi_vaddr;
997 	}
998 
999 	return (0);
1000 }
1001 
1002 static void
1003 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
1004 {
1005 
1006 	/* Free RSS table state shared with the device */
1007 	if (sc->vmx_rss != NULL) {
1008 		iflib_dma_free(&sc->vmx_rss_dma);
1009 		sc->vmx_rss = NULL;
1010 	}
1011 
1012 	/* Free top level state structure shared with the device */
1013 	if (sc->vmx_ds != NULL) {
1014 		iflib_dma_free(&sc->vmx_ds_dma);
1015 		sc->vmx_ds = NULL;
1016 	}
1017 }
1018 
1019 static int
1020 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1021 {
1022 	int error;
1023 
1024 	/* Multicast table state shared with the device */
1025 	error = iflib_dma_alloc_align(sc->vmx_ctx,
1026 	    VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 32, &sc->vmx_mcast_dma, 0);
1027 	if (error)
1028 		device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1029 	else
1030 		sc->vmx_mcast = sc->vmx_mcast_dma.idi_vaddr;
1031 
1032 	return (error);
1033 }
1034 
1035 static void
1036 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1037 {
1038 
1039 	/* Free multicast table state shared with the device */
1040 	if (sc->vmx_mcast != NULL) {
1041 		iflib_dma_free(&sc->vmx_mcast_dma);
1042 		sc->vmx_mcast = NULL;
1043 	}
1044 }
1045 
1046 static void
1047 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1048 {
1049 	struct vmxnet3_driver_shared *ds;
1050 	if_shared_ctx_t sctx;
1051 	if_softc_ctx_t scctx;
1052 	struct vmxnet3_txqueue *txq;
1053 	struct vmxnet3_txq_shared *txs;
1054 	struct vmxnet3_rxqueue *rxq;
1055 	struct vmxnet3_rxq_shared *rxs;
1056 	int i;
1057 
1058 	ds = sc->vmx_ds;
1059 	sctx = sc->vmx_sctx;
1060 	scctx = sc->vmx_scctx;
1061 
1062 	/*
1063 	 * Initialize fields of the shared data that remains the same across
1064 	 * reinits. Note the shared data is zero'd when allocated.
1065 	 */
1066 
1067 	ds->magic = VMXNET3_REV1_MAGIC;
1068 
1069 	/* DriverInfo */
1070 	ds->version = VMXNET3_DRIVER_VERSION;
1071 	ds->guest = VMXNET3_GOS_FREEBSD |
1072 #ifdef __LP64__
1073 	    VMXNET3_GOS_64BIT;
1074 #else
1075 	    VMXNET3_GOS_32BIT;
1076 #endif
1077 	ds->vmxnet3_revision = 1;
1078 	ds->upt_version = 1;
1079 
1080 	/* Misc. conf */
1081 	ds->driver_data = vtophys(sc);
1082 	ds->driver_data_len = sizeof(struct vmxnet3_softc);
1083 	ds->queue_shared = sc->vmx_qs_dma.idi_paddr;
1084 	ds->queue_shared_len = sc->vmx_qs_dma.idi_size;
1085 	ds->nrxsg_max = IFLIB_MAX_RX_SEGS;
1086 
1087 	/* RSS conf */
1088 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1089 		ds->rss.version = 1;
1090 		ds->rss.paddr = sc->vmx_rss_dma.idi_paddr;
1091 		ds->rss.len = sc->vmx_rss_dma.idi_size;
1092 	}
1093 
1094 	/* Interrupt control. */
1095 	ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1096 	/*
1097 	 * Total number of interrupt indexes we are using in the shared
1098 	 * config data, even though we don't actually allocate interrupt
1099 	 * resources for the tx queues.  Some versions of the device will
1100 	 * fail to initialize successfully if interrupt indexes are used in
1101 	 * the shared config that exceed the number of interrupts configured
1102 	 * here.
1103 	 */
1104 	ds->nintr = (scctx->isc_vectors == 1) ?
1105 	    2 : (scctx->isc_nrxqsets + scctx->isc_ntxqsets + 1);
1106 	ds->evintr = sc->vmx_event_intr_idx;
1107 	ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1108 
1109 	for (i = 0; i < ds->nintr; i++)
1110 		ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1111 
1112 	/* Receive filter. */
1113 	ds->mcast_table = sc->vmx_mcast_dma.idi_paddr;
1114 	ds->mcast_tablelen = sc->vmx_mcast_dma.idi_size;
1115 
1116 	/* Tx queues */
1117 	for (i = 0; i < scctx->isc_ntxqsets; i++) {
1118 		txq = &sc->vmx_txq[i];
1119 		txs = txq->vxtxq_ts;
1120 
1121 		txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_paddr;
1122 		txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1123 		txs->comp_ring = txq->vxtxq_comp_ring.vxcr_paddr;
1124 		txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1125 		txs->driver_data = vtophys(txq);
1126 		txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1127 	}
1128 
1129 	/* Rx queues */
1130 	for (i = 0; i < scctx->isc_nrxqsets; i++) {
1131 		rxq = &sc->vmx_rxq[i];
1132 		rxs = rxq->vxrxq_rs;
1133 
1134 		rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_paddr;
1135 		rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1136 		rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_paddr;
1137 		rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1138 		rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_paddr;
1139 		rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1140 		rxs->driver_data = vtophys(rxq);
1141 		rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1142 	}
1143 }
1144 
1145 static void
1146 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
1147 {
1148 	/*
1149 	 * Use the same key as the Linux driver until FreeBSD can do
1150 	 * RSS (presumably Toeplitz) in software.
1151 	 */
1152 	static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
1153 	    0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
1154 	    0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
1155 	    0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
1156 	    0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
1157 	    0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
1158 	};
1159 
1160 	struct vmxnet3_driver_shared *ds;
1161 	if_softc_ctx_t scctx;
1162 	struct vmxnet3_rss_shared *rss;
1163 #ifdef RSS
1164 	uint8_t rss_algo;
1165 #endif
1166 	int i;
1167 
1168 	ds = sc->vmx_ds;
1169 	scctx = sc->vmx_scctx;
1170 	rss = sc->vmx_rss;
1171 
1172 	rss->hash_type =
1173 	    UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
1174 	    UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
1175 	rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
1176 	rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
1177 	rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
1178 #ifdef RSS
1179 	/*
1180 	 * If the software RSS is configured to anything else other than
1181 	 * Toeplitz, then just do Toeplitz in "hardware" for the sake of
1182 	 * the packet distribution, but report the hash as opaque to
1183 	 * disengage from the software RSS.
1184 	 */
1185 	rss_algo = rss_gethashalgo();
1186 	if (rss_algo == RSS_HASH_TOEPLITZ) {
1187 		rss_getkey(rss->hash_key);
1188 		for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) {
1189 			rss->ind_table[i] = rss_get_indirection_to_bucket(i) %
1190 			    scctx->isc_nrxqsets;
1191 		}
1192 		sc->vmx_flags |= VMXNET3_FLAG_SOFT_RSS;
1193 	} else
1194 #endif
1195 	{
1196 		memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
1197 		for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
1198 			rss->ind_table[i] = i % scctx->isc_nrxqsets;
1199 		sc->vmx_flags &= ~VMXNET3_FLAG_SOFT_RSS;
1200 	}
1201 }
1202 
1203 static void
1204 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1205 {
1206 	struct ifnet *ifp;
1207 	struct vmxnet3_driver_shared *ds;
1208 	if_softc_ctx_t scctx;
1209 
1210 	ifp = sc->vmx_ifp;
1211 	ds = sc->vmx_ds;
1212 	scctx = sc->vmx_scctx;
1213 
1214 	ds->mtu = ifp->if_mtu;
1215 	ds->ntxqueue = scctx->isc_ntxqsets;
1216 	ds->nrxqueue = scctx->isc_nrxqsets;
1217 
1218 	ds->upt_features = 0;
1219 	if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
1220 		ds->upt_features |= UPT1_F_CSUM;
1221 	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1222 		ds->upt_features |= UPT1_F_VLAN;
1223 	if (ifp->if_capenable & IFCAP_LRO)
1224 		ds->upt_features |= UPT1_F_LRO;
1225 
1226 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1227 		ds->upt_features |= UPT1_F_RSS;
1228 		vmxnet3_reinit_rss_shared_data(sc);
1229 	}
1230 
1231 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.idi_paddr);
1232 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1233 	    (uint64_t) sc->vmx_ds_dma.idi_paddr >> 32);
1234 }
1235 
1236 static int
1237 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1238 {
1239 	int error;
1240 
1241 	error = vmxnet3_alloc_shared_data(sc);
1242 	if (error)
1243 		return (error);
1244 
1245 	error = vmxnet3_alloc_mcast_table(sc);
1246 	if (error)
1247 		return (error);
1248 
1249 	vmxnet3_init_shared_data(sc);
1250 
1251 	return (0);
1252 }
1253 
1254 static void
1255 vmxnet3_free_data(struct vmxnet3_softc *sc)
1256 {
1257 
1258 	vmxnet3_free_mcast_table(sc);
1259 	vmxnet3_free_shared_data(sc);
1260 }
1261 
1262 static void
1263 vmxnet3_evintr(struct vmxnet3_softc *sc)
1264 {
1265 	device_t dev;
1266 	struct vmxnet3_txq_shared *ts;
1267 	struct vmxnet3_rxq_shared *rs;
1268 	uint32_t event;
1269 
1270 	dev = sc->vmx_dev;
1271 
1272 	/* Clear events. */
1273 	event = sc->vmx_ds->event;
1274 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
1275 
1276 	if (event & VMXNET3_EVENT_LINK)
1277 		vmxnet3_link_status(sc);
1278 
1279 	if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
1280 		vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
1281 		ts = sc->vmx_txq[0].vxtxq_ts;
1282 		if (ts->stopped != 0)
1283 			device_printf(dev, "Tx queue error %#x\n", ts->error);
1284 		rs = sc->vmx_rxq[0].vxrxq_rs;
1285 		if (rs->stopped != 0)
1286 			device_printf(dev, "Rx queue error %#x\n", rs->error);
1287 
1288 		/* XXX - rely on liflib watchdog to reset us? */
1289 		device_printf(dev, "Rx/Tx queue error event ... "
1290 		    "waiting for iflib watchdog reset\n");
1291 	}
1292 
1293 	if (event & VMXNET3_EVENT_DIC)
1294 		device_printf(dev, "device implementation change event\n");
1295 	if (event & VMXNET3_EVENT_DEBUG)
1296 		device_printf(dev, "debug event\n");
1297 }
1298 
1299 static int
1300 vmxnet3_isc_txd_encap(void *vsc, if_pkt_info_t pi)
1301 {
1302 	struct vmxnet3_softc *sc;
1303 	struct vmxnet3_txqueue *txq;
1304 	struct vmxnet3_txring *txr;
1305 	struct vmxnet3_txdesc *txd, *sop;
1306 	bus_dma_segment_t *segs;
1307 	int nsegs;
1308 	int pidx;
1309 	int hdrlen;
1310 	int i;
1311 	int gen;
1312 
1313 	sc = vsc;
1314 	txq = &sc->vmx_txq[pi->ipi_qsidx];
1315 	txr = &txq->vxtxq_cmd_ring;
1316 	segs = pi->ipi_segs;
1317 	nsegs = pi->ipi_nsegs;
1318 	pidx = pi->ipi_pidx;
1319 
1320 	KASSERT(nsegs <= VMXNET3_TX_MAXSEGS,
1321 	    ("%s: packet with too many segments %d", __func__, nsegs));
1322 
1323 	sop = &txr->vxtxr_txd[pidx];
1324 	gen = txr->vxtxr_gen ^ 1;	/* Owned by cpu (yet) */
1325 
1326 	for (i = 0; i < nsegs; i++) {
1327 		txd = &txr->vxtxr_txd[pidx];
1328 
1329 		txd->addr = segs[i].ds_addr;
1330 		txd->len = segs[i].ds_len;
1331 		txd->gen = gen;
1332 		txd->dtype = 0;
1333 		txd->offload_mode = VMXNET3_OM_NONE;
1334 		txd->offload_pos = 0;
1335 		txd->hlen = 0;
1336 		txd->eop = 0;
1337 		txd->compreq = 0;
1338 		txd->vtag_mode = 0;
1339 		txd->vtag = 0;
1340 
1341 		if (++pidx == txr->vxtxr_ndesc) {
1342 			pidx = 0;
1343 			txr->vxtxr_gen ^= 1;
1344 		}
1345 		gen = txr->vxtxr_gen;
1346 	}
1347 	txd->eop = 1;
1348 	txd->compreq = !!(pi->ipi_flags & IPI_TX_INTR);
1349 	pi->ipi_new_pidx = pidx;
1350 
1351 	/*
1352 	 * VLAN
1353 	 */
1354 	if (pi->ipi_mflags & M_VLANTAG) {
1355 		sop->vtag_mode = 1;
1356 		sop->vtag = pi->ipi_vtag;
1357 	}
1358 
1359 	/*
1360 	 * TSO and checksum offloads
1361 	 */
1362 	hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
1363 	if (pi->ipi_csum_flags & CSUM_TSO) {
1364 		sop->offload_mode = VMXNET3_OM_TSO;
1365 		sop->hlen = hdrlen + pi->ipi_tcp_hlen;
1366 		sop->offload_pos = pi->ipi_tso_segsz;
1367 	} else if (pi->ipi_csum_flags & (VMXNET3_CSUM_OFFLOAD |
1368 	    VMXNET3_CSUM_OFFLOAD_IPV6)) {
1369 		sop->offload_mode = VMXNET3_OM_CSUM;
1370 		sop->hlen = hdrlen;
1371 		sop->offload_pos = hdrlen +
1372 		    ((pi->ipi_ipproto == IPPROTO_TCP) ?
1373 			offsetof(struct tcphdr, th_sum) :
1374 			offsetof(struct udphdr, uh_sum));
1375 	}
1376 
1377 	/* Finally, change the ownership. */
1378 	vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
1379 	sop->gen ^= 1;
1380 
1381 	return (0);
1382 }
1383 
1384 static void
1385 vmxnet3_isc_txd_flush(void *vsc, uint16_t txqid, qidx_t pidx)
1386 {
1387 	struct vmxnet3_softc *sc;
1388 	struct vmxnet3_txqueue *txq;
1389 
1390 	sc = vsc;
1391 	txq = &sc->vmx_txq[txqid];
1392 
1393 	/*
1394 	 * pidx is what we last set ipi_new_pidx to in
1395 	 * vmxnet3_isc_txd_encap()
1396 	 */
1397 
1398 	/*
1399 	 * Avoid expensive register updates if the flush request is
1400 	 * redundant.
1401 	 */
1402 	if (txq->vxtxq_last_flush == pidx)
1403 		return;
1404 	txq->vxtxq_last_flush = pidx;
1405 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), pidx);
1406 }
1407 
1408 static int
1409 vmxnet3_isc_txd_credits_update(void *vsc, uint16_t txqid, bool clear)
1410 {
1411 	struct vmxnet3_softc *sc;
1412 	struct vmxnet3_txqueue *txq;
1413 	struct vmxnet3_comp_ring *txc;
1414 	struct vmxnet3_txcompdesc *txcd;
1415 	struct vmxnet3_txring *txr;
1416 	int processed;
1417 
1418 	sc = vsc;
1419 	txq = &sc->vmx_txq[txqid];
1420 	txc = &txq->vxtxq_comp_ring;
1421 	txr = &txq->vxtxq_cmd_ring;
1422 
1423 	/*
1424 	 * If clear is true, we need to report the number of TX command ring
1425 	 * descriptors that have been processed by the device.  If clear is
1426 	 * false, we just need to report whether or not at least one TX
1427 	 * command ring descriptor has been processed by the device.
1428 	 */
1429 	processed = 0;
1430 	for (;;) {
1431 		txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
1432 		if (txcd->gen != txc->vxcr_gen)
1433 			break;
1434 		else if (!clear)
1435 			return (1);
1436 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1437 
1438 		if (++txc->vxcr_next == txc->vxcr_ndesc) {
1439 			txc->vxcr_next = 0;
1440 			txc->vxcr_gen ^= 1;
1441 		}
1442 
1443 		if (txcd->eop_idx < txr->vxtxr_next)
1444 			processed += txr->vxtxr_ndesc -
1445 			    (txr->vxtxr_next - txcd->eop_idx) + 1;
1446 		else
1447 			processed += txcd->eop_idx - txr->vxtxr_next + 1;
1448 		txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
1449 	}
1450 
1451 	return (processed);
1452 }
1453 
1454 static int
1455 vmxnet3_isc_rxd_available(void *vsc, uint16_t rxqid, qidx_t idx, qidx_t budget)
1456 {
1457 	struct vmxnet3_softc *sc;
1458 	struct vmxnet3_rxqueue *rxq;
1459 	struct vmxnet3_comp_ring *rxc;
1460 	struct vmxnet3_rxcompdesc *rxcd;
1461 	int avail;
1462 	int completed_gen;
1463 #ifdef INVARIANTS
1464 	int expect_sop = 1;
1465 #endif
1466 	sc = vsc;
1467 	rxq = &sc->vmx_rxq[rxqid];
1468 	rxc = &rxq->vxrxq_comp_ring;
1469 
1470 	avail = 0;
1471 	completed_gen = rxc->vxcr_gen;
1472 	for (;;) {
1473 		rxcd = &rxc->vxcr_u.rxcd[idx];
1474 		if (rxcd->gen != completed_gen)
1475 			break;
1476 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1477 
1478 #ifdef INVARIANTS
1479 		if (expect_sop)
1480 			KASSERT(rxcd->sop, ("%s: expected sop", __func__));
1481 		else
1482 			KASSERT(!rxcd->sop, ("%s: unexpected sop", __func__));
1483 		expect_sop = rxcd->eop;
1484 #endif
1485 		if (rxcd->eop && (rxcd->len != 0))
1486 			avail++;
1487 		if (avail > budget)
1488 			break;
1489 		if (++idx == rxc->vxcr_ndesc) {
1490 			idx = 0;
1491 			completed_gen ^= 1;
1492 		}
1493 	}
1494 
1495 	return (avail);
1496 }
1497 
1498 static int
1499 vmxnet3_isc_rxd_pkt_get(void *vsc, if_rxd_info_t ri)
1500 {
1501 	struct vmxnet3_softc *sc;
1502 	if_softc_ctx_t scctx;
1503 	struct vmxnet3_rxqueue *rxq;
1504 	struct vmxnet3_comp_ring *rxc;
1505 	struct vmxnet3_rxcompdesc *rxcd;
1506 	struct vmxnet3_rxring *rxr;
1507 	struct vmxnet3_rxdesc *rxd;
1508 	if_rxd_frag_t frag;
1509 	int cqidx;
1510 	uint16_t total_len;
1511 	uint8_t nfrags;
1512 	uint8_t i;
1513 	uint8_t flid;
1514 
1515 	sc = vsc;
1516 	scctx = sc->vmx_scctx;
1517 	rxq = &sc->vmx_rxq[ri->iri_qsidx];
1518 	rxc = &rxq->vxrxq_comp_ring;
1519 
1520 	/*
1521 	 * Get a single packet starting at the given index in the completion
1522 	 * queue.  That we have been called indicates that
1523 	 * vmxnet3_isc_rxd_available() has already verified that either
1524 	 * there is a complete packet available starting at the given index,
1525 	 * or there are one or more zero length packets starting at the
1526 	 * given index followed by a complete packet, so no verification of
1527 	 * ownership of the descriptors (and no associated read barrier) is
1528 	 * required here.
1529 	 */
1530 	cqidx = ri->iri_cidx;
1531 	rxcd = &rxc->vxcr_u.rxcd[cqidx];
1532 	while (rxcd->len == 0) {
1533 		KASSERT(rxcd->sop && rxcd->eop,
1534 		    ("%s: zero-length packet without both sop and eop set",
1535 			__func__));
1536 		rxc->vxcr_zero_length++;
1537 		if (++cqidx == rxc->vxcr_ndesc) {
1538 			cqidx = 0;
1539 			rxc->vxcr_gen ^= 1;
1540 		}
1541 		rxcd = &rxc->vxcr_u.rxcd[cqidx];
1542 	}
1543 	KASSERT(rxcd->sop, ("%s: expected sop", __func__));
1544 
1545 	/*
1546 	 * RSS and flow ID.
1547 	 * Types other than M_HASHTYPE_NONE and M_HASHTYPE_OPAQUE_HASH should
1548 	 * be used only if the software RSS is enabled and it uses the same
1549 	 * algorithm and the hash key as the "hardware".  If the software RSS
1550 	 * is not enabled, then it's simply pointless to use those types.
1551 	 * If it's enabled but with different parameters, then hash values will
1552 	 * not match.
1553 	 */
1554 	ri->iri_flowid = rxcd->rss_hash;
1555 #ifdef RSS
1556 	if ((sc->vmx_flags & VMXNET3_FLAG_SOFT_RSS) != 0) {
1557 		switch (rxcd->rss_type) {
1558 		case VMXNET3_RCD_RSS_TYPE_NONE:
1559 			ri->iri_flowid = ri->iri_qsidx;
1560 			ri->iri_rsstype = M_HASHTYPE_NONE;
1561 			break;
1562 		case VMXNET3_RCD_RSS_TYPE_IPV4:
1563 			ri->iri_rsstype = M_HASHTYPE_RSS_IPV4;
1564 			break;
1565 		case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
1566 			ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV4;
1567 			break;
1568 		case VMXNET3_RCD_RSS_TYPE_IPV6:
1569 			ri->iri_rsstype = M_HASHTYPE_RSS_IPV6;
1570 			break;
1571 		case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
1572 			ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV6;
1573 			break;
1574 		default:
1575 			ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;
1576 			break;
1577 		}
1578 	} else
1579 #endif
1580 	{
1581 		switch (rxcd->rss_type) {
1582 		case VMXNET3_RCD_RSS_TYPE_NONE:
1583 			ri->iri_flowid = ri->iri_qsidx;
1584 			ri->iri_rsstype = M_HASHTYPE_NONE;
1585 			break;
1586 		default:
1587 			ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;
1588 			break;
1589 		}
1590 	}
1591 
1592 	/*
1593 	 * The queue numbering scheme used for rxcd->qid is as follows:
1594 	 *  - All of the command ring 0s are numbered [0, nrxqsets - 1]
1595 	 *  - All of the command ring 1s are numbered [nrxqsets, 2*nrxqsets - 1]
1596 	 *
1597 	 * Thus, rxcd->qid less than nrxqsets indicates command ring (and
1598 	 * flid) 0, and rxcd->qid greater than or equal to nrxqsets
1599 	 * indicates command ring (and flid) 1.
1600 	 */
1601 	nfrags = 0;
1602 	total_len = 0;
1603 	do {
1604 		rxcd = &rxc->vxcr_u.rxcd[cqidx];
1605 		KASSERT(rxcd->gen == rxc->vxcr_gen,
1606 		    ("%s: generation mismatch", __func__));
1607 		flid = (rxcd->qid >= scctx->isc_nrxqsets) ? 1 : 0;
1608 		rxr = &rxq->vxrxq_cmd_ring[flid];
1609 		rxd = &rxr->vxrxr_rxd[rxcd->rxd_idx];
1610 
1611 		frag = &ri->iri_frags[nfrags];
1612 		frag->irf_flid = flid;
1613 		frag->irf_idx = rxcd->rxd_idx;
1614 		frag->irf_len = rxcd->len;
1615 		total_len += rxcd->len;
1616 		nfrags++;
1617 		if (++cqidx == rxc->vxcr_ndesc) {
1618 			cqidx = 0;
1619 			rxc->vxcr_gen ^= 1;
1620 		}
1621 	} while (!rxcd->eop);
1622 
1623 	ri->iri_cidx = cqidx;
1624 	ri->iri_nfrags = nfrags;
1625 	ri->iri_len = total_len;
1626 
1627 	/*
1628 	 * If there's an error, the last descriptor in the packet will
1629 	 * have the error indicator set.  In this case, set all
1630 	 * fragment lengths to zero.  This will cause iflib to discard
1631 	 * the packet, but process all associated descriptors through
1632 	 * the refill mechanism.
1633 	 */
1634 	if (__predict_false(rxcd->error)) {
1635 		rxc->vxcr_pkt_errors++;
1636 		for (i = 0; i < nfrags; i++) {
1637 			frag = &ri->iri_frags[i];
1638 			frag->irf_len = 0;
1639 		}
1640 	} else {
1641 		/* Checksum offload information is in the last descriptor. */
1642 		if (!rxcd->no_csum) {
1643 			uint32_t csum_flags = 0;
1644 
1645 			if (rxcd->ipv4) {
1646 				csum_flags |= CSUM_IP_CHECKED;
1647 				if (rxcd->ipcsum_ok)
1648 					csum_flags |= CSUM_IP_VALID;
1649 			}
1650 			if (!rxcd->fragment && (rxcd->tcp || rxcd->udp)) {
1651 				csum_flags |= CSUM_L4_CALC;
1652 				if (rxcd->csum_ok) {
1653 					csum_flags |= CSUM_L4_VALID;
1654 					ri->iri_csum_data = 0xffff;
1655 				}
1656 			}
1657 			ri->iri_csum_flags = csum_flags;
1658 		}
1659 
1660 		/* VLAN information is in the last descriptor. */
1661 		if (rxcd->vlan) {
1662 			ri->iri_flags |= M_VLANTAG;
1663 			ri->iri_vtag = rxcd->vtag;
1664 		}
1665 	}
1666 
1667 	return (0);
1668 }
1669 
1670 static void
1671 vmxnet3_isc_rxd_refill(void *vsc, if_rxd_update_t iru)
1672 {
1673 	struct vmxnet3_softc *sc;
1674 	struct vmxnet3_rxqueue *rxq;
1675 	struct vmxnet3_rxring *rxr;
1676 	struct vmxnet3_rxdesc *rxd;
1677 	uint64_t *paddrs;
1678 	int count;
1679 	int len;
1680 	int idx;
1681 	int i;
1682 	uint8_t flid;
1683 	uint8_t btype;
1684 
1685 	count = iru->iru_count;
1686 	len = iru->iru_buf_size;
1687 	flid = iru->iru_flidx;
1688 	paddrs = iru->iru_paddrs;
1689 
1690 	sc = vsc;
1691 	rxq = &sc->vmx_rxq[iru->iru_qsidx];
1692 	rxr = &rxq->vxrxq_cmd_ring[flid];
1693 	rxd = rxr->vxrxr_rxd;
1694 
1695 	/*
1696 	 * Command ring 0 is filled with BTYPE_HEAD descriptors, and
1697 	 * command ring 1 is filled with BTYPE_BODY descriptors.
1698 	 */
1699 	btype = (flid == 0) ? VMXNET3_BTYPE_HEAD : VMXNET3_BTYPE_BODY;
1700 	/*
1701 	 * The refill entries from iflib will advance monotonically,
1702 	 * but the refilled descriptors may not be contiguous due to
1703 	 * earlier skipping of descriptors by the device.  The refill
1704 	 * entries from iflib need an entire state update, while the
1705 	 * descriptors previously skipped by the device only need to
1706 	 * have their generation numbers updated.
1707 	 */
1708 	idx = rxr->vxrxr_refill_start;
1709 	i = 0;
1710 	do {
1711 		if (idx == iru->iru_idxs[i]) {
1712 			rxd[idx].addr = paddrs[i];
1713 			rxd[idx].len = len;
1714 			rxd[idx].btype = btype;
1715 			i++;
1716 		} else
1717 			rxr->vxrxr_desc_skips++;
1718 		rxd[idx].gen = rxr->vxrxr_gen;
1719 
1720 		if (++idx == rxr->vxrxr_ndesc) {
1721 			idx = 0;
1722 			rxr->vxrxr_gen ^= 1;
1723 		}
1724 	} while (i != count);
1725 	rxr->vxrxr_refill_start = idx;
1726 }
1727 
1728 static void
1729 vmxnet3_isc_rxd_flush(void *vsc, uint16_t rxqid, uint8_t flid, qidx_t pidx)
1730 {
1731 	struct vmxnet3_softc *sc;
1732 	struct vmxnet3_rxqueue *rxq;
1733 	struct vmxnet3_rxring *rxr;
1734 	bus_size_t r;
1735 
1736 	sc = vsc;
1737 	rxq = &sc->vmx_rxq[rxqid];
1738 	rxr = &rxq->vxrxq_cmd_ring[flid];
1739 
1740 	if (flid == 0)
1741 		r = VMXNET3_BAR0_RXH1(rxqid);
1742 	else
1743 		r = VMXNET3_BAR0_RXH2(rxqid);
1744 
1745 	vmxnet3_write_bar0(sc, r, pidx);
1746 }
1747 
1748 static int
1749 vmxnet3_legacy_intr(void *xsc)
1750 {
1751 	struct vmxnet3_softc *sc;
1752 	if_softc_ctx_t scctx;
1753 	if_ctx_t ctx;
1754 
1755 	sc = xsc;
1756 	scctx = sc->vmx_scctx;
1757 	ctx = sc->vmx_ctx;
1758 
1759 	/*
1760 	 * When there is only a single interrupt configured, this routine
1761 	 * runs in fast interrupt context, following which the rxq 0 task
1762 	 * will be enqueued.
1763 	 */
1764 	if (scctx->isc_intr == IFLIB_INTR_LEGACY) {
1765 		if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
1766 			return (FILTER_HANDLED);
1767 	}
1768 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1769 		vmxnet3_intr_disable_all(ctx);
1770 
1771 	if (sc->vmx_ds->event != 0)
1772 		iflib_admin_intr_deferred(ctx);
1773 
1774 	/*
1775 	 * XXX - When there is both rxq and event activity, do we care
1776 	 * whether the rxq 0 task or the admin task re-enables the interrupt
1777 	 * first?
1778 	 */
1779 	return (FILTER_SCHEDULE_THREAD);
1780 }
1781 
1782 static int
1783 vmxnet3_rxq_intr(void *vrxq)
1784 {
1785 	struct vmxnet3_softc *sc;
1786 	struct vmxnet3_rxqueue *rxq;
1787 
1788 	rxq = vrxq;
1789 	sc = rxq->vxrxq_sc;
1790 
1791 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1792 		vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
1793 
1794 	return (FILTER_SCHEDULE_THREAD);
1795 }
1796 
1797 static int
1798 vmxnet3_event_intr(void *vsc)
1799 {
1800 	struct vmxnet3_softc *sc;
1801 
1802 	sc = vsc;
1803 
1804 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1805 		vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
1806 
1807 	/*
1808 	 * The work will be done via vmxnet3_update_admin_status(), and the
1809 	 * interrupt will be re-enabled in vmxnet3_link_intr_enable().
1810 	 *
1811 	 * The interrupt will be re-enabled by vmxnet3_link_intr_enable().
1812 	 */
1813 	return (FILTER_SCHEDULE_THREAD);
1814 }
1815 
1816 static void
1817 vmxnet3_stop(if_ctx_t ctx)
1818 {
1819 	struct vmxnet3_softc *sc;
1820 
1821 	sc = iflib_get_softc(ctx);
1822 
1823 	sc->vmx_link_active = 0;
1824 	vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
1825 	vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
1826 }
1827 
1828 static void
1829 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
1830 {
1831 	struct vmxnet3_txring *txr;
1832 	struct vmxnet3_comp_ring *txc;
1833 
1834 	txq->vxtxq_last_flush = -1;
1835 
1836 	txr = &txq->vxtxq_cmd_ring;
1837 	txr->vxtxr_next = 0;
1838 	txr->vxtxr_gen = VMXNET3_INIT_GEN;
1839 	/*
1840 	 * iflib has zeroed out the descriptor array during the prior attach
1841 	 * or stop
1842 	 */
1843 
1844 	txc = &txq->vxtxq_comp_ring;
1845 	txc->vxcr_next = 0;
1846 	txc->vxcr_gen = VMXNET3_INIT_GEN;
1847 	/*
1848 	 * iflib has zeroed out the descriptor array during the prior attach
1849 	 * or stop
1850 	 */
1851 }
1852 
1853 static void
1854 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
1855 {
1856 	struct vmxnet3_rxring *rxr;
1857 	struct vmxnet3_comp_ring *rxc;
1858 	int i;
1859 
1860 	/*
1861 	 * The descriptors will be populated with buffers during a
1862 	 * subsequent invocation of vmxnet3_isc_rxd_refill()
1863 	 */
1864 	for (i = 0; i < sc->vmx_sctx->isc_nrxqs - 1; i++) {
1865 		rxr = &rxq->vxrxq_cmd_ring[i];
1866 		rxr->vxrxr_gen = VMXNET3_INIT_GEN;
1867 		rxr->vxrxr_desc_skips = 0;
1868 		rxr->vxrxr_refill_start = 0;
1869 		/*
1870 		 * iflib has zeroed out the descriptor array during the
1871 		 * prior attach or stop
1872 		 */
1873 	}
1874 
1875 	for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
1876 		rxr = &rxq->vxrxq_cmd_ring[i];
1877 		rxr->vxrxr_gen = 0;
1878 		rxr->vxrxr_desc_skips = 0;
1879 		rxr->vxrxr_refill_start = 0;
1880 		bzero(rxr->vxrxr_rxd,
1881 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
1882 	}
1883 
1884 	rxc = &rxq->vxrxq_comp_ring;
1885 	rxc->vxcr_next = 0;
1886 	rxc->vxcr_gen = VMXNET3_INIT_GEN;
1887 	rxc->vxcr_zero_length = 0;
1888 	rxc->vxcr_pkt_errors = 0;
1889 	/*
1890 	 * iflib has zeroed out the descriptor array during the prior attach
1891 	 * or stop
1892 	 */
1893 }
1894 
1895 static void
1896 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
1897 {
1898 	if_softc_ctx_t scctx;
1899 	int q;
1900 
1901 	scctx = sc->vmx_scctx;
1902 
1903 	for (q = 0; q < scctx->isc_ntxqsets; q++)
1904 		vmxnet3_txinit(sc, &sc->vmx_txq[q]);
1905 
1906 	for (q = 0; q < scctx->isc_nrxqsets; q++)
1907 		vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
1908 }
1909 
1910 static int
1911 vmxnet3_enable_device(struct vmxnet3_softc *sc)
1912 {
1913 	if_softc_ctx_t scctx;
1914 	int q;
1915 
1916 	scctx = sc->vmx_scctx;
1917 
1918 	if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
1919 		device_printf(sc->vmx_dev, "device enable command failed!\n");
1920 		return (1);
1921 	}
1922 
1923 	/* Reset the Rx queue heads. */
1924 	for (q = 0; q < scctx->isc_nrxqsets; q++) {
1925 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
1926 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
1927 	}
1928 
1929 	return (0);
1930 }
1931 
1932 static void
1933 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
1934 {
1935 	struct ifnet *ifp;
1936 
1937 	ifp = sc->vmx_ifp;
1938 
1939 	vmxnet3_set_rxfilter(sc, if_getflags(ifp));
1940 
1941 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1942 		bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter,
1943 		    sizeof(sc->vmx_ds->vlan_filter));
1944 	else
1945 		bzero(sc->vmx_ds->vlan_filter,
1946 		    sizeof(sc->vmx_ds->vlan_filter));
1947 	vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
1948 }
1949 
1950 static void
1951 vmxnet3_init(if_ctx_t ctx)
1952 {
1953 	struct vmxnet3_softc *sc;
1954 
1955 	sc = iflib_get_softc(ctx);
1956 
1957 	/* Use the current MAC address. */
1958 	bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN);
1959 	vmxnet3_set_lladdr(sc);
1960 
1961 	vmxnet3_reinit_shared_data(sc);
1962 	vmxnet3_reinit_queues(sc);
1963 
1964 	vmxnet3_enable_device(sc);
1965 
1966 	vmxnet3_reinit_rxfilters(sc);
1967 	vmxnet3_link_status(sc);
1968 }
1969 
1970 static void
1971 vmxnet3_multi_set(if_ctx_t ctx)
1972 {
1973 
1974 	vmxnet3_set_rxfilter(iflib_get_softc(ctx),
1975 	    if_getflags(iflib_get_ifp(ctx)));
1976 }
1977 
1978 static int
1979 vmxnet3_mtu_set(if_ctx_t ctx, uint32_t mtu)
1980 {
1981 	struct vmxnet3_softc *sc;
1982 	if_softc_ctx_t scctx;
1983 
1984 	sc = iflib_get_softc(ctx);
1985 	scctx = sc->vmx_scctx;
1986 
1987 	if (mtu > VMXNET3_TX_MAXSIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN +
1988 		ETHER_CRC_LEN))
1989 		return (EINVAL);
1990 
1991 	/*
1992 	 * Update the max frame size so that the rx mbuf size is
1993 	 * chosen based on the new mtu during the interface init that
1994 	 * will occur after this routine returns.
1995 	 */
1996 	scctx->isc_max_frame_size = mtu +
1997 		ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN;
1998 	/* RX completion queue - n/a */
1999 	scctx->isc_rxd_buf_size[0] = 0;
2000 	/*
2001 	 * For header-type descriptors (used for first segment of
2002 	 * packet), let iflib determine the buffer size based on the
2003 	 * max frame size.
2004 	 */
2005 	scctx->isc_rxd_buf_size[1] = 0;
2006 	/*
2007 	 * For body-type descriptors (used for jumbo frames and LRO),
2008 	 * always use page-sized buffers.
2009 	 */
2010 	scctx->isc_rxd_buf_size[2] = MJUMPAGESIZE;
2011 
2012 	return (0);
2013 }
2014 
2015 static void
2016 vmxnet3_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
2017 {
2018 	struct vmxnet3_softc *sc;
2019 
2020 	sc = iflib_get_softc(ctx);
2021 
2022 	ifmr->ifm_status = IFM_AVALID;
2023 	ifmr->ifm_active = IFM_ETHER;
2024 
2025 	if (vmxnet3_link_is_up(sc) != 0) {
2026 		ifmr->ifm_status |= IFM_ACTIVE;
2027 		ifmr->ifm_active |= IFM_AUTO;
2028 	} else
2029 		ifmr->ifm_active |= IFM_NONE;
2030 }
2031 
2032 static int
2033 vmxnet3_media_change(if_ctx_t ctx)
2034 {
2035 
2036 	/* Ignore. */
2037 	return (0);
2038 }
2039 
2040 static int
2041 vmxnet3_promisc_set(if_ctx_t ctx, int flags)
2042 {
2043 
2044 	vmxnet3_set_rxfilter(iflib_get_softc(ctx), flags);
2045 
2046 	return (0);
2047 }
2048 
2049 static uint64_t
2050 vmxnet3_get_counter(if_ctx_t ctx, ift_counter cnt)
2051 {
2052 	if_t ifp = iflib_get_ifp(ctx);
2053 
2054 	if (cnt < IFCOUNTERS)
2055 		return if_get_counter_default(ifp, cnt);
2056 
2057 	return (0);
2058 }
2059 
2060 static void
2061 vmxnet3_update_admin_status(if_ctx_t ctx)
2062 {
2063 	struct vmxnet3_softc *sc;
2064 
2065 	sc = iflib_get_softc(ctx);
2066 	if (sc->vmx_ds->event != 0)
2067 		vmxnet3_evintr(sc);
2068 
2069 	vmxnet3_refresh_host_stats(sc);
2070 }
2071 
2072 static void
2073 vmxnet3_txq_timer(if_ctx_t ctx, uint16_t qid)
2074 {
2075 	/* Host stats refresh is global, so just trigger it on txq 0 */
2076 	if (qid == 0)
2077 		vmxnet3_refresh_host_stats(iflib_get_softc(ctx));
2078 }
2079 
2080 static void
2081 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
2082 {
2083 	int idx, bit;
2084 
2085 	if (tag == 0 || tag > 4095)
2086 		return;
2087 
2088 	idx = (tag >> 5) & 0x7F;
2089 	bit = tag & 0x1F;
2090 
2091 	/* Update our private VLAN bitvector. */
2092 	if (add)
2093 		sc->vmx_vlan_filter[idx] |= (1 << bit);
2094 	else
2095 		sc->vmx_vlan_filter[idx] &= ~(1 << bit);
2096 }
2097 
2098 static void
2099 vmxnet3_vlan_register(if_ctx_t ctx, uint16_t tag)
2100 {
2101 
2102 	vmxnet3_update_vlan_filter(iflib_get_softc(ctx), 1, tag);
2103 }
2104 
2105 static void
2106 vmxnet3_vlan_unregister(if_ctx_t ctx, uint16_t tag)
2107 {
2108 
2109 	vmxnet3_update_vlan_filter(iflib_get_softc(ctx), 0, tag);
2110 }
2111 
2112 static u_int
2113 vmxnet3_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int count)
2114 {
2115 	struct vmxnet3_softc *sc = arg;
2116 
2117 	if (count < VMXNET3_MULTICAST_MAX)
2118 		bcopy(LLADDR(sdl), &sc->vmx_mcast[count * ETHER_ADDR_LEN],
2119 		    ETHER_ADDR_LEN);
2120 
2121 	return (1);
2122 }
2123 
2124 static void
2125 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc, int flags)
2126 {
2127 	struct ifnet *ifp;
2128 	struct vmxnet3_driver_shared *ds;
2129 	u_int mode;
2130 
2131 	ifp = sc->vmx_ifp;
2132 	ds = sc->vmx_ds;
2133 
2134 	mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST;
2135 	if (flags & IFF_PROMISC)
2136 		mode |= VMXNET3_RXMODE_PROMISC;
2137 	if (flags & IFF_ALLMULTI)
2138 		mode |= VMXNET3_RXMODE_ALLMULTI;
2139 	else {
2140 		int cnt;
2141 
2142 		cnt = if_foreach_llmaddr(ifp, vmxnet3_hash_maddr, sc);
2143 		if (cnt >= VMXNET3_MULTICAST_MAX) {
2144 			cnt = 0;
2145 			mode |= VMXNET3_RXMODE_ALLMULTI;
2146 		} else if (cnt > 0)
2147 			mode |= VMXNET3_RXMODE_MCAST;
2148 		ds->mcast_tablelen = cnt * ETHER_ADDR_LEN;
2149 	}
2150 
2151 	ds->rxmode = mode;
2152 
2153 	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
2154 	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
2155 }
2156 
2157 static void
2158 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
2159 {
2160 
2161 	vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
2162 }
2163 
2164 static int
2165 vmxnet3_link_is_up(struct vmxnet3_softc *sc)
2166 {
2167 	uint32_t status;
2168 
2169 	status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
2170 	return !!(status & 0x1);
2171 }
2172 
2173 static void
2174 vmxnet3_link_status(struct vmxnet3_softc *sc)
2175 {
2176 	if_ctx_t ctx;
2177 	uint64_t speed;
2178 	int link;
2179 
2180 	ctx = sc->vmx_ctx;
2181 	link = vmxnet3_link_is_up(sc);
2182 	speed = IF_Gbps(10);
2183 
2184 	if (link != 0 && sc->vmx_link_active == 0) {
2185 		sc->vmx_link_active = 1;
2186 		iflib_link_state_change(ctx, LINK_STATE_UP, speed);
2187 	} else if (link == 0 && sc->vmx_link_active != 0) {
2188 		sc->vmx_link_active = 0;
2189 		iflib_link_state_change(ctx, LINK_STATE_DOWN, speed);
2190 	}
2191 }
2192 
2193 static void
2194 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
2195 {
2196 	uint32_t ml, mh;
2197 
2198 	ml  = sc->vmx_lladdr[0];
2199 	ml |= sc->vmx_lladdr[1] << 8;
2200 	ml |= sc->vmx_lladdr[2] << 16;
2201 	ml |= sc->vmx_lladdr[3] << 24;
2202 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
2203 
2204 	mh  = sc->vmx_lladdr[4];
2205 	mh |= sc->vmx_lladdr[5] << 8;
2206 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
2207 }
2208 
2209 static void
2210 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
2211 {
2212 	uint32_t ml, mh;
2213 
2214 	ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
2215 	mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
2216 
2217 	sc->vmx_lladdr[0] = ml;
2218 	sc->vmx_lladdr[1] = ml >> 8;
2219 	sc->vmx_lladdr[2] = ml >> 16;
2220 	sc->vmx_lladdr[3] = ml >> 24;
2221 	sc->vmx_lladdr[4] = mh;
2222 	sc->vmx_lladdr[5] = mh >> 8;
2223 }
2224 
2225 static void
2226 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq,
2227     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2228 {
2229 	struct sysctl_oid *node, *txsnode;
2230 	struct sysctl_oid_list *list, *txslist;
2231 	struct UPT1_TxStats *txstats;
2232 	char namebuf[16];
2233 
2234 	txstats = &txq->vxtxq_ts->stats;
2235 
2236 	snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id);
2237 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
2238 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Transmit Queue");
2239 	txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node);
2240 
2241 	/*
2242 	 * Add statistics reported by the host. These are updated by the
2243 	 * iflib txq timer on txq 0.
2244 	 */
2245 	txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats",
2246 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
2247 	txslist = SYSCTL_CHILDREN(txsnode);
2248 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD,
2249 	    &txstats->TSO_packets, "TSO packets");
2250 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD,
2251 	    &txstats->TSO_bytes, "TSO bytes");
2252 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
2253 	    &txstats->ucast_packets, "Unicast packets");
2254 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
2255 	    &txstats->ucast_bytes, "Unicast bytes");
2256 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
2257 	    &txstats->mcast_packets, "Multicast packets");
2258 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
2259 	    &txstats->mcast_bytes, "Multicast bytes");
2260 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD,
2261 	    &txstats->error, "Errors");
2262 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD,
2263 	    &txstats->discard, "Discards");
2264 }
2265 
2266 static void
2267 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq,
2268     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2269 {
2270 	struct sysctl_oid *node, *rxsnode;
2271 	struct sysctl_oid_list *list, *rxslist;
2272 	struct UPT1_RxStats *rxstats;
2273 	char namebuf[16];
2274 
2275 	rxstats = &rxq->vxrxq_rs->stats;
2276 
2277 	snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id);
2278 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
2279 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Receive Queue");
2280 	rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node);
2281 
2282 	/*
2283 	 * Add statistics reported by the host. These are updated by the
2284 	 * iflib txq timer on txq 0.
2285 	 */
2286 	rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats",
2287 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
2288 	rxslist = SYSCTL_CHILDREN(rxsnode);
2289 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD,
2290 	    &rxstats->LRO_packets, "LRO packets");
2291 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD,
2292 	    &rxstats->LRO_bytes, "LRO bytes");
2293 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
2294 	    &rxstats->ucast_packets, "Unicast packets");
2295 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
2296 	    &rxstats->ucast_bytes, "Unicast bytes");
2297 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
2298 	    &rxstats->mcast_packets, "Multicast packets");
2299 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
2300 	    &rxstats->mcast_bytes, "Multicast bytes");
2301 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD,
2302 	    &rxstats->bcast_packets, "Broadcast packets");
2303 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD,
2304 	    &rxstats->bcast_bytes, "Broadcast bytes");
2305 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD,
2306 	    &rxstats->nobuffer, "No buffer");
2307 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD,
2308 	    &rxstats->error, "Errors");
2309 }
2310 
2311 static void
2312 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc,
2313     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2314 {
2315 	if_softc_ctx_t scctx;
2316 	struct sysctl_oid *node;
2317 	struct sysctl_oid_list *list;
2318 	int i;
2319 
2320 	scctx = sc->vmx_scctx;
2321 
2322 	for (i = 0; i < scctx->isc_ntxqsets; i++) {
2323 		struct vmxnet3_txqueue *txq = &sc->vmx_txq[i];
2324 
2325 		node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO,
2326 		    "debug", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
2327 		list = SYSCTL_CHILDREN(node);
2328 
2329 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD,
2330 		    &txq->vxtxq_cmd_ring.vxtxr_next, 0, "");
2331 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD,
2332 		    &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, "");
2333 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD,
2334 		    &txq->vxtxq_cmd_ring.vxtxr_gen, 0, "");
2335 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
2336 		    &txq->vxtxq_comp_ring.vxcr_next, 0, "");
2337 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
2338 		    &txq->vxtxq_comp_ring.vxcr_ndesc, 0,"");
2339 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
2340 		    &txq->vxtxq_comp_ring.vxcr_gen, 0, "");
2341 	}
2342 
2343 	for (i = 0; i < scctx->isc_nrxqsets; i++) {
2344 		struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i];
2345 
2346 		node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
2347 		    "debug", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
2348 		list = SYSCTL_CHILDREN(node);
2349 
2350 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
2351 		    &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
2352 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
2353 		    &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
2354 		SYSCTL_ADD_U64(ctx, list, OID_AUTO, "cmd0_desc_skips", CTLFLAG_RD,
2355 		    &rxq->vxrxq_cmd_ring[0].vxrxr_desc_skips, 0, "");
2356 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
2357 		    &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
2358 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
2359 		    &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
2360 		SYSCTL_ADD_U64(ctx, list, OID_AUTO, "cmd1_desc_skips", CTLFLAG_RD,
2361 		    &rxq->vxrxq_cmd_ring[1].vxrxr_desc_skips, 0, "");
2362 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
2363 		    &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
2364 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
2365 		    &rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
2366 		SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_zero_length", CTLFLAG_RD,
2367 		    &rxq->vxrxq_comp_ring.vxcr_zero_length, 0, "");
2368 		SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_pkt_errors", CTLFLAG_RD,
2369 		    &rxq->vxrxq_comp_ring.vxcr_pkt_errors, 0, "");
2370 	}
2371 }
2372 
2373 static void
2374 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
2375     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2376 {
2377 	if_softc_ctx_t scctx;
2378 	int i;
2379 
2380 	scctx = sc->vmx_scctx;
2381 
2382 	for (i = 0; i < scctx->isc_ntxqsets; i++)
2383 		vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child);
2384 	for (i = 0; i < scctx->isc_nrxqsets; i++)
2385 		vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child);
2386 
2387 	vmxnet3_setup_debug_sysctl(sc, ctx, child);
2388 }
2389 
2390 static void
2391 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
2392 {
2393 	device_t dev;
2394 	struct sysctl_ctx_list *ctx;
2395 	struct sysctl_oid *tree;
2396 	struct sysctl_oid_list *child;
2397 
2398 	dev = sc->vmx_dev;
2399 	ctx = device_get_sysctl_ctx(dev);
2400 	tree = device_get_sysctl_tree(dev);
2401 	child = SYSCTL_CHILDREN(tree);
2402 
2403 	vmxnet3_setup_queue_sysctl(sc, ctx, child);
2404 }
2405 
2406 static void
2407 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
2408 {
2409 
2410 	bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
2411 }
2412 
2413 static uint32_t
2414 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
2415 {
2416 
2417 	return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
2418 }
2419 
2420 static void
2421 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
2422 {
2423 
2424 	bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
2425 }
2426 
2427 static void
2428 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
2429 {
2430 
2431 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
2432 }
2433 
2434 static uint32_t
2435 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
2436 {
2437 
2438 	vmxnet3_write_cmd(sc, cmd);
2439 	bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0,
2440 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
2441 	return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
2442 }
2443 
2444 static void
2445 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
2446 {
2447 
2448 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
2449 }
2450 
2451 static void
2452 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
2453 {
2454 
2455 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
2456 }
2457 
2458 static int
2459 vmxnet3_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
2460 {
2461 	/* Not using interrupts for TX */
2462 	return (0);
2463 }
2464 
2465 static int
2466 vmxnet3_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
2467 {
2468 	struct vmxnet3_softc *sc;
2469 
2470 	sc = iflib_get_softc(ctx);
2471 	vmxnet3_enable_intr(sc, sc->vmx_rxq[qid].vxrxq_intr_idx);
2472 	return (0);
2473 }
2474 
2475 static void
2476 vmxnet3_link_intr_enable(if_ctx_t ctx)
2477 {
2478 	struct vmxnet3_softc *sc;
2479 
2480 	sc = iflib_get_softc(ctx);
2481 	vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2482 }
2483 
2484 static void
2485 vmxnet3_intr_enable_all(if_ctx_t ctx)
2486 {
2487 	struct vmxnet3_softc *sc;
2488 	if_softc_ctx_t scctx;
2489 	int i;
2490 
2491 	sc = iflib_get_softc(ctx);
2492 	scctx = sc->vmx_scctx;
2493 	sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
2494 	for (i = 0; i < scctx->isc_vectors; i++)
2495 		vmxnet3_enable_intr(sc, i);
2496 }
2497 
2498 static void
2499 vmxnet3_intr_disable_all(if_ctx_t ctx)
2500 {
2501 	struct vmxnet3_softc *sc;
2502 	int i;
2503 
2504 	sc = iflib_get_softc(ctx);
2505 	/*
2506 	 * iflib may invoke this routine before vmxnet3_attach_post() has
2507 	 * run, which is before the top level shared data area is
2508 	 * initialized and the device made aware of it.
2509 	 */
2510 	if (sc->vmx_ds != NULL)
2511 		sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
2512 	for (i = 0; i < VMXNET3_MAX_INTRS; i++)
2513 		vmxnet3_disable_intr(sc, i);
2514 }
2515 
2516 /*
2517  * Since this is a purely paravirtualized device, we do not have
2518  * to worry about DMA coherency. But at times, we must make sure
2519  * both the compiler and CPU do not reorder memory operations.
2520  */
2521 static inline void
2522 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
2523 {
2524 
2525 	switch (type) {
2526 	case VMXNET3_BARRIER_RD:
2527 		rmb();
2528 		break;
2529 	case VMXNET3_BARRIER_WR:
2530 		wmb();
2531 		break;
2532 	case VMXNET3_BARRIER_RDWR:
2533 		mb();
2534 		break;
2535 	default:
2536 		panic("%s: bad barrier type %d", __func__, type);
2537 	}
2538 }
2539