xref: /freebsd/sys/dev/mxge/if_mxge.c (revision c756fb6ebbdac854916fb08524d68d871f1f888c)
16d87a65dSAndrew Gallatin /******************************************************************************
2718cf2ccSPedro F. Giffuni SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3b2fc195eSAndrew Gallatin 
4cabc512fSAndrew Gallatin Copyright (c) 2006-2013, Myricom Inc.
5b2fc195eSAndrew Gallatin All rights reserved.
6b2fc195eSAndrew Gallatin 
7b2fc195eSAndrew Gallatin Redistribution and use in source and binary forms, with or without
8b2fc195eSAndrew Gallatin modification, are permitted provided that the following conditions are met:
9b2fc195eSAndrew Gallatin 
10b2fc195eSAndrew Gallatin  1. Redistributions of source code must retain the above copyright notice,
11b2fc195eSAndrew Gallatin     this list of conditions and the following disclaimer.
12b2fc195eSAndrew Gallatin 
13eb8e82f5SAndrew Gallatin  2. Neither the name of the Myricom Inc, nor the names of its
14b2fc195eSAndrew Gallatin     contributors may be used to endorse or promote products derived from
15b2fc195eSAndrew Gallatin     this software without specific prior written permission.
16b2fc195eSAndrew Gallatin 
17b2fc195eSAndrew Gallatin THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18b2fc195eSAndrew Gallatin AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19b2fc195eSAndrew Gallatin IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20b2fc195eSAndrew Gallatin ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21b2fc195eSAndrew Gallatin LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22b2fc195eSAndrew Gallatin CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23b2fc195eSAndrew Gallatin SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24b2fc195eSAndrew Gallatin INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25b2fc195eSAndrew Gallatin CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26b2fc195eSAndrew Gallatin ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27b2fc195eSAndrew Gallatin POSSIBILITY OF SUCH DAMAGE.
28b2fc195eSAndrew Gallatin 
29b2fc195eSAndrew Gallatin ***************************************************************************/
30b2fc195eSAndrew Gallatin 
31b2fc195eSAndrew Gallatin #include <sys/cdefs.h>
32b2fc195eSAndrew Gallatin __FBSDID("$FreeBSD$");
33b2fc195eSAndrew Gallatin 
34b2fc195eSAndrew Gallatin #include <sys/param.h>
35b2fc195eSAndrew Gallatin #include <sys/systm.h>
36b2fc195eSAndrew Gallatin #include <sys/linker.h>
37b2fc195eSAndrew Gallatin #include <sys/firmware.h>
38b2fc195eSAndrew Gallatin #include <sys/endian.h>
39b2fc195eSAndrew Gallatin #include <sys/sockio.h>
40b2fc195eSAndrew Gallatin #include <sys/mbuf.h>
41b2fc195eSAndrew Gallatin #include <sys/malloc.h>
42b2fc195eSAndrew Gallatin #include <sys/kdb.h>
43b2fc195eSAndrew Gallatin #include <sys/kernel.h>
444e7f640dSJohn Baldwin #include <sys/lock.h>
45b2fc195eSAndrew Gallatin #include <sys/module.h>
46b2fc195eSAndrew Gallatin #include <sys/socket.h>
47b2fc195eSAndrew Gallatin #include <sys/sysctl.h>
48b2fc195eSAndrew Gallatin #include <sys/sx.h>
4972c042dfSAndrew Gallatin #include <sys/taskqueue.h>
50d9db5225SCraig Rodrigues #include <sys/zlib.h>
51b2fc195eSAndrew Gallatin 
52b2fc195eSAndrew Gallatin #include <net/if.h>
5376039bc8SGleb Smirnoff #include <net/if_var.h>
54b2fc195eSAndrew Gallatin #include <net/if_arp.h>
55b2fc195eSAndrew Gallatin #include <net/ethernet.h>
56b2fc195eSAndrew Gallatin #include <net/if_dl.h>
57b2fc195eSAndrew Gallatin #include <net/if_media.h>
58b2fc195eSAndrew Gallatin 
59b2fc195eSAndrew Gallatin #include <net/bpf.h>
60b2fc195eSAndrew Gallatin 
61b2fc195eSAndrew Gallatin #include <net/if_types.h>
62b2fc195eSAndrew Gallatin #include <net/if_vlan_var.h>
63b2fc195eSAndrew Gallatin 
64b2fc195eSAndrew Gallatin #include <netinet/in_systm.h>
65b2fc195eSAndrew Gallatin #include <netinet/in.h>
66b2fc195eSAndrew Gallatin #include <netinet/ip.h>
670a7a780eSAndrew Gallatin #include <netinet/ip6.h>
68aed8e389SAndrew Gallatin #include <netinet/tcp.h>
6926dd49c6SAndrew Gallatin #include <netinet/tcp_lro.h>
700a7a780eSAndrew Gallatin #include <netinet6/ip6_var.h>
71b2fc195eSAndrew Gallatin 
72b2fc195eSAndrew Gallatin #include <machine/bus.h>
73053e637fSAndrew Gallatin #include <machine/in_cksum.h>
74b2fc195eSAndrew Gallatin #include <machine/resource.h>
75b2fc195eSAndrew Gallatin #include <sys/bus.h>
76b2fc195eSAndrew Gallatin #include <sys/rman.h>
771e413cf9SAndrew Gallatin #include <sys/smp.h>
78b2fc195eSAndrew Gallatin 
79b2fc195eSAndrew Gallatin #include <dev/pci/pcireg.h>
80b2fc195eSAndrew Gallatin #include <dev/pci/pcivar.h>
81e749ef6bSAndrew Gallatin #include <dev/pci/pci_private.h> /* XXX for pci_cfg_restore */
82b2fc195eSAndrew Gallatin 
83b2fc195eSAndrew Gallatin #include <vm/vm.h>		/* for pmap_mapdev() */
84b2fc195eSAndrew Gallatin #include <vm/pmap.h>
85b2fc195eSAndrew Gallatin 
86c2c14a69SAndrew Gallatin #if defined(__i386) || defined(__amd64)
87c2c14a69SAndrew Gallatin #include <machine/specialreg.h>
88c2c14a69SAndrew Gallatin #endif
89c2c14a69SAndrew Gallatin 
906d87a65dSAndrew Gallatin #include <dev/mxge/mxge_mcp.h>
916d87a65dSAndrew Gallatin #include <dev/mxge/mcp_gen_header.h>
921e413cf9SAndrew Gallatin /*#define MXGE_FAKE_IFP*/
936d87a65dSAndrew Gallatin #include <dev/mxge/if_mxge_var.h>
94869c7348SAndrew Gallatin #ifdef IFNET_BUF_RING
95869c7348SAndrew Gallatin #include <sys/buf_ring.h>
96869c7348SAndrew Gallatin #endif
97b2fc195eSAndrew Gallatin 
98eb6219e3SAndrew Gallatin #include "opt_inet.h"
990a7a780eSAndrew Gallatin #include "opt_inet6.h"
100eb6219e3SAndrew Gallatin 
101b2fc195eSAndrew Gallatin /* tunable params */
1026d87a65dSAndrew Gallatin static int mxge_nvidia_ecrc_enable = 1;
103d91b1b49SAndrew Gallatin static int mxge_force_firmware = 0;
1046d87a65dSAndrew Gallatin static int mxge_intr_coal_delay = 30;
1055e7d8541SAndrew Gallatin static int mxge_deassert_wait = 1;
1066d87a65dSAndrew Gallatin static int mxge_flow_control = 1;
1075e7d8541SAndrew Gallatin static int mxge_verbose = 0;
108dce01b9bSAndrew Gallatin static int mxge_ticks;
1091e413cf9SAndrew Gallatin static int mxge_max_slices = 1;
1105769c5efSAndrew Gallatin static int mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
1111e413cf9SAndrew Gallatin static int mxge_always_promisc = 0;
112f9453025SAndrew Gallatin static int mxge_initial_mtu = ETHERMTU_JUMBO;
11365c69066SAndrew Gallatin static int mxge_throttle = 0;
1146d87a65dSAndrew Gallatin static char *mxge_fw_unaligned = "mxge_ethp_z8e";
1156d87a65dSAndrew Gallatin static char *mxge_fw_aligned = "mxge_eth_z8e";
1161e413cf9SAndrew Gallatin static char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e";
1171e413cf9SAndrew Gallatin static char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e";
118b2fc195eSAndrew Gallatin 
1196d87a65dSAndrew Gallatin static int mxge_probe(device_t dev);
1206d87a65dSAndrew Gallatin static int mxge_attach(device_t dev);
1216d87a65dSAndrew Gallatin static int mxge_detach(device_t dev);
1226d87a65dSAndrew Gallatin static int mxge_shutdown(device_t dev);
1236d87a65dSAndrew Gallatin static void mxge_intr(void *arg);
124b2fc195eSAndrew Gallatin 
1256d87a65dSAndrew Gallatin static device_method_t mxge_methods[] =
126b2fc195eSAndrew Gallatin {
127b2fc195eSAndrew Gallatin   /* Device interface */
1286d87a65dSAndrew Gallatin   DEVMETHOD(device_probe, mxge_probe),
1296d87a65dSAndrew Gallatin   DEVMETHOD(device_attach, mxge_attach),
1306d87a65dSAndrew Gallatin   DEVMETHOD(device_detach, mxge_detach),
1316d87a65dSAndrew Gallatin   DEVMETHOD(device_shutdown, mxge_shutdown),
13261bfd867SSofian Brabez 
13361bfd867SSofian Brabez   DEVMETHOD_END
134b2fc195eSAndrew Gallatin };
135b2fc195eSAndrew Gallatin 
1366d87a65dSAndrew Gallatin static driver_t mxge_driver =
137b2fc195eSAndrew Gallatin {
1386d87a65dSAndrew Gallatin   "mxge",
1396d87a65dSAndrew Gallatin   mxge_methods,
1406d87a65dSAndrew Gallatin   sizeof(mxge_softc_t),
141b2fc195eSAndrew Gallatin };
142b2fc195eSAndrew Gallatin 
1436d87a65dSAndrew Gallatin static devclass_t mxge_devclass;
144b2fc195eSAndrew Gallatin 
145b2fc195eSAndrew Gallatin /* Declare ourselves to be a child of the PCI bus.*/
1466d87a65dSAndrew Gallatin DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, 0, 0);
1476d87a65dSAndrew Gallatin MODULE_DEPEND(mxge, firmware, 1, 1, 1);
148f9ae0280SAndrew Gallatin MODULE_DEPEND(mxge, zlib, 1, 1, 1);
149b2fc195eSAndrew Gallatin 
1501e413cf9SAndrew Gallatin static int mxge_load_firmware(mxge_softc_t *sc, int adopt);
1518fe615baSAndrew Gallatin static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data);
152a393336bSAndrew Gallatin static int mxge_close(mxge_softc_t *sc, int down);
153276edd10SAndrew Gallatin static int mxge_open(mxge_softc_t *sc);
154276edd10SAndrew Gallatin static void mxge_tick(void *arg);
1558fe615baSAndrew Gallatin 
156b2fc195eSAndrew Gallatin static int
1576d87a65dSAndrew Gallatin mxge_probe(device_t dev)
158b2fc195eSAndrew Gallatin {
15901638550SAndrew Gallatin 	int rev;
16001638550SAndrew Gallatin 
16101638550SAndrew Gallatin 
1626d87a65dSAndrew Gallatin 	if ((pci_get_vendor(dev) == MXGE_PCI_VENDOR_MYRICOM) &&
163f1544498SAndrew Gallatin 	    ((pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E) ||
164f1544498SAndrew Gallatin 	     (pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E_9))) {
16501638550SAndrew Gallatin 		rev = pci_get_revid(dev);
16601638550SAndrew Gallatin 		switch (rev) {
16701638550SAndrew Gallatin 		case MXGE_PCI_REV_Z8E:
168b2fc195eSAndrew Gallatin 			device_set_desc(dev, "Myri10G-PCIE-8A");
16901638550SAndrew Gallatin 			break;
17001638550SAndrew Gallatin 		case MXGE_PCI_REV_Z8ES:
17101638550SAndrew Gallatin 			device_set_desc(dev, "Myri10G-PCIE-8B");
17201638550SAndrew Gallatin 			break;
17301638550SAndrew Gallatin 		default:
17401638550SAndrew Gallatin 			device_set_desc(dev, "Myri10G-PCIE-8??");
17501638550SAndrew Gallatin 			device_printf(dev, "Unrecognized rev %d NIC\n",
17601638550SAndrew Gallatin 				      rev);
17701638550SAndrew Gallatin 			break;
17801638550SAndrew Gallatin 		}
179b2fc195eSAndrew Gallatin 		return 0;
180b2fc195eSAndrew Gallatin 	}
181b2fc195eSAndrew Gallatin 	return ENXIO;
182b2fc195eSAndrew Gallatin }
183b2fc195eSAndrew Gallatin 
184b2fc195eSAndrew Gallatin static void
1856d87a65dSAndrew Gallatin mxge_enable_wc(mxge_softc_t *sc)
186b2fc195eSAndrew Gallatin {
187f9ae0280SAndrew Gallatin #if defined(__i386) || defined(__amd64)
188b2fc195eSAndrew Gallatin 	vm_offset_t len;
18947c2e987SAndrew Gallatin 	int err;
190b2fc195eSAndrew Gallatin 
1914d69a9d0SAndrew Gallatin 	sc->wc = 1;
192b2fc195eSAndrew Gallatin 	len = rman_get_size(sc->mem_res);
193c2c14a69SAndrew Gallatin 	err = pmap_change_attr((vm_offset_t) sc->sram,
194c2c14a69SAndrew Gallatin 			       len, PAT_WRITE_COMBINING);
19547c2e987SAndrew Gallatin 	if (err != 0) {
196c2c14a69SAndrew Gallatin 		device_printf(sc->dev, "pmap_change_attr failed, %d\n",
197c2c14a69SAndrew Gallatin 			      err);
1984d69a9d0SAndrew Gallatin 		sc->wc = 0;
199b2fc195eSAndrew Gallatin 	}
200f9ae0280SAndrew Gallatin #endif
201b2fc195eSAndrew Gallatin }
202b2fc195eSAndrew Gallatin 
203b2fc195eSAndrew Gallatin 
204b2fc195eSAndrew Gallatin /* callback to get our DMA address */
205b2fc195eSAndrew Gallatin static void
2066d87a65dSAndrew Gallatin mxge_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
207b2fc195eSAndrew Gallatin 			 int error)
208b2fc195eSAndrew Gallatin {
209b2fc195eSAndrew Gallatin 	if (error == 0) {
210b2fc195eSAndrew Gallatin 		*(bus_addr_t *) arg = segs->ds_addr;
211b2fc195eSAndrew Gallatin 	}
212b2fc195eSAndrew Gallatin }
213b2fc195eSAndrew Gallatin 
214b2fc195eSAndrew Gallatin static int
2156d87a65dSAndrew Gallatin mxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes,
216b2fc195eSAndrew Gallatin 		   bus_size_t alignment)
217b2fc195eSAndrew Gallatin {
218b2fc195eSAndrew Gallatin 	int err;
219b2fc195eSAndrew Gallatin 	device_t dev = sc->dev;
2201e413cf9SAndrew Gallatin 	bus_size_t boundary, maxsegsize;
2211e413cf9SAndrew Gallatin 
2221e413cf9SAndrew Gallatin 	if (bytes > 4096 && alignment == 4096) {
2231e413cf9SAndrew Gallatin 		boundary = 0;
2241e413cf9SAndrew Gallatin 		maxsegsize = bytes;
2251e413cf9SAndrew Gallatin 	} else {
2261e413cf9SAndrew Gallatin 		boundary = 4096;
2271e413cf9SAndrew Gallatin 		maxsegsize = 4096;
2281e413cf9SAndrew Gallatin 	}
229b2fc195eSAndrew Gallatin 
230b2fc195eSAndrew Gallatin 	/* allocate DMAable memory tags */
231b2fc195eSAndrew Gallatin 	err = bus_dma_tag_create(sc->parent_dmat,	/* parent */
232b2fc195eSAndrew Gallatin 				 alignment,		/* alignment */
2331e413cf9SAndrew Gallatin 				 boundary,		/* boundary */
234b2fc195eSAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* low */
235b2fc195eSAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* high */
236b2fc195eSAndrew Gallatin 				 NULL, NULL,		/* filter */
237b2fc195eSAndrew Gallatin 				 bytes,			/* maxsize */
238b2fc195eSAndrew Gallatin 				 1,			/* num segs */
2391e413cf9SAndrew Gallatin 				 maxsegsize,		/* maxsegsize */
240b2fc195eSAndrew Gallatin 				 BUS_DMA_COHERENT,	/* flags */
241b2fc195eSAndrew Gallatin 				 NULL, NULL,		/* lock */
242b2fc195eSAndrew Gallatin 				 &dma->dmat);		/* tag */
243b2fc195eSAndrew Gallatin 	if (err != 0) {
244b2fc195eSAndrew Gallatin 		device_printf(dev, "couldn't alloc tag (err = %d)\n", err);
245b2fc195eSAndrew Gallatin 		return err;
246b2fc195eSAndrew Gallatin 	}
247b2fc195eSAndrew Gallatin 
248b2fc195eSAndrew Gallatin 	/* allocate DMAable memory & map */
249b2fc195eSAndrew Gallatin 	err = bus_dmamem_alloc(dma->dmat, &dma->addr,
250b2fc195eSAndrew Gallatin 			       (BUS_DMA_WAITOK | BUS_DMA_COHERENT
251b2fc195eSAndrew Gallatin 				| BUS_DMA_ZERO),  &dma->map);
252b2fc195eSAndrew Gallatin 	if (err != 0) {
253b2fc195eSAndrew Gallatin 		device_printf(dev, "couldn't alloc mem (err = %d)\n", err);
254b2fc195eSAndrew Gallatin 		goto abort_with_dmat;
255b2fc195eSAndrew Gallatin 	}
256b2fc195eSAndrew Gallatin 
257b2fc195eSAndrew Gallatin 	/* load the memory */
258b2fc195eSAndrew Gallatin 	err = bus_dmamap_load(dma->dmat, dma->map, dma->addr, bytes,
2596d87a65dSAndrew Gallatin 			      mxge_dmamap_callback,
260b2fc195eSAndrew Gallatin 			      (void *)&dma->bus_addr, 0);
261b2fc195eSAndrew Gallatin 	if (err != 0) {
262b2fc195eSAndrew Gallatin 		device_printf(dev, "couldn't load map (err = %d)\n", err);
263b2fc195eSAndrew Gallatin 		goto abort_with_mem;
264b2fc195eSAndrew Gallatin 	}
265b2fc195eSAndrew Gallatin 	return 0;
266b2fc195eSAndrew Gallatin 
267b2fc195eSAndrew Gallatin abort_with_mem:
268b2fc195eSAndrew Gallatin 	bus_dmamem_free(dma->dmat, dma->addr, dma->map);
269b2fc195eSAndrew Gallatin abort_with_dmat:
270b2fc195eSAndrew Gallatin 	(void)bus_dma_tag_destroy(dma->dmat);
271b2fc195eSAndrew Gallatin 	return err;
272b2fc195eSAndrew Gallatin }
273b2fc195eSAndrew Gallatin 
274b2fc195eSAndrew Gallatin 
275b2fc195eSAndrew Gallatin static void
2766d87a65dSAndrew Gallatin mxge_dma_free(mxge_dma_t *dma)
277b2fc195eSAndrew Gallatin {
278b2fc195eSAndrew Gallatin 	bus_dmamap_unload(dma->dmat, dma->map);
279b2fc195eSAndrew Gallatin 	bus_dmamem_free(dma->dmat, dma->addr, dma->map);
280b2fc195eSAndrew Gallatin 	(void)bus_dma_tag_destroy(dma->dmat);
281b2fc195eSAndrew Gallatin }
282b2fc195eSAndrew Gallatin 
283b2fc195eSAndrew Gallatin /*
284b2fc195eSAndrew Gallatin  * The eeprom strings on the lanaiX have the format
285b2fc195eSAndrew Gallatin  * SN=x\0
286b2fc195eSAndrew Gallatin  * MAC=x:x:x:x:x:x\0
287b2fc195eSAndrew Gallatin  * PC=text\0
288b2fc195eSAndrew Gallatin  */
289b2fc195eSAndrew Gallatin 
290b2fc195eSAndrew Gallatin static int
2916d87a65dSAndrew Gallatin mxge_parse_strings(mxge_softc_t *sc)
292b2fc195eSAndrew Gallatin {
293dedbe836SAndrew Gallatin 	char *ptr;
294a4b233ddSAndrew Gallatin 	int i, found_mac, found_sn2;
295dedbe836SAndrew Gallatin 	char *endptr;
296b2fc195eSAndrew Gallatin 
297b2fc195eSAndrew Gallatin 	ptr = sc->eeprom_strings;
298b2fc195eSAndrew Gallatin 	found_mac = 0;
299a4b233ddSAndrew Gallatin 	found_sn2 = 0;
300dedbe836SAndrew Gallatin 	while (*ptr != '\0') {
301dedbe836SAndrew Gallatin 		if (strncmp(ptr, "MAC=", 4) == 0) {
302dedbe836SAndrew Gallatin 			ptr += 4;
303dedbe836SAndrew Gallatin 			for (i = 0;;) {
304dedbe836SAndrew Gallatin 				sc->mac_addr[i] = strtoul(ptr, &endptr, 16);
305dedbe836SAndrew Gallatin 				if (endptr - ptr != 2)
306b2fc195eSAndrew Gallatin 					goto abort;
307dedbe836SAndrew Gallatin 				ptr = endptr;
308dedbe836SAndrew Gallatin 				if (++i == 6)
309dedbe836SAndrew Gallatin 					break;
310dedbe836SAndrew Gallatin 				if (*ptr++ != ':')
311dedbe836SAndrew Gallatin 					goto abort;
312b2fc195eSAndrew Gallatin 			}
313dedbe836SAndrew Gallatin 			found_mac = 1;
314dedbe836SAndrew Gallatin 		} else if (strncmp(ptr, "PC=", 3) == 0) {
3155e7d8541SAndrew Gallatin 			ptr += 3;
316dedbe836SAndrew Gallatin 			strlcpy(sc->product_code_string, ptr,
317dedbe836SAndrew Gallatin 			    sizeof(sc->product_code_string));
318dedbe836SAndrew Gallatin 		} else if (!found_sn2 && (strncmp(ptr, "SN=", 3) == 0)) {
3195e7d8541SAndrew Gallatin 			ptr += 3;
320dedbe836SAndrew Gallatin 			strlcpy(sc->serial_number_string, ptr,
321dedbe836SAndrew Gallatin 			    sizeof(sc->serial_number_string));
322dedbe836SAndrew Gallatin 		} else if (strncmp(ptr, "SN2=", 4) == 0) {
323a4b233ddSAndrew Gallatin 			/* SN2 takes precedence over SN */
324a4b233ddSAndrew Gallatin 			ptr += 4;
325a4b233ddSAndrew Gallatin 			found_sn2 = 1;
326dedbe836SAndrew Gallatin 			strlcpy(sc->serial_number_string, ptr,
327dedbe836SAndrew Gallatin 			    sizeof(sc->serial_number_string));
328b2fc195eSAndrew Gallatin 		}
329dedbe836SAndrew Gallatin 		while (*ptr++ != '\0') {}
330b2fc195eSAndrew Gallatin 	}
331b2fc195eSAndrew Gallatin 
332b2fc195eSAndrew Gallatin 	if (found_mac)
333b2fc195eSAndrew Gallatin 		return 0;
334b2fc195eSAndrew Gallatin 
335b2fc195eSAndrew Gallatin  abort:
336b2fc195eSAndrew Gallatin 	device_printf(sc->dev, "failed to parse eeprom_strings\n");
337b2fc195eSAndrew Gallatin 
338b2fc195eSAndrew Gallatin 	return ENXIO;
339b2fc195eSAndrew Gallatin }
340b2fc195eSAndrew Gallatin 
3410d151103SRoman Divacky #if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__
3428fe615baSAndrew Gallatin static void
3438fe615baSAndrew Gallatin mxge_enable_nvidia_ecrc(mxge_softc_t *sc)
344b2fc195eSAndrew Gallatin {
345b2fc195eSAndrew Gallatin 	uint32_t val;
3468fe615baSAndrew Gallatin 	unsigned long base, off;
347b2fc195eSAndrew Gallatin 	char *va, *cfgptr;
3488fe615baSAndrew Gallatin 	device_t pdev, mcp55;
3498fe615baSAndrew Gallatin 	uint16_t vendor_id, device_id, word;
350b2fc195eSAndrew Gallatin 	uintptr_t bus, slot, func, ivend, idev;
351b2fc195eSAndrew Gallatin 	uint32_t *ptr32;
352b2fc195eSAndrew Gallatin 
3538fe615baSAndrew Gallatin 
3548fe615baSAndrew Gallatin 	if (!mxge_nvidia_ecrc_enable)
3558fe615baSAndrew Gallatin 		return;
3568fe615baSAndrew Gallatin 
3578fe615baSAndrew Gallatin 	pdev = device_get_parent(device_get_parent(sc->dev));
3588fe615baSAndrew Gallatin 	if (pdev == NULL) {
3598fe615baSAndrew Gallatin 		device_printf(sc->dev, "could not find parent?\n");
3608fe615baSAndrew Gallatin 		return;
3618fe615baSAndrew Gallatin 	}
3628fe615baSAndrew Gallatin 	vendor_id = pci_read_config(pdev, PCIR_VENDOR, 2);
3638fe615baSAndrew Gallatin 	device_id = pci_read_config(pdev, PCIR_DEVICE, 2);
3648fe615baSAndrew Gallatin 
3658fe615baSAndrew Gallatin 	if (vendor_id != 0x10de)
3668fe615baSAndrew Gallatin 		return;
3678fe615baSAndrew Gallatin 
3688fe615baSAndrew Gallatin 	base = 0;
3698fe615baSAndrew Gallatin 
3708fe615baSAndrew Gallatin 	if (device_id == 0x005d) {
3718fe615baSAndrew Gallatin 		/* ck804, base address is magic */
3728fe615baSAndrew Gallatin 		base = 0xe0000000UL;
3738fe615baSAndrew Gallatin 	} else if (device_id >= 0x0374 && device_id <= 0x378) {
3748fe615baSAndrew Gallatin 		/* mcp55, base address stored in chipset */
3758fe615baSAndrew Gallatin 		mcp55 = pci_find_bsf(0, 0, 0);
3768fe615baSAndrew Gallatin 		if (mcp55 &&
3778fe615baSAndrew Gallatin 		    0x10de == pci_read_config(mcp55, PCIR_VENDOR, 2) &&
3788fe615baSAndrew Gallatin 		    0x0369 == pci_read_config(mcp55, PCIR_DEVICE, 2)) {
3798fe615baSAndrew Gallatin 			word = pci_read_config(mcp55, 0x90, 2);
3808fe615baSAndrew Gallatin 			base = ((unsigned long)word & 0x7ffeU) << 25;
3818fe615baSAndrew Gallatin 		}
3828fe615baSAndrew Gallatin 	}
3838fe615baSAndrew Gallatin 	if (!base)
3848fe615baSAndrew Gallatin 		return;
3858fe615baSAndrew Gallatin 
386b2fc195eSAndrew Gallatin 	/* XXXX
387b2fc195eSAndrew Gallatin 	   Test below is commented because it is believed that doing
388b2fc195eSAndrew Gallatin 	   config read/write beyond 0xff will access the config space
389b2fc195eSAndrew Gallatin 	   for the next larger function.  Uncomment this and remove
390b2fc195eSAndrew Gallatin 	   the hacky pmap_mapdev() way of accessing config space when
391b2fc195eSAndrew Gallatin 	   FreeBSD grows support for extended pcie config space access
392b2fc195eSAndrew Gallatin 	*/
393b2fc195eSAndrew Gallatin #if 0
394b2fc195eSAndrew Gallatin 	/* See if we can, by some miracle, access the extended
395b2fc195eSAndrew Gallatin 	   config space */
396b2fc195eSAndrew Gallatin 	val = pci_read_config(pdev, 0x178, 4);
397b2fc195eSAndrew Gallatin 	if (val != 0xffffffff) {
398b2fc195eSAndrew Gallatin 		val |= 0x40;
399b2fc195eSAndrew Gallatin 		pci_write_config(pdev, 0x178, val, 4);
4008fe615baSAndrew Gallatin 		return;
401b2fc195eSAndrew Gallatin 	}
402b2fc195eSAndrew Gallatin #endif
403b2fc195eSAndrew Gallatin 	/* Rather than using normal pci config space writes, we must
404b2fc195eSAndrew Gallatin 	 * map the Nvidia config space ourselves.  This is because on
405b2fc195eSAndrew Gallatin 	 * opteron/nvidia class machine the 0xe000000 mapping is
406b2fc195eSAndrew Gallatin 	 * handled by the nvidia chipset, that means the internal PCI
407b2fc195eSAndrew Gallatin 	 * device (the on-chip northbridge), or the amd-8131 bridge
408b2fc195eSAndrew Gallatin 	 * and things behind them are not visible by this method.
409b2fc195eSAndrew Gallatin 	 */
410b2fc195eSAndrew Gallatin 
411b2fc195eSAndrew Gallatin 	BUS_READ_IVAR(device_get_parent(pdev), pdev,
412b2fc195eSAndrew Gallatin 		      PCI_IVAR_BUS, &bus);
413b2fc195eSAndrew Gallatin 	BUS_READ_IVAR(device_get_parent(pdev), pdev,
414b2fc195eSAndrew Gallatin 		      PCI_IVAR_SLOT, &slot);
415b2fc195eSAndrew Gallatin 	BUS_READ_IVAR(device_get_parent(pdev), pdev,
416b2fc195eSAndrew Gallatin 		      PCI_IVAR_FUNCTION, &func);
417b2fc195eSAndrew Gallatin 	BUS_READ_IVAR(device_get_parent(pdev), pdev,
418b2fc195eSAndrew Gallatin 		      PCI_IVAR_VENDOR, &ivend);
419b2fc195eSAndrew Gallatin 	BUS_READ_IVAR(device_get_parent(pdev), pdev,
420b2fc195eSAndrew Gallatin 		      PCI_IVAR_DEVICE, &idev);
421b2fc195eSAndrew Gallatin 
4228fe615baSAndrew Gallatin 	off =  base
423b2fc195eSAndrew Gallatin 		+ 0x00100000UL * (unsigned long)bus
424b2fc195eSAndrew Gallatin 		+ 0x00001000UL * (unsigned long)(func
425b2fc195eSAndrew Gallatin 						 + 8 * slot);
426b2fc195eSAndrew Gallatin 
427b2fc195eSAndrew Gallatin 	/* map it into the kernel */
428b2fc195eSAndrew Gallatin 	va = pmap_mapdev(trunc_page((vm_paddr_t)off), PAGE_SIZE);
429b2fc195eSAndrew Gallatin 
430b2fc195eSAndrew Gallatin 
431b2fc195eSAndrew Gallatin 	if (va == NULL) {
432b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "pmap_kenter_temporary didn't\n");
4338fe615baSAndrew Gallatin 		return;
434b2fc195eSAndrew Gallatin 	}
435b2fc195eSAndrew Gallatin 	/* get a pointer to the config space mapped into the kernel */
436b2fc195eSAndrew Gallatin 	cfgptr = va + (off & PAGE_MASK);
437b2fc195eSAndrew Gallatin 
438b2fc195eSAndrew Gallatin 	/* make sure that we can really access it */
439b2fc195eSAndrew Gallatin 	vendor_id = *(uint16_t *)(cfgptr + PCIR_VENDOR);
440b2fc195eSAndrew Gallatin 	device_id = *(uint16_t *)(cfgptr + PCIR_DEVICE);
441b2fc195eSAndrew Gallatin 	if (! (vendor_id == ivend && device_id == idev)) {
442b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "mapping failed: 0x%x:0x%x\n",
443b2fc195eSAndrew Gallatin 			      vendor_id, device_id);
444b2fc195eSAndrew Gallatin 		pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
4458fe615baSAndrew Gallatin 		return;
446b2fc195eSAndrew Gallatin 	}
447b2fc195eSAndrew Gallatin 
448b2fc195eSAndrew Gallatin 	ptr32 = (uint32_t*)(cfgptr + 0x178);
449b2fc195eSAndrew Gallatin 	val = *ptr32;
450b2fc195eSAndrew Gallatin 
451b2fc195eSAndrew Gallatin 	if (val == 0xffffffff) {
452b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "extended mapping failed\n");
453b2fc195eSAndrew Gallatin 		pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
4548fe615baSAndrew Gallatin 		return;
455b2fc195eSAndrew Gallatin 	}
456b2fc195eSAndrew Gallatin 	*ptr32 = val | 0x40;
457b2fc195eSAndrew Gallatin 	pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
4585e7d8541SAndrew Gallatin 	if (mxge_verbose)
459b2fc195eSAndrew Gallatin 		device_printf(sc->dev,
4605e7d8541SAndrew Gallatin 			      "Enabled ECRC on upstream Nvidia bridge "
4615e7d8541SAndrew Gallatin 			      "at %d:%d:%d\n",
462b2fc195eSAndrew Gallatin 			      (int)bus, (int)slot, (int)func);
4638fe615baSAndrew Gallatin 	return;
464b2fc195eSAndrew Gallatin }
465b2fc195eSAndrew Gallatin #else
4668fe615baSAndrew Gallatin static void
467f9ae0280SAndrew Gallatin mxge_enable_nvidia_ecrc(mxge_softc_t *sc)
468b2fc195eSAndrew Gallatin {
469b2fc195eSAndrew Gallatin 	device_printf(sc->dev,
470b2fc195eSAndrew Gallatin 		      "Nforce 4 chipset on non-x86/amd64!?!?!\n");
4718fe615baSAndrew Gallatin 	return;
472b2fc195eSAndrew Gallatin }
473b2fc195eSAndrew Gallatin #endif
4748fe615baSAndrew Gallatin 
4758fe615baSAndrew Gallatin 
4768fe615baSAndrew Gallatin static int
4778fe615baSAndrew Gallatin mxge_dma_test(mxge_softc_t *sc, int test_type)
4788fe615baSAndrew Gallatin {
4798fe615baSAndrew Gallatin 	mxge_cmd_t cmd;
4808fe615baSAndrew Gallatin 	bus_addr_t dmatest_bus = sc->dmabench_dma.bus_addr;
4818fe615baSAndrew Gallatin 	int status;
4828fe615baSAndrew Gallatin 	uint32_t len;
4838fe615baSAndrew Gallatin 	char *test = " ";
4848fe615baSAndrew Gallatin 
4858fe615baSAndrew Gallatin 
4868fe615baSAndrew Gallatin 	/* Run a small DMA test.
4878fe615baSAndrew Gallatin 	 * The magic multipliers to the length tell the firmware
4888fe615baSAndrew Gallatin 	 * to do DMA read, write, or read+write tests.  The
4898fe615baSAndrew Gallatin 	 * results are returned in cmd.data0.  The upper 16
4908fe615baSAndrew Gallatin 	 * bits of the return is the number of transfers completed.
4918fe615baSAndrew Gallatin 	 * The lower 16 bits is the time in 0.5us ticks that the
4928fe615baSAndrew Gallatin 	 * transfers took to complete.
4938fe615baSAndrew Gallatin 	 */
4948fe615baSAndrew Gallatin 
4951e413cf9SAndrew Gallatin 	len = sc->tx_boundary;
4968fe615baSAndrew Gallatin 
4978fe615baSAndrew Gallatin 	cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
4988fe615baSAndrew Gallatin 	cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
4998fe615baSAndrew Gallatin 	cmd.data2 = len * 0x10000;
5008fe615baSAndrew Gallatin 	status = mxge_send_cmd(sc, test_type, &cmd);
5018fe615baSAndrew Gallatin 	if (status != 0) {
5028fe615baSAndrew Gallatin 		test = "read";
5038fe615baSAndrew Gallatin 		goto abort;
5048fe615baSAndrew Gallatin 	}
5058fe615baSAndrew Gallatin 	sc->read_dma = ((cmd.data0>>16) * len * 2) /
5068fe615baSAndrew Gallatin 		(cmd.data0 & 0xffff);
5078fe615baSAndrew Gallatin 	cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
5088fe615baSAndrew Gallatin 	cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
5098fe615baSAndrew Gallatin 	cmd.data2 = len * 0x1;
5108fe615baSAndrew Gallatin 	status = mxge_send_cmd(sc, test_type, &cmd);
5118fe615baSAndrew Gallatin 	if (status != 0) {
5128fe615baSAndrew Gallatin 		test = "write";
5138fe615baSAndrew Gallatin 		goto abort;
5148fe615baSAndrew Gallatin 	}
5158fe615baSAndrew Gallatin 	sc->write_dma = ((cmd.data0>>16) * len * 2) /
5168fe615baSAndrew Gallatin 		(cmd.data0 & 0xffff);
5178fe615baSAndrew Gallatin 
5188fe615baSAndrew Gallatin 	cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
5198fe615baSAndrew Gallatin 	cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
5208fe615baSAndrew Gallatin 	cmd.data2 = len * 0x10001;
5218fe615baSAndrew Gallatin 	status = mxge_send_cmd(sc, test_type, &cmd);
5228fe615baSAndrew Gallatin 	if (status != 0) {
5238fe615baSAndrew Gallatin 		test = "read/write";
5248fe615baSAndrew Gallatin 		goto abort;
5258fe615baSAndrew Gallatin 	}
5268fe615baSAndrew Gallatin 	sc->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) /
5278fe615baSAndrew Gallatin 		(cmd.data0 & 0xffff);
5288fe615baSAndrew Gallatin 
5298fe615baSAndrew Gallatin abort:
5308fe615baSAndrew Gallatin 	if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST)
5318fe615baSAndrew Gallatin 		device_printf(sc->dev, "DMA %s benchmark failed: %d\n",
5328fe615baSAndrew Gallatin 			      test, status);
5338fe615baSAndrew Gallatin 
5348fe615baSAndrew Gallatin 	return status;
5358fe615baSAndrew Gallatin }
5368fe615baSAndrew Gallatin 
537b2fc195eSAndrew Gallatin /*
538b2fc195eSAndrew Gallatin  * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput
539b2fc195eSAndrew Gallatin  * when the PCI-E Completion packets are aligned on an 8-byte
540b2fc195eSAndrew Gallatin  * boundary.  Some PCI-E chip sets always align Completion packets; on
541b2fc195eSAndrew Gallatin  * the ones that do not, the alignment can be enforced by enabling
542b2fc195eSAndrew Gallatin  * ECRC generation (if supported).
543b2fc195eSAndrew Gallatin  *
544b2fc195eSAndrew Gallatin  * When PCI-E Completion packets are not aligned, it is actually more
545b2fc195eSAndrew Gallatin  * efficient to limit Read-DMA transactions to 2KB, rather than 4KB.
546b2fc195eSAndrew Gallatin  *
547b2fc195eSAndrew Gallatin  * If the driver can neither enable ECRC nor verify that it has
548b2fc195eSAndrew Gallatin  * already been enabled, then it must use a firmware image which works
549b2fc195eSAndrew Gallatin  * around unaligned completion packets (ethp_z8e.dat), and it should
550b2fc195eSAndrew Gallatin  * also ensure that it never gives the device a Read-DMA which is
5511e413cf9SAndrew Gallatin  * larger than 2KB by setting the tx_boundary to 2KB.  If ECRC is
552b2fc195eSAndrew Gallatin  * enabled, then the driver should use the aligned (eth_z8e.dat)
5531e413cf9SAndrew Gallatin  * firmware image, and set tx_boundary to 4KB.
554b2fc195eSAndrew Gallatin  */
555b2fc195eSAndrew Gallatin 
5568fe615baSAndrew Gallatin static int
5578fe615baSAndrew Gallatin mxge_firmware_probe(mxge_softc_t *sc)
5588fe615baSAndrew Gallatin {
5598fe615baSAndrew Gallatin 	device_t dev = sc->dev;
5608fe615baSAndrew Gallatin 	int reg, status;
5618fe615baSAndrew Gallatin 	uint16_t pectl;
5628fe615baSAndrew Gallatin 
5631e413cf9SAndrew Gallatin 	sc->tx_boundary = 4096;
5648fe615baSAndrew Gallatin 	/*
5658fe615baSAndrew Gallatin 	 * Verify the max read request size was set to 4KB
5668fe615baSAndrew Gallatin 	 * before trying the test with 4KB.
5678fe615baSAndrew Gallatin 	 */
5683b0a4aefSJohn Baldwin 	if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
5698fe615baSAndrew Gallatin 		pectl = pci_read_config(dev, reg + 0x8, 2);
5708fe615baSAndrew Gallatin 		if ((pectl & (5 << 12)) != (5 << 12)) {
5718fe615baSAndrew Gallatin 			device_printf(dev, "Max Read Req. size != 4k (0x%x\n",
5728fe615baSAndrew Gallatin 				      pectl);
5731e413cf9SAndrew Gallatin 			sc->tx_boundary = 2048;
5748fe615baSAndrew Gallatin 		}
5758fe615baSAndrew Gallatin 	}
5768fe615baSAndrew Gallatin 
5778fe615baSAndrew Gallatin 	/*
5788fe615baSAndrew Gallatin 	 * load the optimized firmware (which assumes aligned PCIe
5798fe615baSAndrew Gallatin 	 * completions) in order to see if it works on this host.
5808fe615baSAndrew Gallatin 	 */
5818fe615baSAndrew Gallatin 	sc->fw_name = mxge_fw_aligned;
5821e413cf9SAndrew Gallatin 	status = mxge_load_firmware(sc, 1);
5838fe615baSAndrew Gallatin 	if (status != 0) {
5848fe615baSAndrew Gallatin 		return status;
5858fe615baSAndrew Gallatin 	}
5868fe615baSAndrew Gallatin 
5878fe615baSAndrew Gallatin 	/*
5888fe615baSAndrew Gallatin 	 * Enable ECRC if possible
5898fe615baSAndrew Gallatin 	 */
5908fe615baSAndrew Gallatin 	mxge_enable_nvidia_ecrc(sc);
5918fe615baSAndrew Gallatin 
5928fe615baSAndrew Gallatin 	/*
5938fe615baSAndrew Gallatin 	 * Run a DMA test which watches for unaligned completions and
594a4b233ddSAndrew Gallatin 	 * aborts on the first one seen.  Not required on Z8ES or newer.
5958fe615baSAndrew Gallatin 	 */
596a4b233ddSAndrew Gallatin 	if (pci_get_revid(sc->dev) >= MXGE_PCI_REV_Z8ES)
597a4b233ddSAndrew Gallatin 		return 0;
5988fe615baSAndrew Gallatin 	status = mxge_dma_test(sc, MXGEFW_CMD_UNALIGNED_TEST);
5998fe615baSAndrew Gallatin 	if (status == 0)
6008fe615baSAndrew Gallatin 		return 0; /* keep the aligned firmware */
6018fe615baSAndrew Gallatin 
6028fe615baSAndrew Gallatin 	if (status != E2BIG)
6038fe615baSAndrew Gallatin 		device_printf(dev, "DMA test failed: %d\n", status);
6048fe615baSAndrew Gallatin 	if (status == ENOSYS)
6058fe615baSAndrew Gallatin 		device_printf(dev, "Falling back to ethp! "
6068fe615baSAndrew Gallatin 			      "Please install up to date fw\n");
6078fe615baSAndrew Gallatin 	return status;
6088fe615baSAndrew Gallatin }
6098fe615baSAndrew Gallatin 
6108fe615baSAndrew Gallatin static int
6116d87a65dSAndrew Gallatin mxge_select_firmware(mxge_softc_t *sc)
612b2fc195eSAndrew Gallatin {
6138fe615baSAndrew Gallatin 	int aligned = 0;
61465c69066SAndrew Gallatin 	int force_firmware = mxge_force_firmware;
615b2fc195eSAndrew Gallatin 
61665c69066SAndrew Gallatin 	if (sc->throttle)
61765c69066SAndrew Gallatin 		force_firmware = sc->throttle;
618d91b1b49SAndrew Gallatin 
61965c69066SAndrew Gallatin 	if (force_firmware != 0) {
62065c69066SAndrew Gallatin 		if (force_firmware == 1)
621d91b1b49SAndrew Gallatin 			aligned = 1;
622d91b1b49SAndrew Gallatin 		else
623d91b1b49SAndrew Gallatin 			aligned = 0;
624d91b1b49SAndrew Gallatin 		if (mxge_verbose)
625d91b1b49SAndrew Gallatin 			device_printf(sc->dev,
626d91b1b49SAndrew Gallatin 				      "Assuming %s completions (forced)\n",
627d91b1b49SAndrew Gallatin 				      aligned ? "aligned" : "unaligned");
628d91b1b49SAndrew Gallatin 		goto abort;
629d91b1b49SAndrew Gallatin 	}
630d91b1b49SAndrew Gallatin 
631d91b1b49SAndrew Gallatin 	/* if the PCIe link width is 4 or less, we can use the aligned
632d91b1b49SAndrew Gallatin 	   firmware and skip any checks */
633d91b1b49SAndrew Gallatin 	if (sc->link_width != 0 && sc->link_width <= 4) {
634d91b1b49SAndrew Gallatin 		device_printf(sc->dev,
635d91b1b49SAndrew Gallatin 			      "PCIe x%d Link, expect reduced performance\n",
636d91b1b49SAndrew Gallatin 			      sc->link_width);
637d91b1b49SAndrew Gallatin 		aligned = 1;
638d91b1b49SAndrew Gallatin 		goto abort;
639d91b1b49SAndrew Gallatin 	}
640d91b1b49SAndrew Gallatin 
6418fe615baSAndrew Gallatin 	if (0 == mxge_firmware_probe(sc))
6428fe615baSAndrew Gallatin 		return 0;
643b2fc195eSAndrew Gallatin 
644b2fc195eSAndrew Gallatin abort:
645b2fc195eSAndrew Gallatin 	if (aligned) {
6466d87a65dSAndrew Gallatin 		sc->fw_name = mxge_fw_aligned;
6471e413cf9SAndrew Gallatin 		sc->tx_boundary = 4096;
648b2fc195eSAndrew Gallatin 	} else {
6496d87a65dSAndrew Gallatin 		sc->fw_name = mxge_fw_unaligned;
6501e413cf9SAndrew Gallatin 		sc->tx_boundary = 2048;
651b2fc195eSAndrew Gallatin 	}
6521e413cf9SAndrew Gallatin 	return (mxge_load_firmware(sc, 0));
653b2fc195eSAndrew Gallatin }
654b2fc195eSAndrew Gallatin 
6554da0d523SAndrew Gallatin static int
6564da0d523SAndrew Gallatin mxge_validate_firmware(mxge_softc_t *sc, const mcp_gen_header_t *hdr)
6574da0d523SAndrew Gallatin {
658b824b7d8SAndrew Gallatin 
6594da0d523SAndrew Gallatin 
6604da0d523SAndrew Gallatin 	if (be32toh(hdr->mcp_type) != MCP_TYPE_ETH) {
6614da0d523SAndrew Gallatin 		device_printf(sc->dev, "Bad firmware type: 0x%x\n",
6624da0d523SAndrew Gallatin 			      be32toh(hdr->mcp_type));
6634da0d523SAndrew Gallatin 		return EIO;
6644da0d523SAndrew Gallatin 	}
6654da0d523SAndrew Gallatin 
6664da0d523SAndrew Gallatin 	/* save firmware version for sysctl */
667dedbe836SAndrew Gallatin 	strlcpy(sc->fw_version, hdr->version, sizeof(sc->fw_version));
6684da0d523SAndrew Gallatin 	if (mxge_verbose)
6694da0d523SAndrew Gallatin 		device_printf(sc->dev, "firmware id: %s\n", hdr->version);
6704da0d523SAndrew Gallatin 
671b824b7d8SAndrew Gallatin 	sscanf(sc->fw_version, "%d.%d.%d", &sc->fw_ver_major,
672b824b7d8SAndrew Gallatin 	       &sc->fw_ver_minor, &sc->fw_ver_tiny);
6734da0d523SAndrew Gallatin 
674b824b7d8SAndrew Gallatin 	if (!(sc->fw_ver_major == MXGEFW_VERSION_MAJOR
675b824b7d8SAndrew Gallatin 	      && sc->fw_ver_minor == MXGEFW_VERSION_MINOR)) {
6764da0d523SAndrew Gallatin 		device_printf(sc->dev, "Found firmware version %s\n",
6774da0d523SAndrew Gallatin 			      sc->fw_version);
6784da0d523SAndrew Gallatin 		device_printf(sc->dev, "Driver needs %d.%d\n",
6794da0d523SAndrew Gallatin 			      MXGEFW_VERSION_MAJOR, MXGEFW_VERSION_MINOR);
6804da0d523SAndrew Gallatin 		return EINVAL;
6814da0d523SAndrew Gallatin 	}
6824da0d523SAndrew Gallatin 	return 0;
6834da0d523SAndrew Gallatin 
6844da0d523SAndrew Gallatin }
685b2fc195eSAndrew Gallatin 
686f9ae0280SAndrew Gallatin static void *
687f9ae0280SAndrew Gallatin z_alloc(void *nil, u_int items, u_int size)
688f9ae0280SAndrew Gallatin {
689f9ae0280SAndrew Gallatin 	void *ptr;
690f9ae0280SAndrew Gallatin 
691ac2fffa4SPedro F. Giffuni 	ptr = malloc(items * size, M_TEMP, M_NOWAIT);
692f9ae0280SAndrew Gallatin 	return ptr;
693f9ae0280SAndrew Gallatin }
694f9ae0280SAndrew Gallatin 
695f9ae0280SAndrew Gallatin static void
696f9ae0280SAndrew Gallatin z_free(void *nil, void *ptr)
697f9ae0280SAndrew Gallatin {
698f9ae0280SAndrew Gallatin 	free(ptr, M_TEMP);
699f9ae0280SAndrew Gallatin }
700f9ae0280SAndrew Gallatin 
701f9ae0280SAndrew Gallatin 
702b2fc195eSAndrew Gallatin static int
7036d87a65dSAndrew Gallatin mxge_load_firmware_helper(mxge_softc_t *sc, uint32_t *limit)
704b2fc195eSAndrew Gallatin {
705f9ae0280SAndrew Gallatin 	z_stream zs;
706f9ae0280SAndrew Gallatin 	char *inflate_buffer;
70733d54970SLuigi Rizzo 	const struct firmware *fw;
708b2fc195eSAndrew Gallatin 	const mcp_gen_header_t *hdr;
709b2fc195eSAndrew Gallatin 	unsigned hdr_offset;
710b2fc195eSAndrew Gallatin 	int status;
7114da0d523SAndrew Gallatin 	unsigned int i;
7124da0d523SAndrew Gallatin 	char dummy;
713f9ae0280SAndrew Gallatin 	size_t fw_len;
714b2fc195eSAndrew Gallatin 
715b2fc195eSAndrew Gallatin 	fw = firmware_get(sc->fw_name);
716b2fc195eSAndrew Gallatin 	if (fw == NULL) {
717b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Could not find firmware image %s\n",
718b2fc195eSAndrew Gallatin 			      sc->fw_name);
719b2fc195eSAndrew Gallatin 		return ENOENT;
720b2fc195eSAndrew Gallatin 	}
721b2fc195eSAndrew Gallatin 
722f9ae0280SAndrew Gallatin 
723f9ae0280SAndrew Gallatin 
724f9ae0280SAndrew Gallatin 	/* setup zlib and decompress f/w */
725f9ae0280SAndrew Gallatin 	bzero(&zs, sizeof (zs));
726f9ae0280SAndrew Gallatin 	zs.zalloc = z_alloc;
727f9ae0280SAndrew Gallatin 	zs.zfree = z_free;
728f9ae0280SAndrew Gallatin 	status = inflateInit(&zs);
729f9ae0280SAndrew Gallatin 	if (status != Z_OK) {
730b2fc195eSAndrew Gallatin 		status = EIO;
731b2fc195eSAndrew Gallatin 		goto abort_with_fw;
732b2fc195eSAndrew Gallatin 	}
733f9ae0280SAndrew Gallatin 
734f9ae0280SAndrew Gallatin 	/* the uncompressed size is stored as the firmware version,
735f9ae0280SAndrew Gallatin 	   which would otherwise go unused */
736f9ae0280SAndrew Gallatin 	fw_len = (size_t) fw->version;
737f9ae0280SAndrew Gallatin 	inflate_buffer = malloc(fw_len, M_TEMP, M_NOWAIT);
738f9ae0280SAndrew Gallatin 	if (inflate_buffer == NULL)
739f9ae0280SAndrew Gallatin 		goto abort_with_zs;
740f9ae0280SAndrew Gallatin 	zs.avail_in = fw->datasize;
741f9ae0280SAndrew Gallatin 	zs.next_in = __DECONST(char *, fw->data);
742f9ae0280SAndrew Gallatin 	zs.avail_out = fw_len;
743f9ae0280SAndrew Gallatin 	zs.next_out = inflate_buffer;
744f9ae0280SAndrew Gallatin 	status = inflate(&zs, Z_FINISH);
745f9ae0280SAndrew Gallatin 	if (status != Z_STREAM_END) {
746f9ae0280SAndrew Gallatin 		device_printf(sc->dev, "zlib %d\n", status);
747f9ae0280SAndrew Gallatin 		status = EIO;
748f9ae0280SAndrew Gallatin 		goto abort_with_buffer;
749f9ae0280SAndrew Gallatin 	}
750f9ae0280SAndrew Gallatin 
751f9ae0280SAndrew Gallatin 	/* check id */
752f9ae0280SAndrew Gallatin 	hdr_offset = htobe32(*(const uint32_t *)
753f9ae0280SAndrew Gallatin 			     (inflate_buffer + MCP_HEADER_PTR_OFFSET));
754f9ae0280SAndrew Gallatin 	if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw_len) {
755f9ae0280SAndrew Gallatin 		device_printf(sc->dev, "Bad firmware file");
756f9ae0280SAndrew Gallatin 		status = EIO;
757f9ae0280SAndrew Gallatin 		goto abort_with_buffer;
758f9ae0280SAndrew Gallatin 	}
759f9ae0280SAndrew Gallatin 	hdr = (const void*)(inflate_buffer + hdr_offset);
760b2fc195eSAndrew Gallatin 
7614da0d523SAndrew Gallatin 	status = mxge_validate_firmware(sc, hdr);
7624da0d523SAndrew Gallatin 	if (status != 0)
763f9ae0280SAndrew Gallatin 		goto abort_with_buffer;
764b2fc195eSAndrew Gallatin 
765b2fc195eSAndrew Gallatin 	/* Copy the inflated firmware to NIC SRAM. */
766f9ae0280SAndrew Gallatin 	for (i = 0; i < fw_len; i += 256) {
7674da0d523SAndrew Gallatin 		mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i,
768f9ae0280SAndrew Gallatin 			      inflate_buffer + i,
769f9ae0280SAndrew Gallatin 			      min(256U, (unsigned)(fw_len - i)));
77073c7c83fSAndrew Gallatin 		wmb();
7714da0d523SAndrew Gallatin 		dummy = *sc->sram;
77273c7c83fSAndrew Gallatin 		wmb();
7734da0d523SAndrew Gallatin 	}
774b2fc195eSAndrew Gallatin 
775f9ae0280SAndrew Gallatin 	*limit = fw_len;
776b2fc195eSAndrew Gallatin 	status = 0;
777f9ae0280SAndrew Gallatin abort_with_buffer:
778f9ae0280SAndrew Gallatin 	free(inflate_buffer, M_TEMP);
779f9ae0280SAndrew Gallatin abort_with_zs:
780f9ae0280SAndrew Gallatin 	inflateEnd(&zs);
781b2fc195eSAndrew Gallatin abort_with_fw:
782b2fc195eSAndrew Gallatin 	firmware_put(fw, FIRMWARE_UNLOAD);
783b2fc195eSAndrew Gallatin 	return status;
784b2fc195eSAndrew Gallatin }
785b2fc195eSAndrew Gallatin 
786b2fc195eSAndrew Gallatin /*
787b2fc195eSAndrew Gallatin  * Enable or disable periodic RDMAs from the host to make certain
788b2fc195eSAndrew Gallatin  * chipsets resend dropped PCIe messages
789b2fc195eSAndrew Gallatin  */
790b2fc195eSAndrew Gallatin 
791b2fc195eSAndrew Gallatin static void
7926d87a65dSAndrew Gallatin mxge_dummy_rdma(mxge_softc_t *sc, int enable)
793b2fc195eSAndrew Gallatin {
794b2fc195eSAndrew Gallatin 	char buf_bytes[72];
795b2fc195eSAndrew Gallatin 	volatile uint32_t *confirm;
796b2fc195eSAndrew Gallatin 	volatile char *submit;
797b2fc195eSAndrew Gallatin 	uint32_t *buf, dma_low, dma_high;
798b2fc195eSAndrew Gallatin 	int i;
799b2fc195eSAndrew Gallatin 
800b2fc195eSAndrew Gallatin 	buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
801b2fc195eSAndrew Gallatin 
802b2fc195eSAndrew Gallatin 	/* clear confirmation addr */
803b2fc195eSAndrew Gallatin 	confirm = (volatile uint32_t *)sc->cmd;
804b2fc195eSAndrew Gallatin 	*confirm = 0;
80573c7c83fSAndrew Gallatin 	wmb();
806b2fc195eSAndrew Gallatin 
807b2fc195eSAndrew Gallatin 	/* send an rdma command to the PCIe engine, and wait for the
808b2fc195eSAndrew Gallatin 	   response in the confirmation address.  The firmware should
809b2fc195eSAndrew Gallatin 	   write a -1 there to indicate it is alive and well
810b2fc195eSAndrew Gallatin 	*/
811b2fc195eSAndrew Gallatin 
8126d87a65dSAndrew Gallatin 	dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
8136d87a65dSAndrew Gallatin 	dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
814b2fc195eSAndrew Gallatin 	buf[0] = htobe32(dma_high);		/* confirm addr MSW */
815b2fc195eSAndrew Gallatin 	buf[1] = htobe32(dma_low);		/* confirm addr LSW */
816b2fc195eSAndrew Gallatin 	buf[2] = htobe32(0xffffffff);		/* confirm data */
8176d87a65dSAndrew Gallatin 	dma_low = MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr);
8186d87a65dSAndrew Gallatin 	dma_high = MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr);
819b2fc195eSAndrew Gallatin 	buf[3] = htobe32(dma_high); 		/* dummy addr MSW */
820b2fc195eSAndrew Gallatin 	buf[4] = htobe32(dma_low); 		/* dummy addr LSW */
821b2fc195eSAndrew Gallatin 	buf[5] = htobe32(enable);			/* enable? */
822b2fc195eSAndrew Gallatin 
823b2fc195eSAndrew Gallatin 
8240fa7f681SAndrew Gallatin 	submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA);
825b2fc195eSAndrew Gallatin 
8266d87a65dSAndrew Gallatin 	mxge_pio_copy(submit, buf, 64);
82773c7c83fSAndrew Gallatin 	wmb();
828b2fc195eSAndrew Gallatin 	DELAY(1000);
82973c7c83fSAndrew Gallatin 	wmb();
830b2fc195eSAndrew Gallatin 	i = 0;
831b2fc195eSAndrew Gallatin 	while (*confirm != 0xffffffff && i < 20) {
832b2fc195eSAndrew Gallatin 		DELAY(1000);
833b2fc195eSAndrew Gallatin 		i++;
834b2fc195eSAndrew Gallatin 	}
835b2fc195eSAndrew Gallatin 	if (*confirm != 0xffffffff) {
836b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "dummy rdma %s failed (%p = 0x%x)",
837b2fc195eSAndrew Gallatin 			      (enable ? "enable" : "disable"), confirm,
838b2fc195eSAndrew Gallatin 			      *confirm);
839b2fc195eSAndrew Gallatin 	}
840b2fc195eSAndrew Gallatin 	return;
841b2fc195eSAndrew Gallatin }
842b2fc195eSAndrew Gallatin 
843b2fc195eSAndrew Gallatin static int
8446d87a65dSAndrew Gallatin mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data)
845b2fc195eSAndrew Gallatin {
846b2fc195eSAndrew Gallatin 	mcp_cmd_t *buf;
847b2fc195eSAndrew Gallatin 	char buf_bytes[sizeof(*buf) + 8];
848b2fc195eSAndrew Gallatin 	volatile mcp_cmd_response_t *response = sc->cmd;
8490fa7f681SAndrew Gallatin 	volatile char *cmd_addr = sc->sram + MXGEFW_ETH_CMD;
850b2fc195eSAndrew Gallatin 	uint32_t dma_low, dma_high;
851e0501fd0SAndrew Gallatin 	int err, sleep_total = 0;
852b2fc195eSAndrew Gallatin 
853b2fc195eSAndrew Gallatin 	/* ensure buf is aligned to 8 bytes */
854b2fc195eSAndrew Gallatin 	buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
855b2fc195eSAndrew Gallatin 
856b2fc195eSAndrew Gallatin 	buf->data0 = htobe32(data->data0);
857b2fc195eSAndrew Gallatin 	buf->data1 = htobe32(data->data1);
858b2fc195eSAndrew Gallatin 	buf->data2 = htobe32(data->data2);
859b2fc195eSAndrew Gallatin 	buf->cmd = htobe32(cmd);
8606d87a65dSAndrew Gallatin 	dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
8616d87a65dSAndrew Gallatin 	dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
862b2fc195eSAndrew Gallatin 
863b2fc195eSAndrew Gallatin 	buf->response_addr.low = htobe32(dma_low);
864b2fc195eSAndrew Gallatin 	buf->response_addr.high = htobe32(dma_high);
865a98d6cd7SAndrew Gallatin 	mtx_lock(&sc->cmd_mtx);
866b2fc195eSAndrew Gallatin 	response->result = 0xffffffff;
86773c7c83fSAndrew Gallatin 	wmb();
8686d87a65dSAndrew Gallatin 	mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf));
869b2fc195eSAndrew Gallatin 
8705e7d8541SAndrew Gallatin 	/* wait up to 20ms */
871e0501fd0SAndrew Gallatin 	err = EAGAIN;
8725e7d8541SAndrew Gallatin 	for (sleep_total = 0; sleep_total <  20; sleep_total++) {
873b2fc195eSAndrew Gallatin 		bus_dmamap_sync(sc->cmd_dma.dmat,
874b2fc195eSAndrew Gallatin 				sc->cmd_dma.map, BUS_DMASYNC_POSTREAD);
87573c7c83fSAndrew Gallatin 		wmb();
876e0501fd0SAndrew Gallatin 		switch (be32toh(response->result)) {
877e0501fd0SAndrew Gallatin 		case 0:
878b2fc195eSAndrew Gallatin 			data->data0 = be32toh(response->data);
879e0501fd0SAndrew Gallatin 			err = 0;
880e0501fd0SAndrew Gallatin 			break;
881e0501fd0SAndrew Gallatin 		case 0xffffffff:
882e0501fd0SAndrew Gallatin 			DELAY(1000);
883e0501fd0SAndrew Gallatin 			break;
884e0501fd0SAndrew Gallatin 		case MXGEFW_CMD_UNKNOWN:
885e0501fd0SAndrew Gallatin 			err = ENOSYS;
886e0501fd0SAndrew Gallatin 			break;
887e0501fd0SAndrew Gallatin 		case MXGEFW_CMD_ERROR_UNALIGNED:
888e0501fd0SAndrew Gallatin 			err = E2BIG;
889e0501fd0SAndrew Gallatin 			break;
890c587e59fSAndrew Gallatin 		case MXGEFW_CMD_ERROR_BUSY:
891c587e59fSAndrew Gallatin 			err = EBUSY;
892c587e59fSAndrew Gallatin 			break;
893c406ad2eSAndrew Gallatin 		case MXGEFW_CMD_ERROR_I2C_ABSENT:
894c406ad2eSAndrew Gallatin 			err = ENXIO;
895c406ad2eSAndrew Gallatin 			break;
896e0501fd0SAndrew Gallatin 		default:
897b2fc195eSAndrew Gallatin 			device_printf(sc->dev,
8986d87a65dSAndrew Gallatin 				      "mxge: command %d "
899b2fc195eSAndrew Gallatin 				      "failed, result = %d\n",
900b2fc195eSAndrew Gallatin 				      cmd, be32toh(response->result));
901e0501fd0SAndrew Gallatin 			err = ENXIO;
902e0501fd0SAndrew Gallatin 			break;
903b2fc195eSAndrew Gallatin 		}
904e0501fd0SAndrew Gallatin 		if (err != EAGAIN)
905e0501fd0SAndrew Gallatin 			break;
906b2fc195eSAndrew Gallatin 	}
907e0501fd0SAndrew Gallatin 	if (err == EAGAIN)
9086d87a65dSAndrew Gallatin 		device_printf(sc->dev, "mxge: command %d timed out"
909b2fc195eSAndrew Gallatin 			      "result = %d\n",
910b2fc195eSAndrew Gallatin 			      cmd, be32toh(response->result));
911e0501fd0SAndrew Gallatin 	mtx_unlock(&sc->cmd_mtx);
912e0501fd0SAndrew Gallatin 	return err;
913b2fc195eSAndrew Gallatin }
914b2fc195eSAndrew Gallatin 
9154da0d523SAndrew Gallatin static int
9164da0d523SAndrew Gallatin mxge_adopt_running_firmware(mxge_softc_t *sc)
9174da0d523SAndrew Gallatin {
9184da0d523SAndrew Gallatin 	struct mcp_gen_header *hdr;
9194da0d523SAndrew Gallatin 	const size_t bytes = sizeof (struct mcp_gen_header);
9204da0d523SAndrew Gallatin 	size_t hdr_offset;
9214da0d523SAndrew Gallatin 	int status;
9224da0d523SAndrew Gallatin 
9234da0d523SAndrew Gallatin 	/* find running firmware header */
9244da0d523SAndrew Gallatin 	hdr_offset = htobe32(*(volatile uint32_t *)
9254da0d523SAndrew Gallatin 			     (sc->sram + MCP_HEADER_PTR_OFFSET));
9264da0d523SAndrew Gallatin 
9274da0d523SAndrew Gallatin 	if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > sc->sram_size) {
9284da0d523SAndrew Gallatin 		device_printf(sc->dev,
9294da0d523SAndrew Gallatin 			      "Running firmware has bad header offset (%d)\n",
9304da0d523SAndrew Gallatin 			      (int)hdr_offset);
9314da0d523SAndrew Gallatin 		return EIO;
9324da0d523SAndrew Gallatin 	}
9334da0d523SAndrew Gallatin 
9344da0d523SAndrew Gallatin 	/* copy header of running firmware from SRAM to host memory to
9354da0d523SAndrew Gallatin 	 * validate firmware */
9364da0d523SAndrew Gallatin 	hdr = malloc(bytes, M_DEVBUF, M_NOWAIT);
9374da0d523SAndrew Gallatin 	if (hdr == NULL) {
9384da0d523SAndrew Gallatin 		device_printf(sc->dev, "could not malloc firmware hdr\n");
9394da0d523SAndrew Gallatin 		return ENOMEM;
9404da0d523SAndrew Gallatin 	}
9414da0d523SAndrew Gallatin 	bus_space_read_region_1(rman_get_bustag(sc->mem_res),
9424da0d523SAndrew Gallatin 				rman_get_bushandle(sc->mem_res),
9434da0d523SAndrew Gallatin 				hdr_offset, (char *)hdr, bytes);
9444da0d523SAndrew Gallatin 	status = mxge_validate_firmware(sc, hdr);
9454da0d523SAndrew Gallatin 	free(hdr, M_DEVBUF);
946b824b7d8SAndrew Gallatin 
947b824b7d8SAndrew Gallatin 	/*
948b824b7d8SAndrew Gallatin 	 * check to see if adopted firmware has bug where adopting
949b824b7d8SAndrew Gallatin 	 * it will cause broadcasts to be filtered unless the NIC
950b824b7d8SAndrew Gallatin 	 * is kept in ALLMULTI mode
951b824b7d8SAndrew Gallatin 	 */
952b824b7d8SAndrew Gallatin 	if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 &&
953b824b7d8SAndrew Gallatin 	    sc->fw_ver_tiny >= 4 && sc->fw_ver_tiny <= 11) {
954b824b7d8SAndrew Gallatin 		sc->adopted_rx_filter_bug = 1;
955b824b7d8SAndrew Gallatin 		device_printf(sc->dev, "Adopting fw %d.%d.%d: "
956b824b7d8SAndrew Gallatin 			      "working around rx filter bug\n",
957b824b7d8SAndrew Gallatin 			      sc->fw_ver_major, sc->fw_ver_minor,
958b824b7d8SAndrew Gallatin 			      sc->fw_ver_tiny);
959b824b7d8SAndrew Gallatin 	}
960b824b7d8SAndrew Gallatin 
9614da0d523SAndrew Gallatin 	return status;
9624da0d523SAndrew Gallatin }
9634da0d523SAndrew Gallatin 
964b2fc195eSAndrew Gallatin 
965b2fc195eSAndrew Gallatin static int
9661e413cf9SAndrew Gallatin mxge_load_firmware(mxge_softc_t *sc, int adopt)
967b2fc195eSAndrew Gallatin {
968b2fc195eSAndrew Gallatin 	volatile uint32_t *confirm;
969b2fc195eSAndrew Gallatin 	volatile char *submit;
970b2fc195eSAndrew Gallatin 	char buf_bytes[72];
971b2fc195eSAndrew Gallatin 	uint32_t *buf, size, dma_low, dma_high;
972b2fc195eSAndrew Gallatin 	int status, i;
973b2fc195eSAndrew Gallatin 
974b2fc195eSAndrew Gallatin 	buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
975b2fc195eSAndrew Gallatin 
976b2fc195eSAndrew Gallatin 	size = sc->sram_size;
9776d87a65dSAndrew Gallatin 	status = mxge_load_firmware_helper(sc, &size);
978b2fc195eSAndrew Gallatin 	if (status) {
9791e413cf9SAndrew Gallatin 		if (!adopt)
9801e413cf9SAndrew Gallatin 			return status;
9814da0d523SAndrew Gallatin 		/* Try to use the currently running firmware, if
9824da0d523SAndrew Gallatin 		   it is new enough */
9834da0d523SAndrew Gallatin 		status = mxge_adopt_running_firmware(sc);
9844da0d523SAndrew Gallatin 		if (status) {
9854da0d523SAndrew Gallatin 			device_printf(sc->dev,
9864da0d523SAndrew Gallatin 				      "failed to adopt running firmware\n");
987b2fc195eSAndrew Gallatin 			return status;
988b2fc195eSAndrew Gallatin 		}
9894da0d523SAndrew Gallatin 		device_printf(sc->dev,
9904da0d523SAndrew Gallatin 			      "Successfully adopted running firmware\n");
9911e413cf9SAndrew Gallatin 		if (sc->tx_boundary == 4096) {
9924da0d523SAndrew Gallatin 			device_printf(sc->dev,
9934da0d523SAndrew Gallatin 				"Using firmware currently running on NIC"
9944da0d523SAndrew Gallatin 				 ".  For optimal\n");
9954da0d523SAndrew Gallatin 			device_printf(sc->dev,
9964da0d523SAndrew Gallatin 				 "performance consider loading optimized "
9974da0d523SAndrew Gallatin 				 "firmware\n");
9984da0d523SAndrew Gallatin 		}
999d91b1b49SAndrew Gallatin 		sc->fw_name = mxge_fw_unaligned;
10001e413cf9SAndrew Gallatin 		sc->tx_boundary = 2048;
1001d91b1b49SAndrew Gallatin 		return 0;
10024da0d523SAndrew Gallatin 	}
1003b2fc195eSAndrew Gallatin 	/* clear confirmation addr */
1004b2fc195eSAndrew Gallatin 	confirm = (volatile uint32_t *)sc->cmd;
1005b2fc195eSAndrew Gallatin 	*confirm = 0;
100673c7c83fSAndrew Gallatin 	wmb();
1007b2fc195eSAndrew Gallatin 	/* send a reload command to the bootstrap MCP, and wait for the
1008b2fc195eSAndrew Gallatin 	   response in the confirmation address.  The firmware should
1009b2fc195eSAndrew Gallatin 	   write a -1 there to indicate it is alive and well
1010b2fc195eSAndrew Gallatin 	*/
1011b2fc195eSAndrew Gallatin 
10126d87a65dSAndrew Gallatin 	dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
10136d87a65dSAndrew Gallatin 	dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
1014b2fc195eSAndrew Gallatin 
1015b2fc195eSAndrew Gallatin 	buf[0] = htobe32(dma_high);	/* confirm addr MSW */
1016b2fc195eSAndrew Gallatin 	buf[1] = htobe32(dma_low);	/* confirm addr LSW */
1017b2fc195eSAndrew Gallatin 	buf[2] = htobe32(0xffffffff);	/* confirm data */
1018b2fc195eSAndrew Gallatin 
1019b2fc195eSAndrew Gallatin 	/* FIX: All newest firmware should un-protect the bottom of
1020b2fc195eSAndrew Gallatin 	   the sram before handoff. However, the very first interfaces
1021b2fc195eSAndrew Gallatin 	   do not. Therefore the handoff copy must skip the first 8 bytes
1022b2fc195eSAndrew Gallatin 	*/
1023b2fc195eSAndrew Gallatin 					/* where the code starts*/
10246d87a65dSAndrew Gallatin 	buf[3] = htobe32(MXGE_FW_OFFSET + 8);
1025b2fc195eSAndrew Gallatin 	buf[4] = htobe32(size - 8); 	/* length of code */
1026b2fc195eSAndrew Gallatin 	buf[5] = htobe32(8);		/* where to copy to */
1027b2fc195eSAndrew Gallatin 	buf[6] = htobe32(0);		/* where to jump to */
1028b2fc195eSAndrew Gallatin 
10290fa7f681SAndrew Gallatin 	submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF);
10306d87a65dSAndrew Gallatin 	mxge_pio_copy(submit, buf, 64);
103173c7c83fSAndrew Gallatin 	wmb();
1032b2fc195eSAndrew Gallatin 	DELAY(1000);
103373c7c83fSAndrew Gallatin 	wmb();
1034b2fc195eSAndrew Gallatin 	i = 0;
1035b2fc195eSAndrew Gallatin 	while (*confirm != 0xffffffff && i < 20) {
1036b2fc195eSAndrew Gallatin 		DELAY(1000*10);
1037b2fc195eSAndrew Gallatin 		i++;
1038b2fc195eSAndrew Gallatin 		bus_dmamap_sync(sc->cmd_dma.dmat,
1039b2fc195eSAndrew Gallatin 				sc->cmd_dma.map, BUS_DMASYNC_POSTREAD);
1040b2fc195eSAndrew Gallatin 	}
1041b2fc195eSAndrew Gallatin 	if (*confirm != 0xffffffff) {
1042b2fc195eSAndrew Gallatin 		device_printf(sc->dev,"handoff failed (%p = 0x%x)",
1043b2fc195eSAndrew Gallatin 			confirm, *confirm);
1044b2fc195eSAndrew Gallatin 
1045b2fc195eSAndrew Gallatin 		return ENXIO;
1046b2fc195eSAndrew Gallatin 	}
1047b2fc195eSAndrew Gallatin 	return 0;
1048b2fc195eSAndrew Gallatin }
1049b2fc195eSAndrew Gallatin 
1050b2fc195eSAndrew Gallatin static int
10516d87a65dSAndrew Gallatin mxge_update_mac_address(mxge_softc_t *sc)
1052b2fc195eSAndrew Gallatin {
10536d87a65dSAndrew Gallatin 	mxge_cmd_t cmd;
1054b2fc195eSAndrew Gallatin 	uint8_t *addr = sc->mac_addr;
1055b2fc195eSAndrew Gallatin 	int status;
1056b2fc195eSAndrew Gallatin 
1057b2fc195eSAndrew Gallatin 
1058b2fc195eSAndrew Gallatin 	cmd.data0 = ((addr[0] << 24) | (addr[1] << 16)
1059b2fc195eSAndrew Gallatin 		     | (addr[2] << 8) | addr[3]);
1060b2fc195eSAndrew Gallatin 
1061b2fc195eSAndrew Gallatin 	cmd.data1 = ((addr[4] << 8) | (addr[5]));
1062b2fc195eSAndrew Gallatin 
10635e7d8541SAndrew Gallatin 	status = mxge_send_cmd(sc, MXGEFW_SET_MAC_ADDRESS, &cmd);
1064b2fc195eSAndrew Gallatin 	return status;
1065b2fc195eSAndrew Gallatin }
1066b2fc195eSAndrew Gallatin 
1067b2fc195eSAndrew Gallatin static int
10686d87a65dSAndrew Gallatin mxge_change_pause(mxge_softc_t *sc, int pause)
1069b2fc195eSAndrew Gallatin {
10706d87a65dSAndrew Gallatin 	mxge_cmd_t cmd;
1071b2fc195eSAndrew Gallatin 	int status;
1072b2fc195eSAndrew Gallatin 
1073b2fc195eSAndrew Gallatin 	if (pause)
10745e7d8541SAndrew Gallatin 		status = mxge_send_cmd(sc, MXGEFW_ENABLE_FLOW_CONTROL,
1075b2fc195eSAndrew Gallatin 				       &cmd);
1076b2fc195eSAndrew Gallatin 	else
10775e7d8541SAndrew Gallatin 		status = mxge_send_cmd(sc, MXGEFW_DISABLE_FLOW_CONTROL,
1078b2fc195eSAndrew Gallatin 				       &cmd);
1079b2fc195eSAndrew Gallatin 
1080b2fc195eSAndrew Gallatin 	if (status) {
1081b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Failed to set flow control mode\n");
1082b2fc195eSAndrew Gallatin 		return ENXIO;
1083b2fc195eSAndrew Gallatin 	}
1084b2fc195eSAndrew Gallatin 	sc->pause = pause;
1085b2fc195eSAndrew Gallatin 	return 0;
1086b2fc195eSAndrew Gallatin }
1087b2fc195eSAndrew Gallatin 
1088b2fc195eSAndrew Gallatin static void
10896d87a65dSAndrew Gallatin mxge_change_promisc(mxge_softc_t *sc, int promisc)
1090b2fc195eSAndrew Gallatin {
10916d87a65dSAndrew Gallatin 	mxge_cmd_t cmd;
1092b2fc195eSAndrew Gallatin 	int status;
1093b2fc195eSAndrew Gallatin 
10941e413cf9SAndrew Gallatin 	if (mxge_always_promisc)
10951e413cf9SAndrew Gallatin 		promisc = 1;
10961e413cf9SAndrew Gallatin 
1097b2fc195eSAndrew Gallatin 	if (promisc)
10985e7d8541SAndrew Gallatin 		status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC,
1099b2fc195eSAndrew Gallatin 				       &cmd);
1100b2fc195eSAndrew Gallatin 	else
11015e7d8541SAndrew Gallatin 		status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC,
1102b2fc195eSAndrew Gallatin 				       &cmd);
1103b2fc195eSAndrew Gallatin 
1104b2fc195eSAndrew Gallatin 	if (status) {
1105b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Failed to set promisc mode\n");
1106b2fc195eSAndrew Gallatin 	}
1107b2fc195eSAndrew Gallatin }
1108b2fc195eSAndrew Gallatin 
11090fa7f681SAndrew Gallatin static void
11100fa7f681SAndrew Gallatin mxge_set_multicast_list(mxge_softc_t *sc)
11110fa7f681SAndrew Gallatin {
11120fa7f681SAndrew Gallatin 	mxge_cmd_t cmd;
11130fa7f681SAndrew Gallatin 	struct ifmultiaddr *ifma;
11140fa7f681SAndrew Gallatin 	struct ifnet *ifp = sc->ifp;
11150fa7f681SAndrew Gallatin 	int err;
11160fa7f681SAndrew Gallatin 
11170fa7f681SAndrew Gallatin 	/* This firmware is known to not support multicast */
11180fa7f681SAndrew Gallatin 	if (!sc->fw_multicast_support)
11190fa7f681SAndrew Gallatin 		return;
11200fa7f681SAndrew Gallatin 
11210fa7f681SAndrew Gallatin 	/* Disable multicast filtering while we play with the lists*/
11220fa7f681SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_ENABLE_ALLMULTI, &cmd);
11230fa7f681SAndrew Gallatin 	if (err != 0) {
11240fa7f681SAndrew Gallatin 		device_printf(sc->dev, "Failed MXGEFW_ENABLE_ALLMULTI,"
11250fa7f681SAndrew Gallatin 		       " error status: %d\n", err);
11260fa7f681SAndrew Gallatin 		return;
11270fa7f681SAndrew Gallatin 	}
11280fa7f681SAndrew Gallatin 
1129b824b7d8SAndrew Gallatin 	if (sc->adopted_rx_filter_bug)
1130b824b7d8SAndrew Gallatin 		return;
11310fa7f681SAndrew Gallatin 
11320fa7f681SAndrew Gallatin 	if (ifp->if_flags & IFF_ALLMULTI)
11330fa7f681SAndrew Gallatin 		/* request to disable multicast filtering, so quit here */
11340fa7f681SAndrew Gallatin 		return;
11350fa7f681SAndrew Gallatin 
11360fa7f681SAndrew Gallatin 	/* Flush all the filters */
11370fa7f681SAndrew Gallatin 
11380fa7f681SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, &cmd);
11390fa7f681SAndrew Gallatin 	if (err != 0) {
11400fa7f681SAndrew Gallatin 		device_printf(sc->dev,
11410fa7f681SAndrew Gallatin 			      "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS"
11420fa7f681SAndrew Gallatin 			      ", error status: %d\n", err);
11430fa7f681SAndrew Gallatin 		return;
11440fa7f681SAndrew Gallatin 	}
11450fa7f681SAndrew Gallatin 
11460fa7f681SAndrew Gallatin 	/* Walk the multicast list, and add each address */
11470fa7f681SAndrew Gallatin 
1148eb956cd0SRobert Watson 	if_maddr_rlock(ifp);
11490fa7f681SAndrew Gallatin 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
11500fa7f681SAndrew Gallatin 		if (ifma->ifma_addr->sa_family != AF_LINK)
11510fa7f681SAndrew Gallatin 			continue;
11520fa7f681SAndrew Gallatin 		bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
11530fa7f681SAndrew Gallatin 		      &cmd.data0, 4);
11540fa7f681SAndrew Gallatin 		bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr) + 4,
11550fa7f681SAndrew Gallatin 		      &cmd.data1, 2);
11560fa7f681SAndrew Gallatin 		cmd.data0 = htonl(cmd.data0);
11570fa7f681SAndrew Gallatin 		cmd.data1 = htonl(cmd.data1);
11580fa7f681SAndrew Gallatin 		err = mxge_send_cmd(sc, MXGEFW_JOIN_MULTICAST_GROUP, &cmd);
11590fa7f681SAndrew Gallatin 		if (err != 0) {
11600fa7f681SAndrew Gallatin 			device_printf(sc->dev, "Failed "
11610fa7f681SAndrew Gallatin 			       "MXGEFW_JOIN_MULTICAST_GROUP, error status:"
11620fa7f681SAndrew Gallatin 			       "%d\t", err);
11630fa7f681SAndrew Gallatin 			/* abort, leaving multicast filtering off */
1164eb956cd0SRobert Watson 			if_maddr_runlock(ifp);
11650fa7f681SAndrew Gallatin 			return;
11660fa7f681SAndrew Gallatin 		}
11670fa7f681SAndrew Gallatin 	}
1168eb956cd0SRobert Watson 	if_maddr_runlock(ifp);
11690fa7f681SAndrew Gallatin 	/* Enable multicast filtering */
11700fa7f681SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_DISABLE_ALLMULTI, &cmd);
11710fa7f681SAndrew Gallatin 	if (err != 0) {
11720fa7f681SAndrew Gallatin 		device_printf(sc->dev, "Failed MXGEFW_DISABLE_ALLMULTI"
11730fa7f681SAndrew Gallatin 		       ", error status: %d\n", err);
11740fa7f681SAndrew Gallatin 	}
11750fa7f681SAndrew Gallatin }
11760fa7f681SAndrew Gallatin 
1177b2fc195eSAndrew Gallatin static int
1178053e637fSAndrew Gallatin mxge_max_mtu(mxge_softc_t *sc)
1179053e637fSAndrew Gallatin {
1180053e637fSAndrew Gallatin 	mxge_cmd_t cmd;
1181053e637fSAndrew Gallatin 	int status;
1182053e637fSAndrew Gallatin 
1183c792928fSAndrew Gallatin 	if (MJUMPAGESIZE - MXGEFW_PAD >  MXGEFW_MAX_MTU)
1184c792928fSAndrew Gallatin 		return  MXGEFW_MAX_MTU - MXGEFW_PAD;
1185053e637fSAndrew Gallatin 
1186053e637fSAndrew Gallatin 	/* try to set nbufs to see if it we can
1187053e637fSAndrew Gallatin 	   use virtually contiguous jumbos */
1188053e637fSAndrew Gallatin 	cmd.data0 = 0;
1189053e637fSAndrew Gallatin 	status = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
1190053e637fSAndrew Gallatin 			       &cmd);
1191053e637fSAndrew Gallatin 	if (status == 0)
1192c792928fSAndrew Gallatin 		return  MXGEFW_MAX_MTU - MXGEFW_PAD;
1193053e637fSAndrew Gallatin 
1194053e637fSAndrew Gallatin 	/* otherwise, we're limited to MJUMPAGESIZE */
1195053e637fSAndrew Gallatin 	return MJUMPAGESIZE - MXGEFW_PAD;
1196053e637fSAndrew Gallatin }
1197053e637fSAndrew Gallatin 
1198053e637fSAndrew Gallatin static int
1199adae7080SAndrew Gallatin mxge_reset(mxge_softc_t *sc, int interrupts_setup)
1200b2fc195eSAndrew Gallatin {
12011e413cf9SAndrew Gallatin 	struct mxge_slice_state *ss;
12021e413cf9SAndrew Gallatin 	mxge_rx_done_t *rx_done;
12031e413cf9SAndrew Gallatin 	volatile uint32_t *irq_claim;
12046d87a65dSAndrew Gallatin 	mxge_cmd_t cmd;
12051e413cf9SAndrew Gallatin 	int slice, status;
1206b2fc195eSAndrew Gallatin 
1207b2fc195eSAndrew Gallatin 	/* try to send a reset command to the card to see if it
1208b2fc195eSAndrew Gallatin 	   is alive */
1209b2fc195eSAndrew Gallatin 	memset(&cmd, 0, sizeof (cmd));
12105e7d8541SAndrew Gallatin 	status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
1211b2fc195eSAndrew Gallatin 	if (status != 0) {
1212b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "failed reset\n");
1213b2fc195eSAndrew Gallatin 		return ENXIO;
1214b2fc195eSAndrew Gallatin 	}
1215b2fc195eSAndrew Gallatin 
1216091feecdSAndrew Gallatin 	mxge_dummy_rdma(sc, 1);
1217091feecdSAndrew Gallatin 
12181e413cf9SAndrew Gallatin 
12191e413cf9SAndrew Gallatin 	/* set the intrq size */
12201e413cf9SAndrew Gallatin 	cmd.data0 = sc->rx_ring_size;
12211e413cf9SAndrew Gallatin 	status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
12221e413cf9SAndrew Gallatin 
12231e413cf9SAndrew Gallatin 	/*
12241e413cf9SAndrew Gallatin 	 * Even though we already know how many slices are supported
12251e413cf9SAndrew Gallatin 	 * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES
12261e413cf9SAndrew Gallatin 	 * has magic side effects, and must be called after a reset.
12271e413cf9SAndrew Gallatin 	 * It must be called prior to calling any RSS related cmds,
12281e413cf9SAndrew Gallatin 	 * including assigning an interrupt queue for anything but
12291e413cf9SAndrew Gallatin 	 * slice 0.  It must also be called *after*
12301e413cf9SAndrew Gallatin 	 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by
12311e413cf9SAndrew Gallatin 	 * the firmware to compute offsets.
12321e413cf9SAndrew Gallatin 	 */
12331e413cf9SAndrew Gallatin 
12341e413cf9SAndrew Gallatin 	if (sc->num_slices > 1) {
12351e413cf9SAndrew Gallatin 		/* ask the maximum number of slices it supports */
12361e413cf9SAndrew Gallatin 		status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
12371e413cf9SAndrew Gallatin 					   &cmd);
12381e413cf9SAndrew Gallatin 		if (status != 0) {
12391e413cf9SAndrew Gallatin 			device_printf(sc->dev,
12401e413cf9SAndrew Gallatin 				      "failed to get number of slices\n");
12411e413cf9SAndrew Gallatin 			return status;
12421e413cf9SAndrew Gallatin 		}
12431e413cf9SAndrew Gallatin 		/*
12441e413cf9SAndrew Gallatin 		 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior
12451e413cf9SAndrew Gallatin 		 * to setting up the interrupt queue DMA
12461e413cf9SAndrew Gallatin 		 */
12471e413cf9SAndrew Gallatin 		cmd.data0 = sc->num_slices;
12481e413cf9SAndrew Gallatin 		cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
1249c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
1250c6cb3e3fSAndrew Gallatin 		cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
1251c6cb3e3fSAndrew Gallatin #endif
12521e413cf9SAndrew Gallatin 		status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES,
12531e413cf9SAndrew Gallatin 					   &cmd);
12541e413cf9SAndrew Gallatin 		if (status != 0) {
12551e413cf9SAndrew Gallatin 			device_printf(sc->dev,
12561e413cf9SAndrew Gallatin 				      "failed to set number of slices\n");
12571e413cf9SAndrew Gallatin 			return status;
12581e413cf9SAndrew Gallatin 		}
12591e413cf9SAndrew Gallatin 	}
12601e413cf9SAndrew Gallatin 
12611e413cf9SAndrew Gallatin 
1262adae7080SAndrew Gallatin 	if (interrupts_setup) {
1263b2fc195eSAndrew Gallatin 		/* Now exchange information about interrupts  */
12641e413cf9SAndrew Gallatin 		for (slice = 0; slice < sc->num_slices; slice++) {
12651e413cf9SAndrew Gallatin 			rx_done = &sc->ss[slice].rx_done;
12661e413cf9SAndrew Gallatin 			memset(rx_done->entry, 0, sc->rx_ring_size);
12671e413cf9SAndrew Gallatin 			cmd.data0 = MXGE_LOWPART_TO_U32(rx_done->dma.bus_addr);
12681e413cf9SAndrew Gallatin 			cmd.data1 = MXGE_HIGHPART_TO_U32(rx_done->dma.bus_addr);
12691e413cf9SAndrew Gallatin 			cmd.data2 = slice;
12701e413cf9SAndrew Gallatin 			status |= mxge_send_cmd(sc,
12711e413cf9SAndrew Gallatin 						MXGEFW_CMD_SET_INTRQ_DMA,
12721e413cf9SAndrew Gallatin 						&cmd);
12731e413cf9SAndrew Gallatin 		}
1274adae7080SAndrew Gallatin 	}
1275b2fc195eSAndrew Gallatin 
12766d87a65dSAndrew Gallatin 	status |= mxge_send_cmd(sc,
12775e7d8541SAndrew Gallatin 				MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd);
12785e7d8541SAndrew Gallatin 
12795e7d8541SAndrew Gallatin 
12805e7d8541SAndrew Gallatin 	sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0);
12815e7d8541SAndrew Gallatin 
12825e7d8541SAndrew Gallatin 	status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd);
12831e413cf9SAndrew Gallatin 	irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0);
12845e7d8541SAndrew Gallatin 
12855e7d8541SAndrew Gallatin 
12865e7d8541SAndrew Gallatin 	status |= mxge_send_cmd(sc,  MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET,
12876d87a65dSAndrew Gallatin 				&cmd);
12885e7d8541SAndrew Gallatin 	sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0);
1289b2fc195eSAndrew Gallatin 	if (status != 0) {
1290b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "failed set interrupt parameters\n");
1291b2fc195eSAndrew Gallatin 		return status;
1292b2fc195eSAndrew Gallatin 	}
1293b2fc195eSAndrew Gallatin 
12945e7d8541SAndrew Gallatin 
12955e7d8541SAndrew Gallatin 	*sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay);
12965e7d8541SAndrew Gallatin 
12975e7d8541SAndrew Gallatin 
12985e7d8541SAndrew Gallatin 	/* run a DMA benchmark */
12998fe615baSAndrew Gallatin 	(void) mxge_dma_test(sc, MXGEFW_DMA_TEST);
13005e7d8541SAndrew Gallatin 
13011e413cf9SAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++) {
13021e413cf9SAndrew Gallatin 		ss = &sc->ss[slice];
13031e413cf9SAndrew Gallatin 
13041e413cf9SAndrew Gallatin 		ss->irq_claim = irq_claim + (2 * slice);
1305b2fc195eSAndrew Gallatin 		/* reset mcp/driver shared state back to 0 */
13061e413cf9SAndrew Gallatin 		ss->rx_done.idx = 0;
13071e413cf9SAndrew Gallatin 		ss->rx_done.cnt = 0;
13081e413cf9SAndrew Gallatin 		ss->tx.req = 0;
13091e413cf9SAndrew Gallatin 		ss->tx.done = 0;
13101e413cf9SAndrew Gallatin 		ss->tx.pkt_done = 0;
1311c6cb3e3fSAndrew Gallatin 		ss->tx.queue_active = 0;
1312c6cb3e3fSAndrew Gallatin 		ss->tx.activate = 0;
1313c6cb3e3fSAndrew Gallatin 		ss->tx.deactivate = 0;
13141e413cf9SAndrew Gallatin 		ss->tx.wake = 0;
13151e413cf9SAndrew Gallatin 		ss->tx.defrag = 0;
13161e413cf9SAndrew Gallatin 		ss->tx.stall = 0;
13171e413cf9SAndrew Gallatin 		ss->rx_big.cnt = 0;
13181e413cf9SAndrew Gallatin 		ss->rx_small.cnt = 0;
131926dd49c6SAndrew Gallatin 		ss->lc.lro_bad_csum = 0;
132026dd49c6SAndrew Gallatin 		ss->lc.lro_queued = 0;
132126dd49c6SAndrew Gallatin 		ss->lc.lro_flushed = 0;
13221e413cf9SAndrew Gallatin 		if (ss->fw_stats != NULL) {
1323a393336bSAndrew Gallatin 			bzero(ss->fw_stats, sizeof *ss->fw_stats);
13241e413cf9SAndrew Gallatin 		}
13251e413cf9SAndrew Gallatin 	}
1326b2fc195eSAndrew Gallatin 	sc->rdma_tags_available = 15;
13276d87a65dSAndrew Gallatin 	status = mxge_update_mac_address(sc);
1328bb8ddc66SAndrew Gallatin 	mxge_change_promisc(sc, sc->ifp->if_flags & IFF_PROMISC);
13296d87a65dSAndrew Gallatin 	mxge_change_pause(sc, sc->pause);
13300fa7f681SAndrew Gallatin 	mxge_set_multicast_list(sc);
133165c69066SAndrew Gallatin 	if (sc->throttle) {
133265c69066SAndrew Gallatin 		cmd.data0 = sc->throttle;
133365c69066SAndrew Gallatin 		if (mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR,
133465c69066SAndrew Gallatin 				  &cmd)) {
133565c69066SAndrew Gallatin 			device_printf(sc->dev,
133665c69066SAndrew Gallatin 				      "can't enable throttle\n");
133765c69066SAndrew Gallatin 		}
133865c69066SAndrew Gallatin 	}
1339b2fc195eSAndrew Gallatin 	return status;
1340b2fc195eSAndrew Gallatin }
1341b2fc195eSAndrew Gallatin 
1342b2fc195eSAndrew Gallatin static int
134365c69066SAndrew Gallatin mxge_change_throttle(SYSCTL_HANDLER_ARGS)
134465c69066SAndrew Gallatin {
134565c69066SAndrew Gallatin 	mxge_cmd_t cmd;
134665c69066SAndrew Gallatin 	mxge_softc_t *sc;
134765c69066SAndrew Gallatin 	int err;
134865c69066SAndrew Gallatin 	unsigned int throttle;
134965c69066SAndrew Gallatin 
135065c69066SAndrew Gallatin 	sc = arg1;
135165c69066SAndrew Gallatin 	throttle = sc->throttle;
135265c69066SAndrew Gallatin 	err = sysctl_handle_int(oidp, &throttle, arg2, req);
135365c69066SAndrew Gallatin 	if (err != 0) {
135465c69066SAndrew Gallatin 		return err;
135565c69066SAndrew Gallatin 	}
135665c69066SAndrew Gallatin 
135765c69066SAndrew Gallatin 	if (throttle == sc->throttle)
135865c69066SAndrew Gallatin 		return 0;
135965c69066SAndrew Gallatin 
136065c69066SAndrew Gallatin 	if (throttle < MXGE_MIN_THROTTLE || throttle > MXGE_MAX_THROTTLE)
136165c69066SAndrew Gallatin 		return EINVAL;
136265c69066SAndrew Gallatin 
136365c69066SAndrew Gallatin 	mtx_lock(&sc->driver_mtx);
136465c69066SAndrew Gallatin 	cmd.data0 = throttle;
136565c69066SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd);
136665c69066SAndrew Gallatin 	if (err == 0)
136765c69066SAndrew Gallatin 		sc->throttle = throttle;
136865c69066SAndrew Gallatin 	mtx_unlock(&sc->driver_mtx);
136965c69066SAndrew Gallatin 	return err;
137065c69066SAndrew Gallatin }
137165c69066SAndrew Gallatin 
137265c69066SAndrew Gallatin static int
13736d87a65dSAndrew Gallatin mxge_change_intr_coal(SYSCTL_HANDLER_ARGS)
1374b2fc195eSAndrew Gallatin {
13756d87a65dSAndrew Gallatin 	mxge_softc_t *sc;
1376b2fc195eSAndrew Gallatin 	unsigned int intr_coal_delay;
1377b2fc195eSAndrew Gallatin 	int err;
1378b2fc195eSAndrew Gallatin 
1379b2fc195eSAndrew Gallatin 	sc = arg1;
1380b2fc195eSAndrew Gallatin 	intr_coal_delay = sc->intr_coal_delay;
1381b2fc195eSAndrew Gallatin 	err = sysctl_handle_int(oidp, &intr_coal_delay, arg2, req);
1382b2fc195eSAndrew Gallatin 	if (err != 0) {
1383b2fc195eSAndrew Gallatin 		return err;
1384b2fc195eSAndrew Gallatin 	}
1385b2fc195eSAndrew Gallatin 	if (intr_coal_delay == sc->intr_coal_delay)
1386b2fc195eSAndrew Gallatin 		return 0;
1387b2fc195eSAndrew Gallatin 
1388b2fc195eSAndrew Gallatin 	if (intr_coal_delay == 0 || intr_coal_delay > 1000*1000)
1389b2fc195eSAndrew Gallatin 		return EINVAL;
1390b2fc195eSAndrew Gallatin 
1391a98d6cd7SAndrew Gallatin 	mtx_lock(&sc->driver_mtx);
13925e7d8541SAndrew Gallatin 	*sc->intr_coal_delay_ptr = htobe32(intr_coal_delay);
1393b2fc195eSAndrew Gallatin 	sc->intr_coal_delay = intr_coal_delay;
13945e7d8541SAndrew Gallatin 
1395a98d6cd7SAndrew Gallatin 	mtx_unlock(&sc->driver_mtx);
1396b2fc195eSAndrew Gallatin 	return err;
1397b2fc195eSAndrew Gallatin }
1398b2fc195eSAndrew Gallatin 
1399b2fc195eSAndrew Gallatin static int
14006d87a65dSAndrew Gallatin mxge_change_flow_control(SYSCTL_HANDLER_ARGS)
1401b2fc195eSAndrew Gallatin {
14026d87a65dSAndrew Gallatin 	mxge_softc_t *sc;
1403b2fc195eSAndrew Gallatin 	unsigned int enabled;
1404b2fc195eSAndrew Gallatin 	int err;
1405b2fc195eSAndrew Gallatin 
1406b2fc195eSAndrew Gallatin 	sc = arg1;
1407b2fc195eSAndrew Gallatin 	enabled = sc->pause;
1408b2fc195eSAndrew Gallatin 	err = sysctl_handle_int(oidp, &enabled, arg2, req);
1409b2fc195eSAndrew Gallatin 	if (err != 0) {
1410b2fc195eSAndrew Gallatin 		return err;
1411b2fc195eSAndrew Gallatin 	}
1412b2fc195eSAndrew Gallatin 	if (enabled == sc->pause)
1413b2fc195eSAndrew Gallatin 		return 0;
1414b2fc195eSAndrew Gallatin 
1415a98d6cd7SAndrew Gallatin 	mtx_lock(&sc->driver_mtx);
14166d87a65dSAndrew Gallatin 	err = mxge_change_pause(sc, enabled);
1417a98d6cd7SAndrew Gallatin 	mtx_unlock(&sc->driver_mtx);
1418b2fc195eSAndrew Gallatin 	return err;
1419b2fc195eSAndrew Gallatin }
1420b2fc195eSAndrew Gallatin 
1421b2fc195eSAndrew Gallatin static int
14226d87a65dSAndrew Gallatin mxge_handle_be32(SYSCTL_HANDLER_ARGS)
1423b2fc195eSAndrew Gallatin {
1424b2fc195eSAndrew Gallatin 	int err;
1425b2fc195eSAndrew Gallatin 
1426b2fc195eSAndrew Gallatin 	if (arg1 == NULL)
1427b2fc195eSAndrew Gallatin 		return EFAULT;
1428b2fc195eSAndrew Gallatin 	arg2 = be32toh(*(int *)arg1);
1429b2fc195eSAndrew Gallatin 	arg1 = NULL;
1430b2fc195eSAndrew Gallatin 	err = sysctl_handle_int(oidp, arg1, arg2, req);
1431b2fc195eSAndrew Gallatin 
1432b2fc195eSAndrew Gallatin 	return err;
1433b2fc195eSAndrew Gallatin }
1434b2fc195eSAndrew Gallatin 
1435b2fc195eSAndrew Gallatin static void
14361e413cf9SAndrew Gallatin mxge_rem_sysctls(mxge_softc_t *sc)
14371e413cf9SAndrew Gallatin {
14381e413cf9SAndrew Gallatin 	struct mxge_slice_state *ss;
14391e413cf9SAndrew Gallatin 	int slice;
14401e413cf9SAndrew Gallatin 
14411e413cf9SAndrew Gallatin 	if (sc->slice_sysctl_tree == NULL)
14421e413cf9SAndrew Gallatin 		return;
14431e413cf9SAndrew Gallatin 
14441e413cf9SAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++) {
14451e413cf9SAndrew Gallatin 		ss = &sc->ss[slice];
14461e413cf9SAndrew Gallatin 		if (ss == NULL || ss->sysctl_tree == NULL)
14471e413cf9SAndrew Gallatin 			continue;
14481e413cf9SAndrew Gallatin 		sysctl_ctx_free(&ss->sysctl_ctx);
14491e413cf9SAndrew Gallatin 		ss->sysctl_tree = NULL;
14501e413cf9SAndrew Gallatin 	}
14511e413cf9SAndrew Gallatin 	sysctl_ctx_free(&sc->slice_sysctl_ctx);
14521e413cf9SAndrew Gallatin 	sc->slice_sysctl_tree = NULL;
14531e413cf9SAndrew Gallatin }
14541e413cf9SAndrew Gallatin 
14551e413cf9SAndrew Gallatin static void
14566d87a65dSAndrew Gallatin mxge_add_sysctls(mxge_softc_t *sc)
1457b2fc195eSAndrew Gallatin {
1458b2fc195eSAndrew Gallatin 	struct sysctl_ctx_list *ctx;
1459b2fc195eSAndrew Gallatin 	struct sysctl_oid_list *children;
14605e7d8541SAndrew Gallatin 	mcp_irq_data_t *fw;
14611e413cf9SAndrew Gallatin 	struct mxge_slice_state *ss;
14621e413cf9SAndrew Gallatin 	int slice;
14631e413cf9SAndrew Gallatin 	char slice_num[8];
1464b2fc195eSAndrew Gallatin 
1465b2fc195eSAndrew Gallatin 	ctx = device_get_sysctl_ctx(sc->dev);
1466b2fc195eSAndrew Gallatin 	children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
14671e413cf9SAndrew Gallatin 	fw = sc->ss[0].fw_stats;
1468b2fc195eSAndrew Gallatin 
14695e7d8541SAndrew Gallatin 	/* random information */
14705e7d8541SAndrew Gallatin 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
14715e7d8541SAndrew Gallatin 		       "firmware_version",
1472f0188618SHans Petter Selasky 		       CTLFLAG_RD, sc->fw_version,
14735e7d8541SAndrew Gallatin 		       0, "firmware version");
14745e7d8541SAndrew Gallatin 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
14755e7d8541SAndrew Gallatin 		       "serial_number",
1476f0188618SHans Petter Selasky 		       CTLFLAG_RD, sc->serial_number_string,
14775e7d8541SAndrew Gallatin 		       0, "serial number");
14785e7d8541SAndrew Gallatin 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
14795e7d8541SAndrew Gallatin 		       "product_code",
1480f0188618SHans Petter Selasky 		       CTLFLAG_RD, sc->product_code_string,
14815e7d8541SAndrew Gallatin 		       0, "product_code");
14825e7d8541SAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1483d91b1b49SAndrew Gallatin 		       "pcie_link_width",
1484d91b1b49SAndrew Gallatin 		       CTLFLAG_RD, &sc->link_width,
1485d91b1b49SAndrew Gallatin 		       0, "tx_boundary");
1486d91b1b49SAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
14875e7d8541SAndrew Gallatin 		       "tx_boundary",
14881e413cf9SAndrew Gallatin 		       CTLFLAG_RD, &sc->tx_boundary,
14895e7d8541SAndrew Gallatin 		       0, "tx_boundary");
14905e7d8541SAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1491091feecdSAndrew Gallatin 		       "write_combine",
1492091feecdSAndrew Gallatin 		       CTLFLAG_RD, &sc->wc,
1493091feecdSAndrew Gallatin 		       0, "write combining PIO?");
1494091feecdSAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
14955e7d8541SAndrew Gallatin 		       "read_dma_MBs",
14965e7d8541SAndrew Gallatin 		       CTLFLAG_RD, &sc->read_dma,
14975e7d8541SAndrew Gallatin 		       0, "DMA Read speed in MB/s");
14985e7d8541SAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
14995e7d8541SAndrew Gallatin 		       "write_dma_MBs",
15005e7d8541SAndrew Gallatin 		       CTLFLAG_RD, &sc->write_dma,
15015e7d8541SAndrew Gallatin 		       0, "DMA Write speed in MB/s");
15025e7d8541SAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
15035e7d8541SAndrew Gallatin 		       "read_write_dma_MBs",
15045e7d8541SAndrew Gallatin 		       CTLFLAG_RD, &sc->read_write_dma,
15055e7d8541SAndrew Gallatin 		       0, "DMA concurrent Read/Write speed in MB/s");
1506a393336bSAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1507a393336bSAndrew Gallatin 		       "watchdog_resets",
1508a393336bSAndrew Gallatin 		       CTLFLAG_RD, &sc->watchdog_resets,
1509a393336bSAndrew Gallatin 		       0, "Number of times NIC was reset");
15105e7d8541SAndrew Gallatin 
15115e7d8541SAndrew Gallatin 
15125e7d8541SAndrew Gallatin 	/* performance related tunables */
1513b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1514b2fc195eSAndrew Gallatin 			"intr_coal_delay",
1515b2fc195eSAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RW, sc,
15166d87a65dSAndrew Gallatin 			0, mxge_change_intr_coal,
1517b2fc195eSAndrew Gallatin 			"I", "interrupt coalescing delay in usecs");
1518b2fc195eSAndrew Gallatin 
1519b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
152065c69066SAndrew Gallatin 			"throttle",
152165c69066SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RW, sc,
152265c69066SAndrew Gallatin 			0, mxge_change_throttle,
152365c69066SAndrew Gallatin 			"I", "transmit throttling");
152465c69066SAndrew Gallatin 
152565c69066SAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1526b2fc195eSAndrew Gallatin 			"flow_control_enabled",
1527b2fc195eSAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RW, sc,
15286d87a65dSAndrew Gallatin 			0, mxge_change_flow_control,
1529b2fc195eSAndrew Gallatin 			"I", "interrupt coalescing delay in usecs");
1530b2fc195eSAndrew Gallatin 
1531b2fc195eSAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
15325e7d8541SAndrew Gallatin 		       "deassert_wait",
15335e7d8541SAndrew Gallatin 		       CTLFLAG_RW, &mxge_deassert_wait,
15345e7d8541SAndrew Gallatin 		       0, "Wait for IRQ line to go low in ihandler");
1535b2fc195eSAndrew Gallatin 
1536b2fc195eSAndrew Gallatin 	/* stats block from firmware is in network byte order.
1537b2fc195eSAndrew Gallatin 	   Need to swap it */
1538b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1539b2fc195eSAndrew Gallatin 			"link_up",
1540b2fc195eSAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD, &fw->link_up,
15416d87a65dSAndrew Gallatin 			0, mxge_handle_be32,
1542b2fc195eSAndrew Gallatin 			"I", "link up");
1543b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1544b2fc195eSAndrew Gallatin 			"rdma_tags_available",
1545b2fc195eSAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD, &fw->rdma_tags_available,
15466d87a65dSAndrew Gallatin 			0, mxge_handle_be32,
1547b2fc195eSAndrew Gallatin 			"I", "rdma_tags_available");
1548b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1549adae7080SAndrew Gallatin 			"dropped_bad_crc32",
1550adae7080SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD,
1551adae7080SAndrew Gallatin 			&fw->dropped_bad_crc32,
15526d87a65dSAndrew Gallatin 			0, mxge_handle_be32,
1553adae7080SAndrew Gallatin 			"I", "dropped_bad_crc32");
1554adae7080SAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1555adae7080SAndrew Gallatin 			"dropped_bad_phy",
1556adae7080SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD,
1557adae7080SAndrew Gallatin 			&fw->dropped_bad_phy,
1558adae7080SAndrew Gallatin 			0, mxge_handle_be32,
1559adae7080SAndrew Gallatin 			"I", "dropped_bad_phy");
1560b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1561b2fc195eSAndrew Gallatin 			"dropped_link_error_or_filtered",
1562b2fc195eSAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD,
1563b2fc195eSAndrew Gallatin 			&fw->dropped_link_error_or_filtered,
15646d87a65dSAndrew Gallatin 			0, mxge_handle_be32,
1565b2fc195eSAndrew Gallatin 			"I", "dropped_link_error_or_filtered");
1566b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1567adae7080SAndrew Gallatin 			"dropped_link_overflow",
1568adae7080SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_overflow,
1569adae7080SAndrew Gallatin 			0, mxge_handle_be32,
1570adae7080SAndrew Gallatin 			"I", "dropped_link_overflow");
1571adae7080SAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
15720fa7f681SAndrew Gallatin 			"dropped_multicast_filtered",
15730fa7f681SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD,
15740fa7f681SAndrew Gallatin 			&fw->dropped_multicast_filtered,
15750fa7f681SAndrew Gallatin 			0, mxge_handle_be32,
15760fa7f681SAndrew Gallatin 			"I", "dropped_multicast_filtered");
15770fa7f681SAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1578adae7080SAndrew Gallatin 			"dropped_no_big_buffer",
1579adae7080SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_big_buffer,
15806d87a65dSAndrew Gallatin 			0, mxge_handle_be32,
1581adae7080SAndrew Gallatin 			"I", "dropped_no_big_buffer");
1582b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1583b2fc195eSAndrew Gallatin 			"dropped_no_small_buffer",
1584b2fc195eSAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD,
1585b2fc195eSAndrew Gallatin 			&fw->dropped_no_small_buffer,
15866d87a65dSAndrew Gallatin 			0, mxge_handle_be32,
1587b2fc195eSAndrew Gallatin 			"I", "dropped_no_small_buffer");
1588b2fc195eSAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1589adae7080SAndrew Gallatin 			"dropped_overrun",
1590adae7080SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_overrun,
15916d87a65dSAndrew Gallatin 			0, mxge_handle_be32,
1592adae7080SAndrew Gallatin 			"I", "dropped_overrun");
1593adae7080SAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1594adae7080SAndrew Gallatin 			"dropped_pause",
1595adae7080SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD,
1596adae7080SAndrew Gallatin 			&fw->dropped_pause,
1597adae7080SAndrew Gallatin 			0, mxge_handle_be32,
1598adae7080SAndrew Gallatin 			"I", "dropped_pause");
1599adae7080SAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1600adae7080SAndrew Gallatin 			"dropped_runt",
1601adae7080SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_runt,
1602adae7080SAndrew Gallatin 			0, mxge_handle_be32,
1603adae7080SAndrew Gallatin 			"I", "dropped_runt");
1604b2fc195eSAndrew Gallatin 
1605a0394e33SAndrew Gallatin 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1606a0394e33SAndrew Gallatin 			"dropped_unicast_filtered",
1607a0394e33SAndrew Gallatin 			CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered,
1608a0394e33SAndrew Gallatin 			0, mxge_handle_be32,
1609a0394e33SAndrew Gallatin 			"I", "dropped_unicast_filtered");
1610a0394e33SAndrew Gallatin 
16115e7d8541SAndrew Gallatin 	/* verbose printing? */
1612b2fc195eSAndrew Gallatin 	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
16135e7d8541SAndrew Gallatin 		       "verbose",
16145e7d8541SAndrew Gallatin 		       CTLFLAG_RW, &mxge_verbose,
16155e7d8541SAndrew Gallatin 		       0, "verbose printing");
1616b2fc195eSAndrew Gallatin 
16171e413cf9SAndrew Gallatin 	/* add counters exported for debugging from all slices */
16181e413cf9SAndrew Gallatin 	sysctl_ctx_init(&sc->slice_sysctl_ctx);
16191e413cf9SAndrew Gallatin 	sc->slice_sysctl_tree =
16201e413cf9SAndrew Gallatin 		SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx, children, OID_AUTO,
16211e413cf9SAndrew Gallatin 				"slice", CTLFLAG_RD, 0, "");
16221e413cf9SAndrew Gallatin 
16231e413cf9SAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++) {
16241e413cf9SAndrew Gallatin 		ss = &sc->ss[slice];
16251e413cf9SAndrew Gallatin 		sysctl_ctx_init(&ss->sysctl_ctx);
16261e413cf9SAndrew Gallatin 		ctx = &ss->sysctl_ctx;
16271e413cf9SAndrew Gallatin 		children = SYSCTL_CHILDREN(sc->slice_sysctl_tree);
16281e413cf9SAndrew Gallatin 		sprintf(slice_num, "%d", slice);
16291e413cf9SAndrew Gallatin 		ss->sysctl_tree =
16301e413cf9SAndrew Gallatin 			SYSCTL_ADD_NODE(ctx, children, OID_AUTO, slice_num,
16311e413cf9SAndrew Gallatin 					CTLFLAG_RD, 0, "");
16321e413cf9SAndrew Gallatin 		children = SYSCTL_CHILDREN(ss->sysctl_tree);
1633053e637fSAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
16341e413cf9SAndrew Gallatin 			       "rx_small_cnt",
16351e413cf9SAndrew Gallatin 			       CTLFLAG_RD, &ss->rx_small.cnt,
16361e413cf9SAndrew Gallatin 			       0, "rx_small_cnt");
16371e413cf9SAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
16381e413cf9SAndrew Gallatin 			       "rx_big_cnt",
16391e413cf9SAndrew Gallatin 			       CTLFLAG_RD, &ss->rx_big.cnt,
16401e413cf9SAndrew Gallatin 			       0, "rx_small_cnt");
1641e936121dSHans Petter Selasky 		SYSCTL_ADD_U64(ctx, children, OID_AUTO,
164226dd49c6SAndrew Gallatin 			       "lro_flushed", CTLFLAG_RD, &ss->lc.lro_flushed,
1643053e637fSAndrew Gallatin 			       0, "number of lro merge queues flushed");
1644053e637fSAndrew Gallatin 
1645e936121dSHans Petter Selasky 		SYSCTL_ADD_U64(ctx, children, OID_AUTO,
164626dd49c6SAndrew Gallatin 			       "lro_bad_csum", CTLFLAG_RD, &ss->lc.lro_bad_csum,
164726dd49c6SAndrew Gallatin 			       0, "number of bad csums preventing LRO");
164826dd49c6SAndrew Gallatin 
1649e936121dSHans Petter Selasky 		SYSCTL_ADD_U64(ctx, children, OID_AUTO,
165026dd49c6SAndrew Gallatin 			       "lro_queued", CTLFLAG_RD, &ss->lc.lro_queued,
16511e413cf9SAndrew Gallatin 			       0, "number of frames appended to lro merge"
16521e413cf9SAndrew Gallatin 			       "queues");
1653053e637fSAndrew Gallatin 
1654c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING
16551e413cf9SAndrew Gallatin 		/* only transmit from slice 0 for now */
16561e413cf9SAndrew Gallatin 		if (slice > 0)
16571e413cf9SAndrew Gallatin 			continue;
1658c6cb3e3fSAndrew Gallatin #endif
1659c6cb3e3fSAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1660c6cb3e3fSAndrew Gallatin 			       "tx_req",
1661c6cb3e3fSAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.req,
1662c6cb3e3fSAndrew Gallatin 			       0, "tx_req");
16631e413cf9SAndrew Gallatin 
16641e413cf9SAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
16651e413cf9SAndrew Gallatin 			       "tx_done",
16661e413cf9SAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.done,
16671e413cf9SAndrew Gallatin 			       0, "tx_done");
16681e413cf9SAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
16691e413cf9SAndrew Gallatin 			       "tx_pkt_done",
16701e413cf9SAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.pkt_done,
16711e413cf9SAndrew Gallatin 			       0, "tx_done");
16721e413cf9SAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
16731e413cf9SAndrew Gallatin 			       "tx_stall",
16741e413cf9SAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.stall,
16751e413cf9SAndrew Gallatin 			       0, "tx_stall");
16761e413cf9SAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
16771e413cf9SAndrew Gallatin 			       "tx_wake",
16781e413cf9SAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.wake,
16791e413cf9SAndrew Gallatin 			       0, "tx_wake");
16801e413cf9SAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
16811e413cf9SAndrew Gallatin 			       "tx_defrag",
16821e413cf9SAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.defrag,
16831e413cf9SAndrew Gallatin 			       0, "tx_defrag");
1684c6cb3e3fSAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1685c6cb3e3fSAndrew Gallatin 			       "tx_queue_active",
1686c6cb3e3fSAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.queue_active,
1687c6cb3e3fSAndrew Gallatin 			       0, "tx_queue_active");
1688c6cb3e3fSAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1689c6cb3e3fSAndrew Gallatin 			       "tx_activate",
1690c6cb3e3fSAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.activate,
1691c6cb3e3fSAndrew Gallatin 			       0, "tx_activate");
1692c6cb3e3fSAndrew Gallatin 		SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1693c6cb3e3fSAndrew Gallatin 			       "tx_deactivate",
1694c6cb3e3fSAndrew Gallatin 			       CTLFLAG_RD, &ss->tx.deactivate,
1695c6cb3e3fSAndrew Gallatin 			       0, "tx_deactivate");
16961e413cf9SAndrew Gallatin 	}
1697b2fc195eSAndrew Gallatin }
1698b2fc195eSAndrew Gallatin 
1699b2fc195eSAndrew Gallatin /* copy an array of mcp_kreq_ether_send_t's to the mcp.  Copy
1700b2fc195eSAndrew Gallatin    backwards one at a time and handle ring wraps */
1701b2fc195eSAndrew Gallatin 
1702b2fc195eSAndrew Gallatin static inline void
17031e413cf9SAndrew Gallatin mxge_submit_req_backwards(mxge_tx_ring_t *tx,
1704b2fc195eSAndrew Gallatin 			    mcp_kreq_ether_send_t *src, int cnt)
1705b2fc195eSAndrew Gallatin {
1706b2fc195eSAndrew Gallatin 	int idx, starting_slot;
1707b2fc195eSAndrew Gallatin 	starting_slot = tx->req;
1708b2fc195eSAndrew Gallatin 	while (cnt > 1) {
1709b2fc195eSAndrew Gallatin 		cnt--;
1710b2fc195eSAndrew Gallatin 		idx = (starting_slot + cnt) & tx->mask;
17116d87a65dSAndrew Gallatin 		mxge_pio_copy(&tx->lanai[idx],
1712b2fc195eSAndrew Gallatin 			      &src[cnt], sizeof(*src));
171373c7c83fSAndrew Gallatin 		wmb();
1714b2fc195eSAndrew Gallatin 	}
1715b2fc195eSAndrew Gallatin }
1716b2fc195eSAndrew Gallatin 
1717b2fc195eSAndrew Gallatin /*
1718b2fc195eSAndrew Gallatin  * copy an array of mcp_kreq_ether_send_t's to the mcp.  Copy
1719b2fc195eSAndrew Gallatin  * at most 32 bytes at a time, so as to avoid involving the software
1720b2fc195eSAndrew Gallatin  * pio handler in the nic.   We re-write the first segment's flags
1721b2fc195eSAndrew Gallatin  * to mark them valid only after writing the entire chain
1722b2fc195eSAndrew Gallatin  */
1723b2fc195eSAndrew Gallatin 
1724b2fc195eSAndrew Gallatin static inline void
17251e413cf9SAndrew Gallatin mxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src,
1726b2fc195eSAndrew Gallatin 		  int cnt)
1727b2fc195eSAndrew Gallatin {
1728b2fc195eSAndrew Gallatin 	int idx, i;
1729b2fc195eSAndrew Gallatin 	uint32_t *src_ints;
1730b2fc195eSAndrew Gallatin 	volatile uint32_t *dst_ints;
1731b2fc195eSAndrew Gallatin 	mcp_kreq_ether_send_t *srcp;
1732b2fc195eSAndrew Gallatin 	volatile mcp_kreq_ether_send_t *dstp, *dst;
17335e7d8541SAndrew Gallatin 	uint8_t last_flags;
1734b2fc195eSAndrew Gallatin 
1735b2fc195eSAndrew Gallatin 	idx = tx->req & tx->mask;
1736b2fc195eSAndrew Gallatin 
17375e7d8541SAndrew Gallatin 	last_flags = src->flags;
17385e7d8541SAndrew Gallatin 	src->flags = 0;
173973c7c83fSAndrew Gallatin 	wmb();
1740b2fc195eSAndrew Gallatin 	dst = dstp = &tx->lanai[idx];
1741b2fc195eSAndrew Gallatin 	srcp = src;
1742b2fc195eSAndrew Gallatin 
1743b2fc195eSAndrew Gallatin 	if ((idx + cnt) < tx->mask) {
1744b2fc195eSAndrew Gallatin 		for (i = 0; i < (cnt - 1); i += 2) {
17456d87a65dSAndrew Gallatin 			mxge_pio_copy(dstp, srcp, 2 * sizeof(*src));
174673c7c83fSAndrew Gallatin 			wmb(); /* force write every 32 bytes */
1747b2fc195eSAndrew Gallatin 			srcp += 2;
1748b2fc195eSAndrew Gallatin 			dstp += 2;
1749b2fc195eSAndrew Gallatin 		}
1750b2fc195eSAndrew Gallatin 	} else {
1751b2fc195eSAndrew Gallatin 		/* submit all but the first request, and ensure
1752b2fc195eSAndrew Gallatin 		   that it is submitted below */
17536d87a65dSAndrew Gallatin 		mxge_submit_req_backwards(tx, src, cnt);
1754b2fc195eSAndrew Gallatin 		i = 0;
1755b2fc195eSAndrew Gallatin 	}
1756b2fc195eSAndrew Gallatin 	if (i < cnt) {
1757b2fc195eSAndrew Gallatin 		/* submit the first request */
17586d87a65dSAndrew Gallatin 		mxge_pio_copy(dstp, srcp, sizeof(*src));
175973c7c83fSAndrew Gallatin 		wmb(); /* barrier before setting valid flag */
1760b2fc195eSAndrew Gallatin 	}
1761b2fc195eSAndrew Gallatin 
1762b2fc195eSAndrew Gallatin 	/* re-write the last 32-bits with the valid flags */
17635e7d8541SAndrew Gallatin 	src->flags = last_flags;
1764b2fc195eSAndrew Gallatin 	src_ints = (uint32_t *)src;
1765b2fc195eSAndrew Gallatin 	src_ints+=3;
1766b2fc195eSAndrew Gallatin 	dst_ints = (volatile uint32_t *)dst;
1767b2fc195eSAndrew Gallatin 	dst_ints+=3;
1768b2fc195eSAndrew Gallatin 	*dst_ints =  *src_ints;
1769b2fc195eSAndrew Gallatin 	tx->req += cnt;
177073c7c83fSAndrew Gallatin 	wmb();
1771b2fc195eSAndrew Gallatin }
1772b2fc195eSAndrew Gallatin 
17730a7a780eSAndrew Gallatin static int
17740a7a780eSAndrew Gallatin mxge_parse_tx(struct mxge_slice_state *ss, struct mbuf *m,
17750a7a780eSAndrew Gallatin     struct mxge_pkt_info *pi)
17760a7a780eSAndrew Gallatin {
17770a7a780eSAndrew Gallatin 	struct ether_vlan_header *eh;
17780a7a780eSAndrew Gallatin 	uint16_t etype;
17790a7a780eSAndrew Gallatin 	int tso = m->m_pkthdr.csum_flags & (CSUM_TSO);
17800a7a780eSAndrew Gallatin #if IFCAP_TSO6 && defined(INET6)
17810a7a780eSAndrew Gallatin 	int nxt;
17820a7a780eSAndrew Gallatin #endif
17830a7a780eSAndrew Gallatin 
17840a7a780eSAndrew Gallatin 	eh = mtod(m, struct ether_vlan_header *);
17850a7a780eSAndrew Gallatin 	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
17860a7a780eSAndrew Gallatin 		etype = ntohs(eh->evl_proto);
17870a7a780eSAndrew Gallatin 		pi->ip_off = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
17880a7a780eSAndrew Gallatin 	} else {
17890a7a780eSAndrew Gallatin 		etype = ntohs(eh->evl_encap_proto);
17900a7a780eSAndrew Gallatin 		pi->ip_off = ETHER_HDR_LEN;
17910a7a780eSAndrew Gallatin 	}
17920a7a780eSAndrew Gallatin 
17930a7a780eSAndrew Gallatin 	switch (etype) {
17940a7a780eSAndrew Gallatin 	case ETHERTYPE_IP:
17950a7a780eSAndrew Gallatin 		/*
17960a7a780eSAndrew Gallatin 		 * ensure ip header is in first mbuf, copy it to a
17970a7a780eSAndrew Gallatin 		 * scratch buffer if not
17980a7a780eSAndrew Gallatin 		 */
17990a7a780eSAndrew Gallatin 		pi->ip = (struct ip *)(m->m_data + pi->ip_off);
18000a7a780eSAndrew Gallatin 		pi->ip6 = NULL;
18010a7a780eSAndrew Gallatin 		if (__predict_false(m->m_len < pi->ip_off + sizeof(*pi->ip))) {
18020a7a780eSAndrew Gallatin 			m_copydata(m, 0, pi->ip_off + sizeof(*pi->ip),
18030a7a780eSAndrew Gallatin 			    ss->scratch);
18040a7a780eSAndrew Gallatin 			pi->ip = (struct ip *)(ss->scratch + pi->ip_off);
18050a7a780eSAndrew Gallatin 		}
18060a7a780eSAndrew Gallatin 		pi->ip_hlen = pi->ip->ip_hl << 2;
18070a7a780eSAndrew Gallatin 		if (!tso)
18080a7a780eSAndrew Gallatin 			return 0;
18090a7a780eSAndrew Gallatin 
18100a7a780eSAndrew Gallatin 		if (__predict_false(m->m_len < pi->ip_off + pi->ip_hlen +
18110a7a780eSAndrew Gallatin 		    sizeof(struct tcphdr))) {
18120a7a780eSAndrew Gallatin 			m_copydata(m, 0, pi->ip_off + pi->ip_hlen +
18130a7a780eSAndrew Gallatin 			    sizeof(struct tcphdr), ss->scratch);
18140a7a780eSAndrew Gallatin 			pi->ip = (struct ip *)(ss->scratch + pi->ip_off);
18150a7a780eSAndrew Gallatin 		}
18160a7a780eSAndrew Gallatin 		pi->tcp = (struct tcphdr *)((char *)pi->ip + pi->ip_hlen);
18170a7a780eSAndrew Gallatin 		break;
18180a7a780eSAndrew Gallatin #if IFCAP_TSO6 && defined(INET6)
18190a7a780eSAndrew Gallatin 	case ETHERTYPE_IPV6:
18200a7a780eSAndrew Gallatin 		pi->ip6 = (struct ip6_hdr *)(m->m_data + pi->ip_off);
18210a7a780eSAndrew Gallatin 		if (__predict_false(m->m_len < pi->ip_off + sizeof(*pi->ip6))) {
18220a7a780eSAndrew Gallatin 			m_copydata(m, 0, pi->ip_off + sizeof(*pi->ip6),
18230a7a780eSAndrew Gallatin 			    ss->scratch);
18240a7a780eSAndrew Gallatin 			pi->ip6 = (struct ip6_hdr *)(ss->scratch + pi->ip_off);
18250a7a780eSAndrew Gallatin 		}
18260a7a780eSAndrew Gallatin 		nxt = 0;
18270a7a780eSAndrew Gallatin 		pi->ip_hlen = ip6_lasthdr(m, pi->ip_off, IPPROTO_IPV6, &nxt);
18280a7a780eSAndrew Gallatin 		pi->ip_hlen -= pi->ip_off;
18290a7a780eSAndrew Gallatin 		if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP)
18300a7a780eSAndrew Gallatin 			return EINVAL;
18310a7a780eSAndrew Gallatin 
18320a7a780eSAndrew Gallatin 		if (!tso)
18330a7a780eSAndrew Gallatin 			return 0;
18340a7a780eSAndrew Gallatin 
18350a7a780eSAndrew Gallatin 		if (pi->ip_off + pi->ip_hlen > ss->sc->max_tso6_hlen)
18360a7a780eSAndrew Gallatin 			return EINVAL;
18370a7a780eSAndrew Gallatin 
18380a7a780eSAndrew Gallatin 		if (__predict_false(m->m_len < pi->ip_off + pi->ip_hlen +
18390a7a780eSAndrew Gallatin 		    sizeof(struct tcphdr))) {
18400a7a780eSAndrew Gallatin 			m_copydata(m, 0, pi->ip_off + pi->ip_hlen +
18410a7a780eSAndrew Gallatin 			    sizeof(struct tcphdr), ss->scratch);
18420a7a780eSAndrew Gallatin 			pi->ip6 = (struct ip6_hdr *)(ss->scratch + pi->ip_off);
18430a7a780eSAndrew Gallatin 		}
18440a7a780eSAndrew Gallatin 		pi->tcp = (struct tcphdr *)((char *)pi->ip6 + pi->ip_hlen);
18450a7a780eSAndrew Gallatin 		break;
18460a7a780eSAndrew Gallatin #endif
18470a7a780eSAndrew Gallatin 	default:
18480a7a780eSAndrew Gallatin 		return EINVAL;
18490a7a780eSAndrew Gallatin 	}
18500a7a780eSAndrew Gallatin 	return 0;
18510a7a780eSAndrew Gallatin }
18520a7a780eSAndrew Gallatin 
185337d89b0cSAndrew Gallatin #if IFCAP_TSO4
185437d89b0cSAndrew Gallatin 
1855b2fc195eSAndrew Gallatin static void
18561e413cf9SAndrew Gallatin mxge_encap_tso(struct mxge_slice_state *ss, struct mbuf *m,
18570a7a780eSAndrew Gallatin 	       int busdma_seg_cnt, struct mxge_pkt_info *pi)
1858aed8e389SAndrew Gallatin {
18591e413cf9SAndrew Gallatin 	mxge_tx_ring_t *tx;
1860aed8e389SAndrew Gallatin 	mcp_kreq_ether_send_t *req;
1861aed8e389SAndrew Gallatin 	bus_dma_segment_t *seg;
1862aed8e389SAndrew Gallatin 	uint32_t low, high_swapped;
1863aed8e389SAndrew Gallatin 	int len, seglen, cum_len, cum_len_next;
1864aed8e389SAndrew Gallatin 	int next_is_first, chop, cnt, rdma_count, small;
18650a7a780eSAndrew Gallatin 	uint16_t pseudo_hdr_offset, cksum_offset, mss, sum;
1866aed8e389SAndrew Gallatin 	uint8_t flags, flags_next;
1867aed8e389SAndrew Gallatin 	static int once;
1868aed8e389SAndrew Gallatin 
1869aed8e389SAndrew Gallatin 	mss = m->m_pkthdr.tso_segsz;
1870aed8e389SAndrew Gallatin 
1871aed8e389SAndrew Gallatin 	/* negative cum_len signifies to the
1872aed8e389SAndrew Gallatin 	 * send loop that we are still in the
1873aed8e389SAndrew Gallatin 	 * header portion of the TSO packet.
1874aed8e389SAndrew Gallatin 	 */
1875aed8e389SAndrew Gallatin 
18760a7a780eSAndrew Gallatin 	cksum_offset = pi->ip_off + pi->ip_hlen;
18770a7a780eSAndrew Gallatin 	cum_len = -(cksum_offset + (pi->tcp->th_off << 2));
1878aed8e389SAndrew Gallatin 
1879aed8e389SAndrew Gallatin 	/* TSO implies checksum offload on this hardware */
18800a7a780eSAndrew Gallatin 	if (__predict_false((m->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) == 0)) {
18814ed8ca8fSAndrew Gallatin 		/*
18824ed8ca8fSAndrew Gallatin 		 * If packet has full TCP csum, replace it with pseudo hdr
18834ed8ca8fSAndrew Gallatin 		 * sum that the NIC expects, otherwise the NIC will emit
18844ed8ca8fSAndrew Gallatin 		 * packets with bad TCP checksums.
18854ed8ca8fSAndrew Gallatin 		 */
18864ed8ca8fSAndrew Gallatin 		m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
18870a7a780eSAndrew Gallatin 		if (pi->ip6) {
18880a7a780eSAndrew Gallatin #if (CSUM_TCP_IPV6 != 0) && defined(INET6)
18890a7a780eSAndrew Gallatin 			m->m_pkthdr.csum_flags |= CSUM_TCP_IPV6;
18900a7a780eSAndrew Gallatin 			sum = in6_cksum_pseudo(pi->ip6,
18910a7a780eSAndrew Gallatin 			    m->m_pkthdr.len - cksum_offset,
18920a7a780eSAndrew Gallatin 			    IPPROTO_TCP, 0);
18930a7a780eSAndrew Gallatin #endif
18940a7a780eSAndrew Gallatin 		} else {
1895abc5b96bSAndrew Gallatin #ifdef INET
18960a7a780eSAndrew Gallatin 			m->m_pkthdr.csum_flags |= CSUM_TCP;
18970a7a780eSAndrew Gallatin 			sum = in_pseudo(pi->ip->ip_src.s_addr,
18980a7a780eSAndrew Gallatin 			    pi->ip->ip_dst.s_addr,
18990a7a780eSAndrew Gallatin 			    htons(IPPROTO_TCP + (m->m_pkthdr.len -
19000a7a780eSAndrew Gallatin 				    cksum_offset)));
1901abc5b96bSAndrew Gallatin #endif
19020a7a780eSAndrew Gallatin 		}
19030a7a780eSAndrew Gallatin 		m_copyback(m, offsetof(struct tcphdr, th_sum) +
19040a7a780eSAndrew Gallatin 		    cksum_offset, sizeof(sum), (caddr_t)&sum);
19054ed8ca8fSAndrew Gallatin 	}
1906aed8e389SAndrew Gallatin 	flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST;
1907aed8e389SAndrew Gallatin 
1908aed8e389SAndrew Gallatin 
1909aed8e389SAndrew Gallatin 	/* for TSO, pseudo_hdr_offset holds mss.
1910aed8e389SAndrew Gallatin 	 * The firmware figures out where to put
1911aed8e389SAndrew Gallatin 	 * the checksum by parsing the header. */
1912aed8e389SAndrew Gallatin 	pseudo_hdr_offset = htobe16(mss);
1913aed8e389SAndrew Gallatin 
19140a7a780eSAndrew Gallatin 	if (pi->ip6) {
19150a7a780eSAndrew Gallatin 		/*
19160a7a780eSAndrew Gallatin 		 * for IPv6 TSO, the "checksum offset" is re-purposed
19170a7a780eSAndrew Gallatin 		 * to store the TCP header len
19180a7a780eSAndrew Gallatin 		 */
19190a7a780eSAndrew Gallatin 		cksum_offset = (pi->tcp->th_off << 2);
19200a7a780eSAndrew Gallatin 	}
19210a7a780eSAndrew Gallatin 
19221e413cf9SAndrew Gallatin 	tx = &ss->tx;
1923aed8e389SAndrew Gallatin 	req = tx->req_list;
1924aed8e389SAndrew Gallatin 	seg = tx->seg_list;
1925aed8e389SAndrew Gallatin 	cnt = 0;
1926aed8e389SAndrew Gallatin 	rdma_count = 0;
1927aed8e389SAndrew Gallatin 	/* "rdma_count" is the number of RDMAs belonging to the
1928aed8e389SAndrew Gallatin 	 * current packet BEFORE the current send request. For
1929aed8e389SAndrew Gallatin 	 * non-TSO packets, this is equal to "count".
1930aed8e389SAndrew Gallatin 	 * For TSO packets, rdma_count needs to be reset
1931aed8e389SAndrew Gallatin 	 * to 0 after a segment cut.
1932aed8e389SAndrew Gallatin 	 *
1933aed8e389SAndrew Gallatin 	 * The rdma_count field of the send request is
1934aed8e389SAndrew Gallatin 	 * the number of RDMAs of the packet starting at
1935aed8e389SAndrew Gallatin 	 * that request. For TSO send requests with one ore more cuts
1936aed8e389SAndrew Gallatin 	 * in the middle, this is the number of RDMAs starting
1937aed8e389SAndrew Gallatin 	 * after the last cut in the request. All previous
1938aed8e389SAndrew Gallatin 	 * segments before the last cut implicitly have 1 RDMA.
1939aed8e389SAndrew Gallatin 	 *
1940aed8e389SAndrew Gallatin 	 * Since the number of RDMAs is not known beforehand,
1941aed8e389SAndrew Gallatin 	 * it must be filled-in retroactively - after each
1942aed8e389SAndrew Gallatin 	 * segmentation cut or at the end of the entire packet.
1943aed8e389SAndrew Gallatin 	 */
1944aed8e389SAndrew Gallatin 
1945aed8e389SAndrew Gallatin 	while (busdma_seg_cnt) {
1946aed8e389SAndrew Gallatin 		/* Break the busdma segment up into pieces*/
1947aed8e389SAndrew Gallatin 		low = MXGE_LOWPART_TO_U32(seg->ds_addr);
1948aed8e389SAndrew Gallatin 		high_swapped = 	htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
1949e39a0a37SAndrew Gallatin 		len = seg->ds_len;
1950aed8e389SAndrew Gallatin 
1951aed8e389SAndrew Gallatin 		while (len) {
1952aed8e389SAndrew Gallatin 			flags_next = flags & ~MXGEFW_FLAGS_FIRST;
1953e39a0a37SAndrew Gallatin 			seglen = len;
1954aed8e389SAndrew Gallatin 			cum_len_next = cum_len + seglen;
1955aed8e389SAndrew Gallatin 			(req-rdma_count)->rdma_count = rdma_count + 1;
1956aed8e389SAndrew Gallatin 			if (__predict_true(cum_len >= 0)) {
1957aed8e389SAndrew Gallatin 				/* payload */
1958aed8e389SAndrew Gallatin 				chop = (cum_len_next > mss);
1959aed8e389SAndrew Gallatin 				cum_len_next = cum_len_next % mss;
1960aed8e389SAndrew Gallatin 				next_is_first = (cum_len_next == 0);
1961aed8e389SAndrew Gallatin 				flags |= chop * MXGEFW_FLAGS_TSO_CHOP;
1962aed8e389SAndrew Gallatin 				flags_next |= next_is_first *
1963aed8e389SAndrew Gallatin 					MXGEFW_FLAGS_FIRST;
1964aed8e389SAndrew Gallatin 				rdma_count |= -(chop | next_is_first);
1965aed8e389SAndrew Gallatin 				rdma_count += chop & !next_is_first;
1966aed8e389SAndrew Gallatin 			} else if (cum_len_next >= 0) {
1967aed8e389SAndrew Gallatin 				/* header ends */
1968aed8e389SAndrew Gallatin 				rdma_count = -1;
1969aed8e389SAndrew Gallatin 				cum_len_next = 0;
1970aed8e389SAndrew Gallatin 				seglen = -cum_len;
1971aed8e389SAndrew Gallatin 				small = (mss <= MXGEFW_SEND_SMALL_SIZE);
1972aed8e389SAndrew Gallatin 				flags_next = MXGEFW_FLAGS_TSO_PLD |
1973aed8e389SAndrew Gallatin 					MXGEFW_FLAGS_FIRST |
1974aed8e389SAndrew Gallatin 					(small * MXGEFW_FLAGS_SMALL);
1975aed8e389SAndrew Gallatin 			    }
1976aed8e389SAndrew Gallatin 
1977aed8e389SAndrew Gallatin 			req->addr_high = high_swapped;
1978aed8e389SAndrew Gallatin 			req->addr_low = htobe32(low);
1979aed8e389SAndrew Gallatin 			req->pseudo_hdr_offset = pseudo_hdr_offset;
1980aed8e389SAndrew Gallatin 			req->pad = 0;
1981aed8e389SAndrew Gallatin 			req->rdma_count = 1;
1982aed8e389SAndrew Gallatin 			req->length = htobe16(seglen);
1983aed8e389SAndrew Gallatin 			req->cksum_offset = cksum_offset;
1984aed8e389SAndrew Gallatin 			req->flags = flags | ((cum_len & 1) *
1985aed8e389SAndrew Gallatin 					      MXGEFW_FLAGS_ALIGN_ODD);
1986aed8e389SAndrew Gallatin 			low += seglen;
1987aed8e389SAndrew Gallatin 			len -= seglen;
1988aed8e389SAndrew Gallatin 			cum_len = cum_len_next;
1989aed8e389SAndrew Gallatin 			flags = flags_next;
1990aed8e389SAndrew Gallatin 			req++;
1991aed8e389SAndrew Gallatin 			cnt++;
1992aed8e389SAndrew Gallatin 			rdma_count++;
19930a7a780eSAndrew Gallatin 			if (cksum_offset != 0 && !pi->ip6) {
1994aed8e389SAndrew Gallatin 				if (__predict_false(cksum_offset > seglen))
1995aed8e389SAndrew Gallatin 					cksum_offset -= seglen;
1996aed8e389SAndrew Gallatin 				else
1997aed8e389SAndrew Gallatin 					cksum_offset = 0;
19980a7a780eSAndrew Gallatin 			}
1999adae7080SAndrew Gallatin 			if (__predict_false(cnt > tx->max_desc))
2000aed8e389SAndrew Gallatin 				goto drop;
2001aed8e389SAndrew Gallatin 		}
2002aed8e389SAndrew Gallatin 		busdma_seg_cnt--;
2003aed8e389SAndrew Gallatin 		seg++;
2004aed8e389SAndrew Gallatin 	}
2005aed8e389SAndrew Gallatin 	(req-rdma_count)->rdma_count = rdma_count;
2006aed8e389SAndrew Gallatin 
2007aed8e389SAndrew Gallatin 	do {
2008aed8e389SAndrew Gallatin 		req--;
2009aed8e389SAndrew Gallatin 		req->flags |= MXGEFW_FLAGS_TSO_LAST;
2010aed8e389SAndrew Gallatin 	} while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | MXGEFW_FLAGS_FIRST)));
2011aed8e389SAndrew Gallatin 
2012aed8e389SAndrew Gallatin 	tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1;
2013aed8e389SAndrew Gallatin 	mxge_submit_req(tx, tx->req_list, cnt);
2014c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
2015c6cb3e3fSAndrew Gallatin 	if ((ss->sc->num_slices > 1) && tx->queue_active == 0) {
2016c6cb3e3fSAndrew Gallatin 		/* tell the NIC to start polling this slice */
2017c6cb3e3fSAndrew Gallatin 		*tx->send_go = 1;
2018c6cb3e3fSAndrew Gallatin 		tx->queue_active = 1;
2019c6cb3e3fSAndrew Gallatin 		tx->activate++;
2020c6cb3e3fSAndrew Gallatin 		wmb();
2021c6cb3e3fSAndrew Gallatin 	}
2022c6cb3e3fSAndrew Gallatin #endif
2023aed8e389SAndrew Gallatin 	return;
2024aed8e389SAndrew Gallatin 
2025aed8e389SAndrew Gallatin drop:
2026e39a0a37SAndrew Gallatin 	bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map);
2027aed8e389SAndrew Gallatin 	m_freem(m);
2028c6cb3e3fSAndrew Gallatin 	ss->oerrors++;
2029aed8e389SAndrew Gallatin 	if (!once) {
2030adae7080SAndrew Gallatin 		printf("tx->max_desc exceeded via TSO!\n");
2031adae7080SAndrew Gallatin 		printf("mss = %d, %ld, %d!\n", mss,
2032adae7080SAndrew Gallatin 		       (long)seg - (long)tx->seg_list, tx->max_desc);
2033aed8e389SAndrew Gallatin 		once = 1;
2034aed8e389SAndrew Gallatin 	}
2035aed8e389SAndrew Gallatin 	return;
2036aed8e389SAndrew Gallatin 
2037aed8e389SAndrew Gallatin }
2038aed8e389SAndrew Gallatin 
203937d89b0cSAndrew Gallatin #endif /* IFCAP_TSO4 */
204037d89b0cSAndrew Gallatin 
204137d89b0cSAndrew Gallatin #ifdef MXGE_NEW_VLAN_API
2042c792928fSAndrew Gallatin /*
2043c792928fSAndrew Gallatin  * We reproduce the software vlan tag insertion from
2044c792928fSAndrew Gallatin  * net/if_vlan.c:vlan_start() here so that we can advertise "hardware"
2045c792928fSAndrew Gallatin  * vlan tag insertion. We need to advertise this in order to have the
2046c792928fSAndrew Gallatin  * vlan interface respect our csum offload flags.
2047c792928fSAndrew Gallatin  */
2048c792928fSAndrew Gallatin static struct mbuf *
2049c792928fSAndrew Gallatin mxge_vlan_tag_insert(struct mbuf *m)
2050c792928fSAndrew Gallatin {
2051c792928fSAndrew Gallatin 	struct ether_vlan_header *evl;
2052c792928fSAndrew Gallatin 
2053c6499eccSGleb Smirnoff 	M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_NOWAIT);
2054c792928fSAndrew Gallatin 	if (__predict_false(m == NULL))
2055c792928fSAndrew Gallatin 		return NULL;
2056c792928fSAndrew Gallatin 	if (m->m_len < sizeof(*evl)) {
2057c792928fSAndrew Gallatin 		m = m_pullup(m, sizeof(*evl));
2058c792928fSAndrew Gallatin 		if (__predict_false(m == NULL))
2059c792928fSAndrew Gallatin 			return NULL;
2060c792928fSAndrew Gallatin 	}
2061c792928fSAndrew Gallatin 	/*
2062c792928fSAndrew Gallatin 	 * Transform the Ethernet header into an Ethernet header
2063c792928fSAndrew Gallatin 	 * with 802.1Q encapsulation.
2064c792928fSAndrew Gallatin 	 */
2065c792928fSAndrew Gallatin 	evl = mtod(m, struct ether_vlan_header *);
2066c792928fSAndrew Gallatin 	bcopy((char *)evl + ETHER_VLAN_ENCAP_LEN,
2067c792928fSAndrew Gallatin 	      (char *)evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
2068c792928fSAndrew Gallatin 	evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
2069c792928fSAndrew Gallatin 	evl->evl_tag = htons(m->m_pkthdr.ether_vtag);
2070c792928fSAndrew Gallatin 	m->m_flags &= ~M_VLANTAG;
2071c792928fSAndrew Gallatin 	return m;
2072c792928fSAndrew Gallatin }
207337d89b0cSAndrew Gallatin #endif /* MXGE_NEW_VLAN_API */
2074c792928fSAndrew Gallatin 
2075aed8e389SAndrew Gallatin static void
20761e413cf9SAndrew Gallatin mxge_encap(struct mxge_slice_state *ss, struct mbuf *m)
2077b2fc195eSAndrew Gallatin {
20780a7a780eSAndrew Gallatin 	struct mxge_pkt_info pi = {0,0,0,0};
20791e413cf9SAndrew Gallatin 	mxge_softc_t *sc;
2080b2fc195eSAndrew Gallatin 	mcp_kreq_ether_send_t *req;
2081b2fc195eSAndrew Gallatin 	bus_dma_segment_t *seg;
2082b2fc195eSAndrew Gallatin 	struct mbuf *m_tmp;
2083b2fc195eSAndrew Gallatin 	struct ifnet *ifp;
20841e413cf9SAndrew Gallatin 	mxge_tx_ring_t *tx;
20850a7a780eSAndrew Gallatin 	int cnt, cum_len, err, i, idx, odd_flag;
2086aed8e389SAndrew Gallatin 	uint16_t pseudo_hdr_offset;
2087aed8e389SAndrew Gallatin 	uint8_t flags, cksum_offset;
2088b2fc195eSAndrew Gallatin 
2089b2fc195eSAndrew Gallatin 
20901e413cf9SAndrew Gallatin 	sc = ss->sc;
2091b2fc195eSAndrew Gallatin 	ifp = sc->ifp;
20921e413cf9SAndrew Gallatin 	tx = &ss->tx;
2093b2fc195eSAndrew Gallatin 
209437d89b0cSAndrew Gallatin #ifdef MXGE_NEW_VLAN_API
2095c792928fSAndrew Gallatin 	if (m->m_flags & M_VLANTAG) {
2096c792928fSAndrew Gallatin 		m = mxge_vlan_tag_insert(m);
2097c792928fSAndrew Gallatin 		if (__predict_false(m == NULL))
20980a7a780eSAndrew Gallatin 			goto drop_without_m;
2099c792928fSAndrew Gallatin 	}
210037d89b0cSAndrew Gallatin #endif
21010a7a780eSAndrew Gallatin 	if (m->m_pkthdr.csum_flags &
21020a7a780eSAndrew Gallatin 	    (CSUM_TSO | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) {
21030a7a780eSAndrew Gallatin 		if (mxge_parse_tx(ss, m, &pi))
21040a7a780eSAndrew Gallatin 			goto drop;
21050a7a780eSAndrew Gallatin 	}
21060a7a780eSAndrew Gallatin 
2107b2fc195eSAndrew Gallatin 	/* (try to) map the frame for DMA */
2108b2fc195eSAndrew Gallatin 	idx = tx->req & tx->mask;
2109b2fc195eSAndrew Gallatin 	err = bus_dmamap_load_mbuf_sg(tx->dmat, tx->info[idx].map,
2110aed8e389SAndrew Gallatin 				      m, tx->seg_list, &cnt,
2111b2fc195eSAndrew Gallatin 				      BUS_DMA_NOWAIT);
2112adae7080SAndrew Gallatin 	if (__predict_false(err == EFBIG)) {
2113b2fc195eSAndrew Gallatin 		/* Too many segments in the chain.  Try
2114b2fc195eSAndrew Gallatin 		   to defrag */
2115b2fc195eSAndrew Gallatin 		m_tmp = m_defrag(m, M_NOWAIT);
2116b2fc195eSAndrew Gallatin 		if (m_tmp == NULL) {
2117b2fc195eSAndrew Gallatin 			goto drop;
2118b2fc195eSAndrew Gallatin 		}
21191e413cf9SAndrew Gallatin 		ss->tx.defrag++;
2120b2fc195eSAndrew Gallatin 		m = m_tmp;
2121b2fc195eSAndrew Gallatin 		err = bus_dmamap_load_mbuf_sg(tx->dmat,
2122b2fc195eSAndrew Gallatin 					      tx->info[idx].map,
2123aed8e389SAndrew Gallatin 					      m, tx->seg_list, &cnt,
2124b2fc195eSAndrew Gallatin 					      BUS_DMA_NOWAIT);
2125b2fc195eSAndrew Gallatin 	}
2126adae7080SAndrew Gallatin 	if (__predict_false(err != 0)) {
2127aed8e389SAndrew Gallatin 		device_printf(sc->dev, "bus_dmamap_load_mbuf_sg returned %d"
2128aed8e389SAndrew Gallatin 			      " packet len = %d\n", err, m->m_pkthdr.len);
2129b2fc195eSAndrew Gallatin 		goto drop;
2130b2fc195eSAndrew Gallatin 	}
2131b2fc195eSAndrew Gallatin 	bus_dmamap_sync(tx->dmat, tx->info[idx].map,
2132b2fc195eSAndrew Gallatin 			BUS_DMASYNC_PREWRITE);
21335e7d8541SAndrew Gallatin 	tx->info[idx].m = m;
2134b2fc195eSAndrew Gallatin 
213537d89b0cSAndrew Gallatin #if IFCAP_TSO4
2136aed8e389SAndrew Gallatin 	/* TSO is different enough, we handle it in another routine */
2137aed8e389SAndrew Gallatin 	if (m->m_pkthdr.csum_flags & (CSUM_TSO)) {
21380a7a780eSAndrew Gallatin 		mxge_encap_tso(ss, m, cnt, &pi);
2139aed8e389SAndrew Gallatin 		return;
2140aed8e389SAndrew Gallatin 	}
214137d89b0cSAndrew Gallatin #endif
2142aed8e389SAndrew Gallatin 
2143b2fc195eSAndrew Gallatin 	req = tx->req_list;
2144b2fc195eSAndrew Gallatin 	cksum_offset = 0;
21455e7d8541SAndrew Gallatin 	pseudo_hdr_offset = 0;
21465e7d8541SAndrew Gallatin 	flags = MXGEFW_FLAGS_NO_TSO;
2147b2fc195eSAndrew Gallatin 
2148b2fc195eSAndrew Gallatin 	/* checksum offloading? */
21490a7a780eSAndrew Gallatin 	if (m->m_pkthdr.csum_flags &
21500a7a780eSAndrew Gallatin 	    (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) {
2151aed8e389SAndrew Gallatin 		/* ensure ip header is in first mbuf, copy
2152aed8e389SAndrew Gallatin 		   it to a scratch buffer if not */
21530a7a780eSAndrew Gallatin 		cksum_offset = pi.ip_off + pi.ip_hlen;
2154b2fc195eSAndrew Gallatin 		pseudo_hdr_offset = cksum_offset +  m->m_pkthdr.csum_data;
21555e7d8541SAndrew Gallatin 		pseudo_hdr_offset = htobe16(pseudo_hdr_offset);
2156b2fc195eSAndrew Gallatin 		req->cksum_offset = cksum_offset;
21575e7d8541SAndrew Gallatin 		flags |= MXGEFW_FLAGS_CKSUM;
2158aed8e389SAndrew Gallatin 		odd_flag = MXGEFW_FLAGS_ALIGN_ODD;
2159aed8e389SAndrew Gallatin 	} else {
2160aed8e389SAndrew Gallatin 		odd_flag = 0;
2161b2fc195eSAndrew Gallatin 	}
21625e7d8541SAndrew Gallatin 	if (m->m_pkthdr.len < MXGEFW_SEND_SMALL_SIZE)
21635e7d8541SAndrew Gallatin 		flags |= MXGEFW_FLAGS_SMALL;
2164b2fc195eSAndrew Gallatin 
2165b2fc195eSAndrew Gallatin 	/* convert segments into a request list */
2166b2fc195eSAndrew Gallatin 	cum_len = 0;
2167aed8e389SAndrew Gallatin 	seg = tx->seg_list;
21685e7d8541SAndrew Gallatin 	req->flags = MXGEFW_FLAGS_FIRST;
2169b2fc195eSAndrew Gallatin 	for (i = 0; i < cnt; i++) {
2170b2fc195eSAndrew Gallatin 		req->addr_low =
21716d87a65dSAndrew Gallatin 			htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr));
2172b2fc195eSAndrew Gallatin 		req->addr_high =
21736d87a65dSAndrew Gallatin 			htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
2174b2fc195eSAndrew Gallatin 		req->length = htobe16(seg->ds_len);
2175b2fc195eSAndrew Gallatin 		req->cksum_offset = cksum_offset;
2176b2fc195eSAndrew Gallatin 		if (cksum_offset > seg->ds_len)
2177b2fc195eSAndrew Gallatin 			cksum_offset -= seg->ds_len;
2178b2fc195eSAndrew Gallatin 		else
2179b2fc195eSAndrew Gallatin 			cksum_offset = 0;
21805e7d8541SAndrew Gallatin 		req->pseudo_hdr_offset = pseudo_hdr_offset;
21815e7d8541SAndrew Gallatin 		req->pad = 0; /* complete solid 16-byte block */
21825e7d8541SAndrew Gallatin 		req->rdma_count = 1;
2183aed8e389SAndrew Gallatin 		req->flags |= flags | ((cum_len & 1) * odd_flag);
2184b2fc195eSAndrew Gallatin 		cum_len += seg->ds_len;
2185b2fc195eSAndrew Gallatin 		seg++;
2186b2fc195eSAndrew Gallatin 		req++;
2187b2fc195eSAndrew Gallatin 		req->flags = 0;
2188b2fc195eSAndrew Gallatin 	}
2189b2fc195eSAndrew Gallatin 	req--;
2190b2fc195eSAndrew Gallatin 	/* pad runts to 60 bytes */
2191b2fc195eSAndrew Gallatin 	if (cum_len < 60) {
2192b2fc195eSAndrew Gallatin 		req++;
2193b2fc195eSAndrew Gallatin 		req->addr_low =
21946d87a65dSAndrew Gallatin 			htobe32(MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr));
2195b2fc195eSAndrew Gallatin 		req->addr_high =
21966d87a65dSAndrew Gallatin 			htobe32(MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr));
2197b2fc195eSAndrew Gallatin 		req->length = htobe16(60 - cum_len);
21985e7d8541SAndrew Gallatin 		req->cksum_offset = 0;
21995e7d8541SAndrew Gallatin 		req->pseudo_hdr_offset = pseudo_hdr_offset;
22005e7d8541SAndrew Gallatin 		req->pad = 0; /* complete solid 16-byte block */
22015e7d8541SAndrew Gallatin 		req->rdma_count = 1;
2202aed8e389SAndrew Gallatin 		req->flags |= flags | ((cum_len & 1) * odd_flag);
2203b2fc195eSAndrew Gallatin 		cnt++;
2204b2fc195eSAndrew Gallatin 	}
22055e7d8541SAndrew Gallatin 
22065e7d8541SAndrew Gallatin 	tx->req_list[0].rdma_count = cnt;
22075e7d8541SAndrew Gallatin #if 0
22085e7d8541SAndrew Gallatin 	/* print what the firmware will see */
22095e7d8541SAndrew Gallatin 	for (i = 0; i < cnt; i++) {
22105e7d8541SAndrew Gallatin 		printf("%d: addr: 0x%x 0x%x len:%d pso%d,"
22115e7d8541SAndrew Gallatin 		    "cso:%d, flags:0x%x, rdma:%d\n",
22125e7d8541SAndrew Gallatin 		    i, (int)ntohl(tx->req_list[i].addr_high),
22135e7d8541SAndrew Gallatin 		    (int)ntohl(tx->req_list[i].addr_low),
22145e7d8541SAndrew Gallatin 		    (int)ntohs(tx->req_list[i].length),
22155e7d8541SAndrew Gallatin 		    (int)ntohs(tx->req_list[i].pseudo_hdr_offset),
22165e7d8541SAndrew Gallatin 		    tx->req_list[i].cksum_offset, tx->req_list[i].flags,
22175e7d8541SAndrew Gallatin 		    tx->req_list[i].rdma_count);
22185e7d8541SAndrew Gallatin 	}
22195e7d8541SAndrew Gallatin 	printf("--------------\n");
22205e7d8541SAndrew Gallatin #endif
22215e7d8541SAndrew Gallatin 	tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1;
22226d87a65dSAndrew Gallatin 	mxge_submit_req(tx, tx->req_list, cnt);
2223c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
2224c6cb3e3fSAndrew Gallatin 	if ((ss->sc->num_slices > 1) && tx->queue_active == 0) {
2225c6cb3e3fSAndrew Gallatin 		/* tell the NIC to start polling this slice */
2226c6cb3e3fSAndrew Gallatin 		*tx->send_go = 1;
2227c6cb3e3fSAndrew Gallatin 		tx->queue_active = 1;
2228c6cb3e3fSAndrew Gallatin 		tx->activate++;
2229c6cb3e3fSAndrew Gallatin 		wmb();
2230c6cb3e3fSAndrew Gallatin 	}
2231c6cb3e3fSAndrew Gallatin #endif
2232b2fc195eSAndrew Gallatin 	return;
2233b2fc195eSAndrew Gallatin 
2234b2fc195eSAndrew Gallatin drop:
2235b2fc195eSAndrew Gallatin 	m_freem(m);
22360a7a780eSAndrew Gallatin drop_without_m:
2237c6cb3e3fSAndrew Gallatin 	ss->oerrors++;
2238b2fc195eSAndrew Gallatin 	return;
2239b2fc195eSAndrew Gallatin }
2240b2fc195eSAndrew Gallatin 
2241c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
2242c6cb3e3fSAndrew Gallatin static void
2243c6cb3e3fSAndrew Gallatin mxge_qflush(struct ifnet *ifp)
2244c6cb3e3fSAndrew Gallatin {
2245c6cb3e3fSAndrew Gallatin 	mxge_softc_t *sc = ifp->if_softc;
2246c6cb3e3fSAndrew Gallatin 	mxge_tx_ring_t *tx;
2247c6cb3e3fSAndrew Gallatin 	struct mbuf *m;
2248c6cb3e3fSAndrew Gallatin 	int slice;
2249b2fc195eSAndrew Gallatin 
2250c6cb3e3fSAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++) {
2251c6cb3e3fSAndrew Gallatin 		tx = &sc->ss[slice].tx;
2252c6cb3e3fSAndrew Gallatin 		mtx_lock(&tx->mtx);
2253c6cb3e3fSAndrew Gallatin 		while ((m = buf_ring_dequeue_sc(tx->br)) != NULL)
2254c6cb3e3fSAndrew Gallatin 			m_freem(m);
2255c6cb3e3fSAndrew Gallatin 		mtx_unlock(&tx->mtx);
2256c6cb3e3fSAndrew Gallatin 	}
2257c6cb3e3fSAndrew Gallatin 	if_qflush(ifp);
2258c6cb3e3fSAndrew Gallatin }
22596d914a32SAndrew Gallatin 
2260c6cb3e3fSAndrew Gallatin static inline void
2261c6cb3e3fSAndrew Gallatin mxge_start_locked(struct mxge_slice_state *ss)
2262c6cb3e3fSAndrew Gallatin {
2263c6cb3e3fSAndrew Gallatin 	mxge_softc_t *sc;
2264c6cb3e3fSAndrew Gallatin 	struct mbuf *m;
2265c6cb3e3fSAndrew Gallatin 	struct ifnet *ifp;
2266c6cb3e3fSAndrew Gallatin 	mxge_tx_ring_t *tx;
2267c6cb3e3fSAndrew Gallatin 
2268c6cb3e3fSAndrew Gallatin 	sc = ss->sc;
2269c6cb3e3fSAndrew Gallatin 	ifp = sc->ifp;
2270c6cb3e3fSAndrew Gallatin 	tx = &ss->tx;
2271c6cb3e3fSAndrew Gallatin 
2272c6cb3e3fSAndrew Gallatin 	while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) {
2273c6cb3e3fSAndrew Gallatin 		m = drbr_dequeue(ifp, tx->br);
2274c6cb3e3fSAndrew Gallatin 		if (m == NULL) {
2275c6cb3e3fSAndrew Gallatin 			return;
2276c6cb3e3fSAndrew Gallatin 		}
2277c6cb3e3fSAndrew Gallatin 		/* let BPF see it */
2278c6cb3e3fSAndrew Gallatin 		BPF_MTAP(ifp, m);
2279c6cb3e3fSAndrew Gallatin 
2280c6cb3e3fSAndrew Gallatin 		/* give it to the nic */
2281c6cb3e3fSAndrew Gallatin 		mxge_encap(ss, m);
2282c6cb3e3fSAndrew Gallatin 	}
2283c6cb3e3fSAndrew Gallatin 	/* ran out of transmit slots */
2284c6cb3e3fSAndrew Gallatin 	if (((ss->if_drv_flags & IFF_DRV_OACTIVE) == 0)
2285c6cb3e3fSAndrew Gallatin 	    && (!drbr_empty(ifp, tx->br))) {
2286c6cb3e3fSAndrew Gallatin 		ss->if_drv_flags |= IFF_DRV_OACTIVE;
2287c6cb3e3fSAndrew Gallatin 		tx->stall++;
2288c6cb3e3fSAndrew Gallatin 	}
2289c6cb3e3fSAndrew Gallatin }
2290c6cb3e3fSAndrew Gallatin 
2291c6cb3e3fSAndrew Gallatin static int
2292c6cb3e3fSAndrew Gallatin mxge_transmit_locked(struct mxge_slice_state *ss, struct mbuf *m)
2293c6cb3e3fSAndrew Gallatin {
2294c6cb3e3fSAndrew Gallatin 	mxge_softc_t *sc;
2295c6cb3e3fSAndrew Gallatin 	struct ifnet *ifp;
2296c6cb3e3fSAndrew Gallatin 	mxge_tx_ring_t *tx;
2297c6cb3e3fSAndrew Gallatin 	int err;
2298c6cb3e3fSAndrew Gallatin 
2299c6cb3e3fSAndrew Gallatin 	sc = ss->sc;
2300c6cb3e3fSAndrew Gallatin 	ifp = sc->ifp;
2301c6cb3e3fSAndrew Gallatin 	tx = &ss->tx;
2302c6cb3e3fSAndrew Gallatin 
2303c6cb3e3fSAndrew Gallatin 	if ((ss->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
2304c6cb3e3fSAndrew Gallatin 	    IFF_DRV_RUNNING) {
2305c6cb3e3fSAndrew Gallatin 		err = drbr_enqueue(ifp, tx->br, m);
2306c6cb3e3fSAndrew Gallatin 		return (err);
2307c6cb3e3fSAndrew Gallatin 	}
2308c6cb3e3fSAndrew Gallatin 
2309193cbc4dSMax Laier 	if (!drbr_needs_enqueue(ifp, tx->br) &&
2310c6cb3e3fSAndrew Gallatin 	    ((tx->mask - (tx->req - tx->done)) > tx->max_desc)) {
2311c6cb3e3fSAndrew Gallatin 		/* let BPF see it */
2312c6cb3e3fSAndrew Gallatin 		BPF_MTAP(ifp, m);
2313c6cb3e3fSAndrew Gallatin 		/* give it to the nic */
2314c6cb3e3fSAndrew Gallatin 		mxge_encap(ss, m);
2315c6cb3e3fSAndrew Gallatin 	} else if ((err = drbr_enqueue(ifp, tx->br, m)) != 0) {
2316c6cb3e3fSAndrew Gallatin 		return (err);
2317c6cb3e3fSAndrew Gallatin 	}
2318c6cb3e3fSAndrew Gallatin 	if (!drbr_empty(ifp, tx->br))
2319c6cb3e3fSAndrew Gallatin 		mxge_start_locked(ss);
2320c6cb3e3fSAndrew Gallatin 	return (0);
2321c6cb3e3fSAndrew Gallatin }
2322c6cb3e3fSAndrew Gallatin 
2323c6cb3e3fSAndrew Gallatin static int
2324c6cb3e3fSAndrew Gallatin mxge_transmit(struct ifnet *ifp, struct mbuf *m)
2325c6cb3e3fSAndrew Gallatin {
2326c6cb3e3fSAndrew Gallatin 	mxge_softc_t *sc = ifp->if_softc;
2327c6cb3e3fSAndrew Gallatin 	struct mxge_slice_state *ss;
2328c6cb3e3fSAndrew Gallatin 	mxge_tx_ring_t *tx;
2329c6cb3e3fSAndrew Gallatin 	int err = 0;
2330c6cb3e3fSAndrew Gallatin 	int slice;
2331c6cb3e3fSAndrew Gallatin 
2332c6cb3e3fSAndrew Gallatin 	slice = m->m_pkthdr.flowid;
2333c6cb3e3fSAndrew Gallatin 	slice &= (sc->num_slices - 1);  /* num_slices always power of 2 */
2334c6cb3e3fSAndrew Gallatin 
2335c6cb3e3fSAndrew Gallatin 	ss = &sc->ss[slice];
2336c6cb3e3fSAndrew Gallatin 	tx = &ss->tx;
2337c6cb3e3fSAndrew Gallatin 
2338c6cb3e3fSAndrew Gallatin 	if (mtx_trylock(&tx->mtx)) {
2339c6cb3e3fSAndrew Gallatin 		err = mxge_transmit_locked(ss, m);
2340c6cb3e3fSAndrew Gallatin 		mtx_unlock(&tx->mtx);
2341c6cb3e3fSAndrew Gallatin 	} else {
2342c6cb3e3fSAndrew Gallatin 		err = drbr_enqueue(ifp, tx->br, m);
2343c6cb3e3fSAndrew Gallatin 	}
2344c6cb3e3fSAndrew Gallatin 
2345c6cb3e3fSAndrew Gallatin 	return (err);
2346c6cb3e3fSAndrew Gallatin }
2347c6cb3e3fSAndrew Gallatin 
2348c6cb3e3fSAndrew Gallatin #else
23496d914a32SAndrew Gallatin 
23506d914a32SAndrew Gallatin static inline void
23511e413cf9SAndrew Gallatin mxge_start_locked(struct mxge_slice_state *ss)
2352b2fc195eSAndrew Gallatin {
23531e413cf9SAndrew Gallatin 	mxge_softc_t *sc;
2354b2fc195eSAndrew Gallatin 	struct mbuf *m;
2355b2fc195eSAndrew Gallatin 	struct ifnet *ifp;
23561e413cf9SAndrew Gallatin 	mxge_tx_ring_t *tx;
2357b2fc195eSAndrew Gallatin 
23581e413cf9SAndrew Gallatin 	sc = ss->sc;
2359b2fc195eSAndrew Gallatin 	ifp = sc->ifp;
23601e413cf9SAndrew Gallatin 	tx = &ss->tx;
2361adae7080SAndrew Gallatin 	while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) {
23626d914a32SAndrew Gallatin 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
23636d914a32SAndrew Gallatin 		if (m == NULL) {
23646d914a32SAndrew Gallatin 			return;
23656d914a32SAndrew Gallatin 		}
2366b2fc195eSAndrew Gallatin 		/* let BPF see it */
2367b2fc195eSAndrew Gallatin 		BPF_MTAP(ifp, m);
2368b2fc195eSAndrew Gallatin 
2369b2fc195eSAndrew Gallatin 		/* give it to the nic */
23701e413cf9SAndrew Gallatin 		mxge_encap(ss, m);
23716d914a32SAndrew Gallatin 	}
23726d914a32SAndrew Gallatin 	/* ran out of transmit slots */
2373a82c2581SAndrew Gallatin 	if ((sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
2374b2fc195eSAndrew Gallatin 		sc->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2375adae7080SAndrew Gallatin 		tx->stall++;
2376a82c2581SAndrew Gallatin 	}
2377b2fc195eSAndrew Gallatin }
2378c6cb3e3fSAndrew Gallatin #endif
2379b2fc195eSAndrew Gallatin static void
23806d87a65dSAndrew Gallatin mxge_start(struct ifnet *ifp)
2381b2fc195eSAndrew Gallatin {
23826d87a65dSAndrew Gallatin 	mxge_softc_t *sc = ifp->if_softc;
23831e413cf9SAndrew Gallatin 	struct mxge_slice_state *ss;
2384b2fc195eSAndrew Gallatin 
23851e413cf9SAndrew Gallatin 	/* only use the first slice for now */
23861e413cf9SAndrew Gallatin 	ss = &sc->ss[0];
23871e413cf9SAndrew Gallatin 	mtx_lock(&ss->tx.mtx);
23881e413cf9SAndrew Gallatin 	mxge_start_locked(ss);
23891e413cf9SAndrew Gallatin 	mtx_unlock(&ss->tx.mtx);
2390b2fc195eSAndrew Gallatin }
2391b2fc195eSAndrew Gallatin 
23925e7d8541SAndrew Gallatin /*
23935e7d8541SAndrew Gallatin  * copy an array of mcp_kreq_ether_recv_t's to the mcp.  Copy
23945e7d8541SAndrew Gallatin  * at most 32 bytes at a time, so as to avoid involving the software
23955e7d8541SAndrew Gallatin  * pio handler in the nic.   We re-write the first segment's low
23965e7d8541SAndrew Gallatin  * DMA address to mark it valid only after we write the entire chunk
23975e7d8541SAndrew Gallatin  * in a burst
23985e7d8541SAndrew Gallatin  */
23995e7d8541SAndrew Gallatin static inline void
24005e7d8541SAndrew Gallatin mxge_submit_8rx(volatile mcp_kreq_ether_recv_t *dst,
24015e7d8541SAndrew Gallatin 		mcp_kreq_ether_recv_t *src)
24025e7d8541SAndrew Gallatin {
24035e7d8541SAndrew Gallatin 	uint32_t low;
24045e7d8541SAndrew Gallatin 
24055e7d8541SAndrew Gallatin 	low = src->addr_low;
24065e7d8541SAndrew Gallatin 	src->addr_low = 0xffffffff;
2407a1480dfbSAndrew Gallatin 	mxge_pio_copy(dst, src, 4 * sizeof (*src));
240873c7c83fSAndrew Gallatin 	wmb();
2409a1480dfbSAndrew Gallatin 	mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src));
241073c7c83fSAndrew Gallatin 	wmb();
241140385a5fSAndrew Gallatin 	src->addr_low = low;
24125e7d8541SAndrew Gallatin 	dst->addr_low = low;
241373c7c83fSAndrew Gallatin 	wmb();
24145e7d8541SAndrew Gallatin }
24155e7d8541SAndrew Gallatin 
2416b2fc195eSAndrew Gallatin static int
24171e413cf9SAndrew Gallatin mxge_get_buf_small(struct mxge_slice_state *ss, bus_dmamap_t map, int idx)
2418b2fc195eSAndrew Gallatin {
2419b2fc195eSAndrew Gallatin 	bus_dma_segment_t seg;
2420b2fc195eSAndrew Gallatin 	struct mbuf *m;
24211e413cf9SAndrew Gallatin 	mxge_rx_ring_t *rx = &ss->rx_small;
2422b2fc195eSAndrew Gallatin 	int cnt, err;
2423b2fc195eSAndrew Gallatin 
2424c6499eccSGleb Smirnoff 	m = m_gethdr(M_NOWAIT, MT_DATA);
2425b2fc195eSAndrew Gallatin 	if (m == NULL) {
2426b2fc195eSAndrew Gallatin 		rx->alloc_fail++;
2427b2fc195eSAndrew Gallatin 		err = ENOBUFS;
2428b2fc195eSAndrew Gallatin 		goto done;
2429b2fc195eSAndrew Gallatin 	}
2430b2fc195eSAndrew Gallatin 	m->m_len = MHLEN;
2431b2fc195eSAndrew Gallatin 	err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m,
2432b2fc195eSAndrew Gallatin 				      &seg, &cnt, BUS_DMA_NOWAIT);
2433b2fc195eSAndrew Gallatin 	if (err != 0) {
2434b2fc195eSAndrew Gallatin 		m_free(m);
2435b2fc195eSAndrew Gallatin 		goto done;
2436b2fc195eSAndrew Gallatin 	}
2437b2fc195eSAndrew Gallatin 	rx->info[idx].m = m;
2438b2fc195eSAndrew Gallatin 	rx->shadow[idx].addr_low =
24396d87a65dSAndrew Gallatin 		htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr));
2440b2fc195eSAndrew Gallatin 	rx->shadow[idx].addr_high =
24416d87a65dSAndrew Gallatin 		htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr));
2442b2fc195eSAndrew Gallatin 
2443b2fc195eSAndrew Gallatin done:
2444adae7080SAndrew Gallatin 	if ((idx & 7) == 7)
2445adae7080SAndrew Gallatin 		mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]);
2446b2fc195eSAndrew Gallatin 	return err;
2447b2fc195eSAndrew Gallatin }
2448b2fc195eSAndrew Gallatin 
2449b2fc195eSAndrew Gallatin static int
24501e413cf9SAndrew Gallatin mxge_get_buf_big(struct mxge_slice_state *ss, bus_dmamap_t map, int idx)
2451b2fc195eSAndrew Gallatin {
2452053e637fSAndrew Gallatin 	bus_dma_segment_t seg[3];
2453b2fc195eSAndrew Gallatin 	struct mbuf *m;
24541e413cf9SAndrew Gallatin 	mxge_rx_ring_t *rx = &ss->rx_big;
2455053e637fSAndrew Gallatin 	int cnt, err, i;
2456b2fc195eSAndrew Gallatin 
2457c6499eccSGleb Smirnoff 	m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx->cl_size);
2458b2fc195eSAndrew Gallatin 	if (m == NULL) {
2459b2fc195eSAndrew Gallatin 		rx->alloc_fail++;
2460b2fc195eSAndrew Gallatin 		err = ENOBUFS;
2461b2fc195eSAndrew Gallatin 		goto done;
2462b2fc195eSAndrew Gallatin 	}
24634d9a5852SAndrew Gallatin 	m->m_len = rx->mlen;
2464b2fc195eSAndrew Gallatin 	err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m,
2465053e637fSAndrew Gallatin 				      seg, &cnt, BUS_DMA_NOWAIT);
2466b2fc195eSAndrew Gallatin 	if (err != 0) {
2467b2fc195eSAndrew Gallatin 		m_free(m);
2468b2fc195eSAndrew Gallatin 		goto done;
2469b2fc195eSAndrew Gallatin 	}
2470b2fc195eSAndrew Gallatin 	rx->info[idx].m = m;
2471b0f7b922SAndrew Gallatin 	rx->shadow[idx].addr_low =
2472b0f7b922SAndrew Gallatin 		htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr));
2473b0f7b922SAndrew Gallatin 	rx->shadow[idx].addr_high =
2474b0f7b922SAndrew Gallatin 		htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
2475053e637fSAndrew Gallatin 
2476b0f7b922SAndrew Gallatin #if MXGE_VIRT_JUMBOS
2477b0f7b922SAndrew Gallatin 	for (i = 1; i < cnt; i++) {
2478053e637fSAndrew Gallatin 		rx->shadow[idx + i].addr_low =
2479053e637fSAndrew Gallatin 			htobe32(MXGE_LOWPART_TO_U32(seg[i].ds_addr));
2480053e637fSAndrew Gallatin 		rx->shadow[idx + i].addr_high =
2481053e637fSAndrew Gallatin 			htobe32(MXGE_HIGHPART_TO_U32(seg[i].ds_addr));
2482053e637fSAndrew Gallatin        }
2483b0f7b922SAndrew Gallatin #endif
2484b2fc195eSAndrew Gallatin 
2485b2fc195eSAndrew Gallatin done:
2486053e637fSAndrew Gallatin        for (i = 0; i < rx->nbufs; i++) {
2487b2fc195eSAndrew Gallatin 		if ((idx & 7) == 7) {
24885e7d8541SAndrew Gallatin 			mxge_submit_8rx(&rx->lanai[idx - 7],
24895e7d8541SAndrew Gallatin 					&rx->shadow[idx - 7]);
2490b2fc195eSAndrew Gallatin 		}
2491053e637fSAndrew Gallatin 		idx++;
2492053e637fSAndrew Gallatin 	}
2493b2fc195eSAndrew Gallatin 	return err;
2494b2fc195eSAndrew Gallatin }
2495b2fc195eSAndrew Gallatin 
249626dd49c6SAndrew Gallatin #ifdef INET6
249726dd49c6SAndrew Gallatin 
249826dd49c6SAndrew Gallatin static uint16_t
249926dd49c6SAndrew Gallatin mxge_csum_generic(uint16_t *raw, int len)
250026dd49c6SAndrew Gallatin {
250126dd49c6SAndrew Gallatin 	uint32_t csum;
250226dd49c6SAndrew Gallatin 
250326dd49c6SAndrew Gallatin 
250426dd49c6SAndrew Gallatin 	csum = 0;
250526dd49c6SAndrew Gallatin 	while (len > 0) {
250626dd49c6SAndrew Gallatin 		csum += *raw;
250726dd49c6SAndrew Gallatin 		raw++;
250826dd49c6SAndrew Gallatin 		len -= 2;
250926dd49c6SAndrew Gallatin 	}
251026dd49c6SAndrew Gallatin 	csum = (csum >> 16) + (csum & 0xffff);
251126dd49c6SAndrew Gallatin 	csum = (csum >> 16) + (csum & 0xffff);
251226dd49c6SAndrew Gallatin 	return (uint16_t)csum;
251326dd49c6SAndrew Gallatin }
251426dd49c6SAndrew Gallatin 
251526dd49c6SAndrew Gallatin static inline uint16_t
251626dd49c6SAndrew Gallatin mxge_rx_csum6(void *p, struct mbuf *m, uint32_t csum)
251726dd49c6SAndrew Gallatin {
251826dd49c6SAndrew Gallatin 	uint32_t partial;
251926dd49c6SAndrew Gallatin 	int nxt, cksum_offset;
252026dd49c6SAndrew Gallatin 	struct ip6_hdr *ip6 = p;
252126dd49c6SAndrew Gallatin 	uint16_t c;
252226dd49c6SAndrew Gallatin 
252326dd49c6SAndrew Gallatin 	nxt = ip6->ip6_nxt;
252426dd49c6SAndrew Gallatin 	cksum_offset = sizeof (*ip6) + ETHER_HDR_LEN;
252526dd49c6SAndrew Gallatin 	if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) {
252626dd49c6SAndrew Gallatin 		cksum_offset = ip6_lasthdr(m, ETHER_HDR_LEN,
252726dd49c6SAndrew Gallatin 					   IPPROTO_IPV6, &nxt);
252826dd49c6SAndrew Gallatin 		if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP)
252926dd49c6SAndrew Gallatin 			return (1);
253026dd49c6SAndrew Gallatin 	}
253126dd49c6SAndrew Gallatin 
253226dd49c6SAndrew Gallatin 	/*
253326dd49c6SAndrew Gallatin 	 * IPv6 headers do not contain a checksum, and hence
253426dd49c6SAndrew Gallatin 	 * do not checksum to zero, so they don't "fall out"
253526dd49c6SAndrew Gallatin 	 * of the partial checksum calculation like IPv4
253626dd49c6SAndrew Gallatin 	 * headers do.  We need to fix the partial checksum by
253726dd49c6SAndrew Gallatin 	 * subtracting the checksum of the IPv6 header.
253826dd49c6SAndrew Gallatin 	 */
253926dd49c6SAndrew Gallatin 
254026dd49c6SAndrew Gallatin 	partial = mxge_csum_generic((uint16_t *)ip6, cksum_offset -
254126dd49c6SAndrew Gallatin 				    ETHER_HDR_LEN);
254226dd49c6SAndrew Gallatin 	csum += ~partial;
254326dd49c6SAndrew Gallatin 	csum +=	 (csum < ~partial);
254426dd49c6SAndrew Gallatin 	csum = (csum >> 16) + (csum & 0xFFFF);
254526dd49c6SAndrew Gallatin 	csum = (csum >> 16) + (csum & 0xFFFF);
254626dd49c6SAndrew Gallatin 	c = in6_cksum_pseudo(ip6, m->m_pkthdr.len - cksum_offset, nxt,
254726dd49c6SAndrew Gallatin 			     csum);
254826dd49c6SAndrew Gallatin 	c ^= 0xffff;
254926dd49c6SAndrew Gallatin 	return (c);
255026dd49c6SAndrew Gallatin }
255126dd49c6SAndrew Gallatin #endif /* INET6 */
25529b03b0f3SAndrew Gallatin /*
25539b03b0f3SAndrew Gallatin  *  Myri10GE hardware checksums are not valid if the sender
25549b03b0f3SAndrew Gallatin  *  padded the frame with non-zero padding.  This is because
25559b03b0f3SAndrew Gallatin  *  the firmware just does a simple 16-bit 1s complement
25569b03b0f3SAndrew Gallatin  *  checksum across the entire frame, excluding the first 14
2557053e637fSAndrew Gallatin  *  bytes.  It is best to simply to check the checksum and
2558053e637fSAndrew Gallatin  *  tell the stack about it only if the checksum is good
25599b03b0f3SAndrew Gallatin  */
25609b03b0f3SAndrew Gallatin 
2561053e637fSAndrew Gallatin static inline uint16_t
2562053e637fSAndrew Gallatin mxge_rx_csum(struct mbuf *m, int csum)
2563053e637fSAndrew Gallatin {
2564053e637fSAndrew Gallatin 	struct ether_header *eh;
256526dd49c6SAndrew Gallatin #ifdef INET
2566053e637fSAndrew Gallatin 	struct ip *ip;
256726dd49c6SAndrew Gallatin #endif
2568abc5b96bSAndrew Gallatin #if defined(INET) || defined(INET6)
2569abc5b96bSAndrew Gallatin 	int cap = m->m_pkthdr.rcvif->if_capenable;
2570abc5b96bSAndrew Gallatin #endif
257126dd49c6SAndrew Gallatin 	uint16_t c, etype;
257226dd49c6SAndrew Gallatin 
2573053e637fSAndrew Gallatin 
2574053e637fSAndrew Gallatin 	eh = mtod(m, struct ether_header *);
257526dd49c6SAndrew Gallatin 	etype = ntohs(eh->ether_type);
257626dd49c6SAndrew Gallatin 	switch (etype) {
2577eb6219e3SAndrew Gallatin #ifdef INET
257826dd49c6SAndrew Gallatin 	case ETHERTYPE_IP:
257926dd49c6SAndrew Gallatin 		if ((cap & IFCAP_RXCSUM) == 0)
258026dd49c6SAndrew Gallatin 			return (1);
258126dd49c6SAndrew Gallatin 		ip = (struct ip *)(eh + 1);
258226dd49c6SAndrew Gallatin 		if (ip->ip_p != IPPROTO_TCP && ip->ip_p != IPPROTO_UDP)
258326dd49c6SAndrew Gallatin 			return (1);
2584053e637fSAndrew Gallatin 		c = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
258526dd49c6SAndrew Gallatin 			      htonl(ntohs(csum) + ntohs(ip->ip_len) -
258626dd49c6SAndrew Gallatin 				    (ip->ip_hl << 2) + ip->ip_p));
2587053e637fSAndrew Gallatin 		c ^= 0xffff;
258826dd49c6SAndrew Gallatin 		break;
258926dd49c6SAndrew Gallatin #endif
259026dd49c6SAndrew Gallatin #ifdef INET6
259126dd49c6SAndrew Gallatin 	case ETHERTYPE_IPV6:
259226dd49c6SAndrew Gallatin 		if ((cap & IFCAP_RXCSUM_IPV6) == 0)
259326dd49c6SAndrew Gallatin 			return (1);
259426dd49c6SAndrew Gallatin 		c = mxge_rx_csum6((eh + 1), m, csum);
259526dd49c6SAndrew Gallatin 		break;
259626dd49c6SAndrew Gallatin #endif
259726dd49c6SAndrew Gallatin 	default:
259826dd49c6SAndrew Gallatin 		c = 1;
259926dd49c6SAndrew Gallatin 	}
2600053e637fSAndrew Gallatin 	return (c);
26015e7d8541SAndrew Gallatin }
2602053e637fSAndrew Gallatin 
2603c792928fSAndrew Gallatin static void
2604c792928fSAndrew Gallatin mxge_vlan_tag_remove(struct mbuf *m, uint32_t *csum)
2605c792928fSAndrew Gallatin {
2606c792928fSAndrew Gallatin 	struct ether_vlan_header *evl;
2607c792928fSAndrew Gallatin 	struct ether_header *eh;
2608c792928fSAndrew Gallatin 	uint32_t partial;
2609c792928fSAndrew Gallatin 
2610c792928fSAndrew Gallatin 	evl = mtod(m, struct ether_vlan_header *);
2611c792928fSAndrew Gallatin 	eh = mtod(m, struct ether_header *);
2612c792928fSAndrew Gallatin 
2613c792928fSAndrew Gallatin 	/*
2614c792928fSAndrew Gallatin 	 * fix checksum by subtracting ETHER_VLAN_ENCAP_LEN bytes
2615c792928fSAndrew Gallatin 	 * after what the firmware thought was the end of the ethernet
2616c792928fSAndrew Gallatin 	 * header.
2617c792928fSAndrew Gallatin 	 */
2618c792928fSAndrew Gallatin 
2619c792928fSAndrew Gallatin 	/* put checksum into host byte order */
2620c792928fSAndrew Gallatin 	*csum = ntohs(*csum);
2621c792928fSAndrew Gallatin 	partial = ntohl(*(uint32_t *)(mtod(m, char *) + ETHER_HDR_LEN));
2622c792928fSAndrew Gallatin 	(*csum) += ~partial;
2623c792928fSAndrew Gallatin 	(*csum) +=  ((*csum) < ~partial);
2624c792928fSAndrew Gallatin 	(*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF);
2625c792928fSAndrew Gallatin 	(*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF);
2626c792928fSAndrew Gallatin 
2627c792928fSAndrew Gallatin 	/* restore checksum to network byte order;
2628c792928fSAndrew Gallatin 	   later consumers expect this */
2629c792928fSAndrew Gallatin 	*csum = htons(*csum);
2630c792928fSAndrew Gallatin 
2631c792928fSAndrew Gallatin 	/* save the tag */
263237d89b0cSAndrew Gallatin #ifdef MXGE_NEW_VLAN_API
2633c792928fSAndrew Gallatin 	m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag);
263437d89b0cSAndrew Gallatin #else
263537d89b0cSAndrew Gallatin 	{
263637d89b0cSAndrew Gallatin 		struct m_tag *mtag;
263737d89b0cSAndrew Gallatin 		mtag = m_tag_alloc(MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int),
263837d89b0cSAndrew Gallatin 				   M_NOWAIT);
263937d89b0cSAndrew Gallatin 		if (mtag == NULL)
264037d89b0cSAndrew Gallatin 			return;
264137d89b0cSAndrew Gallatin 		VLAN_TAG_VALUE(mtag) = ntohs(evl->evl_tag);
264237d89b0cSAndrew Gallatin 		m_tag_prepend(m, mtag);
264337d89b0cSAndrew Gallatin 	}
264437d89b0cSAndrew Gallatin 
264537d89b0cSAndrew Gallatin #endif
264637d89b0cSAndrew Gallatin 	m->m_flags |= M_VLANTAG;
2647c792928fSAndrew Gallatin 
2648c792928fSAndrew Gallatin 	/*
2649c792928fSAndrew Gallatin 	 * Remove the 802.1q header by copying the Ethernet
2650c792928fSAndrew Gallatin 	 * addresses over it and adjusting the beginning of
2651c792928fSAndrew Gallatin 	 * the data in the mbuf.  The encapsulated Ethernet
2652c792928fSAndrew Gallatin 	 * type field is already in place.
2653c792928fSAndrew Gallatin 	 */
2654c792928fSAndrew Gallatin 	bcopy((char *)evl, (char *)evl + ETHER_VLAN_ENCAP_LEN,
2655c792928fSAndrew Gallatin 	      ETHER_HDR_LEN - ETHER_TYPE_LEN);
2656c792928fSAndrew Gallatin 	m_adj(m, ETHER_VLAN_ENCAP_LEN);
2657c792928fSAndrew Gallatin }
2658c792928fSAndrew Gallatin 
26595e7d8541SAndrew Gallatin 
26605e7d8541SAndrew Gallatin static inline void
266126dd49c6SAndrew Gallatin mxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len,
266226dd49c6SAndrew Gallatin 		 uint32_t csum, int lro)
2663b2fc195eSAndrew Gallatin {
26641e413cf9SAndrew Gallatin 	mxge_softc_t *sc;
2665b2fc195eSAndrew Gallatin 	struct ifnet *ifp;
2666053e637fSAndrew Gallatin 	struct mbuf *m;
2667c792928fSAndrew Gallatin 	struct ether_header *eh;
26681e413cf9SAndrew Gallatin 	mxge_rx_ring_t *rx;
2669053e637fSAndrew Gallatin 	bus_dmamap_t old_map;
2670b2fc195eSAndrew Gallatin 	int idx;
2671b2fc195eSAndrew Gallatin 
26721e413cf9SAndrew Gallatin 	sc = ss->sc;
2673b2fc195eSAndrew Gallatin 	ifp = sc->ifp;
26741e413cf9SAndrew Gallatin 	rx = &ss->rx_big;
2675b2fc195eSAndrew Gallatin 	idx = rx->cnt & rx->mask;
2676053e637fSAndrew Gallatin 	rx->cnt += rx->nbufs;
2677b2fc195eSAndrew Gallatin 	/* save a pointer to the received mbuf */
2678b2fc195eSAndrew Gallatin 	m = rx->info[idx].m;
2679b2fc195eSAndrew Gallatin 	/* try to replace the received mbuf */
26801e413cf9SAndrew Gallatin 	if (mxge_get_buf_big(ss, rx->extra_map, idx)) {
2681053e637fSAndrew Gallatin 		/* drop the frame -- the old mbuf is re-cycled */
2682f3f040d9SGleb Smirnoff 		if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
2683053e637fSAndrew Gallatin 		return;
2684b2fc195eSAndrew Gallatin 	}
2685053e637fSAndrew Gallatin 
2686b2fc195eSAndrew Gallatin 	/* unmap the received buffer */
2687b2fc195eSAndrew Gallatin 	old_map = rx->info[idx].map;
2688b2fc195eSAndrew Gallatin 	bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);
2689b2fc195eSAndrew Gallatin 	bus_dmamap_unload(rx->dmat, old_map);
2690b2fc195eSAndrew Gallatin 
2691b2fc195eSAndrew Gallatin 	/* swap the bus_dmamap_t's */
2692b2fc195eSAndrew Gallatin 	rx->info[idx].map = rx->extra_map;
2693b2fc195eSAndrew Gallatin 	rx->extra_map = old_map;
2694b2fc195eSAndrew Gallatin 
2695053e637fSAndrew Gallatin 	/* mcp implicitly skips 1st 2 bytes so that packet is properly
2696053e637fSAndrew Gallatin 	 * aligned */
26975e7d8541SAndrew Gallatin 	m->m_data += MXGEFW_PAD;
2698b2fc195eSAndrew Gallatin 
2699053e637fSAndrew Gallatin 	m->m_pkthdr.rcvif = ifp;
2700053e637fSAndrew Gallatin 	m->m_len = m->m_pkthdr.len = len;
27011e413cf9SAndrew Gallatin 	ss->ipackets++;
2702c792928fSAndrew Gallatin 	eh = mtod(m, struct ether_header *);
2703c792928fSAndrew Gallatin 	if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2704c792928fSAndrew Gallatin 		mxge_vlan_tag_remove(m, &csum);
2705c792928fSAndrew Gallatin 	}
2706eacb70baSSepherosa Ziehau 	/* flowid only valid if RSS hashing is enabled */
2707eacb70baSSepherosa Ziehau 	if (sc->num_slices > 1) {
2708eacb70baSSepherosa Ziehau 		m->m_pkthdr.flowid = (ss - sc->ss);
2709eacb70baSSepherosa Ziehau 		M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2710eacb70baSSepherosa Ziehau 	}
2711b2fc195eSAndrew Gallatin 	/* if the checksum is valid, mark it in the mbuf header */
271226dd49c6SAndrew Gallatin 	if ((ifp->if_capenable & (IFCAP_RXCSUM_IPV6 | IFCAP_RXCSUM)) &&
271326dd49c6SAndrew Gallatin 	    (0 == mxge_rx_csum(m, csum))) {
271426dd49c6SAndrew Gallatin 		/* Tell the stack that the  checksum is good */
2715053e637fSAndrew Gallatin 		m->m_pkthdr.csum_data = 0xffff;
271626dd49c6SAndrew Gallatin 		m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR |
271726dd49c6SAndrew Gallatin 			CSUM_DATA_VALID;
271826dd49c6SAndrew Gallatin 
271926dd49c6SAndrew Gallatin #if defined(INET) || defined (INET6)
272026dd49c6SAndrew Gallatin 		if (lro && (0 == tcp_lro_rx(&ss->lc, m, 0)))
272126dd49c6SAndrew Gallatin 			return;
272226dd49c6SAndrew Gallatin #endif
2723b2fc195eSAndrew Gallatin 	}
2724053e637fSAndrew Gallatin 	/* pass the frame up the stack */
2725053e637fSAndrew Gallatin 	(*ifp->if_input)(ifp, m);
2726b2fc195eSAndrew Gallatin }
2727b2fc195eSAndrew Gallatin 
2728b2fc195eSAndrew Gallatin static inline void
272926dd49c6SAndrew Gallatin mxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len,
273026dd49c6SAndrew Gallatin 		   uint32_t csum, int lro)
2731b2fc195eSAndrew Gallatin {
27321e413cf9SAndrew Gallatin 	mxge_softc_t *sc;
2733b2fc195eSAndrew Gallatin 	struct ifnet *ifp;
2734c792928fSAndrew Gallatin 	struct ether_header *eh;
2735b2fc195eSAndrew Gallatin 	struct mbuf *m;
27361e413cf9SAndrew Gallatin 	mxge_rx_ring_t *rx;
2737b2fc195eSAndrew Gallatin 	bus_dmamap_t old_map;
2738b2fc195eSAndrew Gallatin 	int idx;
2739b2fc195eSAndrew Gallatin 
27401e413cf9SAndrew Gallatin 	sc = ss->sc;
2741b2fc195eSAndrew Gallatin 	ifp = sc->ifp;
27421e413cf9SAndrew Gallatin 	rx = &ss->rx_small;
2743b2fc195eSAndrew Gallatin 	idx = rx->cnt & rx->mask;
2744b2fc195eSAndrew Gallatin 	rx->cnt++;
2745b2fc195eSAndrew Gallatin 	/* save a pointer to the received mbuf */
2746b2fc195eSAndrew Gallatin 	m = rx->info[idx].m;
2747b2fc195eSAndrew Gallatin 	/* try to replace the received mbuf */
27481e413cf9SAndrew Gallatin 	if (mxge_get_buf_small(ss, rx->extra_map, idx)) {
2749b2fc195eSAndrew Gallatin 		/* drop the frame -- the old mbuf is re-cycled */
2750f3f040d9SGleb Smirnoff 		if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
2751b2fc195eSAndrew Gallatin 		return;
2752b2fc195eSAndrew Gallatin 	}
2753b2fc195eSAndrew Gallatin 
2754b2fc195eSAndrew Gallatin 	/* unmap the received buffer */
2755b2fc195eSAndrew Gallatin 	old_map = rx->info[idx].map;
2756b2fc195eSAndrew Gallatin 	bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);
2757b2fc195eSAndrew Gallatin 	bus_dmamap_unload(rx->dmat, old_map);
2758b2fc195eSAndrew Gallatin 
2759b2fc195eSAndrew Gallatin 	/* swap the bus_dmamap_t's */
2760b2fc195eSAndrew Gallatin 	rx->info[idx].map = rx->extra_map;
2761b2fc195eSAndrew Gallatin 	rx->extra_map = old_map;
2762b2fc195eSAndrew Gallatin 
2763b2fc195eSAndrew Gallatin 	/* mcp implicitly skips 1st 2 bytes so that packet is properly
2764b2fc195eSAndrew Gallatin 	 * aligned */
27655e7d8541SAndrew Gallatin 	m->m_data += MXGEFW_PAD;
2766b2fc195eSAndrew Gallatin 
27679b03b0f3SAndrew Gallatin 	m->m_pkthdr.rcvif = ifp;
27689b03b0f3SAndrew Gallatin 	m->m_len = m->m_pkthdr.len = len;
27691e413cf9SAndrew Gallatin 	ss->ipackets++;
2770c792928fSAndrew Gallatin 	eh = mtod(m, struct ether_header *);
2771c792928fSAndrew Gallatin 	if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2772c792928fSAndrew Gallatin 		mxge_vlan_tag_remove(m, &csum);
2773c792928fSAndrew Gallatin 	}
2774eacb70baSSepherosa Ziehau 	/* flowid only valid if RSS hashing is enabled */
2775eacb70baSSepherosa Ziehau 	if (sc->num_slices > 1) {
2776eacb70baSSepherosa Ziehau 		m->m_pkthdr.flowid = (ss - sc->ss);
2777eacb70baSSepherosa Ziehau 		M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2778eacb70baSSepherosa Ziehau 	}
2779b2fc195eSAndrew Gallatin 	/* if the checksum is valid, mark it in the mbuf header */
278026dd49c6SAndrew Gallatin 	if ((ifp->if_capenable & (IFCAP_RXCSUM_IPV6 | IFCAP_RXCSUM)) &&
278126dd49c6SAndrew Gallatin 	    (0 == mxge_rx_csum(m, csum))) {
278226dd49c6SAndrew Gallatin 		/* Tell the stack that the  checksum is good */
2783053e637fSAndrew Gallatin 		m->m_pkthdr.csum_data = 0xffff;
278426dd49c6SAndrew Gallatin 		m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR |
278526dd49c6SAndrew Gallatin 			CSUM_DATA_VALID;
278626dd49c6SAndrew Gallatin 
278726dd49c6SAndrew Gallatin #if defined(INET) || defined (INET6)
278826dd49c6SAndrew Gallatin 		if (lro && (0 == tcp_lro_rx(&ss->lc, m, csum)))
278926dd49c6SAndrew Gallatin 			return;
279026dd49c6SAndrew Gallatin #endif
2791053e637fSAndrew Gallatin 	}
2792b2fc195eSAndrew Gallatin 	/* pass the frame up the stack */
2793b2fc195eSAndrew Gallatin 	(*ifp->if_input)(ifp, m);
2794b2fc195eSAndrew Gallatin }
2795b2fc195eSAndrew Gallatin 
2796b2fc195eSAndrew Gallatin static inline void
27971e413cf9SAndrew Gallatin mxge_clean_rx_done(struct mxge_slice_state *ss)
27985e7d8541SAndrew Gallatin {
27991e413cf9SAndrew Gallatin 	mxge_rx_done_t *rx_done = &ss->rx_done;
28005e7d8541SAndrew Gallatin 	int limit = 0;
28015e7d8541SAndrew Gallatin 	uint16_t length;
28025e7d8541SAndrew Gallatin 	uint16_t checksum;
280326dd49c6SAndrew Gallatin 	int lro;
28045e7d8541SAndrew Gallatin 
280526dd49c6SAndrew Gallatin 	lro = ss->sc->ifp->if_capenable & IFCAP_LRO;
28065e7d8541SAndrew Gallatin 	while (rx_done->entry[rx_done->idx].length != 0) {
28075e7d8541SAndrew Gallatin 		length = ntohs(rx_done->entry[rx_done->idx].length);
28085e7d8541SAndrew Gallatin 		rx_done->entry[rx_done->idx].length = 0;
2809053e637fSAndrew Gallatin 		checksum = rx_done->entry[rx_done->idx].checksum;
2810b4db9009SAndrew Gallatin 		if (length <= (MHLEN - MXGEFW_PAD))
281126dd49c6SAndrew Gallatin 			mxge_rx_done_small(ss, length, checksum, lro);
28125e7d8541SAndrew Gallatin 		else
281326dd49c6SAndrew Gallatin 			mxge_rx_done_big(ss, length, checksum, lro);
28145e7d8541SAndrew Gallatin 		rx_done->cnt++;
2815adae7080SAndrew Gallatin 		rx_done->idx = rx_done->cnt & rx_done->mask;
28165e7d8541SAndrew Gallatin 
28175e7d8541SAndrew Gallatin 		/* limit potential for livelock */
2818f616ebc7SAndrew Gallatin 		if (__predict_false(++limit > rx_done->mask / 2))
28195e7d8541SAndrew Gallatin 			break;
2820053e637fSAndrew Gallatin 	}
282126dd49c6SAndrew Gallatin #if defined(INET)  || defined (INET6)
28226dd38b87SSepherosa Ziehau 	tcp_lro_flush_all(&ss->lc);
2823eb6219e3SAndrew Gallatin #endif
28245e7d8541SAndrew Gallatin }
28255e7d8541SAndrew Gallatin 
28265e7d8541SAndrew Gallatin 
28275e7d8541SAndrew Gallatin static inline void
28281e413cf9SAndrew Gallatin mxge_tx_done(struct mxge_slice_state *ss, uint32_t mcp_idx)
2829b2fc195eSAndrew Gallatin {
2830b2fc195eSAndrew Gallatin 	struct ifnet *ifp;
28311e413cf9SAndrew Gallatin 	mxge_tx_ring_t *tx;
2832b2fc195eSAndrew Gallatin 	struct mbuf *m;
2833b2fc195eSAndrew Gallatin 	bus_dmamap_t map;
2834f616ebc7SAndrew Gallatin 	int idx;
2835c6cb3e3fSAndrew Gallatin 	int *flags;
2836b2fc195eSAndrew Gallatin 
28371e413cf9SAndrew Gallatin 	tx = &ss->tx;
28381e413cf9SAndrew Gallatin 	ifp = ss->sc->ifp;
28395e7d8541SAndrew Gallatin 	while (tx->pkt_done != mcp_idx) {
2840b2fc195eSAndrew Gallatin 		idx = tx->done & tx->mask;
2841b2fc195eSAndrew Gallatin 		tx->done++;
2842b2fc195eSAndrew Gallatin 		m = tx->info[idx].m;
2843b2fc195eSAndrew Gallatin 		/* mbuf and DMA map only attached to the first
2844b2fc195eSAndrew Gallatin 		   segment per-mbuf */
2845b2fc195eSAndrew Gallatin 		if (m != NULL) {
284671032832SAndrew Gallatin 			ss->obytes += m->m_pkthdr.len;
284771032832SAndrew Gallatin 			if (m->m_flags & M_MCAST)
284871032832SAndrew Gallatin 				ss->omcasts++;
2849c6cb3e3fSAndrew Gallatin 			ss->opackets++;
2850b2fc195eSAndrew Gallatin 			tx->info[idx].m = NULL;
2851b2fc195eSAndrew Gallatin 			map = tx->info[idx].map;
2852b2fc195eSAndrew Gallatin 			bus_dmamap_unload(tx->dmat, map);
2853b2fc195eSAndrew Gallatin 			m_freem(m);
2854b2fc195eSAndrew Gallatin 		}
28555e7d8541SAndrew Gallatin 		if (tx->info[idx].flag) {
28565e7d8541SAndrew Gallatin 			tx->info[idx].flag = 0;
28575e7d8541SAndrew Gallatin 			tx->pkt_done++;
28585e7d8541SAndrew Gallatin 		}
2859b2fc195eSAndrew Gallatin 	}
2860b2fc195eSAndrew Gallatin 
2861b2fc195eSAndrew Gallatin 	/* If we have space, clear IFF_OACTIVE to tell the stack that
2862b2fc195eSAndrew Gallatin 	   its OK to send packets */
2863c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
2864c6cb3e3fSAndrew Gallatin 	flags = &ss->if_drv_flags;
2865c6cb3e3fSAndrew Gallatin #else
2866c6cb3e3fSAndrew Gallatin 	flags = &ifp->if_drv_flags;
2867c6cb3e3fSAndrew Gallatin #endif
28681e413cf9SAndrew Gallatin 	mtx_lock(&ss->tx.mtx);
2869c6cb3e3fSAndrew Gallatin 	if ((*flags) & IFF_DRV_OACTIVE &&
2870c6cb3e3fSAndrew Gallatin 	    tx->req - tx->done < (tx->mask + 1)/4) {
2871c6cb3e3fSAndrew Gallatin 		*(flags) &= ~IFF_DRV_OACTIVE;
28721e413cf9SAndrew Gallatin 		ss->tx.wake++;
28731e413cf9SAndrew Gallatin 		mxge_start_locked(ss);
2874b2fc195eSAndrew Gallatin 	}
2875c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
2876c6cb3e3fSAndrew Gallatin 	if ((ss->sc->num_slices > 1) && (tx->req == tx->done)) {
2877c6cb3e3fSAndrew Gallatin 		/* let the NIC stop polling this queue, since there
2878c6cb3e3fSAndrew Gallatin 		 * are no more transmits pending */
2879c6cb3e3fSAndrew Gallatin 		if (tx->req == tx->done) {
2880c6cb3e3fSAndrew Gallatin 			*tx->send_stop = 1;
2881c6cb3e3fSAndrew Gallatin 			tx->queue_active = 0;
2882c6cb3e3fSAndrew Gallatin 			tx->deactivate++;
2883c6cb3e3fSAndrew Gallatin 			wmb();
2884c6cb3e3fSAndrew Gallatin 		}
2885c6cb3e3fSAndrew Gallatin 	}
2886c6cb3e3fSAndrew Gallatin #endif
2887c6cb3e3fSAndrew Gallatin 	mtx_unlock(&ss->tx.mtx);
2888c6cb3e3fSAndrew Gallatin 
2889b2fc195eSAndrew Gallatin }
2890b2fc195eSAndrew Gallatin 
289101638550SAndrew Gallatin static struct mxge_media_type mxge_xfp_media_types[] =
2892c587e59fSAndrew Gallatin {
2893c587e59fSAndrew Gallatin 	{IFM_10G_CX4,	0x7f, 		"10GBASE-CX4 (module)"},
2894c587e59fSAndrew Gallatin 	{IFM_10G_SR, 	(1 << 7),	"10GBASE-SR"},
2895c587e59fSAndrew Gallatin 	{IFM_10G_LR, 	(1 << 6),	"10GBASE-LR"},
2896c587e59fSAndrew Gallatin 	{0,		(1 << 5),	"10GBASE-ER"},
289701638550SAndrew Gallatin 	{IFM_10G_LRM,	(1 << 4),	"10GBASE-LRM"},
2898c587e59fSAndrew Gallatin 	{0,		(1 << 3),	"10GBASE-SW"},
2899c587e59fSAndrew Gallatin 	{0,		(1 << 2),	"10GBASE-LW"},
2900c587e59fSAndrew Gallatin 	{0,		(1 << 1),	"10GBASE-EW"},
2901c587e59fSAndrew Gallatin 	{0,		(1 << 0),	"Reserved"}
2902c587e59fSAndrew Gallatin };
290301638550SAndrew Gallatin static struct mxge_media_type mxge_sfp_media_types[] =
290401638550SAndrew Gallatin {
290551bc2092SAndrew Gallatin 	{IFM_10G_TWINAX,      0,	"10GBASE-Twinax"},
29064ae3322fSAndrew Gallatin 	{0,		(1 << 7),	"Reserved"},
290701638550SAndrew Gallatin 	{IFM_10G_LRM,	(1 << 6),	"10GBASE-LRM"},
290801638550SAndrew Gallatin 	{IFM_10G_LR, 	(1 << 5),	"10GBASE-LR"},
290956b67858SAndrew Gallatin 	{IFM_10G_SR,	(1 << 4),	"10GBASE-SR"},
291056b67858SAndrew Gallatin 	{IFM_10G_TWINAX,(1 << 0),	"10GBASE-Twinax"}
291101638550SAndrew Gallatin };
2912c587e59fSAndrew Gallatin 
2913c587e59fSAndrew Gallatin static void
2914c406ad2eSAndrew Gallatin mxge_media_set(mxge_softc_t *sc, int media_type)
2915c587e59fSAndrew Gallatin {
2916c406ad2eSAndrew Gallatin 
2917c406ad2eSAndrew Gallatin 
2918c406ad2eSAndrew Gallatin 	ifmedia_add(&sc->media, IFM_ETHER | IFM_FDX | media_type,
2919c406ad2eSAndrew Gallatin 		    0, NULL);
2920c406ad2eSAndrew Gallatin 	ifmedia_set(&sc->media, IFM_ETHER | IFM_FDX | media_type);
2921c406ad2eSAndrew Gallatin 	sc->current_media = media_type;
2922c406ad2eSAndrew Gallatin 	sc->media.ifm_media = sc->media.ifm_cur->ifm_media;
2923c587e59fSAndrew Gallatin }
2924c587e59fSAndrew Gallatin 
2925c587e59fSAndrew Gallatin static void
2926c406ad2eSAndrew Gallatin mxge_media_init(mxge_softc_t *sc)
2927c587e59fSAndrew Gallatin {
2928c587e59fSAndrew Gallatin 	char *ptr;
2929c406ad2eSAndrew Gallatin 	int i;
2930c587e59fSAndrew Gallatin 
2931c406ad2eSAndrew Gallatin 	ifmedia_removeall(&sc->media);
2932c406ad2eSAndrew Gallatin 	mxge_media_set(sc, IFM_AUTO);
2933c587e59fSAndrew Gallatin 
2934c587e59fSAndrew Gallatin 	/*
2935c587e59fSAndrew Gallatin 	 * parse the product code to deterimine the interface type
2936c587e59fSAndrew Gallatin 	 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character
2937c587e59fSAndrew Gallatin 	 * after the 3rd dash in the driver's cached copy of the
2938c587e59fSAndrew Gallatin 	 * EEPROM's product code string.
2939c587e59fSAndrew Gallatin 	 */
2940c587e59fSAndrew Gallatin 	ptr = sc->product_code_string;
2941c587e59fSAndrew Gallatin 	if (ptr == NULL) {
2942c587e59fSAndrew Gallatin 		device_printf(sc->dev, "Missing product code\n");
2943c406ad2eSAndrew Gallatin 		return;
2944c587e59fSAndrew Gallatin 	}
2945c587e59fSAndrew Gallatin 
2946c587e59fSAndrew Gallatin 	for (i = 0; i < 3; i++, ptr++) {
2947dc15eac0SEd Schouten 		ptr = strchr(ptr, '-');
2948c587e59fSAndrew Gallatin 		if (ptr == NULL) {
2949c587e59fSAndrew Gallatin 			device_printf(sc->dev,
2950c587e59fSAndrew Gallatin 				      "only %d dashes in PC?!?\n", i);
2951c587e59fSAndrew Gallatin 			return;
2952c587e59fSAndrew Gallatin 		}
2953c587e59fSAndrew Gallatin 	}
295430882b10SAndrew Gallatin 	if (*ptr == 'C' || *(ptr +1) == 'C') {
295501638550SAndrew Gallatin 		/* -C is CX4 */
2956c406ad2eSAndrew Gallatin 		sc->connector = MXGE_CX4;
2957c406ad2eSAndrew Gallatin 		mxge_media_set(sc, IFM_10G_CX4);
2958c406ad2eSAndrew Gallatin 	} else if (*ptr == 'Q') {
295901638550SAndrew Gallatin 		/* -Q is Quad Ribbon Fiber */
2960c406ad2eSAndrew Gallatin 		sc->connector = MXGE_QRF;
2961c587e59fSAndrew Gallatin 		device_printf(sc->dev, "Quad Ribbon Fiber Media\n");
2962c587e59fSAndrew Gallatin 		/* FreeBSD has no media type for Quad ribbon fiber */
2963c406ad2eSAndrew Gallatin 	} else if (*ptr == 'R') {
2964c406ad2eSAndrew Gallatin 		/* -R is XFP */
2965c406ad2eSAndrew Gallatin 		sc->connector = MXGE_XFP;
2966c406ad2eSAndrew Gallatin 	} else if (*ptr == 'S' || *(ptr +1) == 'S') {
2967c406ad2eSAndrew Gallatin 		/* -S or -2S is SFP+ */
2968c406ad2eSAndrew Gallatin 		sc->connector = MXGE_SFP;
2969c406ad2eSAndrew Gallatin 	} else {
2970c406ad2eSAndrew Gallatin 		device_printf(sc->dev, "Unknown media type: %c\n", *ptr);
2971c406ad2eSAndrew Gallatin 	}
2972c587e59fSAndrew Gallatin }
2973c587e59fSAndrew Gallatin 
2974c406ad2eSAndrew Gallatin /*
2975c406ad2eSAndrew Gallatin  * Determine the media type for a NIC.  Some XFPs will identify
2976c406ad2eSAndrew Gallatin  * themselves only when their link is up, so this is initiated via a
2977c406ad2eSAndrew Gallatin  * link up interrupt.  However, this can potentially take up to
2978c406ad2eSAndrew Gallatin  * several milliseconds, so it is run via the watchdog routine, rather
2979c406ad2eSAndrew Gallatin  * than in the interrupt handler itself.
2980c406ad2eSAndrew Gallatin  */
2981c406ad2eSAndrew Gallatin static void
2982c406ad2eSAndrew Gallatin mxge_media_probe(mxge_softc_t *sc)
2983c406ad2eSAndrew Gallatin {
2984c406ad2eSAndrew Gallatin 	mxge_cmd_t cmd;
2985c406ad2eSAndrew Gallatin 	char *cage_type;
2986c406ad2eSAndrew Gallatin 
2987c406ad2eSAndrew Gallatin 	struct mxge_media_type *mxge_media_types = NULL;
2988c406ad2eSAndrew Gallatin 	int i, err, ms, mxge_media_type_entries;
2989c406ad2eSAndrew Gallatin 	uint32_t byte;
2990c406ad2eSAndrew Gallatin 
2991c406ad2eSAndrew Gallatin 	sc->need_media_probe = 0;
2992c406ad2eSAndrew Gallatin 
2993c406ad2eSAndrew Gallatin 	if (sc->connector == MXGE_XFP) {
299401638550SAndrew Gallatin 		/* -R is XFP */
299501638550SAndrew Gallatin 		mxge_media_types = mxge_xfp_media_types;
299601638550SAndrew Gallatin 		mxge_media_type_entries =
299773a1170aSPedro F. Giffuni 			nitems(mxge_xfp_media_types);
299801638550SAndrew Gallatin 		byte = MXGE_XFP_COMPLIANCE_BYTE;
299901638550SAndrew Gallatin 		cage_type = "XFP";
3000c406ad2eSAndrew Gallatin 	} else 	if (sc->connector == MXGE_SFP) {
300101638550SAndrew Gallatin 		/* -S or -2S is SFP+ */
300201638550SAndrew Gallatin 		mxge_media_types = mxge_sfp_media_types;
300301638550SAndrew Gallatin 		mxge_media_type_entries =
300473a1170aSPedro F. Giffuni 			nitems(mxge_sfp_media_types);
300501638550SAndrew Gallatin 		cage_type = "SFP+";
300601638550SAndrew Gallatin 		byte = 3;
3007c406ad2eSAndrew Gallatin 	} else {
3008c406ad2eSAndrew Gallatin 		/* nothing to do; media type cannot change */
3009c587e59fSAndrew Gallatin 		return;
3010c587e59fSAndrew Gallatin 	}
3011c587e59fSAndrew Gallatin 
3012c587e59fSAndrew Gallatin 	/*
3013c587e59fSAndrew Gallatin 	 * At this point we know the NIC has an XFP cage, so now we
3014c587e59fSAndrew Gallatin 	 * try to determine what is in the cage by using the
3015c587e59fSAndrew Gallatin 	 * firmware's XFP I2C commands to read the XFP 10GbE compilance
3016c587e59fSAndrew Gallatin 	 * register.  We read just one byte, which may take over
3017c587e59fSAndrew Gallatin 	 * a millisecond
3018c587e59fSAndrew Gallatin 	 */
3019c587e59fSAndrew Gallatin 
3020c587e59fSAndrew Gallatin 	cmd.data0 = 0;	 /* just fetch 1 byte, not all 256 */
302101638550SAndrew Gallatin 	cmd.data1 = byte;
302201638550SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_READ, &cmd);
302301638550SAndrew Gallatin 	if (err == MXGEFW_CMD_ERROR_I2C_FAILURE) {
3024c587e59fSAndrew Gallatin 		device_printf(sc->dev, "failed to read XFP\n");
3025c587e59fSAndrew Gallatin 	}
302601638550SAndrew Gallatin 	if (err == MXGEFW_CMD_ERROR_I2C_ABSENT) {
302701638550SAndrew Gallatin 		device_printf(sc->dev, "Type R/S with no XFP!?!?\n");
3028c587e59fSAndrew Gallatin 	}
3029c587e59fSAndrew Gallatin 	if (err != MXGEFW_CMD_OK) {
3030c587e59fSAndrew Gallatin 		return;
3031c587e59fSAndrew Gallatin 	}
3032c587e59fSAndrew Gallatin 
3033c587e59fSAndrew Gallatin 	/* now we wait for the data to be cached */
303401638550SAndrew Gallatin 	cmd.data0 = byte;
303501638550SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd);
3036c587e59fSAndrew Gallatin 	for (ms = 0; (err == EBUSY) && (ms < 50); ms++) {
3037c587e59fSAndrew Gallatin 		DELAY(1000);
303801638550SAndrew Gallatin 		cmd.data0 = byte;
303901638550SAndrew Gallatin 		err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd);
3040c587e59fSAndrew Gallatin 	}
3041c587e59fSAndrew Gallatin 	if (err != MXGEFW_CMD_OK) {
304201638550SAndrew Gallatin 		device_printf(sc->dev, "failed to read %s (%d, %dms)\n",
304301638550SAndrew Gallatin 			      cage_type, err, ms);
3044c587e59fSAndrew Gallatin 		return;
3045c587e59fSAndrew Gallatin 	}
3046c587e59fSAndrew Gallatin 
3047c587e59fSAndrew Gallatin 	if (cmd.data0 == mxge_media_types[0].bitmask) {
3048c587e59fSAndrew Gallatin 		if (mxge_verbose)
304901638550SAndrew Gallatin 			device_printf(sc->dev, "%s:%s\n", cage_type,
3050c587e59fSAndrew Gallatin 				      mxge_media_types[0].name);
3051c406ad2eSAndrew Gallatin 		if (sc->current_media != mxge_media_types[0].flag) {
3052c406ad2eSAndrew Gallatin 			mxge_media_init(sc);
3053c406ad2eSAndrew Gallatin 			mxge_media_set(sc, mxge_media_types[0].flag);
3054c406ad2eSAndrew Gallatin 		}
3055c587e59fSAndrew Gallatin 		return;
3056c587e59fSAndrew Gallatin 	}
305701638550SAndrew Gallatin 	for (i = 1; i < mxge_media_type_entries; i++) {
3058c587e59fSAndrew Gallatin 		if (cmd.data0 & mxge_media_types[i].bitmask) {
3059c587e59fSAndrew Gallatin 			if (mxge_verbose)
306001638550SAndrew Gallatin 				device_printf(sc->dev, "%s:%s\n",
306101638550SAndrew Gallatin 					      cage_type,
3062c587e59fSAndrew Gallatin 					      mxge_media_types[i].name);
3063c587e59fSAndrew Gallatin 
3064c406ad2eSAndrew Gallatin 			if (sc->current_media != mxge_media_types[i].flag) {
3065c406ad2eSAndrew Gallatin 				mxge_media_init(sc);
3066c406ad2eSAndrew Gallatin 				mxge_media_set(sc, mxge_media_types[i].flag);
3067c406ad2eSAndrew Gallatin 			}
3068c587e59fSAndrew Gallatin 			return;
3069c587e59fSAndrew Gallatin 		}
3070c587e59fSAndrew Gallatin 	}
3071c406ad2eSAndrew Gallatin 	if (mxge_verbose)
3072c406ad2eSAndrew Gallatin 		device_printf(sc->dev, "%s media 0x%x unknown\n",
3073c406ad2eSAndrew Gallatin 			      cage_type, cmd.data0);
3074c587e59fSAndrew Gallatin 
3075c587e59fSAndrew Gallatin 	return;
3076c587e59fSAndrew Gallatin }
3077c587e59fSAndrew Gallatin 
3078b2fc195eSAndrew Gallatin static void
30796d87a65dSAndrew Gallatin mxge_intr(void *arg)
3080b2fc195eSAndrew Gallatin {
30811e413cf9SAndrew Gallatin 	struct mxge_slice_state *ss = arg;
30821e413cf9SAndrew Gallatin 	mxge_softc_t *sc = ss->sc;
30831e413cf9SAndrew Gallatin 	mcp_irq_data_t *stats = ss->fw_stats;
30841e413cf9SAndrew Gallatin 	mxge_tx_ring_t *tx = &ss->tx;
30851e413cf9SAndrew Gallatin 	mxge_rx_done_t *rx_done = &ss->rx_done;
30865e7d8541SAndrew Gallatin 	uint32_t send_done_count;
30875e7d8541SAndrew Gallatin 	uint8_t valid;
3088b2fc195eSAndrew Gallatin 
3089b2fc195eSAndrew Gallatin 
3090c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING
30911e413cf9SAndrew Gallatin 	/* an interrupt on a non-zero slice is implicitly valid
30921e413cf9SAndrew Gallatin 	   since MSI-X irqs are not shared */
30931e413cf9SAndrew Gallatin 	if (ss != sc->ss) {
30941e413cf9SAndrew Gallatin 		mxge_clean_rx_done(ss);
30951e413cf9SAndrew Gallatin 		*ss->irq_claim = be32toh(3);
30961e413cf9SAndrew Gallatin 		return;
30971e413cf9SAndrew Gallatin 	}
3098c6cb3e3fSAndrew Gallatin #endif
30991e413cf9SAndrew Gallatin 
31005e7d8541SAndrew Gallatin 	/* make sure the DMA has finished */
31015e7d8541SAndrew Gallatin 	if (!stats->valid) {
31025e7d8541SAndrew Gallatin 		return;
3103b2fc195eSAndrew Gallatin 	}
31045e7d8541SAndrew Gallatin 	valid = stats->valid;
3105b2fc195eSAndrew Gallatin 
310691ed8913SAndrew Gallatin 	if (sc->legacy_irq) {
31075e7d8541SAndrew Gallatin 		/* lower legacy IRQ  */
31085e7d8541SAndrew Gallatin 		*sc->irq_deassert = 0;
31095e7d8541SAndrew Gallatin 		if (!mxge_deassert_wait)
31105e7d8541SAndrew Gallatin 			/* don't wait for conf. that irq is low */
31115e7d8541SAndrew Gallatin 			stats->valid = 0;
3112dc8731d4SAndrew Gallatin 	} else {
3113dc8731d4SAndrew Gallatin 		stats->valid = 0;
3114dc8731d4SAndrew Gallatin 	}
3115dc8731d4SAndrew Gallatin 
3116dc8731d4SAndrew Gallatin 	/* loop while waiting for legacy irq deassertion */
31175e7d8541SAndrew Gallatin 	do {
31185e7d8541SAndrew Gallatin 		/* check for transmit completes and receives */
31195e7d8541SAndrew Gallatin 		send_done_count = be32toh(stats->send_done_count);
31205e7d8541SAndrew Gallatin 		while ((send_done_count != tx->pkt_done) ||
31215e7d8541SAndrew Gallatin 		       (rx_done->entry[rx_done->idx].length != 0)) {
3122c6cb3e3fSAndrew Gallatin 			if (send_done_count != tx->pkt_done)
31231e413cf9SAndrew Gallatin 				mxge_tx_done(ss, (int)send_done_count);
31241e413cf9SAndrew Gallatin 			mxge_clean_rx_done(ss);
31255e7d8541SAndrew Gallatin 			send_done_count = be32toh(stats->send_done_count);
3126b2fc195eSAndrew Gallatin 		}
312791ed8913SAndrew Gallatin 		if (sc->legacy_irq && mxge_deassert_wait)
312873c7c83fSAndrew Gallatin 			wmb();
31295e7d8541SAndrew Gallatin 	} while (*((volatile uint8_t *) &stats->valid));
3130b2fc195eSAndrew Gallatin 
3131c6cb3e3fSAndrew Gallatin 	/* fw link & error stats meaningful only on the first slice */
3132c6cb3e3fSAndrew Gallatin 	if (__predict_false((ss == sc->ss) && stats->stats_updated)) {
31335e7d8541SAndrew Gallatin 		if (sc->link_state != stats->link_up) {
31345e7d8541SAndrew Gallatin 			sc->link_state = stats->link_up;
3135b2fc195eSAndrew Gallatin 			if (sc->link_state) {
31365e7d8541SAndrew Gallatin 				if_link_state_change(sc->ifp, LINK_STATE_UP);
31375e7d8541SAndrew Gallatin 				if (mxge_verbose)
31385e7d8541SAndrew Gallatin 					device_printf(sc->dev, "link up\n");
3139b2fc195eSAndrew Gallatin 			} else {
31405e7d8541SAndrew Gallatin 				if_link_state_change(sc->ifp, LINK_STATE_DOWN);
31415e7d8541SAndrew Gallatin 				if (mxge_verbose)
31425e7d8541SAndrew Gallatin 					device_printf(sc->dev, "link down\n");
3143b2fc195eSAndrew Gallatin 			}
3144c587e59fSAndrew Gallatin 			sc->need_media_probe = 1;
3145b2fc195eSAndrew Gallatin 		}
3146b2fc195eSAndrew Gallatin 		if (sc->rdma_tags_available !=
31471e413cf9SAndrew Gallatin 		    be32toh(stats->rdma_tags_available)) {
3148b2fc195eSAndrew Gallatin 			sc->rdma_tags_available =
31491e413cf9SAndrew Gallatin 				be32toh(stats->rdma_tags_available);
31505e7d8541SAndrew Gallatin 			device_printf(sc->dev, "RDMA timed out! %d tags "
31515e7d8541SAndrew Gallatin 				      "left\n", sc->rdma_tags_available);
31525e7d8541SAndrew Gallatin 		}
3153c587e59fSAndrew Gallatin 
3154c587e59fSAndrew Gallatin 		if (stats->link_down) {
31555e7d8541SAndrew Gallatin 			sc->down_cnt += stats->link_down;
3156c587e59fSAndrew Gallatin 			sc->link_state = 0;
3157c587e59fSAndrew Gallatin 			if_link_state_change(sc->ifp, LINK_STATE_DOWN);
3158c587e59fSAndrew Gallatin 		}
3159b2fc195eSAndrew Gallatin 	}
3160b2fc195eSAndrew Gallatin 
31615e7d8541SAndrew Gallatin 	/* check to see if we have rx token to pass back */
31625e7d8541SAndrew Gallatin 	if (valid & 0x1)
31631e413cf9SAndrew Gallatin 	    *ss->irq_claim = be32toh(3);
31641e413cf9SAndrew Gallatin 	*(ss->irq_claim + 1) = be32toh(3);
3165b2fc195eSAndrew Gallatin }
3166b2fc195eSAndrew Gallatin 
3167b2fc195eSAndrew Gallatin static void
31686d87a65dSAndrew Gallatin mxge_init(void *arg)
3169b2fc195eSAndrew Gallatin {
31703cae7311SAndrew Gallatin 	mxge_softc_t *sc = arg;
31713cae7311SAndrew Gallatin 	struct ifnet *ifp = sc->ifp;
31723cae7311SAndrew Gallatin 
31733cae7311SAndrew Gallatin 
31743cae7311SAndrew Gallatin 	mtx_lock(&sc->driver_mtx);
31753cae7311SAndrew Gallatin 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
31763cae7311SAndrew Gallatin 		(void) mxge_open(sc);
31773cae7311SAndrew Gallatin 	mtx_unlock(&sc->driver_mtx);
3178b2fc195eSAndrew Gallatin }
3179b2fc195eSAndrew Gallatin 
3180b2fc195eSAndrew Gallatin 
3181b2fc195eSAndrew Gallatin 
3182b2fc195eSAndrew Gallatin static void
31831e413cf9SAndrew Gallatin mxge_free_slice_mbufs(struct mxge_slice_state *ss)
31841e413cf9SAndrew Gallatin {
31851e413cf9SAndrew Gallatin 	int i;
31861e413cf9SAndrew Gallatin 
318726dd49c6SAndrew Gallatin #if defined(INET) || defined(INET6)
318826dd49c6SAndrew Gallatin 	tcp_lro_free(&ss->lc);
318926dd49c6SAndrew Gallatin #endif
31901e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->rx_big.mask; i++) {
31911e413cf9SAndrew Gallatin 		if (ss->rx_big.info[i].m == NULL)
31921e413cf9SAndrew Gallatin 			continue;
31931e413cf9SAndrew Gallatin 		bus_dmamap_unload(ss->rx_big.dmat,
31941e413cf9SAndrew Gallatin 				  ss->rx_big.info[i].map);
31951e413cf9SAndrew Gallatin 		m_freem(ss->rx_big.info[i].m);
31961e413cf9SAndrew Gallatin 		ss->rx_big.info[i].m = NULL;
31971e413cf9SAndrew Gallatin 	}
31981e413cf9SAndrew Gallatin 
31991e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->rx_small.mask; i++) {
32001e413cf9SAndrew Gallatin 		if (ss->rx_small.info[i].m == NULL)
32011e413cf9SAndrew Gallatin 			continue;
32021e413cf9SAndrew Gallatin 		bus_dmamap_unload(ss->rx_small.dmat,
32031e413cf9SAndrew Gallatin 				  ss->rx_small.info[i].map);
32041e413cf9SAndrew Gallatin 		m_freem(ss->rx_small.info[i].m);
32051e413cf9SAndrew Gallatin 		ss->rx_small.info[i].m = NULL;
32061e413cf9SAndrew Gallatin 	}
32071e413cf9SAndrew Gallatin 
32081e413cf9SAndrew Gallatin 	/* transmit ring used only on the first slice */
32091e413cf9SAndrew Gallatin 	if (ss->tx.info == NULL)
32101e413cf9SAndrew Gallatin 		return;
32111e413cf9SAndrew Gallatin 
32121e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->tx.mask; i++) {
32131e413cf9SAndrew Gallatin 		ss->tx.info[i].flag = 0;
32141e413cf9SAndrew Gallatin 		if (ss->tx.info[i].m == NULL)
32151e413cf9SAndrew Gallatin 			continue;
32161e413cf9SAndrew Gallatin 		bus_dmamap_unload(ss->tx.dmat,
32171e413cf9SAndrew Gallatin 				  ss->tx.info[i].map);
32181e413cf9SAndrew Gallatin 		m_freem(ss->tx.info[i].m);
32191e413cf9SAndrew Gallatin 		ss->tx.info[i].m = NULL;
32201e413cf9SAndrew Gallatin 	}
32211e413cf9SAndrew Gallatin }
32221e413cf9SAndrew Gallatin 
32231e413cf9SAndrew Gallatin static void
32246d87a65dSAndrew Gallatin mxge_free_mbufs(mxge_softc_t *sc)
3225b2fc195eSAndrew Gallatin {
32261e413cf9SAndrew Gallatin 	int slice;
32271e413cf9SAndrew Gallatin 
32281e413cf9SAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++)
32291e413cf9SAndrew Gallatin 		mxge_free_slice_mbufs(&sc->ss[slice]);
32301e413cf9SAndrew Gallatin }
32311e413cf9SAndrew Gallatin 
32321e413cf9SAndrew Gallatin static void
32331e413cf9SAndrew Gallatin mxge_free_slice_rings(struct mxge_slice_state *ss)
32341e413cf9SAndrew Gallatin {
3235b2fc195eSAndrew Gallatin 	int i;
3236b2fc195eSAndrew Gallatin 
3237b2fc195eSAndrew Gallatin 
32381e413cf9SAndrew Gallatin 	if (ss->rx_done.entry != NULL)
32391e413cf9SAndrew Gallatin 		mxge_dma_free(&ss->rx_done.dma);
32401e413cf9SAndrew Gallatin 	ss->rx_done.entry = NULL;
3241b2fc195eSAndrew Gallatin 
32421e413cf9SAndrew Gallatin 	if (ss->tx.req_bytes != NULL)
32431e413cf9SAndrew Gallatin 		free(ss->tx.req_bytes, M_DEVBUF);
32441e413cf9SAndrew Gallatin 	ss->tx.req_bytes = NULL;
32451e413cf9SAndrew Gallatin 
32461e413cf9SAndrew Gallatin 	if (ss->tx.seg_list != NULL)
32471e413cf9SAndrew Gallatin 		free(ss->tx.seg_list, M_DEVBUF);
32481e413cf9SAndrew Gallatin 	ss->tx.seg_list = NULL;
32491e413cf9SAndrew Gallatin 
32501e413cf9SAndrew Gallatin 	if (ss->rx_small.shadow != NULL)
32511e413cf9SAndrew Gallatin 		free(ss->rx_small.shadow, M_DEVBUF);
32521e413cf9SAndrew Gallatin 	ss->rx_small.shadow = NULL;
32531e413cf9SAndrew Gallatin 
32541e413cf9SAndrew Gallatin 	if (ss->rx_big.shadow != NULL)
32551e413cf9SAndrew Gallatin 		free(ss->rx_big.shadow, M_DEVBUF);
32561e413cf9SAndrew Gallatin 	ss->rx_big.shadow = NULL;
32571e413cf9SAndrew Gallatin 
32581e413cf9SAndrew Gallatin 	if (ss->tx.info != NULL) {
32591e413cf9SAndrew Gallatin 		if (ss->tx.dmat != NULL) {
32601e413cf9SAndrew Gallatin 			for (i = 0; i <= ss->tx.mask; i++) {
32611e413cf9SAndrew Gallatin 				bus_dmamap_destroy(ss->tx.dmat,
32621e413cf9SAndrew Gallatin 						   ss->tx.info[i].map);
3263b2fc195eSAndrew Gallatin 			}
32641e413cf9SAndrew Gallatin 			bus_dma_tag_destroy(ss->tx.dmat);
32651e413cf9SAndrew Gallatin 		}
32661e413cf9SAndrew Gallatin 		free(ss->tx.info, M_DEVBUF);
32671e413cf9SAndrew Gallatin 	}
32681e413cf9SAndrew Gallatin 	ss->tx.info = NULL;
32691e413cf9SAndrew Gallatin 
32701e413cf9SAndrew Gallatin 	if (ss->rx_small.info != NULL) {
32711e413cf9SAndrew Gallatin 		if (ss->rx_small.dmat != NULL) {
32721e413cf9SAndrew Gallatin 			for (i = 0; i <= ss->rx_small.mask; i++) {
32731e413cf9SAndrew Gallatin 				bus_dmamap_destroy(ss->rx_small.dmat,
32741e413cf9SAndrew Gallatin 						   ss->rx_small.info[i].map);
32751e413cf9SAndrew Gallatin 			}
32761e413cf9SAndrew Gallatin 			bus_dmamap_destroy(ss->rx_small.dmat,
32771e413cf9SAndrew Gallatin 					   ss->rx_small.extra_map);
32781e413cf9SAndrew Gallatin 			bus_dma_tag_destroy(ss->rx_small.dmat);
32791e413cf9SAndrew Gallatin 		}
32801e413cf9SAndrew Gallatin 		free(ss->rx_small.info, M_DEVBUF);
32811e413cf9SAndrew Gallatin 	}
32821e413cf9SAndrew Gallatin 	ss->rx_small.info = NULL;
32831e413cf9SAndrew Gallatin 
32841e413cf9SAndrew Gallatin 	if (ss->rx_big.info != NULL) {
32851e413cf9SAndrew Gallatin 		if (ss->rx_big.dmat != NULL) {
32861e413cf9SAndrew Gallatin 			for (i = 0; i <= ss->rx_big.mask; i++) {
32871e413cf9SAndrew Gallatin 				bus_dmamap_destroy(ss->rx_big.dmat,
32881e413cf9SAndrew Gallatin 						   ss->rx_big.info[i].map);
32891e413cf9SAndrew Gallatin 			}
32901e413cf9SAndrew Gallatin 			bus_dmamap_destroy(ss->rx_big.dmat,
32911e413cf9SAndrew Gallatin 					   ss->rx_big.extra_map);
32921e413cf9SAndrew Gallatin 			bus_dma_tag_destroy(ss->rx_big.dmat);
32931e413cf9SAndrew Gallatin 		}
32941e413cf9SAndrew Gallatin 		free(ss->rx_big.info, M_DEVBUF);
32951e413cf9SAndrew Gallatin 	}
32961e413cf9SAndrew Gallatin 	ss->rx_big.info = NULL;
3297b2fc195eSAndrew Gallatin }
3298b2fc195eSAndrew Gallatin 
3299b2fc195eSAndrew Gallatin static void
33006d87a65dSAndrew Gallatin mxge_free_rings(mxge_softc_t *sc)
3301b2fc195eSAndrew Gallatin {
33021e413cf9SAndrew Gallatin 	int slice;
3303b2fc195eSAndrew Gallatin 
33041e413cf9SAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++)
33051e413cf9SAndrew Gallatin 		mxge_free_slice_rings(&sc->ss[slice]);
3306c2657176SAndrew Gallatin }
3307b2fc195eSAndrew Gallatin 
3308b2fc195eSAndrew Gallatin static int
33091e413cf9SAndrew Gallatin mxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries,
33101e413cf9SAndrew Gallatin 		       int tx_ring_entries)
3311b2fc195eSAndrew Gallatin {
33121e413cf9SAndrew Gallatin 	mxge_softc_t *sc = ss->sc;
33131e413cf9SAndrew Gallatin 	size_t bytes;
33141e413cf9SAndrew Gallatin 	int err, i;
3315b2fc195eSAndrew Gallatin 
33161e413cf9SAndrew Gallatin 	/* allocate per-slice receive resources */
3317adae7080SAndrew Gallatin 
33181e413cf9SAndrew Gallatin 	ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1;
33191e413cf9SAndrew Gallatin 	ss->rx_done.mask = (2 * rx_ring_entries) - 1;
3320aed8e389SAndrew Gallatin 
3321b2fc195eSAndrew Gallatin 	/* allocate the rx shadow rings */
33221e413cf9SAndrew Gallatin 	bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
33231e413cf9SAndrew Gallatin 	ss->rx_small.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3324b2fc195eSAndrew Gallatin 
33251e413cf9SAndrew Gallatin 	bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
33261e413cf9SAndrew Gallatin 	ss->rx_big.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3327b2fc195eSAndrew Gallatin 
33281e413cf9SAndrew Gallatin 	/* allocate the rx host info rings */
33291e413cf9SAndrew Gallatin 	bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
33301e413cf9SAndrew Gallatin 	ss->rx_small.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3331b2fc195eSAndrew Gallatin 
33321e413cf9SAndrew Gallatin 	bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
33331e413cf9SAndrew Gallatin 	ss->rx_big.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3334b2fc195eSAndrew Gallatin 
33351e413cf9SAndrew Gallatin 	/* allocate the rx busdma resources */
3336b2fc195eSAndrew Gallatin 	err = bus_dma_tag_create(sc->parent_dmat,	/* parent */
3337b2fc195eSAndrew Gallatin 				 1,			/* alignment */
3338b2fc195eSAndrew Gallatin 				 4096,			/* boundary */
3339b2fc195eSAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* low */
3340b2fc195eSAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* high */
3341b2fc195eSAndrew Gallatin 				 NULL, NULL,		/* filter */
3342b2fc195eSAndrew Gallatin 				 MHLEN,			/* maxsize */
3343b2fc195eSAndrew Gallatin 				 1,			/* num segs */
3344b2fc195eSAndrew Gallatin 				 MHLEN,			/* maxsegsize */
3345b2fc195eSAndrew Gallatin 				 BUS_DMA_ALLOCNOW,	/* flags */
3346b2fc195eSAndrew Gallatin 				 NULL, NULL,		/* lock */
33471e413cf9SAndrew Gallatin 				 &ss->rx_small.dmat);	/* tag */
3348b2fc195eSAndrew Gallatin 	if (err != 0) {
3349b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Err %d allocating rx_small dmat\n",
3350b2fc195eSAndrew Gallatin 			      err);
3351c2ede4b3SMartin Blapp 		return err;
3352b2fc195eSAndrew Gallatin 	}
3353b2fc195eSAndrew Gallatin 
3354b2fc195eSAndrew Gallatin 	err = bus_dma_tag_create(sc->parent_dmat,	/* parent */
3355b2fc195eSAndrew Gallatin 				 1,			/* alignment */
3356b0f7b922SAndrew Gallatin #if MXGE_VIRT_JUMBOS
3357b2fc195eSAndrew Gallatin 				 4096,			/* boundary */
3358b0f7b922SAndrew Gallatin #else
3359b0f7b922SAndrew Gallatin 				 0,			/* boundary */
3360b0f7b922SAndrew Gallatin #endif
3361b2fc195eSAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* low */
3362b2fc195eSAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* high */
3363b2fc195eSAndrew Gallatin 				 NULL, NULL,		/* filter */
3364053e637fSAndrew Gallatin 				 3*4096,		/* maxsize */
3365b0f7b922SAndrew Gallatin #if MXGE_VIRT_JUMBOS
3366053e637fSAndrew Gallatin 				 3,			/* num segs */
3367b2fc195eSAndrew Gallatin 				 4096,			/* maxsegsize*/
3368b0f7b922SAndrew Gallatin #else
3369b0f7b922SAndrew Gallatin 				 1,			/* num segs */
3370b0f7b922SAndrew Gallatin 				 MJUM9BYTES,		/* maxsegsize*/
3371b0f7b922SAndrew Gallatin #endif
3372b2fc195eSAndrew Gallatin 				 BUS_DMA_ALLOCNOW,	/* flags */
3373b2fc195eSAndrew Gallatin 				 NULL, NULL,		/* lock */
33741e413cf9SAndrew Gallatin 				 &ss->rx_big.dmat);	/* tag */
3375b2fc195eSAndrew Gallatin 	if (err != 0) {
3376b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Err %d allocating rx_big dmat\n",
3377b2fc195eSAndrew Gallatin 			      err);
3378c2ede4b3SMartin Blapp 		return err;
3379b2fc195eSAndrew Gallatin 	}
33801e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->rx_small.mask; i++) {
33811e413cf9SAndrew Gallatin 		err = bus_dmamap_create(ss->rx_small.dmat, 0,
33821e413cf9SAndrew Gallatin 					&ss->rx_small.info[i].map);
3383b2fc195eSAndrew Gallatin 		if (err != 0) {
3384b2fc195eSAndrew Gallatin 			device_printf(sc->dev, "Err %d  rx_small dmamap\n",
3385b2fc195eSAndrew Gallatin 				      err);
3386c2ede4b3SMartin Blapp 			return err;
3387b2fc195eSAndrew Gallatin 		}
3388b2fc195eSAndrew Gallatin 	}
33891e413cf9SAndrew Gallatin 	err = bus_dmamap_create(ss->rx_small.dmat, 0,
33901e413cf9SAndrew Gallatin 				&ss->rx_small.extra_map);
3391b2fc195eSAndrew Gallatin 	if (err != 0) {
3392b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Err %d extra rx_small dmamap\n",
3393b2fc195eSAndrew Gallatin 			      err);
3394c2ede4b3SMartin Blapp 		return err;
3395b2fc195eSAndrew Gallatin 	}
3396b2fc195eSAndrew Gallatin 
33971e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->rx_big.mask; i++) {
33981e413cf9SAndrew Gallatin 		err = bus_dmamap_create(ss->rx_big.dmat, 0,
33991e413cf9SAndrew Gallatin 					&ss->rx_big.info[i].map);
3400b2fc195eSAndrew Gallatin 		if (err != 0) {
3401b2fc195eSAndrew Gallatin 			device_printf(sc->dev, "Err %d  rx_big dmamap\n",
3402b2fc195eSAndrew Gallatin 				      err);
3403c2ede4b3SMartin Blapp 			return err;
3404b2fc195eSAndrew Gallatin 		}
3405b2fc195eSAndrew Gallatin 	}
34061e413cf9SAndrew Gallatin 	err = bus_dmamap_create(ss->rx_big.dmat, 0,
34071e413cf9SAndrew Gallatin 				&ss->rx_big.extra_map);
3408b2fc195eSAndrew Gallatin 	if (err != 0) {
3409b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Err %d extra rx_big dmamap\n",
3410b2fc195eSAndrew Gallatin 			      err);
3411c2ede4b3SMartin Blapp 		return err;
34121e413cf9SAndrew Gallatin 	}
34131e413cf9SAndrew Gallatin 
3414b78540b1SGabor Kovesdan 	/* now allocate TX resources */
34151e413cf9SAndrew Gallatin 
3416c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING
34171e413cf9SAndrew Gallatin 	/* only use a single TX ring for now */
34181e413cf9SAndrew Gallatin 	if (ss != ss->sc->ss)
34191e413cf9SAndrew Gallatin 		return 0;
3420c6cb3e3fSAndrew Gallatin #endif
34211e413cf9SAndrew Gallatin 
34221e413cf9SAndrew Gallatin 	ss->tx.mask = tx_ring_entries - 1;
34231e413cf9SAndrew Gallatin 	ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4);
34241e413cf9SAndrew Gallatin 
34251e413cf9SAndrew Gallatin 
34261e413cf9SAndrew Gallatin 	/* allocate the tx request copy block */
34271e413cf9SAndrew Gallatin 	bytes = 8 +
34281e413cf9SAndrew Gallatin 		sizeof (*ss->tx.req_list) * (ss->tx.max_desc + 4);
34291e413cf9SAndrew Gallatin 	ss->tx.req_bytes = malloc(bytes, M_DEVBUF, M_WAITOK);
34301e413cf9SAndrew Gallatin 	/* ensure req_list entries are aligned to 8 bytes */
34311e413cf9SAndrew Gallatin 	ss->tx.req_list = (mcp_kreq_ether_send_t *)
34321e413cf9SAndrew Gallatin 		((unsigned long)(ss->tx.req_bytes + 7) & ~7UL);
34331e413cf9SAndrew Gallatin 
34341e413cf9SAndrew Gallatin 	/* allocate the tx busdma segment list */
34351e413cf9SAndrew Gallatin 	bytes = sizeof (*ss->tx.seg_list) * ss->tx.max_desc;
34361e413cf9SAndrew Gallatin 	ss->tx.seg_list = (bus_dma_segment_t *)
34371e413cf9SAndrew Gallatin 		malloc(bytes, M_DEVBUF, M_WAITOK);
34381e413cf9SAndrew Gallatin 
34391e413cf9SAndrew Gallatin 	/* allocate the tx host info ring */
34401e413cf9SAndrew Gallatin 	bytes = tx_ring_entries * sizeof (*ss->tx.info);
34411e413cf9SAndrew Gallatin 	ss->tx.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
34421e413cf9SAndrew Gallatin 
34431e413cf9SAndrew Gallatin 	/* allocate the tx busdma resources */
34441e413cf9SAndrew Gallatin 	err = bus_dma_tag_create(sc->parent_dmat,	/* parent */
34451e413cf9SAndrew Gallatin 				 1,			/* alignment */
34461e413cf9SAndrew Gallatin 				 sc->tx_boundary,	/* boundary */
34471e413cf9SAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* low */
34481e413cf9SAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* high */
34491e413cf9SAndrew Gallatin 				 NULL, NULL,		/* filter */
34501e413cf9SAndrew Gallatin 				 65536 + 256,		/* maxsize */
34511e413cf9SAndrew Gallatin 				 ss->tx.max_desc - 2,	/* num segs */
34521e413cf9SAndrew Gallatin 				 sc->tx_boundary,	/* maxsegsz */
34531e413cf9SAndrew Gallatin 				 BUS_DMA_ALLOCNOW,	/* flags */
34541e413cf9SAndrew Gallatin 				 NULL, NULL,		/* lock */
34551e413cf9SAndrew Gallatin 				 &ss->tx.dmat);		/* tag */
34561e413cf9SAndrew Gallatin 
34571e413cf9SAndrew Gallatin 	if (err != 0) {
34581e413cf9SAndrew Gallatin 		device_printf(sc->dev, "Err %d allocating tx dmat\n",
34591e413cf9SAndrew Gallatin 			      err);
3460c2ede4b3SMartin Blapp 		return err;
34611e413cf9SAndrew Gallatin 	}
34621e413cf9SAndrew Gallatin 
34631e413cf9SAndrew Gallatin 	/* now use these tags to setup dmamaps for each slot
34641e413cf9SAndrew Gallatin 	   in the ring */
34651e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->tx.mask; i++) {
34661e413cf9SAndrew Gallatin 		err = bus_dmamap_create(ss->tx.dmat, 0,
34671e413cf9SAndrew Gallatin 					&ss->tx.info[i].map);
34681e413cf9SAndrew Gallatin 		if (err != 0) {
34691e413cf9SAndrew Gallatin 			device_printf(sc->dev, "Err %d  tx dmamap\n",
34701e413cf9SAndrew Gallatin 				      err);
3471c2ede4b3SMartin Blapp 			return err;
34721e413cf9SAndrew Gallatin 		}
3473b2fc195eSAndrew Gallatin 	}
3474b2fc195eSAndrew Gallatin 	return 0;
3475b2fc195eSAndrew Gallatin 
3476b2fc195eSAndrew Gallatin }
3477b2fc195eSAndrew Gallatin 
34781e413cf9SAndrew Gallatin static int
34791e413cf9SAndrew Gallatin mxge_alloc_rings(mxge_softc_t *sc)
34801e413cf9SAndrew Gallatin {
34811e413cf9SAndrew Gallatin 	mxge_cmd_t cmd;
34821e413cf9SAndrew Gallatin 	int tx_ring_size;
34831e413cf9SAndrew Gallatin 	int tx_ring_entries, rx_ring_entries;
34841e413cf9SAndrew Gallatin 	int err, slice;
34851e413cf9SAndrew Gallatin 
34861e413cf9SAndrew Gallatin 	/* get ring sizes */
34871e413cf9SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd);
34881e413cf9SAndrew Gallatin 	tx_ring_size = cmd.data0;
34891e413cf9SAndrew Gallatin 	if (err != 0) {
34901e413cf9SAndrew Gallatin 		device_printf(sc->dev, "Cannot determine tx ring sizes\n");
34911e413cf9SAndrew Gallatin 		goto abort;
34921e413cf9SAndrew Gallatin 	}
34931e413cf9SAndrew Gallatin 
34941e413cf9SAndrew Gallatin 	tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t);
34951e413cf9SAndrew Gallatin 	rx_ring_entries = sc->rx_ring_size / sizeof (mcp_dma_addr_t);
34961e413cf9SAndrew Gallatin 	IFQ_SET_MAXLEN(&sc->ifp->if_snd, tx_ring_entries - 1);
34971e413cf9SAndrew Gallatin 	sc->ifp->if_snd.ifq_drv_maxlen = sc->ifp->if_snd.ifq_maxlen;
34981e413cf9SAndrew Gallatin 	IFQ_SET_READY(&sc->ifp->if_snd);
34991e413cf9SAndrew Gallatin 
35001e413cf9SAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++) {
35011e413cf9SAndrew Gallatin 		err = mxge_alloc_slice_rings(&sc->ss[slice],
35021e413cf9SAndrew Gallatin 					     rx_ring_entries,
35031e413cf9SAndrew Gallatin 					     tx_ring_entries);
35041e413cf9SAndrew Gallatin 		if (err != 0)
35051e413cf9SAndrew Gallatin 			goto abort;
35061e413cf9SAndrew Gallatin 	}
35071e413cf9SAndrew Gallatin 	return 0;
35081e413cf9SAndrew Gallatin 
35091e413cf9SAndrew Gallatin abort:
35101e413cf9SAndrew Gallatin 	mxge_free_rings(sc);
35111e413cf9SAndrew Gallatin 	return err;
35121e413cf9SAndrew Gallatin 
35131e413cf9SAndrew Gallatin }
35141e413cf9SAndrew Gallatin 
35151e413cf9SAndrew Gallatin 
3516053e637fSAndrew Gallatin static void
3517053e637fSAndrew Gallatin mxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs)
3518053e637fSAndrew Gallatin {
3519c792928fSAndrew Gallatin 	int bufsize = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD;
3520053e637fSAndrew Gallatin 
3521053e637fSAndrew Gallatin 	if (bufsize < MCLBYTES) {
3522053e637fSAndrew Gallatin 		/* easy, everything fits in a single buffer */
3523053e637fSAndrew Gallatin 		*big_buf_size = MCLBYTES;
3524053e637fSAndrew Gallatin 		*cl_size = MCLBYTES;
3525053e637fSAndrew Gallatin 		*nbufs = 1;
3526053e637fSAndrew Gallatin 		return;
3527053e637fSAndrew Gallatin 	}
3528053e637fSAndrew Gallatin 
3529053e637fSAndrew Gallatin 	if (bufsize < MJUMPAGESIZE) {
3530053e637fSAndrew Gallatin 		/* still easy, everything still fits in a single buffer */
3531053e637fSAndrew Gallatin 		*big_buf_size = MJUMPAGESIZE;
3532053e637fSAndrew Gallatin 		*cl_size = MJUMPAGESIZE;
3533053e637fSAndrew Gallatin 		*nbufs = 1;
3534053e637fSAndrew Gallatin 		return;
3535053e637fSAndrew Gallatin 	}
3536b0f7b922SAndrew Gallatin #if MXGE_VIRT_JUMBOS
3537053e637fSAndrew Gallatin 	/* now we need to use virtually contiguous buffers */
3538053e637fSAndrew Gallatin 	*cl_size = MJUM9BYTES;
3539053e637fSAndrew Gallatin 	*big_buf_size = 4096;
3540053e637fSAndrew Gallatin 	*nbufs = mtu / 4096 + 1;
3541053e637fSAndrew Gallatin 	/* needs to be a power of two, so round up */
3542053e637fSAndrew Gallatin 	if (*nbufs == 3)
3543053e637fSAndrew Gallatin 		*nbufs = 4;
3544b0f7b922SAndrew Gallatin #else
3545b0f7b922SAndrew Gallatin 	*cl_size = MJUM9BYTES;
3546b0f7b922SAndrew Gallatin 	*big_buf_size = MJUM9BYTES;
3547b0f7b922SAndrew Gallatin 	*nbufs = 1;
3548b0f7b922SAndrew Gallatin #endif
3549053e637fSAndrew Gallatin }
3550053e637fSAndrew Gallatin 
3551b2fc195eSAndrew Gallatin static int
35521e413cf9SAndrew Gallatin mxge_slice_open(struct mxge_slice_state *ss, int nbufs, int cl_size)
3553b2fc195eSAndrew Gallatin {
35541e413cf9SAndrew Gallatin 	mxge_softc_t *sc;
35556d87a65dSAndrew Gallatin 	mxge_cmd_t cmd;
3556b2fc195eSAndrew Gallatin 	bus_dmamap_t map;
35571e413cf9SAndrew Gallatin 	int err, i, slice;
3558b2fc195eSAndrew Gallatin 
35591e413cf9SAndrew Gallatin 
35601e413cf9SAndrew Gallatin 	sc = ss->sc;
35611e413cf9SAndrew Gallatin 	slice = ss - sc->ss;
35621e413cf9SAndrew Gallatin 
356326dd49c6SAndrew Gallatin #if defined(INET) || defined(INET6)
356426dd49c6SAndrew Gallatin 	(void)tcp_lro_init(&ss->lc);
356526dd49c6SAndrew Gallatin #endif
356626dd49c6SAndrew Gallatin 	ss->lc.ifp = sc->ifp;
3567053e637fSAndrew Gallatin 
35681e413cf9SAndrew Gallatin 	/* get the lanai pointers to the send and receive rings */
35691e413cf9SAndrew Gallatin 
35701e413cf9SAndrew Gallatin 	err = 0;
3571c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING
35721e413cf9SAndrew Gallatin 	/* We currently only send from the first slice */
35731e413cf9SAndrew Gallatin 	if (slice == 0) {
3574c6cb3e3fSAndrew Gallatin #endif
35751e413cf9SAndrew Gallatin 		cmd.data0 = slice;
35761e413cf9SAndrew Gallatin 		err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd);
35771e413cf9SAndrew Gallatin 		ss->tx.lanai =
35781e413cf9SAndrew Gallatin 			(volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0);
3579c6cb3e3fSAndrew Gallatin 		ss->tx.send_go = (volatile uint32_t *)
3580c6cb3e3fSAndrew Gallatin 			(sc->sram + MXGEFW_ETH_SEND_GO + 64 * slice);
3581c6cb3e3fSAndrew Gallatin 		ss->tx.send_stop = (volatile uint32_t *)
3582c6cb3e3fSAndrew Gallatin 		(sc->sram + MXGEFW_ETH_SEND_STOP + 64 * slice);
3583c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING
35841e413cf9SAndrew Gallatin 	}
3585c6cb3e3fSAndrew Gallatin #endif
35861e413cf9SAndrew Gallatin 	cmd.data0 = slice;
35871e413cf9SAndrew Gallatin 	err |= mxge_send_cmd(sc,
35881e413cf9SAndrew Gallatin 			     MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd);
35891e413cf9SAndrew Gallatin 	ss->rx_small.lanai =
35901e413cf9SAndrew Gallatin 		(volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
35911e413cf9SAndrew Gallatin 	cmd.data0 = slice;
35921e413cf9SAndrew Gallatin 	err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd);
35931e413cf9SAndrew Gallatin 	ss->rx_big.lanai =
35941e413cf9SAndrew Gallatin 		(volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
35951e413cf9SAndrew Gallatin 
35961e413cf9SAndrew Gallatin 	if (err != 0) {
35971e413cf9SAndrew Gallatin 		device_printf(sc->dev,
35981e413cf9SAndrew Gallatin 			      "failed to get ring sizes or locations\n");
35991e413cf9SAndrew Gallatin 		return EIO;
36001e413cf9SAndrew Gallatin 	}
36011e413cf9SAndrew Gallatin 
36021e413cf9SAndrew Gallatin 	/* stock receive rings */
36031e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->rx_small.mask; i++) {
36041e413cf9SAndrew Gallatin 		map = ss->rx_small.info[i].map;
36051e413cf9SAndrew Gallatin 		err = mxge_get_buf_small(ss, map, i);
36061e413cf9SAndrew Gallatin 		if (err) {
36071e413cf9SAndrew Gallatin 			device_printf(sc->dev, "alloced %d/%d smalls\n",
36081e413cf9SAndrew Gallatin 				      i, ss->rx_small.mask + 1);
36091e413cf9SAndrew Gallatin 			return ENOMEM;
36101e413cf9SAndrew Gallatin 		}
36111e413cf9SAndrew Gallatin 	}
36121e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->rx_big.mask; i++) {
36131e413cf9SAndrew Gallatin 		ss->rx_big.shadow[i].addr_low = 0xffffffff;
36141e413cf9SAndrew Gallatin 		ss->rx_big.shadow[i].addr_high = 0xffffffff;
36151e413cf9SAndrew Gallatin 	}
36161e413cf9SAndrew Gallatin 	ss->rx_big.nbufs = nbufs;
36171e413cf9SAndrew Gallatin 	ss->rx_big.cl_size = cl_size;
36184d9a5852SAndrew Gallatin 	ss->rx_big.mlen = ss->sc->ifp->if_mtu + ETHER_HDR_LEN +
36194d9a5852SAndrew Gallatin 		ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD;
36201e413cf9SAndrew Gallatin 	for (i = 0; i <= ss->rx_big.mask; i += ss->rx_big.nbufs) {
36211e413cf9SAndrew Gallatin 		map = ss->rx_big.info[i].map;
36221e413cf9SAndrew Gallatin 		err = mxge_get_buf_big(ss, map, i);
36231e413cf9SAndrew Gallatin 		if (err) {
36241e413cf9SAndrew Gallatin 			device_printf(sc->dev, "alloced %d/%d bigs\n",
36251e413cf9SAndrew Gallatin 				      i, ss->rx_big.mask + 1);
36261e413cf9SAndrew Gallatin 			return ENOMEM;
36271e413cf9SAndrew Gallatin 		}
36281e413cf9SAndrew Gallatin 	}
36291e413cf9SAndrew Gallatin 	return 0;
36301e413cf9SAndrew Gallatin }
36311e413cf9SAndrew Gallatin 
36321e413cf9SAndrew Gallatin static int
36331e413cf9SAndrew Gallatin mxge_open(mxge_softc_t *sc)
36341e413cf9SAndrew Gallatin {
36351e413cf9SAndrew Gallatin 	mxge_cmd_t cmd;
36361e413cf9SAndrew Gallatin 	int err, big_bytes, nbufs, slice, cl_size, i;
36371e413cf9SAndrew Gallatin 	bus_addr_t bus;
36381e413cf9SAndrew Gallatin 	volatile uint8_t *itable;
3639c6cb3e3fSAndrew Gallatin 	struct mxge_slice_state *ss;
3640b2fc195eSAndrew Gallatin 
36417d542e2dSAndrew Gallatin 	/* Copy the MAC address in case it was overridden */
36427d542e2dSAndrew Gallatin 	bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN);
36437d542e2dSAndrew Gallatin 
3644adae7080SAndrew Gallatin 	err = mxge_reset(sc, 1);
3645b2fc195eSAndrew Gallatin 	if (err != 0) {
3646b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "failed to reset\n");
3647b2fc195eSAndrew Gallatin 		return EIO;
3648b2fc195eSAndrew Gallatin 	}
3649b2fc195eSAndrew Gallatin 
36501e413cf9SAndrew Gallatin 	if (sc->num_slices > 1) {
36511e413cf9SAndrew Gallatin 		/* setup the indirection table */
36521e413cf9SAndrew Gallatin 		cmd.data0 = sc->num_slices;
36531e413cf9SAndrew Gallatin 		err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE,
36541e413cf9SAndrew Gallatin 				    &cmd);
3655b2fc195eSAndrew Gallatin 
36561e413cf9SAndrew Gallatin 		err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET,
36571e413cf9SAndrew Gallatin 				     &cmd);
36581e413cf9SAndrew Gallatin 		if (err != 0) {
36591e413cf9SAndrew Gallatin 			device_printf(sc->dev,
36601e413cf9SAndrew Gallatin 				      "failed to setup rss tables\n");
36611e413cf9SAndrew Gallatin 			return err;
36621e413cf9SAndrew Gallatin 		}
36631e413cf9SAndrew Gallatin 
36641e413cf9SAndrew Gallatin 		/* just enable an identity mapping */
36651e413cf9SAndrew Gallatin 		itable = sc->sram + cmd.data0;
36661e413cf9SAndrew Gallatin 		for (i = 0; i < sc->num_slices; i++)
36671e413cf9SAndrew Gallatin 			itable[i] = (uint8_t)i;
36681e413cf9SAndrew Gallatin 
36691e413cf9SAndrew Gallatin 		cmd.data0 = 1;
36701e413cf9SAndrew Gallatin 		cmd.data1 = mxge_rss_hash_type;
36711e413cf9SAndrew Gallatin 		err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd);
36721e413cf9SAndrew Gallatin 		if (err != 0) {
36731e413cf9SAndrew Gallatin 			device_printf(sc->dev, "failed to enable slices\n");
36741e413cf9SAndrew Gallatin 			return err;
36751e413cf9SAndrew Gallatin 		}
36761e413cf9SAndrew Gallatin 	}
36771e413cf9SAndrew Gallatin 
36781e413cf9SAndrew Gallatin 
36791e413cf9SAndrew Gallatin 	mxge_choose_params(sc->ifp->if_mtu, &big_bytes, &cl_size, &nbufs);
36801e413cf9SAndrew Gallatin 
36811e413cf9SAndrew Gallatin 	cmd.data0 = nbufs;
3682053e637fSAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
3683053e637fSAndrew Gallatin 			    &cmd);
3684053e637fSAndrew Gallatin 	/* error is only meaningful if we're trying to set
3685053e637fSAndrew Gallatin 	   MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */
36861e413cf9SAndrew Gallatin 	if (err && nbufs > 1) {
3687053e637fSAndrew Gallatin 		device_printf(sc->dev,
3688053e637fSAndrew Gallatin 			      "Failed to set alway-use-n to %d\n",
36891e413cf9SAndrew Gallatin 			      nbufs);
3690053e637fSAndrew Gallatin 		return EIO;
3691053e637fSAndrew Gallatin 	}
3692b2fc195eSAndrew Gallatin 	/* Give the firmware the mtu and the big and small buffer
3693b2fc195eSAndrew Gallatin 	   sizes.  The firmware wants the big buf size to be a power
3694b2fc195eSAndrew Gallatin 	   of two. Luckily, FreeBSD's clusters are powers of two */
3695c792928fSAndrew Gallatin 	cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
36965e7d8541SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd);
3697b4db9009SAndrew Gallatin 	cmd.data0 = MHLEN - MXGEFW_PAD;
36985e7d8541SAndrew Gallatin 	err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE,
3699b2fc195eSAndrew Gallatin 			     &cmd);
3700053e637fSAndrew Gallatin 	cmd.data0 = big_bytes;
37015e7d8541SAndrew Gallatin 	err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd);
37020fa7f681SAndrew Gallatin 
37030fa7f681SAndrew Gallatin 	if (err != 0) {
37040fa7f681SAndrew Gallatin 		device_printf(sc->dev, "failed to setup params\n");
37050fa7f681SAndrew Gallatin 		goto abort;
37060fa7f681SAndrew Gallatin 	}
37070fa7f681SAndrew Gallatin 
3708b2fc195eSAndrew Gallatin 	/* Now give him the pointer to the stats block */
3709c6cb3e3fSAndrew Gallatin 	for (slice = 0;
3710c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
3711c6cb3e3fSAndrew Gallatin 	     slice < sc->num_slices;
3712c6cb3e3fSAndrew Gallatin #else
3713c6cb3e3fSAndrew Gallatin 	     slice < 1;
3714c6cb3e3fSAndrew Gallatin #endif
3715c6cb3e3fSAndrew Gallatin 	     slice++) {
3716c6cb3e3fSAndrew Gallatin 		ss = &sc->ss[slice];
3717c6cb3e3fSAndrew Gallatin 		cmd.data0 =
3718c6cb3e3fSAndrew Gallatin 			MXGE_LOWPART_TO_U32(ss->fw_stats_dma.bus_addr);
3719c6cb3e3fSAndrew Gallatin 		cmd.data1 =
3720c6cb3e3fSAndrew Gallatin 			MXGE_HIGHPART_TO_U32(ss->fw_stats_dma.bus_addr);
37210fa7f681SAndrew Gallatin 		cmd.data2 = sizeof(struct mcp_irq_data);
3722c6cb3e3fSAndrew Gallatin 		cmd.data2 |= (slice << 16);
3723c6cb3e3fSAndrew Gallatin 		err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd);
3724c6cb3e3fSAndrew Gallatin 	}
37250fa7f681SAndrew Gallatin 
37260fa7f681SAndrew Gallatin 	if (err != 0) {
37271e413cf9SAndrew Gallatin 		bus = sc->ss->fw_stats_dma.bus_addr;
37280fa7f681SAndrew Gallatin 		bus += offsetof(struct mcp_irq_data, send_done_count);
37290fa7f681SAndrew Gallatin 		cmd.data0 = MXGE_LOWPART_TO_U32(bus);
37300fa7f681SAndrew Gallatin 		cmd.data1 = MXGE_HIGHPART_TO_U32(bus);
37310fa7f681SAndrew Gallatin 		err = mxge_send_cmd(sc,
37320fa7f681SAndrew Gallatin 				    MXGEFW_CMD_SET_STATS_DMA_OBSOLETE,
37330fa7f681SAndrew Gallatin 				    &cmd);
37340fa7f681SAndrew Gallatin 		/* Firmware cannot support multicast without STATS_DMA_V2 */
37350fa7f681SAndrew Gallatin 		sc->fw_multicast_support = 0;
37360fa7f681SAndrew Gallatin 	} else {
37370fa7f681SAndrew Gallatin 		sc->fw_multicast_support = 1;
37380fa7f681SAndrew Gallatin 	}
3739b2fc195eSAndrew Gallatin 
3740b2fc195eSAndrew Gallatin 	if (err != 0) {
3741b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "failed to setup params\n");
3742b2fc195eSAndrew Gallatin 		goto abort;
3743b2fc195eSAndrew Gallatin 	}
3744b2fc195eSAndrew Gallatin 
37451e413cf9SAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++) {
37461e413cf9SAndrew Gallatin 		err = mxge_slice_open(&sc->ss[slice], nbufs, cl_size);
37471e413cf9SAndrew Gallatin 		if (err != 0) {
37481e413cf9SAndrew Gallatin 			device_printf(sc->dev, "couldn't open slice %d\n",
37491e413cf9SAndrew Gallatin 				      slice);
37501e413cf9SAndrew Gallatin 			goto abort;
37511e413cf9SAndrew Gallatin 		}
37521e413cf9SAndrew Gallatin 	}
37531e413cf9SAndrew Gallatin 
3754b2fc195eSAndrew Gallatin 	/* Finally, start the firmware running */
37555e7d8541SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd);
3756b2fc195eSAndrew Gallatin 	if (err) {
3757b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Couldn't bring up link\n");
3758b2fc195eSAndrew Gallatin 		goto abort;
3759b2fc195eSAndrew Gallatin 	}
3760c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
3761c6cb3e3fSAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++) {
3762c6cb3e3fSAndrew Gallatin 		ss = &sc->ss[slice];
3763c6cb3e3fSAndrew Gallatin 		ss->if_drv_flags |= IFF_DRV_RUNNING;
3764c6cb3e3fSAndrew Gallatin 		ss->if_drv_flags &= ~IFF_DRV_OACTIVE;
3765c6cb3e3fSAndrew Gallatin 	}
3766c6cb3e3fSAndrew Gallatin #endif
3767b2fc195eSAndrew Gallatin 	sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
3768b2fc195eSAndrew Gallatin 	sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3769b2fc195eSAndrew Gallatin 
3770b2fc195eSAndrew Gallatin 	return 0;
3771b2fc195eSAndrew Gallatin 
3772b2fc195eSAndrew Gallatin 
3773b2fc195eSAndrew Gallatin abort:
37746d87a65dSAndrew Gallatin 	mxge_free_mbufs(sc);
3775a98d6cd7SAndrew Gallatin 
3776b2fc195eSAndrew Gallatin 	return err;
3777b2fc195eSAndrew Gallatin }
3778b2fc195eSAndrew Gallatin 
3779b2fc195eSAndrew Gallatin static int
3780a393336bSAndrew Gallatin mxge_close(mxge_softc_t *sc, int down)
3781b2fc195eSAndrew Gallatin {
37826d87a65dSAndrew Gallatin 	mxge_cmd_t cmd;
3783b2fc195eSAndrew Gallatin 	int err, old_down_cnt;
3784c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
3785c6cb3e3fSAndrew Gallatin 	struct mxge_slice_state *ss;
3786c6cb3e3fSAndrew Gallatin 	int slice;
3787c6cb3e3fSAndrew Gallatin #endif
3788b2fc195eSAndrew Gallatin 
3789c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
3790c6cb3e3fSAndrew Gallatin 	for (slice = 0; slice < sc->num_slices; slice++) {
3791c6cb3e3fSAndrew Gallatin 		ss = &sc->ss[slice];
3792c6cb3e3fSAndrew Gallatin 		ss->if_drv_flags &= ~IFF_DRV_RUNNING;
3793c6cb3e3fSAndrew Gallatin 	}
3794c6cb3e3fSAndrew Gallatin #endif
3795b2fc195eSAndrew Gallatin 	sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3796a393336bSAndrew Gallatin 	if (!down) {
3797b2fc195eSAndrew Gallatin 		old_down_cnt = sc->down_cnt;
379873c7c83fSAndrew Gallatin 		wmb();
37995e7d8541SAndrew Gallatin 		err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
3800b2fc195eSAndrew Gallatin 		if (err) {
3801a393336bSAndrew Gallatin 			device_printf(sc->dev,
3802a393336bSAndrew Gallatin 				      "Couldn't bring down link\n");
3803b2fc195eSAndrew Gallatin 		}
3804b2fc195eSAndrew Gallatin 		if (old_down_cnt == sc->down_cnt) {
3805b2fc195eSAndrew Gallatin 			/* wait for down irq */
3806dce01b9bSAndrew Gallatin 			DELAY(10 * sc->intr_coal_delay);
3807b2fc195eSAndrew Gallatin 		}
380873c7c83fSAndrew Gallatin 		wmb();
3809b2fc195eSAndrew Gallatin 		if (old_down_cnt == sc->down_cnt) {
3810b2fc195eSAndrew Gallatin 			device_printf(sc->dev, "never got down irq\n");
3811b2fc195eSAndrew Gallatin 		}
3812a393336bSAndrew Gallatin 	}
38136d87a65dSAndrew Gallatin 	mxge_free_mbufs(sc);
3814a98d6cd7SAndrew Gallatin 
3815b2fc195eSAndrew Gallatin 	return 0;
3816b2fc195eSAndrew Gallatin }
3817b2fc195eSAndrew Gallatin 
3818dce01b9bSAndrew Gallatin static void
3819dce01b9bSAndrew Gallatin mxge_setup_cfg_space(mxge_softc_t *sc)
3820dce01b9bSAndrew Gallatin {
3821dce01b9bSAndrew Gallatin 	device_t dev = sc->dev;
3822dce01b9bSAndrew Gallatin 	int reg;
3823c68534f1SScott Long 	uint16_t lnk, pectl;
3824dce01b9bSAndrew Gallatin 
3825dce01b9bSAndrew Gallatin 	/* find the PCIe link width and set max read request to 4KB*/
38263b0a4aefSJohn Baldwin 	if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
3827dce01b9bSAndrew Gallatin 		lnk = pci_read_config(dev, reg + 0x12, 2);
3828dce01b9bSAndrew Gallatin 		sc->link_width = (lnk >> 4) & 0x3f;
3829dce01b9bSAndrew Gallatin 
383083d54b59SAndrew Gallatin 		if (sc->pectl == 0) {
3831dce01b9bSAndrew Gallatin 			pectl = pci_read_config(dev, reg + 0x8, 2);
3832dce01b9bSAndrew Gallatin 			pectl = (pectl & ~0x7000) | (5 << 12);
3833dce01b9bSAndrew Gallatin 			pci_write_config(dev, reg + 0x8, pectl, 2);
383483d54b59SAndrew Gallatin 			sc->pectl = pectl;
383583d54b59SAndrew Gallatin 		} else {
383683d54b59SAndrew Gallatin 			/* restore saved pectl after watchdog reset */
383783d54b59SAndrew Gallatin 			pci_write_config(dev, reg + 0x8, sc->pectl, 2);
383883d54b59SAndrew Gallatin 		}
3839dce01b9bSAndrew Gallatin 	}
3840dce01b9bSAndrew Gallatin 
3841dce01b9bSAndrew Gallatin 	/* Enable DMA and Memory space access */
3842dce01b9bSAndrew Gallatin 	pci_enable_busmaster(dev);
3843dce01b9bSAndrew Gallatin }
3844dce01b9bSAndrew Gallatin 
3845dce01b9bSAndrew Gallatin static uint32_t
3846dce01b9bSAndrew Gallatin mxge_read_reboot(mxge_softc_t *sc)
3847dce01b9bSAndrew Gallatin {
3848dce01b9bSAndrew Gallatin 	device_t dev = sc->dev;
3849dce01b9bSAndrew Gallatin 	uint32_t vs;
3850dce01b9bSAndrew Gallatin 
3851dce01b9bSAndrew Gallatin 	/* find the vendor specific offset */
38523b0a4aefSJohn Baldwin 	if (pci_find_cap(dev, PCIY_VENDOR, &vs) != 0) {
3853dce01b9bSAndrew Gallatin 		device_printf(sc->dev,
3854dce01b9bSAndrew Gallatin 			      "could not find vendor specific offset\n");
3855dce01b9bSAndrew Gallatin 		return (uint32_t)-1;
3856dce01b9bSAndrew Gallatin 	}
3857dce01b9bSAndrew Gallatin 	/* enable read32 mode */
3858dce01b9bSAndrew Gallatin 	pci_write_config(dev, vs + 0x10, 0x3, 1);
3859dce01b9bSAndrew Gallatin 	/* tell NIC which register to read */
3860dce01b9bSAndrew Gallatin 	pci_write_config(dev, vs + 0x18, 0xfffffff0, 4);
3861dce01b9bSAndrew Gallatin 	return (pci_read_config(dev, vs + 0x14, 4));
3862dce01b9bSAndrew Gallatin }
3863dce01b9bSAndrew Gallatin 
386472c042dfSAndrew Gallatin static void
386572c042dfSAndrew Gallatin mxge_watchdog_reset(mxge_softc_t *sc)
3866dce01b9bSAndrew Gallatin {
3867e749ef6bSAndrew Gallatin 	struct pci_devinfo *dinfo;
3868a393336bSAndrew Gallatin 	struct mxge_slice_state *ss;
3869a393336bSAndrew Gallatin 	int err, running, s, num_tx_slices = 1;
3870dce01b9bSAndrew Gallatin 	uint32_t reboot;
3871dce01b9bSAndrew Gallatin 	uint16_t cmd;
3872dce01b9bSAndrew Gallatin 
3873dce01b9bSAndrew Gallatin 	err = ENXIO;
3874dce01b9bSAndrew Gallatin 
3875dce01b9bSAndrew Gallatin 	device_printf(sc->dev, "Watchdog reset!\n");
3876dce01b9bSAndrew Gallatin 
3877dce01b9bSAndrew Gallatin 	/*
3878dce01b9bSAndrew Gallatin 	 * check to see if the NIC rebooted.  If it did, then all of
3879dce01b9bSAndrew Gallatin 	 * PCI config space has been reset, and things like the
3880dce01b9bSAndrew Gallatin 	 * busmaster bit will be zero.  If this is the case, then we
3881dce01b9bSAndrew Gallatin 	 * must restore PCI config space before the NIC can be used
3882dce01b9bSAndrew Gallatin 	 * again
3883dce01b9bSAndrew Gallatin 	 */
3884dce01b9bSAndrew Gallatin 	cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3885dce01b9bSAndrew Gallatin 	if (cmd == 0xffff) {
3886dce01b9bSAndrew Gallatin 		/*
3887dce01b9bSAndrew Gallatin 		 * maybe the watchdog caught the NIC rebooting; wait
3888dce01b9bSAndrew Gallatin 		 * up to 100ms for it to finish.  If it does not come
3889dce01b9bSAndrew Gallatin 		 * back, then give up
3890dce01b9bSAndrew Gallatin 		 */
3891dce01b9bSAndrew Gallatin 		DELAY(1000*100);
3892dce01b9bSAndrew Gallatin 		cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3893dce01b9bSAndrew Gallatin 		if (cmd == 0xffff) {
3894dce01b9bSAndrew Gallatin 			device_printf(sc->dev, "NIC disappeared!\n");
3895dce01b9bSAndrew Gallatin 		}
3896dce01b9bSAndrew Gallatin 	}
3897dce01b9bSAndrew Gallatin 	if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) {
3898dce01b9bSAndrew Gallatin 		/* print the reboot status */
3899dce01b9bSAndrew Gallatin 		reboot = mxge_read_reboot(sc);
3900dce01b9bSAndrew Gallatin 		device_printf(sc->dev, "NIC rebooted, status = 0x%x\n",
3901dce01b9bSAndrew Gallatin 			      reboot);
3902a393336bSAndrew Gallatin 		running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING;
3903a393336bSAndrew Gallatin 		if (running) {
3904a393336bSAndrew Gallatin 
3905a393336bSAndrew Gallatin 			/*
3906a393336bSAndrew Gallatin 			 * quiesce NIC so that TX routines will not try to
3907a393336bSAndrew Gallatin 			 * xmit after restoration of BAR
3908a393336bSAndrew Gallatin 			 */
3909a393336bSAndrew Gallatin 
3910a393336bSAndrew Gallatin 			/* Mark the link as down */
3911a393336bSAndrew Gallatin 			if (sc->link_state) {
3912a393336bSAndrew Gallatin 				sc->link_state = 0;
3913a393336bSAndrew Gallatin 				if_link_state_change(sc->ifp,
3914a393336bSAndrew Gallatin 						     LINK_STATE_DOWN);
3915a393336bSAndrew Gallatin 			}
3916a393336bSAndrew Gallatin #ifdef IFNET_BUF_RING
3917a393336bSAndrew Gallatin 			num_tx_slices = sc->num_slices;
3918a393336bSAndrew Gallatin #endif
3919a393336bSAndrew Gallatin 			/* grab all TX locks to ensure no tx  */
3920a393336bSAndrew Gallatin 			for (s = 0; s < num_tx_slices; s++) {
3921a393336bSAndrew Gallatin 				ss = &sc->ss[s];
3922a393336bSAndrew Gallatin 				mtx_lock(&ss->tx.mtx);
3923a393336bSAndrew Gallatin 			}
3924a393336bSAndrew Gallatin 			mxge_close(sc, 1);
3925a393336bSAndrew Gallatin 		}
3926dce01b9bSAndrew Gallatin 		/* restore PCI configuration space */
3927e749ef6bSAndrew Gallatin 		dinfo = device_get_ivars(sc->dev);
3928e749ef6bSAndrew Gallatin 		pci_cfg_restore(sc->dev, dinfo);
3929dce01b9bSAndrew Gallatin 
3930dce01b9bSAndrew Gallatin 		/* and redo any changes we made to our config space */
3931dce01b9bSAndrew Gallatin 		mxge_setup_cfg_space(sc);
393210882804SAndrew Gallatin 
3933a393336bSAndrew Gallatin 		/* reload f/w */
3934a393336bSAndrew Gallatin 		err = mxge_load_firmware(sc, 0);
3935a393336bSAndrew Gallatin 		if (err) {
3936a393336bSAndrew Gallatin 			device_printf(sc->dev,
3937a393336bSAndrew Gallatin 				      "Unable to re-load f/w\n");
393810882804SAndrew Gallatin 		}
3939a393336bSAndrew Gallatin 		if (running) {
3940a393336bSAndrew Gallatin 			if (!err)
3941a393336bSAndrew Gallatin 				err = mxge_open(sc);
3942a393336bSAndrew Gallatin 			/* release all TX locks */
3943a393336bSAndrew Gallatin 			for (s = 0; s < num_tx_slices; s++) {
3944a393336bSAndrew Gallatin 				ss = &sc->ss[s];
394583d54b59SAndrew Gallatin #ifdef IFNET_BUF_RING
394683d54b59SAndrew Gallatin 				mxge_start_locked(ss);
394783d54b59SAndrew Gallatin #endif
3948a393336bSAndrew Gallatin 				mtx_unlock(&ss->tx.mtx);
3949a393336bSAndrew Gallatin 			}
3950a393336bSAndrew Gallatin 		}
3951a393336bSAndrew Gallatin 		sc->watchdog_resets++;
3952dce01b9bSAndrew Gallatin 	} else {
3953c6cb3e3fSAndrew Gallatin 		device_printf(sc->dev,
395472c042dfSAndrew Gallatin 			      "NIC did not reboot, not resetting\n");
395572c042dfSAndrew Gallatin 		err = 0;
395672c042dfSAndrew Gallatin 	}
395772c042dfSAndrew Gallatin 	if (err) {
395872c042dfSAndrew Gallatin 		device_printf(sc->dev, "watchdog reset failed\n");
395972c042dfSAndrew Gallatin 	} else {
39606b484a49SAndrew Gallatin 		if (sc->dying == 2)
39616b484a49SAndrew Gallatin 			sc->dying = 0;
39626b484a49SAndrew Gallatin 		callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
396372c042dfSAndrew Gallatin 	}
396472c042dfSAndrew Gallatin }
396572c042dfSAndrew Gallatin 
396672c042dfSAndrew Gallatin static void
396772c042dfSAndrew Gallatin mxge_watchdog_task(void *arg, int pending)
396872c042dfSAndrew Gallatin {
396972c042dfSAndrew Gallatin 	mxge_softc_t *sc = arg;
397072c042dfSAndrew Gallatin 
397172c042dfSAndrew Gallatin 
397272c042dfSAndrew Gallatin 	mtx_lock(&sc->driver_mtx);
397372c042dfSAndrew Gallatin 	mxge_watchdog_reset(sc);
397472c042dfSAndrew Gallatin 	mtx_unlock(&sc->driver_mtx);
397572c042dfSAndrew Gallatin }
397672c042dfSAndrew Gallatin 
397772c042dfSAndrew Gallatin static void
397872c042dfSAndrew Gallatin mxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice)
397972c042dfSAndrew Gallatin {
398072c042dfSAndrew Gallatin 	tx = &sc->ss[slice].tx;
398172c042dfSAndrew Gallatin 	device_printf(sc->dev, "slice %d struck? ring state:\n", slice);
3982c6cb3e3fSAndrew Gallatin 	device_printf(sc->dev,
3983c6cb3e3fSAndrew Gallatin 		      "tx.req=%d tx.done=%d, tx.queue_active=%d\n",
3984c6cb3e3fSAndrew Gallatin 		      tx->req, tx->done, tx->queue_active);
3985c6cb3e3fSAndrew Gallatin 	device_printf(sc->dev, "tx.activate=%d tx.deactivate=%d\n",
3986c6cb3e3fSAndrew Gallatin 			      tx->activate, tx->deactivate);
3987dce01b9bSAndrew Gallatin 	device_printf(sc->dev, "pkt_done=%d fw=%d\n",
3988c6cb3e3fSAndrew Gallatin 		      tx->pkt_done,
39891e413cf9SAndrew Gallatin 		      be32toh(sc->ss->fw_stats->send_done_count));
3990dce01b9bSAndrew Gallatin }
3991dce01b9bSAndrew Gallatin 
3992e749ef6bSAndrew Gallatin static int
3993dce01b9bSAndrew Gallatin mxge_watchdog(mxge_softc_t *sc)
3994dce01b9bSAndrew Gallatin {
3995c6cb3e3fSAndrew Gallatin 	mxge_tx_ring_t *tx;
39961e413cf9SAndrew Gallatin 	uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause);
3997c6cb3e3fSAndrew Gallatin 	int i, err = 0;
3998dce01b9bSAndrew Gallatin 
3999dce01b9bSAndrew Gallatin 	/* see if we have outstanding transmits, which
4000dce01b9bSAndrew Gallatin 	   have been pending for more than mxge_ticks */
4001c6cb3e3fSAndrew Gallatin 	for (i = 0;
4002c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
4003c6cb3e3fSAndrew Gallatin 	     (i < sc->num_slices) && (err == 0);
4004c6cb3e3fSAndrew Gallatin #else
4005c6cb3e3fSAndrew Gallatin 	     (i < 1) && (err == 0);
4006c6cb3e3fSAndrew Gallatin #endif
4007c6cb3e3fSAndrew Gallatin 	     i++) {
4008c6cb3e3fSAndrew Gallatin 		tx = &sc->ss[i].tx;
4009dce01b9bSAndrew Gallatin 		if (tx->req != tx->done &&
4010dce01b9bSAndrew Gallatin 		    tx->watchdog_req != tx->watchdog_done &&
4011c587e59fSAndrew Gallatin 		    tx->done == tx->watchdog_done) {
4012c587e59fSAndrew Gallatin 			/* check for pause blocking before resetting */
401372c042dfSAndrew Gallatin 			if (tx->watchdog_rx_pause == rx_pause) {
401472c042dfSAndrew Gallatin 				mxge_warn_stuck(sc, tx, i);
401572c042dfSAndrew Gallatin 				taskqueue_enqueue(sc->tq, &sc->watchdog_task);
401672c042dfSAndrew Gallatin 				return (ENXIO);
401772c042dfSAndrew Gallatin 			}
4018c587e59fSAndrew Gallatin 			else
4019c587e59fSAndrew Gallatin 				device_printf(sc->dev, "Flow control blocking "
4020c587e59fSAndrew Gallatin 					      "xmits, check link partner\n");
4021c587e59fSAndrew Gallatin 		}
4022dce01b9bSAndrew Gallatin 
4023dce01b9bSAndrew Gallatin 		tx->watchdog_req = tx->req;
4024dce01b9bSAndrew Gallatin 		tx->watchdog_done = tx->done;
4025c587e59fSAndrew Gallatin 		tx->watchdog_rx_pause = rx_pause;
4026c6cb3e3fSAndrew Gallatin 	}
4027c587e59fSAndrew Gallatin 
4028c587e59fSAndrew Gallatin 	if (sc->need_media_probe)
4029c587e59fSAndrew Gallatin 		mxge_media_probe(sc);
4030e749ef6bSAndrew Gallatin 	return (err);
4031dce01b9bSAndrew Gallatin }
4032dce01b9bSAndrew Gallatin 
4033f3f040d9SGleb Smirnoff static uint64_t
4034f3f040d9SGleb Smirnoff mxge_get_counter(struct ifnet *ifp, ift_counter cnt)
40351e413cf9SAndrew Gallatin {
4036f3f040d9SGleb Smirnoff 	struct mxge_softc *sc;
4037f3f040d9SGleb Smirnoff 	uint64_t rv;
40381e413cf9SAndrew Gallatin 
4039f3f040d9SGleb Smirnoff 	sc = if_getsoftc(ifp);
4040f3f040d9SGleb Smirnoff 	rv = 0;
4041f3f040d9SGleb Smirnoff 
4042f3f040d9SGleb Smirnoff 	switch (cnt) {
4043f3f040d9SGleb Smirnoff 	case IFCOUNTER_IPACKETS:
4044f3f040d9SGleb Smirnoff 		for (int s = 0; s < sc->num_slices; s++)
4045f3f040d9SGleb Smirnoff 			rv += sc->ss[s].ipackets;
4046f3f040d9SGleb Smirnoff 		return (rv);
4047f3f040d9SGleb Smirnoff 	case IFCOUNTER_OPACKETS:
4048f3f040d9SGleb Smirnoff 		for (int s = 0; s < sc->num_slices; s++)
4049f3f040d9SGleb Smirnoff 			rv += sc->ss[s].opackets;
4050f3f040d9SGleb Smirnoff 		return (rv);
4051f3f040d9SGleb Smirnoff 	case IFCOUNTER_OERRORS:
4052f3f040d9SGleb Smirnoff 		for (int s = 0; s < sc->num_slices; s++)
4053f3f040d9SGleb Smirnoff 			rv += sc->ss[s].oerrors;
4054f3f040d9SGleb Smirnoff 		return (rv);
405571032832SAndrew Gallatin #ifdef IFNET_BUF_RING
4056f3f040d9SGleb Smirnoff 	case IFCOUNTER_OBYTES:
4057f3f040d9SGleb Smirnoff 		for (int s = 0; s < sc->num_slices; s++)
4058f3f040d9SGleb Smirnoff 			rv += sc->ss[s].obytes;
4059f3f040d9SGleb Smirnoff 		return (rv);
4060f3f040d9SGleb Smirnoff 	case IFCOUNTER_OMCASTS:
4061f3f040d9SGleb Smirnoff 		for (int s = 0; s < sc->num_slices; s++)
4062f3f040d9SGleb Smirnoff 			rv += sc->ss[s].omcasts;
4063f3f040d9SGleb Smirnoff 		return (rv);
4064f3f040d9SGleb Smirnoff 	case IFCOUNTER_OQDROPS:
4065f3f040d9SGleb Smirnoff 		for (int s = 0; s < sc->num_slices; s++)
4066f3f040d9SGleb Smirnoff 			rv += sc->ss[s].tx.br->br_drops;
4067f3f040d9SGleb Smirnoff 		return (rv);
406871032832SAndrew Gallatin #endif
4069f3f040d9SGleb Smirnoff 	default:
4070f3f040d9SGleb Smirnoff 		return (if_get_counter_default(ifp, cnt));
40711e413cf9SAndrew Gallatin 	}
40721e413cf9SAndrew Gallatin }
4073c6cb3e3fSAndrew Gallatin 
40741e413cf9SAndrew Gallatin static void
4075dce01b9bSAndrew Gallatin mxge_tick(void *arg)
4076dce01b9bSAndrew Gallatin {
4077dce01b9bSAndrew Gallatin 	mxge_softc_t *sc = arg;
40786b484a49SAndrew Gallatin 	u_long pkts = 0;
4079e749ef6bSAndrew Gallatin 	int err = 0;
40806b484a49SAndrew Gallatin 	int running, ticks;
40816b484a49SAndrew Gallatin 	uint16_t cmd;
4082dce01b9bSAndrew Gallatin 
40836b484a49SAndrew Gallatin 	ticks = mxge_ticks;
40846b484a49SAndrew Gallatin 	running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING;
40856b484a49SAndrew Gallatin 	if (running) {
40861e413cf9SAndrew Gallatin 		if (!sc->watchdog_countdown) {
4087e749ef6bSAndrew Gallatin 			err = mxge_watchdog(sc);
40881e413cf9SAndrew Gallatin 			sc->watchdog_countdown = 4;
40891e413cf9SAndrew Gallatin 		}
40901e413cf9SAndrew Gallatin 		sc->watchdog_countdown--;
40916b484a49SAndrew Gallatin 	}
40926b484a49SAndrew Gallatin 	if (pkts == 0) {
40936b484a49SAndrew Gallatin 		/* ensure NIC did not suffer h/w fault while idle */
40946b484a49SAndrew Gallatin 		cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
40956b484a49SAndrew Gallatin 		if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) {
40966b484a49SAndrew Gallatin 			sc->dying = 2;
40976b484a49SAndrew Gallatin 			taskqueue_enqueue(sc->tq, &sc->watchdog_task);
40986b484a49SAndrew Gallatin 			err = ENXIO;
40996b484a49SAndrew Gallatin 		}
41006b484a49SAndrew Gallatin 		/* look less often if NIC is idle */
41016b484a49SAndrew Gallatin 		ticks *= 4;
41026b484a49SAndrew Gallatin 	}
41036b484a49SAndrew Gallatin 
4104e749ef6bSAndrew Gallatin 	if (err == 0)
41056b484a49SAndrew Gallatin 		callout_reset(&sc->co_hdl, ticks, mxge_tick, sc);
4106e749ef6bSAndrew Gallatin 
4107dce01b9bSAndrew Gallatin }
4108b2fc195eSAndrew Gallatin 
4109b2fc195eSAndrew Gallatin static int
41106d87a65dSAndrew Gallatin mxge_media_change(struct ifnet *ifp)
4111b2fc195eSAndrew Gallatin {
4112b2fc195eSAndrew Gallatin 	return EINVAL;
4113b2fc195eSAndrew Gallatin }
4114b2fc195eSAndrew Gallatin 
4115b2fc195eSAndrew Gallatin static int
41166d87a65dSAndrew Gallatin mxge_change_mtu(mxge_softc_t *sc, int mtu)
4117b2fc195eSAndrew Gallatin {
4118b2fc195eSAndrew Gallatin 	struct ifnet *ifp = sc->ifp;
4119b2fc195eSAndrew Gallatin 	int real_mtu, old_mtu;
4120b2fc195eSAndrew Gallatin 	int err = 0;
4121b2fc195eSAndrew Gallatin 
4122b2fc195eSAndrew Gallatin 
4123c792928fSAndrew Gallatin 	real_mtu = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
4124053e637fSAndrew Gallatin 	if ((real_mtu > sc->max_mtu) || real_mtu < 60)
4125b2fc195eSAndrew Gallatin 		return EINVAL;
4126a98d6cd7SAndrew Gallatin 	mtx_lock(&sc->driver_mtx);
4127b2fc195eSAndrew Gallatin 	old_mtu = ifp->if_mtu;
4128b2fc195eSAndrew Gallatin 	ifp->if_mtu = mtu;
4129b2fc195eSAndrew Gallatin 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
4130a393336bSAndrew Gallatin 		mxge_close(sc, 0);
41316d87a65dSAndrew Gallatin 		err = mxge_open(sc);
4132b2fc195eSAndrew Gallatin 		if (err != 0) {
4133b2fc195eSAndrew Gallatin 			ifp->if_mtu = old_mtu;
4134a393336bSAndrew Gallatin 			mxge_close(sc, 0);
41356d87a65dSAndrew Gallatin 			(void) mxge_open(sc);
4136b2fc195eSAndrew Gallatin 		}
4137b2fc195eSAndrew Gallatin 	}
4138a98d6cd7SAndrew Gallatin 	mtx_unlock(&sc->driver_mtx);
4139b2fc195eSAndrew Gallatin 	return err;
4140b2fc195eSAndrew Gallatin }
4141b2fc195eSAndrew Gallatin 
4142b2fc195eSAndrew Gallatin static void
41436d87a65dSAndrew Gallatin mxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
4144b2fc195eSAndrew Gallatin {
41456d87a65dSAndrew Gallatin 	mxge_softc_t *sc = ifp->if_softc;
4146b2fc195eSAndrew Gallatin 
4147b2fc195eSAndrew Gallatin 
4148b2fc195eSAndrew Gallatin 	if (sc == NULL)
4149b2fc195eSAndrew Gallatin 		return;
4150b2fc195eSAndrew Gallatin 	ifmr->ifm_status = IFM_AVALID;
4151c406ad2eSAndrew Gallatin 	ifmr->ifm_active = IFM_ETHER | IFM_FDX;
4152c587e59fSAndrew Gallatin 	ifmr->ifm_status |= sc->link_state ? IFM_ACTIVE : 0;
4153c406ad2eSAndrew Gallatin 	ifmr->ifm_active |= sc->current_media;
4154b2fc195eSAndrew Gallatin }
4155b2fc195eSAndrew Gallatin 
4156b2fc195eSAndrew Gallatin static int
41576d87a65dSAndrew Gallatin mxge_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
4158b2fc195eSAndrew Gallatin {
41596d87a65dSAndrew Gallatin 	mxge_softc_t *sc = ifp->if_softc;
4160b2fc195eSAndrew Gallatin 	struct ifreq *ifr = (struct ifreq *)data;
4161b2fc195eSAndrew Gallatin 	int err, mask;
4162b2fc195eSAndrew Gallatin 
4163b2fc195eSAndrew Gallatin 	err = 0;
4164b2fc195eSAndrew Gallatin 	switch (command) {
4165b2fc195eSAndrew Gallatin 	case SIOCSIFMTU:
41666d87a65dSAndrew Gallatin 		err = mxge_change_mtu(sc, ifr->ifr_mtu);
4167b2fc195eSAndrew Gallatin 		break;
4168b2fc195eSAndrew Gallatin 
4169b2fc195eSAndrew Gallatin 	case SIOCSIFFLAGS:
4170a98d6cd7SAndrew Gallatin 		mtx_lock(&sc->driver_mtx);
41718c5d766cSAndrew Gallatin 		if (sc->dying) {
41728c5d766cSAndrew Gallatin 			mtx_unlock(&sc->driver_mtx);
41738c5d766cSAndrew Gallatin 			return EINVAL;
41748c5d766cSAndrew Gallatin 		}
4175b2fc195eSAndrew Gallatin 		if (ifp->if_flags & IFF_UP) {
4176dce01b9bSAndrew Gallatin 			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
41776d87a65dSAndrew Gallatin 				err = mxge_open(sc);
4178dce01b9bSAndrew Gallatin 			} else {
41790fa7f681SAndrew Gallatin 				/* take care of promis can allmulti
41800fa7f681SAndrew Gallatin 				   flag chages */
41810fa7f681SAndrew Gallatin 				mxge_change_promisc(sc,
41820fa7f681SAndrew Gallatin 						    ifp->if_flags & IFF_PROMISC);
41830fa7f681SAndrew Gallatin 				mxge_set_multicast_list(sc);
41840fa7f681SAndrew Gallatin 			}
4185b2fc195eSAndrew Gallatin 		} else {
4186dce01b9bSAndrew Gallatin 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
4187a393336bSAndrew Gallatin 				mxge_close(sc, 0);
4188dce01b9bSAndrew Gallatin 			}
4189b2fc195eSAndrew Gallatin 		}
4190a98d6cd7SAndrew Gallatin 		mtx_unlock(&sc->driver_mtx);
4191b2fc195eSAndrew Gallatin 		break;
4192b2fc195eSAndrew Gallatin 
4193b2fc195eSAndrew Gallatin 	case SIOCADDMULTI:
4194b2fc195eSAndrew Gallatin 	case SIOCDELMULTI:
4195a98d6cd7SAndrew Gallatin 		mtx_lock(&sc->driver_mtx);
41960fa7f681SAndrew Gallatin 		mxge_set_multicast_list(sc);
4197a98d6cd7SAndrew Gallatin 		mtx_unlock(&sc->driver_mtx);
4198b2fc195eSAndrew Gallatin 		break;
4199b2fc195eSAndrew Gallatin 
4200b2fc195eSAndrew Gallatin 	case SIOCSIFCAP:
4201a98d6cd7SAndrew Gallatin 		mtx_lock(&sc->driver_mtx);
4202b2fc195eSAndrew Gallatin 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
4203b2fc195eSAndrew Gallatin 		if (mask & IFCAP_TXCSUM) {
4204b2fc195eSAndrew Gallatin 			if (IFCAP_TXCSUM & ifp->if_capenable) {
4205aed8e389SAndrew Gallatin 				ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
42060a7a780eSAndrew Gallatin 				ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP);
4207b2fc195eSAndrew Gallatin 			} else {
4208b2fc195eSAndrew Gallatin 				ifp->if_capenable |= IFCAP_TXCSUM;
4209b2fc195eSAndrew Gallatin 				ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
4210b2fc195eSAndrew Gallatin 			}
4211b2fc195eSAndrew Gallatin 		} else if (mask & IFCAP_RXCSUM) {
4212b2fc195eSAndrew Gallatin 			if (IFCAP_RXCSUM & ifp->if_capenable) {
4213b2fc195eSAndrew Gallatin 				ifp->if_capenable &= ~IFCAP_RXCSUM;
4214b2fc195eSAndrew Gallatin 			} else {
4215b2fc195eSAndrew Gallatin 				ifp->if_capenable |= IFCAP_RXCSUM;
4216b2fc195eSAndrew Gallatin 			}
4217b2fc195eSAndrew Gallatin 		}
4218aed8e389SAndrew Gallatin 		if (mask & IFCAP_TSO4) {
4219aed8e389SAndrew Gallatin 			if (IFCAP_TSO4 & ifp->if_capenable) {
4220aed8e389SAndrew Gallatin 				ifp->if_capenable &= ~IFCAP_TSO4;
4221aed8e389SAndrew Gallatin 			} else if (IFCAP_TXCSUM & ifp->if_capenable) {
4222aed8e389SAndrew Gallatin 				ifp->if_capenable |= IFCAP_TSO4;
4223aed8e389SAndrew Gallatin 				ifp->if_hwassist |= CSUM_TSO;
4224aed8e389SAndrew Gallatin 			} else {
4225aed8e389SAndrew Gallatin 				printf("mxge requires tx checksum offload"
4226aed8e389SAndrew Gallatin 				       " be enabled to use TSO\n");
4227aed8e389SAndrew Gallatin 				err = EINVAL;
4228aed8e389SAndrew Gallatin 			}
4229aed8e389SAndrew Gallatin 		}
42300a7a780eSAndrew Gallatin #if IFCAP_TSO6
42310a7a780eSAndrew Gallatin 		if (mask & IFCAP_TXCSUM_IPV6) {
42320a7a780eSAndrew Gallatin 			if (IFCAP_TXCSUM_IPV6 & ifp->if_capenable) {
42330a7a780eSAndrew Gallatin 				ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6
42340a7a780eSAndrew Gallatin 						       | IFCAP_TSO6);
42350a7a780eSAndrew Gallatin 				ifp->if_hwassist &= ~(CSUM_TCP_IPV6
42360a7a780eSAndrew Gallatin 						      | CSUM_UDP);
42370a7a780eSAndrew Gallatin 			} else {
42380a7a780eSAndrew Gallatin 				ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
42390a7a780eSAndrew Gallatin 				ifp->if_hwassist |= (CSUM_TCP_IPV6
42400a7a780eSAndrew Gallatin 						     | CSUM_UDP_IPV6);
42410a7a780eSAndrew Gallatin 			}
424226dd49c6SAndrew Gallatin 		} else if (mask & IFCAP_RXCSUM_IPV6) {
424326dd49c6SAndrew Gallatin 			if (IFCAP_RXCSUM_IPV6 & ifp->if_capenable) {
424426dd49c6SAndrew Gallatin 				ifp->if_capenable &= ~IFCAP_RXCSUM_IPV6;
42450a7a780eSAndrew Gallatin 			} else {
424626dd49c6SAndrew Gallatin 				ifp->if_capenable |= IFCAP_RXCSUM_IPV6;
42470a7a780eSAndrew Gallatin 			}
42480a7a780eSAndrew Gallatin 		}
42490a7a780eSAndrew Gallatin 		if (mask & IFCAP_TSO6) {
42500a7a780eSAndrew Gallatin 			if (IFCAP_TSO6 & ifp->if_capenable) {
42510a7a780eSAndrew Gallatin 				ifp->if_capenable &= ~IFCAP_TSO6;
42520a7a780eSAndrew Gallatin 			} else if (IFCAP_TXCSUM_IPV6 & ifp->if_capenable) {
42530a7a780eSAndrew Gallatin 				ifp->if_capenable |= IFCAP_TSO6;
42540a7a780eSAndrew Gallatin 				ifp->if_hwassist |= CSUM_TSO;
42550a7a780eSAndrew Gallatin 			} else {
42560a7a780eSAndrew Gallatin 				printf("mxge requires tx checksum offload"
42570a7a780eSAndrew Gallatin 				       " be enabled to use TSO\n");
42580a7a780eSAndrew Gallatin 				err = EINVAL;
42590a7a780eSAndrew Gallatin 			}
42600a7a780eSAndrew Gallatin 		}
42610a7a780eSAndrew Gallatin #endif /*IFCAP_TSO6 */
42620a7a780eSAndrew Gallatin 
426326dd49c6SAndrew Gallatin 		if (mask & IFCAP_LRO)
426426dd49c6SAndrew Gallatin 			ifp->if_capenable ^= IFCAP_LRO;
4265c792928fSAndrew Gallatin 		if (mask & IFCAP_VLAN_HWTAGGING)
4266c792928fSAndrew Gallatin 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
42670dce6781SAndrew Gallatin 		if (mask & IFCAP_VLAN_HWTSO)
42680dce6781SAndrew Gallatin 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
42690dce6781SAndrew Gallatin 
42700dce6781SAndrew Gallatin 		if (!(ifp->if_capabilities & IFCAP_VLAN_HWTSO) ||
42710dce6781SAndrew Gallatin 		    !(ifp->if_capenable & IFCAP_VLAN_HWTAGGING))
42720dce6781SAndrew Gallatin 			ifp->if_capenable &= ~IFCAP_VLAN_HWTSO;
42730dce6781SAndrew Gallatin 
4274a98d6cd7SAndrew Gallatin 		mtx_unlock(&sc->driver_mtx);
4275c792928fSAndrew Gallatin 		VLAN_CAPABILITIES(ifp);
4276c792928fSAndrew Gallatin 
4277b2fc195eSAndrew Gallatin 		break;
4278b2fc195eSAndrew Gallatin 
4279b2fc195eSAndrew Gallatin 	case SIOCGIFMEDIA:
4280c406ad2eSAndrew Gallatin 		mtx_lock(&sc->driver_mtx);
4281c406ad2eSAndrew Gallatin 		mxge_media_probe(sc);
4282c406ad2eSAndrew Gallatin 		mtx_unlock(&sc->driver_mtx);
4283b2fc195eSAndrew Gallatin 		err = ifmedia_ioctl(ifp, (struct ifreq *)data,
4284b2fc195eSAndrew Gallatin 				    &sc->media, command);
4285b2fc195eSAndrew Gallatin 		break;
4286b2fc195eSAndrew Gallatin 
4287b2fc195eSAndrew Gallatin 	default:
4288*c756fb6eSRavi Pokala 		err = ether_ioctl(ifp, command, data);
4289*c756fb6eSRavi Pokala 		break;
4290b2fc195eSAndrew Gallatin 	}
4291b2fc195eSAndrew Gallatin 	return err;
4292b2fc195eSAndrew Gallatin }
4293b2fc195eSAndrew Gallatin 
4294b2fc195eSAndrew Gallatin static void
42956d87a65dSAndrew Gallatin mxge_fetch_tunables(mxge_softc_t *sc)
4296b2fc195eSAndrew Gallatin {
4297b2fc195eSAndrew Gallatin 
42981e413cf9SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.max_slices", &mxge_max_slices);
42996d87a65dSAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled",
43006d87a65dSAndrew Gallatin 			  &mxge_flow_control);
43016d87a65dSAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay",
43026d87a65dSAndrew Gallatin 			  &mxge_intr_coal_delay);
43036d87a65dSAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable",
43046d87a65dSAndrew Gallatin 			  &mxge_nvidia_ecrc_enable);
4305d91b1b49SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.force_firmware",
4306d91b1b49SAndrew Gallatin 			  &mxge_force_firmware);
43075e7d8541SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.deassert_wait",
43085e7d8541SAndrew Gallatin 			  &mxge_deassert_wait);
43095e7d8541SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.verbose",
43105e7d8541SAndrew Gallatin 			  &mxge_verbose);
4311dce01b9bSAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks);
43121e413cf9SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.always_promisc", &mxge_always_promisc);
43131e413cf9SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.rss_hash_type", &mxge_rss_hash_type);
431494c7d993SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.rss_hashtype", &mxge_rss_hash_type);
4315f9453025SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.initial_mtu", &mxge_initial_mtu);
431665c69066SAndrew Gallatin 	TUNABLE_INT_FETCH("hw.mxge.throttle", &mxge_throttle);
4317b2fc195eSAndrew Gallatin 
43185e7d8541SAndrew Gallatin 	if (bootverbose)
43195e7d8541SAndrew Gallatin 		mxge_verbose = 1;
43206d87a65dSAndrew Gallatin 	if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000)
43216d87a65dSAndrew Gallatin 		mxge_intr_coal_delay = 30;
4322dce01b9bSAndrew Gallatin 	if (mxge_ticks == 0)
43231e413cf9SAndrew Gallatin 		mxge_ticks = hz / 2;
43246d87a65dSAndrew Gallatin 	sc->pause = mxge_flow_control;
43251e413cf9SAndrew Gallatin 	if (mxge_rss_hash_type < MXGEFW_RSS_HASH_TYPE_IPV4
4326bb8ddc66SAndrew Gallatin 	    || mxge_rss_hash_type > MXGEFW_RSS_HASH_TYPE_MAX) {
43275769c5efSAndrew Gallatin 		mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
4328b2fc195eSAndrew Gallatin 	}
4329f9453025SAndrew Gallatin 	if (mxge_initial_mtu > ETHERMTU_JUMBO ||
4330f9453025SAndrew Gallatin 	    mxge_initial_mtu < ETHER_MIN_LEN)
4331f9453025SAndrew Gallatin 		mxge_initial_mtu = ETHERMTU_JUMBO;
433265c69066SAndrew Gallatin 
433365c69066SAndrew Gallatin 	if (mxge_throttle && mxge_throttle > MXGE_MAX_THROTTLE)
433465c69066SAndrew Gallatin 		mxge_throttle = MXGE_MAX_THROTTLE;
433565c69066SAndrew Gallatin 	if (mxge_throttle && mxge_throttle < MXGE_MIN_THROTTLE)
433665c69066SAndrew Gallatin 		mxge_throttle = MXGE_MIN_THROTTLE;
433765c69066SAndrew Gallatin 	sc->throttle = mxge_throttle;
43381e413cf9SAndrew Gallatin }
43391e413cf9SAndrew Gallatin 
43401e413cf9SAndrew Gallatin 
43411e413cf9SAndrew Gallatin static void
43421e413cf9SAndrew Gallatin mxge_free_slices(mxge_softc_t *sc)
43431e413cf9SAndrew Gallatin {
43441e413cf9SAndrew Gallatin 	struct mxge_slice_state *ss;
43451e413cf9SAndrew Gallatin 	int i;
43461e413cf9SAndrew Gallatin 
43471e413cf9SAndrew Gallatin 
43481e413cf9SAndrew Gallatin 	if (sc->ss == NULL)
43491e413cf9SAndrew Gallatin 		return;
43501e413cf9SAndrew Gallatin 
43511e413cf9SAndrew Gallatin 	for (i = 0; i < sc->num_slices; i++) {
43521e413cf9SAndrew Gallatin 		ss = &sc->ss[i];
43531e413cf9SAndrew Gallatin 		if (ss->fw_stats != NULL) {
43541e413cf9SAndrew Gallatin 			mxge_dma_free(&ss->fw_stats_dma);
43551e413cf9SAndrew Gallatin 			ss->fw_stats = NULL;
4356c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
4357c6cb3e3fSAndrew Gallatin 			if (ss->tx.br != NULL) {
4358c6cb3e3fSAndrew Gallatin 				drbr_free(ss->tx.br, M_DEVBUF);
4359c6cb3e3fSAndrew Gallatin 				ss->tx.br = NULL;
4360c6cb3e3fSAndrew Gallatin 			}
4361c6cb3e3fSAndrew Gallatin #endif
43621e413cf9SAndrew Gallatin 			mtx_destroy(&ss->tx.mtx);
43631e413cf9SAndrew Gallatin 		}
43641e413cf9SAndrew Gallatin 		if (ss->rx_done.entry != NULL) {
43651e413cf9SAndrew Gallatin 			mxge_dma_free(&ss->rx_done.dma);
43661e413cf9SAndrew Gallatin 			ss->rx_done.entry = NULL;
43671e413cf9SAndrew Gallatin 		}
43681e413cf9SAndrew Gallatin 	}
43691e413cf9SAndrew Gallatin 	free(sc->ss, M_DEVBUF);
43701e413cf9SAndrew Gallatin 	sc->ss = NULL;
43711e413cf9SAndrew Gallatin }
43721e413cf9SAndrew Gallatin 
43731e413cf9SAndrew Gallatin static int
43741e413cf9SAndrew Gallatin mxge_alloc_slices(mxge_softc_t *sc)
43751e413cf9SAndrew Gallatin {
43761e413cf9SAndrew Gallatin 	mxge_cmd_t cmd;
43771e413cf9SAndrew Gallatin 	struct mxge_slice_state *ss;
43781e413cf9SAndrew Gallatin 	size_t bytes;
43791e413cf9SAndrew Gallatin 	int err, i, max_intr_slots;
43801e413cf9SAndrew Gallatin 
43811e413cf9SAndrew Gallatin 	err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
43821e413cf9SAndrew Gallatin 	if (err != 0) {
43831e413cf9SAndrew Gallatin 		device_printf(sc->dev, "Cannot determine rx ring size\n");
43841e413cf9SAndrew Gallatin 		return err;
43851e413cf9SAndrew Gallatin 	}
43861e413cf9SAndrew Gallatin 	sc->rx_ring_size = cmd.data0;
43871e413cf9SAndrew Gallatin 	max_intr_slots = 2 * (sc->rx_ring_size / sizeof (mcp_dma_addr_t));
43881e413cf9SAndrew Gallatin 
4389ac2fffa4SPedro F. Giffuni 	bytes = sizeof (*sc->ss) * sc->num_slices;
4390ac2fffa4SPedro F. Giffuni 	sc->ss = malloc(bytes, M_DEVBUF, M_NOWAIT | M_ZERO);
43911e413cf9SAndrew Gallatin 	if (sc->ss == NULL)
43921e413cf9SAndrew Gallatin 		return (ENOMEM);
43931e413cf9SAndrew Gallatin 	for (i = 0; i < sc->num_slices; i++) {
43941e413cf9SAndrew Gallatin 		ss = &sc->ss[i];
43951e413cf9SAndrew Gallatin 
43961e413cf9SAndrew Gallatin 		ss->sc = sc;
43971e413cf9SAndrew Gallatin 
43981e413cf9SAndrew Gallatin 		/* allocate per-slice rx interrupt queues */
43991e413cf9SAndrew Gallatin 
44001e413cf9SAndrew Gallatin 		bytes = max_intr_slots * sizeof (*ss->rx_done.entry);
44011e413cf9SAndrew Gallatin 		err = mxge_dma_alloc(sc, &ss->rx_done.dma, bytes, 4096);
44021e413cf9SAndrew Gallatin 		if (err != 0)
44031e413cf9SAndrew Gallatin 			goto abort;
44041e413cf9SAndrew Gallatin 		ss->rx_done.entry = ss->rx_done.dma.addr;
44051e413cf9SAndrew Gallatin 		bzero(ss->rx_done.entry, bytes);
44061e413cf9SAndrew Gallatin 
44071e413cf9SAndrew Gallatin 		/*
44081e413cf9SAndrew Gallatin 		 * allocate the per-slice firmware stats; stats
44091e413cf9SAndrew Gallatin 		 * (including tx) are used used only on the first
44101e413cf9SAndrew Gallatin 		 * slice for now
44111e413cf9SAndrew Gallatin 		 */
4412c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING
44131e413cf9SAndrew Gallatin 		if (i > 0)
44141e413cf9SAndrew Gallatin 			continue;
4415c6cb3e3fSAndrew Gallatin #endif
44161e413cf9SAndrew Gallatin 
44171e413cf9SAndrew Gallatin 		bytes = sizeof (*ss->fw_stats);
44181e413cf9SAndrew Gallatin 		err = mxge_dma_alloc(sc, &ss->fw_stats_dma,
44191e413cf9SAndrew Gallatin 				     sizeof (*ss->fw_stats), 64);
44201e413cf9SAndrew Gallatin 		if (err != 0)
44211e413cf9SAndrew Gallatin 			goto abort;
44221e413cf9SAndrew Gallatin 		ss->fw_stats = (mcp_irq_data_t *)ss->fw_stats_dma.addr;
44231e413cf9SAndrew Gallatin 		snprintf(ss->tx.mtx_name, sizeof(ss->tx.mtx_name),
44241e413cf9SAndrew Gallatin 			 "%s:tx(%d)", device_get_nameunit(sc->dev), i);
44251e413cf9SAndrew Gallatin 		mtx_init(&ss->tx.mtx, ss->tx.mtx_name, NULL, MTX_DEF);
4426c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
4427c6cb3e3fSAndrew Gallatin 		ss->tx.br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK,
4428c6cb3e3fSAndrew Gallatin 					   &ss->tx.mtx);
4429c6cb3e3fSAndrew Gallatin #endif
44301e413cf9SAndrew Gallatin 	}
44311e413cf9SAndrew Gallatin 
44321e413cf9SAndrew Gallatin 	return (0);
44331e413cf9SAndrew Gallatin 
44341e413cf9SAndrew Gallatin abort:
44351e413cf9SAndrew Gallatin 	mxge_free_slices(sc);
44361e413cf9SAndrew Gallatin 	return (ENOMEM);
44371e413cf9SAndrew Gallatin }
44381e413cf9SAndrew Gallatin 
44391e413cf9SAndrew Gallatin static void
44401e413cf9SAndrew Gallatin mxge_slice_probe(mxge_softc_t *sc)
44411e413cf9SAndrew Gallatin {
44421e413cf9SAndrew Gallatin 	mxge_cmd_t cmd;
44431e413cf9SAndrew Gallatin 	char *old_fw;
44441e413cf9SAndrew Gallatin 	int msix_cnt, status, max_intr_slots;
44451e413cf9SAndrew Gallatin 
44461e413cf9SAndrew Gallatin 	sc->num_slices = 1;
44471e413cf9SAndrew Gallatin 	/*
44481e413cf9SAndrew Gallatin 	 *  don't enable multiple slices if they are not enabled,
44491e413cf9SAndrew Gallatin 	 *  or if this is not an SMP system
44501e413cf9SAndrew Gallatin 	 */
44511e413cf9SAndrew Gallatin 
44521e413cf9SAndrew Gallatin 	if (mxge_max_slices == 0 || mxge_max_slices == 1 || mp_ncpus < 2)
44531e413cf9SAndrew Gallatin 		return;
44541e413cf9SAndrew Gallatin 
44551e413cf9SAndrew Gallatin 	/* see how many MSI-X interrupts are available */
44561e413cf9SAndrew Gallatin 	msix_cnt = pci_msix_count(sc->dev);
44571e413cf9SAndrew Gallatin 	if (msix_cnt < 2)
44581e413cf9SAndrew Gallatin 		return;
44591e413cf9SAndrew Gallatin 
44601e413cf9SAndrew Gallatin 	/* now load the slice aware firmware see what it supports */
44611e413cf9SAndrew Gallatin 	old_fw = sc->fw_name;
44621e413cf9SAndrew Gallatin 	if (old_fw == mxge_fw_aligned)
44631e413cf9SAndrew Gallatin 		sc->fw_name = mxge_fw_rss_aligned;
44641e413cf9SAndrew Gallatin 	else
44651e413cf9SAndrew Gallatin 		sc->fw_name = mxge_fw_rss_unaligned;
44661e413cf9SAndrew Gallatin 	status = mxge_load_firmware(sc, 0);
44671e413cf9SAndrew Gallatin 	if (status != 0) {
44681e413cf9SAndrew Gallatin 		device_printf(sc->dev, "Falling back to a single slice\n");
44691e413cf9SAndrew Gallatin 		return;
44701e413cf9SAndrew Gallatin 	}
44711e413cf9SAndrew Gallatin 
44721e413cf9SAndrew Gallatin 	/* try to send a reset command to the card to see if it
44731e413cf9SAndrew Gallatin 	   is alive */
44741e413cf9SAndrew Gallatin 	memset(&cmd, 0, sizeof (cmd));
44751e413cf9SAndrew Gallatin 	status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
44761e413cf9SAndrew Gallatin 	if (status != 0) {
44771e413cf9SAndrew Gallatin 		device_printf(sc->dev, "failed reset\n");
44781e413cf9SAndrew Gallatin 		goto abort_with_fw;
44791e413cf9SAndrew Gallatin 	}
44801e413cf9SAndrew Gallatin 
44811e413cf9SAndrew Gallatin 	/* get rx ring size */
44821e413cf9SAndrew Gallatin 	status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
44831e413cf9SAndrew Gallatin 	if (status != 0) {
44841e413cf9SAndrew Gallatin 		device_printf(sc->dev, "Cannot determine rx ring size\n");
44851e413cf9SAndrew Gallatin 		goto abort_with_fw;
44861e413cf9SAndrew Gallatin 	}
44871e413cf9SAndrew Gallatin 	max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t));
44881e413cf9SAndrew Gallatin 
44891e413cf9SAndrew Gallatin 	/* tell it the size of the interrupt queues */
44901e413cf9SAndrew Gallatin 	cmd.data0 = max_intr_slots * sizeof (struct mcp_slot);
44911e413cf9SAndrew Gallatin 	status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
44921e413cf9SAndrew Gallatin 	if (status != 0) {
44931e413cf9SAndrew Gallatin 		device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n");
44941e413cf9SAndrew Gallatin 		goto abort_with_fw;
44951e413cf9SAndrew Gallatin 	}
44961e413cf9SAndrew Gallatin 
44971e413cf9SAndrew Gallatin 	/* ask the maximum number of slices it supports */
44981e413cf9SAndrew Gallatin 	status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd);
44991e413cf9SAndrew Gallatin 	if (status != 0) {
45001e413cf9SAndrew Gallatin 		device_printf(sc->dev,
45011e413cf9SAndrew Gallatin 			      "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n");
45021e413cf9SAndrew Gallatin 		goto abort_with_fw;
45031e413cf9SAndrew Gallatin 	}
45041e413cf9SAndrew Gallatin 	sc->num_slices = cmd.data0;
45051e413cf9SAndrew Gallatin 	if (sc->num_slices > msix_cnt)
45061e413cf9SAndrew Gallatin 		sc->num_slices = msix_cnt;
45071e413cf9SAndrew Gallatin 
45081e413cf9SAndrew Gallatin 	if (mxge_max_slices == -1) {
45091e413cf9SAndrew Gallatin 		/* cap to number of CPUs in system */
45101e413cf9SAndrew Gallatin 		if (sc->num_slices > mp_ncpus)
45111e413cf9SAndrew Gallatin 			sc->num_slices = mp_ncpus;
45121e413cf9SAndrew Gallatin 	} else {
45131e413cf9SAndrew Gallatin 		if (sc->num_slices > mxge_max_slices)
45141e413cf9SAndrew Gallatin 			sc->num_slices = mxge_max_slices;
45151e413cf9SAndrew Gallatin 	}
45161e413cf9SAndrew Gallatin 	/* make sure it is a power of two */
45171e413cf9SAndrew Gallatin 	while (sc->num_slices & (sc->num_slices - 1))
45181e413cf9SAndrew Gallatin 		sc->num_slices--;
45191e413cf9SAndrew Gallatin 
45201e413cf9SAndrew Gallatin 	if (mxge_verbose)
45211e413cf9SAndrew Gallatin 		device_printf(sc->dev, "using %d slices\n",
45221e413cf9SAndrew Gallatin 			      sc->num_slices);
45231e413cf9SAndrew Gallatin 
45241e413cf9SAndrew Gallatin 	return;
45251e413cf9SAndrew Gallatin 
45261e413cf9SAndrew Gallatin abort_with_fw:
45271e413cf9SAndrew Gallatin 	sc->fw_name = old_fw;
45281e413cf9SAndrew Gallatin 	(void) mxge_load_firmware(sc, 0);
45291e413cf9SAndrew Gallatin }
45301e413cf9SAndrew Gallatin 
45311e413cf9SAndrew Gallatin static int
45321e413cf9SAndrew Gallatin mxge_add_msix_irqs(mxge_softc_t *sc)
45331e413cf9SAndrew Gallatin {
4534ac2fffa4SPedro F. Giffuni 	size_t bytes;
45351e413cf9SAndrew Gallatin 	int count, err, i, rid;
45361e413cf9SAndrew Gallatin 
45371e413cf9SAndrew Gallatin 	rid = PCIR_BAR(2);
45381e413cf9SAndrew Gallatin 	sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
45391e413cf9SAndrew Gallatin 						    &rid, RF_ACTIVE);
45401e413cf9SAndrew Gallatin 
45411e413cf9SAndrew Gallatin 	if (sc->msix_table_res == NULL) {
45421e413cf9SAndrew Gallatin 		device_printf(sc->dev, "couldn't alloc MSIX table res\n");
45431e413cf9SAndrew Gallatin 		return ENXIO;
45441e413cf9SAndrew Gallatin 	}
45451e413cf9SAndrew Gallatin 
45461e413cf9SAndrew Gallatin 	count = sc->num_slices;
45471e413cf9SAndrew Gallatin 	err = pci_alloc_msix(sc->dev, &count);
45481e413cf9SAndrew Gallatin 	if (err != 0) {
45491e413cf9SAndrew Gallatin 		device_printf(sc->dev, "pci_alloc_msix: failed, wanted %d"
45501e413cf9SAndrew Gallatin 			      "err = %d \n", sc->num_slices, err);
45511e413cf9SAndrew Gallatin 		goto abort_with_msix_table;
45521e413cf9SAndrew Gallatin 	}
45531e413cf9SAndrew Gallatin 	if (count < sc->num_slices) {
45541e413cf9SAndrew Gallatin 		device_printf(sc->dev, "pci_alloc_msix: need %d, got %d\n",
45551e413cf9SAndrew Gallatin 			      count, sc->num_slices);
45561e413cf9SAndrew Gallatin 		device_printf(sc->dev,
45571e413cf9SAndrew Gallatin 			      "Try setting hw.mxge.max_slices to %d\n",
45581e413cf9SAndrew Gallatin 			      count);
45591e413cf9SAndrew Gallatin 		err = ENOSPC;
45601e413cf9SAndrew Gallatin 		goto abort_with_msix;
45611e413cf9SAndrew Gallatin 	}
4562ac2fffa4SPedro F. Giffuni 	bytes = sizeof (*sc->msix_irq_res) * sc->num_slices;
4563ac2fffa4SPedro F. Giffuni 	sc->msix_irq_res = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO);
45641e413cf9SAndrew Gallatin 	if (sc->msix_irq_res == NULL) {
45651e413cf9SAndrew Gallatin 		err = ENOMEM;
45661e413cf9SAndrew Gallatin 		goto abort_with_msix;
45671e413cf9SAndrew Gallatin 	}
45681e413cf9SAndrew Gallatin 
45691e413cf9SAndrew Gallatin 	for (i = 0; i < sc->num_slices; i++) {
45701e413cf9SAndrew Gallatin 		rid = i + 1;
45711e413cf9SAndrew Gallatin 		sc->msix_irq_res[i] = bus_alloc_resource_any(sc->dev,
45721e413cf9SAndrew Gallatin 							  SYS_RES_IRQ,
45731e413cf9SAndrew Gallatin 							  &rid, RF_ACTIVE);
45741e413cf9SAndrew Gallatin 		if (sc->msix_irq_res[i] == NULL) {
45751e413cf9SAndrew Gallatin 			device_printf(sc->dev, "couldn't allocate IRQ res"
45761e413cf9SAndrew Gallatin 				      " for message %d\n", i);
45771e413cf9SAndrew Gallatin 			err = ENXIO;
45781e413cf9SAndrew Gallatin 			goto abort_with_res;
45791e413cf9SAndrew Gallatin 		}
45801e413cf9SAndrew Gallatin 	}
45811e413cf9SAndrew Gallatin 
4582ac2fffa4SPedro F. Giffuni 	bytes = sizeof (*sc->msix_ih) * sc->num_slices;
4583ac2fffa4SPedro F. Giffuni 	sc->msix_ih =  malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO);
45841e413cf9SAndrew Gallatin 
45851e413cf9SAndrew Gallatin 	for (i = 0; i < sc->num_slices; i++) {
45861e413cf9SAndrew Gallatin 		err = bus_setup_intr(sc->dev, sc->msix_irq_res[i],
45871e413cf9SAndrew Gallatin 				     INTR_TYPE_NET | INTR_MPSAFE,
458837d89b0cSAndrew Gallatin #if __FreeBSD_version > 700030
458937d89b0cSAndrew Gallatin 				     NULL,
459037d89b0cSAndrew Gallatin #endif
459137d89b0cSAndrew Gallatin 				     mxge_intr, &sc->ss[i], &sc->msix_ih[i]);
45921e413cf9SAndrew Gallatin 		if (err != 0) {
45931e413cf9SAndrew Gallatin 			device_printf(sc->dev, "couldn't setup intr for "
45941e413cf9SAndrew Gallatin 				      "message %d\n", i);
45951e413cf9SAndrew Gallatin 			goto abort_with_intr;
45961e413cf9SAndrew Gallatin 		}
459721089137SAndrew Gallatin 		bus_describe_intr(sc->dev, sc->msix_irq_res[i],
459821089137SAndrew Gallatin 				  sc->msix_ih[i], "s%d", i);
45991e413cf9SAndrew Gallatin 	}
46001e413cf9SAndrew Gallatin 
46011e413cf9SAndrew Gallatin 	if (mxge_verbose) {
46021e413cf9SAndrew Gallatin 		device_printf(sc->dev, "using %d msix IRQs:",
46031e413cf9SAndrew Gallatin 			      sc->num_slices);
46041e413cf9SAndrew Gallatin 		for (i = 0; i < sc->num_slices; i++)
4605da1b038aSJustin Hibbits 			printf(" %jd", rman_get_start(sc->msix_irq_res[i]));
46061e413cf9SAndrew Gallatin 		printf("\n");
46071e413cf9SAndrew Gallatin 	}
46081e413cf9SAndrew Gallatin 	return (0);
46091e413cf9SAndrew Gallatin 
46101e413cf9SAndrew Gallatin abort_with_intr:
46111e413cf9SAndrew Gallatin 	for (i = 0; i < sc->num_slices; i++) {
46121e413cf9SAndrew Gallatin 		if (sc->msix_ih[i] != NULL) {
46131e413cf9SAndrew Gallatin 			bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
46141e413cf9SAndrew Gallatin 					  sc->msix_ih[i]);
46151e413cf9SAndrew Gallatin 			sc->msix_ih[i] = NULL;
46161e413cf9SAndrew Gallatin 		}
46171e413cf9SAndrew Gallatin 	}
46181e413cf9SAndrew Gallatin 	free(sc->msix_ih, M_DEVBUF);
46191e413cf9SAndrew Gallatin 
46201e413cf9SAndrew Gallatin 
46211e413cf9SAndrew Gallatin abort_with_res:
46221e413cf9SAndrew Gallatin 	for (i = 0; i < sc->num_slices; i++) {
46231e413cf9SAndrew Gallatin 		rid = i + 1;
46241e413cf9SAndrew Gallatin 		if (sc->msix_irq_res[i] != NULL)
46251e413cf9SAndrew Gallatin 			bus_release_resource(sc->dev, SYS_RES_IRQ, rid,
46261e413cf9SAndrew Gallatin 					     sc->msix_irq_res[i]);
46271e413cf9SAndrew Gallatin 		sc->msix_irq_res[i] = NULL;
46281e413cf9SAndrew Gallatin 	}
46291e413cf9SAndrew Gallatin 	free(sc->msix_irq_res, M_DEVBUF);
46301e413cf9SAndrew Gallatin 
46311e413cf9SAndrew Gallatin 
46321e413cf9SAndrew Gallatin abort_with_msix:
46331e413cf9SAndrew Gallatin 	pci_release_msi(sc->dev);
46341e413cf9SAndrew Gallatin 
46351e413cf9SAndrew Gallatin abort_with_msix_table:
46361e413cf9SAndrew Gallatin 	bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2),
46371e413cf9SAndrew Gallatin 			     sc->msix_table_res);
46381e413cf9SAndrew Gallatin 
46391e413cf9SAndrew Gallatin 	return err;
46401e413cf9SAndrew Gallatin }
46411e413cf9SAndrew Gallatin 
46421e413cf9SAndrew Gallatin static int
46431e413cf9SAndrew Gallatin mxge_add_single_irq(mxge_softc_t *sc)
46441e413cf9SAndrew Gallatin {
46451e413cf9SAndrew Gallatin 	int count, err, rid;
46461e413cf9SAndrew Gallatin 
46471e413cf9SAndrew Gallatin 	count = pci_msi_count(sc->dev);
46481e413cf9SAndrew Gallatin 	if (count == 1 && pci_alloc_msi(sc->dev, &count) == 0) {
46491e413cf9SAndrew Gallatin 		rid = 1;
46501e413cf9SAndrew Gallatin 	} else {
46511e413cf9SAndrew Gallatin 		rid = 0;
465291ed8913SAndrew Gallatin 		sc->legacy_irq = 1;
46531e413cf9SAndrew Gallatin 	}
465443cd6160SJustin Hibbits 	sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
465543cd6160SJustin Hibbits 					     RF_SHAREABLE | RF_ACTIVE);
46561e413cf9SAndrew Gallatin 	if (sc->irq_res == NULL) {
46571e413cf9SAndrew Gallatin 		device_printf(sc->dev, "could not alloc interrupt\n");
46581e413cf9SAndrew Gallatin 		return ENXIO;
46591e413cf9SAndrew Gallatin 	}
46601e413cf9SAndrew Gallatin 	if (mxge_verbose)
4661da1b038aSJustin Hibbits 		device_printf(sc->dev, "using %s irq %jd\n",
466291ed8913SAndrew Gallatin 			      sc->legacy_irq ? "INTx" : "MSI",
46631e413cf9SAndrew Gallatin 			      rman_get_start(sc->irq_res));
46641e413cf9SAndrew Gallatin 	err = bus_setup_intr(sc->dev, sc->irq_res,
46651e413cf9SAndrew Gallatin 			     INTR_TYPE_NET | INTR_MPSAFE,
466637d89b0cSAndrew Gallatin #if __FreeBSD_version > 700030
466737d89b0cSAndrew Gallatin 			     NULL,
466837d89b0cSAndrew Gallatin #endif
466937d89b0cSAndrew Gallatin 			     mxge_intr, &sc->ss[0], &sc->ih);
46701e413cf9SAndrew Gallatin 	if (err != 0) {
46711e413cf9SAndrew Gallatin 		bus_release_resource(sc->dev, SYS_RES_IRQ,
467291ed8913SAndrew Gallatin 				     sc->legacy_irq ? 0 : 1, sc->irq_res);
467391ed8913SAndrew Gallatin 		if (!sc->legacy_irq)
46741e413cf9SAndrew Gallatin 			pci_release_msi(sc->dev);
46751e413cf9SAndrew Gallatin 	}
46761e413cf9SAndrew Gallatin 	return err;
46771e413cf9SAndrew Gallatin }
46781e413cf9SAndrew Gallatin 
46791e413cf9SAndrew Gallatin static void
46801e413cf9SAndrew Gallatin mxge_rem_msix_irqs(mxge_softc_t *sc)
46811e413cf9SAndrew Gallatin {
46821e413cf9SAndrew Gallatin 	int i, rid;
46831e413cf9SAndrew Gallatin 
46841e413cf9SAndrew Gallatin 	for (i = 0; i < sc->num_slices; i++) {
46851e413cf9SAndrew Gallatin 		if (sc->msix_ih[i] != NULL) {
46861e413cf9SAndrew Gallatin 			bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
46871e413cf9SAndrew Gallatin 					  sc->msix_ih[i]);
46881e413cf9SAndrew Gallatin 			sc->msix_ih[i] = NULL;
46891e413cf9SAndrew Gallatin 		}
46901e413cf9SAndrew Gallatin 	}
46911e413cf9SAndrew Gallatin 	free(sc->msix_ih, M_DEVBUF);
46921e413cf9SAndrew Gallatin 
46931e413cf9SAndrew Gallatin 	for (i = 0; i < sc->num_slices; i++) {
46941e413cf9SAndrew Gallatin 		rid = i + 1;
46951e413cf9SAndrew Gallatin 		if (sc->msix_irq_res[i] != NULL)
46961e413cf9SAndrew Gallatin 			bus_release_resource(sc->dev, SYS_RES_IRQ, rid,
46971e413cf9SAndrew Gallatin 					     sc->msix_irq_res[i]);
46981e413cf9SAndrew Gallatin 		sc->msix_irq_res[i] = NULL;
46991e413cf9SAndrew Gallatin 	}
47001e413cf9SAndrew Gallatin 	free(sc->msix_irq_res, M_DEVBUF);
47011e413cf9SAndrew Gallatin 
47021e413cf9SAndrew Gallatin 	bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2),
47031e413cf9SAndrew Gallatin 			     sc->msix_table_res);
47041e413cf9SAndrew Gallatin 
47051e413cf9SAndrew Gallatin 	pci_release_msi(sc->dev);
47061e413cf9SAndrew Gallatin 	return;
47071e413cf9SAndrew Gallatin }
47081e413cf9SAndrew Gallatin 
47091e413cf9SAndrew Gallatin static void
47101e413cf9SAndrew Gallatin mxge_rem_single_irq(mxge_softc_t *sc)
47111e413cf9SAndrew Gallatin {
47121e413cf9SAndrew Gallatin 	bus_teardown_intr(sc->dev, sc->irq_res, sc->ih);
47131e413cf9SAndrew Gallatin 	bus_release_resource(sc->dev, SYS_RES_IRQ,
471491ed8913SAndrew Gallatin 			     sc->legacy_irq ? 0 : 1, sc->irq_res);
471591ed8913SAndrew Gallatin 	if (!sc->legacy_irq)
47161e413cf9SAndrew Gallatin 		pci_release_msi(sc->dev);
47171e413cf9SAndrew Gallatin }
47181e413cf9SAndrew Gallatin 
47191e413cf9SAndrew Gallatin static void
47201e413cf9SAndrew Gallatin mxge_rem_irq(mxge_softc_t *sc)
47211e413cf9SAndrew Gallatin {
47221e413cf9SAndrew Gallatin 	if (sc->num_slices > 1)
47231e413cf9SAndrew Gallatin 		mxge_rem_msix_irqs(sc);
47241e413cf9SAndrew Gallatin 	else
47251e413cf9SAndrew Gallatin 		mxge_rem_single_irq(sc);
47261e413cf9SAndrew Gallatin }
47271e413cf9SAndrew Gallatin 
47281e413cf9SAndrew Gallatin static int
47291e413cf9SAndrew Gallatin mxge_add_irq(mxge_softc_t *sc)
47301e413cf9SAndrew Gallatin {
47311e413cf9SAndrew Gallatin 	int err;
47321e413cf9SAndrew Gallatin 
47331e413cf9SAndrew Gallatin 	if (sc->num_slices > 1)
47341e413cf9SAndrew Gallatin 		err = mxge_add_msix_irqs(sc);
47351e413cf9SAndrew Gallatin 	else
47361e413cf9SAndrew Gallatin 		err = mxge_add_single_irq(sc);
47371e413cf9SAndrew Gallatin 
47381e413cf9SAndrew Gallatin 	if (0 && err == 0 && sc->num_slices > 1) {
47391e413cf9SAndrew Gallatin 		mxge_rem_msix_irqs(sc);
47401e413cf9SAndrew Gallatin 		err = mxge_add_msix_irqs(sc);
47411e413cf9SAndrew Gallatin 	}
47421e413cf9SAndrew Gallatin 	return err;
47431e413cf9SAndrew Gallatin }
47441e413cf9SAndrew Gallatin 
4745b2fc195eSAndrew Gallatin 
4746b2fc195eSAndrew Gallatin static int
47476d87a65dSAndrew Gallatin mxge_attach(device_t dev)
4748b2fc195eSAndrew Gallatin {
47490a7a780eSAndrew Gallatin 	mxge_cmd_t cmd;
47506d87a65dSAndrew Gallatin 	mxge_softc_t *sc = device_get_softc(dev);
4751b2fc195eSAndrew Gallatin 	struct ifnet *ifp;
47521e413cf9SAndrew Gallatin 	int err, rid;
4753b2fc195eSAndrew Gallatin 
4754b2fc195eSAndrew Gallatin 	sc->dev = dev;
47556d87a65dSAndrew Gallatin 	mxge_fetch_tunables(sc);
4756b2fc195eSAndrew Gallatin 
475772c042dfSAndrew Gallatin 	TASK_INIT(&sc->watchdog_task, 1, mxge_watchdog_task, sc);
4758a4a75d67SJohn Baldwin 	sc->tq = taskqueue_create("mxge_taskq", M_WAITOK,
4759a4a75d67SJohn Baldwin 				  taskqueue_thread_enqueue, &sc->tq);
476072c042dfSAndrew Gallatin 	if (sc->tq == NULL) {
476172c042dfSAndrew Gallatin 		err = ENOMEM;
476272c042dfSAndrew Gallatin 		goto abort_with_nothing;
476372c042dfSAndrew Gallatin 	}
476472c042dfSAndrew Gallatin 
476562ce43ccSScott Long 	err = bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
4766b2fc195eSAndrew Gallatin 				 1,			/* alignment */
47671e413cf9SAndrew Gallatin 				 0,			/* boundary */
4768b2fc195eSAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* low */
4769b2fc195eSAndrew Gallatin 				 BUS_SPACE_MAXADDR,	/* high */
4770b2fc195eSAndrew Gallatin 				 NULL, NULL,		/* filter */
4771aed8e389SAndrew Gallatin 				 65536 + 256,		/* maxsize */
47725e7d8541SAndrew Gallatin 				 MXGE_MAX_SEND_DESC, 	/* num segs */
47731e413cf9SAndrew Gallatin 				 65536,			/* maxsegsize */
4774b2fc195eSAndrew Gallatin 				 0,			/* flags */
4775b2fc195eSAndrew Gallatin 				 NULL, NULL,		/* lock */
4776b2fc195eSAndrew Gallatin 				 &sc->parent_dmat);	/* tag */
4777b2fc195eSAndrew Gallatin 
4778b2fc195eSAndrew Gallatin 	if (err != 0) {
4779b2fc195eSAndrew Gallatin 		device_printf(sc->dev, "Err %d allocating parent dmat\n",
4780b2fc195eSAndrew Gallatin 			      err);
478172c042dfSAndrew Gallatin 		goto abort_with_tq;
4782b2fc195eSAndrew Gallatin 	}
4783b2fc195eSAndrew Gallatin 
4784b2fc195eSAndrew Gallatin 	ifp = sc->ifp = if_alloc(IFT_ETHER);
4785b2fc195eSAndrew Gallatin 	if (ifp == NULL) {
4786b2fc195eSAndrew Gallatin 		device_printf(dev, "can not if_alloc()\n");
4787b2fc195eSAndrew Gallatin 		err = ENOSPC;
4788b2fc195eSAndrew Gallatin 		goto abort_with_parent_dmat;
4789b2fc195eSAndrew Gallatin 	}
47901e413cf9SAndrew Gallatin 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
47911e413cf9SAndrew Gallatin 
4792a98d6cd7SAndrew Gallatin 	snprintf(sc->cmd_mtx_name, sizeof(sc->cmd_mtx_name), "%s:cmd",
4793a98d6cd7SAndrew Gallatin 		 device_get_nameunit(dev));
4794a98d6cd7SAndrew Gallatin 	mtx_init(&sc->cmd_mtx, sc->cmd_mtx_name, NULL, MTX_DEF);
4795a98d6cd7SAndrew Gallatin 	snprintf(sc->driver_mtx_name, sizeof(sc->driver_mtx_name),
4796a98d6cd7SAndrew Gallatin 		 "%s:drv", device_get_nameunit(dev));
4797a98d6cd7SAndrew Gallatin 	mtx_init(&sc->driver_mtx, sc->driver_mtx_name,
4798b2fc195eSAndrew Gallatin 		 MTX_NETWORK_LOCK, MTX_DEF);
4799b2fc195eSAndrew Gallatin 
4800dce01b9bSAndrew Gallatin 	callout_init_mtx(&sc->co_hdl, &sc->driver_mtx, 0);
4801d91b1b49SAndrew Gallatin 
4802dce01b9bSAndrew Gallatin 	mxge_setup_cfg_space(sc);
4803b2fc195eSAndrew Gallatin 
4804b2fc195eSAndrew Gallatin 	/* Map the board into the kernel */
4805b2fc195eSAndrew Gallatin 	rid = PCIR_BARS;
480643cd6160SJustin Hibbits 	sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
480743cd6160SJustin Hibbits 					     RF_ACTIVE);
4808b2fc195eSAndrew Gallatin 	if (sc->mem_res == NULL) {
4809b2fc195eSAndrew Gallatin 		device_printf(dev, "could not map memory\n");
4810b2fc195eSAndrew Gallatin 		err = ENXIO;
4811b2fc195eSAndrew Gallatin 		goto abort_with_lock;
4812b2fc195eSAndrew Gallatin 	}
4813b2fc195eSAndrew Gallatin 	sc->sram = rman_get_virtual(sc->mem_res);
4814b2fc195eSAndrew Gallatin 	sc->sram_size = 2*1024*1024 - (2*(48*1024)+(32*1024)) - 0x100;
4815b2fc195eSAndrew Gallatin 	if (sc->sram_size > rman_get_size(sc->mem_res)) {
4816da1b038aSJustin Hibbits 		device_printf(dev, "impossible memory region size %jd\n",
4817b2fc195eSAndrew Gallatin 			      rman_get_size(sc->mem_res));
4818b2fc195eSAndrew Gallatin 		err = ENXIO;
4819b2fc195eSAndrew Gallatin 		goto abort_with_mem_res;
4820b2fc195eSAndrew Gallatin 	}
4821b2fc195eSAndrew Gallatin 
4822b2fc195eSAndrew Gallatin 	/* make NULL terminated copy of the EEPROM strings section of
4823b2fc195eSAndrew Gallatin 	   lanai SRAM */
48246d87a65dSAndrew Gallatin 	bzero(sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE);
4825b2fc195eSAndrew Gallatin 	bus_space_read_region_1(rman_get_bustag(sc->mem_res),
4826b2fc195eSAndrew Gallatin 				rman_get_bushandle(sc->mem_res),
48276d87a65dSAndrew Gallatin 				sc->sram_size - MXGE_EEPROM_STRINGS_SIZE,
4828b2fc195eSAndrew Gallatin 				sc->eeprom_strings,
48296d87a65dSAndrew Gallatin 				MXGE_EEPROM_STRINGS_SIZE - 2);
48306d87a65dSAndrew Gallatin 	err = mxge_parse_strings(sc);
4831b2fc195eSAndrew Gallatin 	if (err != 0)
4832b2fc195eSAndrew Gallatin 		goto abort_with_mem_res;
4833b2fc195eSAndrew Gallatin 
4834b2fc195eSAndrew Gallatin 	/* Enable write combining for efficient use of PCIe bus */
48356d87a65dSAndrew Gallatin 	mxge_enable_wc(sc);
4836b2fc195eSAndrew Gallatin 
4837b2fc195eSAndrew Gallatin 	/* Allocate the out of band dma memory */
48386d87a65dSAndrew Gallatin 	err = mxge_dma_alloc(sc, &sc->cmd_dma,
48396d87a65dSAndrew Gallatin 			     sizeof (mxge_cmd_t), 64);
4840b2fc195eSAndrew Gallatin 	if (err != 0)
4841b2fc195eSAndrew Gallatin 		goto abort_with_mem_res;
4842b2fc195eSAndrew Gallatin 	sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr;
48436d87a65dSAndrew Gallatin 	err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64);
4844b2fc195eSAndrew Gallatin 	if (err != 0)
4845b2fc195eSAndrew Gallatin 		goto abort_with_cmd_dma;
4846b2fc195eSAndrew Gallatin 
4847a98d6cd7SAndrew Gallatin 	err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096);
4848a98d6cd7SAndrew Gallatin 	if (err != 0)
48491e413cf9SAndrew Gallatin 		goto abort_with_zeropad_dma;
4850b2fc195eSAndrew Gallatin 
48518fe615baSAndrew Gallatin 	/* select & load the firmware */
48528fe615baSAndrew Gallatin 	err = mxge_select_firmware(sc);
4853b2fc195eSAndrew Gallatin 	if (err != 0)
48541e413cf9SAndrew Gallatin 		goto abort_with_dmabench;
48555e7d8541SAndrew Gallatin 	sc->intr_coal_delay = mxge_intr_coal_delay;
48561e413cf9SAndrew Gallatin 
48571e413cf9SAndrew Gallatin 	mxge_slice_probe(sc);
48581e413cf9SAndrew Gallatin 	err = mxge_alloc_slices(sc);
48591e413cf9SAndrew Gallatin 	if (err != 0)
48601e413cf9SAndrew Gallatin 		goto abort_with_dmabench;
48611e413cf9SAndrew Gallatin 
4862adae7080SAndrew Gallatin 	err = mxge_reset(sc, 0);
4863b2fc195eSAndrew Gallatin 	if (err != 0)
48641e413cf9SAndrew Gallatin 		goto abort_with_slices;
4865b2fc195eSAndrew Gallatin 
4866a98d6cd7SAndrew Gallatin 	err = mxge_alloc_rings(sc);
4867a98d6cd7SAndrew Gallatin 	if (err != 0) {
4868a98d6cd7SAndrew Gallatin 		device_printf(sc->dev, "failed to allocate rings\n");
48692e084798SAndrew Gallatin 		goto abort_with_slices;
4870a98d6cd7SAndrew Gallatin 	}
4871a98d6cd7SAndrew Gallatin 
48721e413cf9SAndrew Gallatin 	err = mxge_add_irq(sc);
4873a98d6cd7SAndrew Gallatin 	if (err != 0) {
48741e413cf9SAndrew Gallatin 		device_printf(sc->dev, "failed to add irq\n");
4875a98d6cd7SAndrew Gallatin 		goto abort_with_rings;
4876a98d6cd7SAndrew Gallatin 	}
48771e413cf9SAndrew Gallatin 
4878b245f96cSGleb Smirnoff 	ifp->if_baudrate = IF_Gbps(10);
4879c792928fSAndrew Gallatin 	ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO4 |
488026dd49c6SAndrew Gallatin 		IFCAP_VLAN_MTU | IFCAP_LINKSTATE | IFCAP_TXCSUM_IPV6 |
488126dd49c6SAndrew Gallatin 		IFCAP_RXCSUM_IPV6;
488226dd49c6SAndrew Gallatin #if defined(INET) || defined(INET6)
4883eb6219e3SAndrew Gallatin 	ifp->if_capabilities |= IFCAP_LRO;
4884eb6219e3SAndrew Gallatin #endif
488537d89b0cSAndrew Gallatin 
488637d89b0cSAndrew Gallatin #ifdef MXGE_NEW_VLAN_API
488737d89b0cSAndrew Gallatin 	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
48880dce6781SAndrew Gallatin 
48890dce6781SAndrew Gallatin 	/* Only FW 1.4.32 and newer can do TSO over vlans */
48900dce6781SAndrew Gallatin 	if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 &&
48910dce6781SAndrew Gallatin 	    sc->fw_ver_tiny >= 32)
48920dce6781SAndrew Gallatin 		ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
489337d89b0cSAndrew Gallatin #endif
4894053e637fSAndrew Gallatin 	sc->max_mtu = mxge_max_mtu(sc);
4895053e637fSAndrew Gallatin 	if (sc->max_mtu >= 9000)
4896053e637fSAndrew Gallatin 		ifp->if_capabilities |= IFCAP_JUMBO_MTU;
4897053e637fSAndrew Gallatin 	else
4898053e637fSAndrew Gallatin 		device_printf(dev, "MTU limited to %d.  Install "
4899adae7080SAndrew Gallatin 			      "latest firmware for 9000 byte jumbo support\n",
4900053e637fSAndrew Gallatin 			      sc->max_mtu - ETHER_HDR_LEN);
4901aed8e389SAndrew Gallatin 	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO;
49020a7a780eSAndrew Gallatin 	ifp->if_hwassist |= CSUM_TCP_IPV6 | CSUM_UDP_IPV6;
49030a7a780eSAndrew Gallatin 	/* check to see if f/w supports TSO for IPv6 */
49040a7a780eSAndrew Gallatin 	if (!mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, &cmd)) {
49050a7a780eSAndrew Gallatin 		if (CSUM_TCP_IPV6)
49060a7a780eSAndrew Gallatin 			ifp->if_capabilities |= IFCAP_TSO6;
49070a7a780eSAndrew Gallatin 		sc->max_tso6_hlen = min(cmd.data0,
49080a7a780eSAndrew Gallatin 					sizeof (sc->ss[0].scratch));
49090a7a780eSAndrew Gallatin 	}
4910b2fc195eSAndrew Gallatin 	ifp->if_capenable = ifp->if_capabilities;
4911f04b33f8SAndrew Gallatin 	if (sc->lro_cnt == 0)
4912f04b33f8SAndrew Gallatin 		ifp->if_capenable &= ~IFCAP_LRO;
49136d87a65dSAndrew Gallatin 	ifp->if_init = mxge_init;
4914b2fc195eSAndrew Gallatin 	ifp->if_softc = sc;
4915b2fc195eSAndrew Gallatin 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
49166d87a65dSAndrew Gallatin 	ifp->if_ioctl = mxge_ioctl;
49176d87a65dSAndrew Gallatin 	ifp->if_start = mxge_start;
4918f3f040d9SGleb Smirnoff 	ifp->if_get_counter = mxge_get_counter;
4919c587e59fSAndrew Gallatin 	/* Initialise the ifmedia structure */
4920c587e59fSAndrew Gallatin 	ifmedia_init(&sc->media, 0, mxge_media_change,
4921c587e59fSAndrew Gallatin 		     mxge_media_status);
4922c406ad2eSAndrew Gallatin 	mxge_media_init(sc);
4923c587e59fSAndrew Gallatin 	mxge_media_probe(sc);
49248c5d766cSAndrew Gallatin 	sc->dying = 0;
4925b2fc195eSAndrew Gallatin 	ether_ifattach(ifp, sc->mac_addr);
4926f9453025SAndrew Gallatin 	/* ether_ifattach sets mtu to ETHERMTU */
4927f9453025SAndrew Gallatin 	if (mxge_initial_mtu != ETHERMTU)
4928f9453025SAndrew Gallatin 		mxge_change_mtu(sc, mxge_initial_mtu);
4929b2fc195eSAndrew Gallatin 
49306d87a65dSAndrew Gallatin 	mxge_add_sysctls(sc);
4931c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING
4932c6cb3e3fSAndrew Gallatin 	ifp->if_transmit = mxge_transmit;
4933c6cb3e3fSAndrew Gallatin 	ifp->if_qflush = mxge_qflush;
4934c6cb3e3fSAndrew Gallatin #endif
49352e084798SAndrew Gallatin 	taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
49362e084798SAndrew Gallatin 				device_get_nameunit(sc->dev));
49376b484a49SAndrew Gallatin 	callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
4938b2fc195eSAndrew Gallatin 	return 0;
4939b2fc195eSAndrew Gallatin 
4940a98d6cd7SAndrew Gallatin abort_with_rings:
4941a98d6cd7SAndrew Gallatin 	mxge_free_rings(sc);
49421e413cf9SAndrew Gallatin abort_with_slices:
49431e413cf9SAndrew Gallatin 	mxge_free_slices(sc);
4944a98d6cd7SAndrew Gallatin abort_with_dmabench:
4945a98d6cd7SAndrew Gallatin 	mxge_dma_free(&sc->dmabench_dma);
4946b2fc195eSAndrew Gallatin abort_with_zeropad_dma:
49476d87a65dSAndrew Gallatin 	mxge_dma_free(&sc->zeropad_dma);
4948b2fc195eSAndrew Gallatin abort_with_cmd_dma:
49496d87a65dSAndrew Gallatin 	mxge_dma_free(&sc->cmd_dma);
4950b2fc195eSAndrew Gallatin abort_with_mem_res:
4951b2fc195eSAndrew Gallatin 	bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
4952b2fc195eSAndrew Gallatin abort_with_lock:
4953b2fc195eSAndrew Gallatin 	pci_disable_busmaster(dev);
4954a98d6cd7SAndrew Gallatin 	mtx_destroy(&sc->cmd_mtx);
4955a98d6cd7SAndrew Gallatin 	mtx_destroy(&sc->driver_mtx);
4956b2fc195eSAndrew Gallatin 	if_free(ifp);
4957b2fc195eSAndrew Gallatin abort_with_parent_dmat:
4958b2fc195eSAndrew Gallatin 	bus_dma_tag_destroy(sc->parent_dmat);
495972c042dfSAndrew Gallatin abort_with_tq:
496072c042dfSAndrew Gallatin 	if (sc->tq != NULL) {
496172c042dfSAndrew Gallatin 		taskqueue_drain(sc->tq, &sc->watchdog_task);
496272c042dfSAndrew Gallatin 		taskqueue_free(sc->tq);
496372c042dfSAndrew Gallatin 		sc->tq = NULL;
496472c042dfSAndrew Gallatin 	}
4965b2fc195eSAndrew Gallatin abort_with_nothing:
4966b2fc195eSAndrew Gallatin 	return err;
4967b2fc195eSAndrew Gallatin }
4968b2fc195eSAndrew Gallatin 
4969b2fc195eSAndrew Gallatin static int
49706d87a65dSAndrew Gallatin mxge_detach(device_t dev)
4971b2fc195eSAndrew Gallatin {
49726d87a65dSAndrew Gallatin 	mxge_softc_t *sc = device_get_softc(dev);
4973b2fc195eSAndrew Gallatin 
497437d89b0cSAndrew Gallatin 	if (mxge_vlans_active(sc)) {
4975c792928fSAndrew Gallatin 		device_printf(sc->dev,
4976c792928fSAndrew Gallatin 			      "Detach vlans before removing module\n");
4977c792928fSAndrew Gallatin 		return EBUSY;
4978c792928fSAndrew Gallatin 	}
4979a98d6cd7SAndrew Gallatin 	mtx_lock(&sc->driver_mtx);
49808c5d766cSAndrew Gallatin 	sc->dying = 1;
4981b2fc195eSAndrew Gallatin 	if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING)
4982a393336bSAndrew Gallatin 		mxge_close(sc, 0);
4983a98d6cd7SAndrew Gallatin 	mtx_unlock(&sc->driver_mtx);
4984b2fc195eSAndrew Gallatin 	ether_ifdetach(sc->ifp);
498572c042dfSAndrew Gallatin 	if (sc->tq != NULL) {
498672c042dfSAndrew Gallatin 		taskqueue_drain(sc->tq, &sc->watchdog_task);
498772c042dfSAndrew Gallatin 		taskqueue_free(sc->tq);
498872c042dfSAndrew Gallatin 		sc->tq = NULL;
498972c042dfSAndrew Gallatin 	}
4990e749ef6bSAndrew Gallatin 	callout_drain(&sc->co_hdl);
4991dce01b9bSAndrew Gallatin 	ifmedia_removeall(&sc->media);
4992091feecdSAndrew Gallatin 	mxge_dummy_rdma(sc, 0);
49931e413cf9SAndrew Gallatin 	mxge_rem_sysctls(sc);
49941e413cf9SAndrew Gallatin 	mxge_rem_irq(sc);
4995a98d6cd7SAndrew Gallatin 	mxge_free_rings(sc);
49961e413cf9SAndrew Gallatin 	mxge_free_slices(sc);
4997a98d6cd7SAndrew Gallatin 	mxge_dma_free(&sc->dmabench_dma);
49986d87a65dSAndrew Gallatin 	mxge_dma_free(&sc->zeropad_dma);
49996d87a65dSAndrew Gallatin 	mxge_dma_free(&sc->cmd_dma);
5000b2fc195eSAndrew Gallatin 	bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
5001b2fc195eSAndrew Gallatin 	pci_disable_busmaster(dev);
5002a98d6cd7SAndrew Gallatin 	mtx_destroy(&sc->cmd_mtx);
5003a98d6cd7SAndrew Gallatin 	mtx_destroy(&sc->driver_mtx);
5004b2fc195eSAndrew Gallatin 	if_free(sc->ifp);
5005b2fc195eSAndrew Gallatin 	bus_dma_tag_destroy(sc->parent_dmat);
5006b2fc195eSAndrew Gallatin 	return 0;
5007b2fc195eSAndrew Gallatin }
5008b2fc195eSAndrew Gallatin 
5009b2fc195eSAndrew Gallatin static int
50106d87a65dSAndrew Gallatin mxge_shutdown(device_t dev)
5011b2fc195eSAndrew Gallatin {
5012b2fc195eSAndrew Gallatin 	return 0;
5013b2fc195eSAndrew Gallatin }
5014b2fc195eSAndrew Gallatin 
5015b2fc195eSAndrew Gallatin /*
5016b2fc195eSAndrew Gallatin   This file uses Myri10GE driver indentation.
5017b2fc195eSAndrew Gallatin 
5018b2fc195eSAndrew Gallatin   Local Variables:
5019b2fc195eSAndrew Gallatin   c-file-style:"linux"
5020b2fc195eSAndrew Gallatin   tab-width:8
5021b2fc195eSAndrew Gallatin   End:
5022b2fc195eSAndrew Gallatin */
5023