xref: /freebsd/sys/dev/ae/if_ae.c (revision 9a0c3479e22feda1bdb2db4b97f9deb1b5fa6269)
1 /*-
2  * Copyright (c) 2008 Stanislav Sedov <stas@FreeBSD.org>.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  *
25  * Driver for Attansic Technology Corp. L2 FastEthernet adapter.
26  *
27  * This driver is heavily based on age(4) Attansic L1 driver by Pyun YongHyeon.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/endian.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
39 #include <sys/mbuf.h>
40 #include <sys/rman.h>
41 #include <sys/module.h>
42 #include <sys/queue.h>
43 #include <sys/socket.h>
44 #include <sys/sockio.h>
45 #include <sys/sysctl.h>
46 #include <sys/taskqueue.h>
47 
48 #include <net/bpf.h>
49 #include <net/if.h>
50 #include <net/if_arp.h>
51 #include <net/ethernet.h>
52 #include <net/if_dl.h>
53 #include <net/if_media.h>
54 #include <net/if_types.h>
55 #include <net/if_vlan_var.h>
56 
57 #include <netinet/in.h>
58 #include <netinet/in_systm.h>
59 #include <netinet/ip.h>
60 #include <netinet/tcp.h>
61 
62 #include <dev/mii/mii.h>
63 #include <dev/mii/miivar.h>
64 #include <dev/pci/pcireg.h>
65 #include <dev/pci/pcivar.h>
66 
67 #include <machine/bus.h>
68 
69 #include "miibus_if.h"
70 
71 #include "if_aereg.h"
72 #include "if_aevar.h"
73 
74 /*
75  * Devices supported by this driver.
76  */
77 static struct ae_dev {
78 	uint16_t	vendorid;
79 	uint16_t	deviceid;
80 	const char	*name;
81 } ae_devs[] = {
82 	{ VENDORID_ATTANSIC, DEVICEID_ATTANSIC_L2,
83 		"Attansic Technology Corp, L2 FastEthernet" },
84 };
85 #define	AE_DEVS_COUNT (sizeof(ae_devs) / sizeof(*ae_devs))
86 
87 static struct resource_spec ae_res_spec_mem[] = {
88 	{ SYS_RES_MEMORY,       PCIR_BAR(0),    RF_ACTIVE },
89 	{ -1,			0,		0 }
90 };
91 static struct resource_spec ae_res_spec_irq[] = {
92 	{ SYS_RES_IRQ,		0,		RF_ACTIVE | RF_SHAREABLE },
93 	{ -1,			0,		0 }
94 };
95 static struct resource_spec ae_res_spec_msi[] = {
96 	{ SYS_RES_IRQ,		1,		RF_ACTIVE },
97 	{ -1,			0,		0 }
98 };
99 
100 static int	ae_probe(device_t dev);
101 static int	ae_attach(device_t dev);
102 static void	ae_pcie_init(ae_softc_t *sc);
103 static void	ae_phy_reset(ae_softc_t *sc);
104 static void	ae_phy_init(ae_softc_t *sc);
105 static int	ae_reset(ae_softc_t *sc);
106 static void	ae_init(void *arg);
107 static int	ae_init_locked(ae_softc_t *sc);
108 static int	ae_detach(device_t dev);
109 static int	ae_miibus_readreg(device_t dev, int phy, int reg);
110 static int	ae_miibus_writereg(device_t dev, int phy, int reg, int val);
111 static void	ae_miibus_statchg(device_t dev);
112 static void	ae_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr);
113 static int	ae_mediachange(struct ifnet *ifp);
114 static void	ae_retrieve_address(ae_softc_t *sc);
115 static void	ae_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs,
116     int error);
117 static int	ae_alloc_rings(ae_softc_t *sc);
118 static void	ae_dma_free(ae_softc_t *sc);
119 static int	ae_shutdown(device_t dev);
120 static int	ae_suspend(device_t dev);
121 static void	ae_powersave_disable(ae_softc_t *sc);
122 static void	ae_powersave_enable(ae_softc_t *sc);
123 static int	ae_resume(device_t dev);
124 static unsigned int	ae_tx_avail_size(ae_softc_t *sc);
125 static int	ae_encap(ae_softc_t *sc, struct mbuf **m_head);
126 static void	ae_start(struct ifnet *ifp);
127 static void	ae_start_locked(struct ifnet *ifp);
128 static void	ae_link_task(void *arg, int pending);
129 static void	ae_stop_rxmac(ae_softc_t *sc);
130 static void	ae_stop_txmac(ae_softc_t *sc);
131 static void	ae_mac_config(ae_softc_t *sc);
132 static int	ae_intr(void *arg);
133 static void	ae_int_task(void *arg, int pending);
134 static void	ae_tx_intr(ae_softc_t *sc);
135 static int	ae_rxeof(ae_softc_t *sc, ae_rxd_t *rxd);
136 static void	ae_rx_intr(ae_softc_t *sc);
137 static void	ae_watchdog(ae_softc_t *sc);
138 static void	ae_tick(void *arg);
139 static void	ae_rxfilter(ae_softc_t *sc);
140 static void	ae_rxvlan(ae_softc_t *sc);
141 static int	ae_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
142 static void	ae_stop(ae_softc_t *sc);
143 static int	ae_check_eeprom_present(ae_softc_t *sc, int *vpdc);
144 static int	ae_vpd_read_word(ae_softc_t *sc, int reg, uint32_t *word);
145 static int	ae_get_vpd_eaddr(ae_softc_t *sc, uint32_t *eaddr);
146 static int	ae_get_reg_eaddr(ae_softc_t *sc, uint32_t *eaddr);
147 static void	ae_update_stats_rx(uint16_t flags, ae_stats_t *stats);
148 static void	ae_update_stats_tx(uint16_t flags, ae_stats_t *stats);
149 static void	ae_init_tunables(ae_softc_t *sc);
150 
151 static device_method_t ae_methods[] = {
152 	/* Device interface. */
153 	DEVMETHOD(device_probe,		ae_probe),
154 	DEVMETHOD(device_attach,	ae_attach),
155 	DEVMETHOD(device_detach,	ae_detach),
156 	DEVMETHOD(device_shutdown,	ae_shutdown),
157 	DEVMETHOD(device_suspend,	ae_suspend),
158 	DEVMETHOD(device_resume,	ae_resume),
159 
160 	/* MII interface. */
161 	DEVMETHOD(miibus_readreg,	ae_miibus_readreg),
162 	DEVMETHOD(miibus_writereg,	ae_miibus_writereg),
163 	DEVMETHOD(miibus_statchg,	ae_miibus_statchg),
164 
165 	{ NULL, NULL }
166 };
167 static driver_t ae_driver = {
168         "ae",
169         ae_methods,
170         sizeof(ae_softc_t)
171 };
172 static devclass_t ae_devclass;
173 
174 DRIVER_MODULE(ae, pci, ae_driver, ae_devclass, 0, 0);
175 DRIVER_MODULE(miibus, ae, miibus_driver, miibus_devclass, 0, 0);
176 MODULE_DEPEND(ae, pci, 1, 1, 1);
177 MODULE_DEPEND(ae, ether, 1, 1, 1);
178 MODULE_DEPEND(ae, miibus, 1, 1, 1);
179 
180 /*
181  * Tunables.
182  */
183 static int msi_disable = 0;
184 TUNABLE_INT("hw.ae.msi_disable", &msi_disable);
185 
186 #define	AE_READ_4(sc, reg) \
187 	bus_read_4((sc)->mem[0], (reg))
188 #define	AE_READ_2(sc, reg) \
189 	bus_read_2((sc)->mem[0], (reg))
190 #define	AE_READ_1(sc, reg) \
191 	bus_read_1((sc)->mem[0], (reg))
192 #define	AE_WRITE_4(sc, reg, val) \
193 	bus_write_4((sc)->mem[0], (reg), (val))
194 #define	AE_WRITE_2(sc, reg, val) \
195 	bus_write_2((sc)->mem[0], (reg), (val))
196 #define	AE_WRITE_1(sc, reg, val) \
197 	bus_write_1((sc)->mem[0], (reg), (val))
198 #define	AE_PHY_READ(sc, reg) \
199 	ae_miibus_readreg(sc->dev, 0, reg)
200 #define	AE_PHY_WRITE(sc, reg, val) \
201 	ae_miibus_writereg(sc->dev, 0, reg, val)
202 #define	AE_CHECK_EADDR_VALID(eaddr) \
203 	((eaddr[0] == 0 && eaddr[1] == 0) || \
204 	(eaddr[0] == 0xffffffff && eaddr[1] == 0xffff))
205 #define	AE_RXD_VLAN(vtag) \
206 	(((vtag) >> 4) | (((vtag) & 0x07) << 13) | (((vtag) & 0x08) << 9))
207 #define	AE_TXD_VLAN(vtag) \
208 	(((vtag) << 4) | (((vtag) >> 13) & 0x07) | (((vtag) >> 9) & 0x08))
209 
210 static int
211 ae_probe(device_t dev)
212 {
213 	uint16_t deviceid, vendorid;
214 	int i;
215 
216 	vendorid = pci_get_vendor(dev);
217 	deviceid = pci_get_device(dev);
218 
219 	/*
220 	 * Search through the list of supported devs for matching one.
221 	 */
222 	for (i = 0; i < AE_DEVS_COUNT; i++) {
223 		if (vendorid == ae_devs[i].vendorid &&
224 		    deviceid == ae_devs[i].deviceid) {
225 			device_set_desc(dev, ae_devs[i].name);
226 			return (BUS_PROBE_DEFAULT);
227 		}
228 	}
229 	return (ENXIO);
230 }
231 
232 static int
233 ae_attach(device_t dev)
234 {
235 	ae_softc_t *sc;
236 	struct ifnet *ifp;
237 	uint8_t chiprev;
238 	uint32_t pcirev;
239 	int nmsi, pmc;
240 	int error;
241 
242 	sc = device_get_softc(dev); /* Automatically allocated and zeroed
243 				       on attach. */
244 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
245 	sc->dev = dev;
246 
247 	/*
248 	 * Initialize mutexes and tasks.
249 	 */
250 	mtx_init(&sc->mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, MTX_DEF);
251 	callout_init_mtx(&sc->tick_ch, &sc->mtx, 0);
252 	TASK_INIT(&sc->int_task, 0, ae_int_task, sc);
253 	TASK_INIT(&sc->link_task, 0, ae_link_task, sc);
254 
255 	pci_enable_busmaster(dev);		/* Enable bus mastering. */
256 
257 	sc->spec_mem = ae_res_spec_mem;
258 
259 	/*
260 	 * Allocate memory-mapped registers.
261 	 */
262 	error = bus_alloc_resources(dev, sc->spec_mem, sc->mem);
263 	if (error != 0) {
264 		device_printf(dev, "could not allocate memory resources.\n");
265 		sc->spec_mem = NULL;
266 		goto fail;
267 	}
268 
269 	/*
270 	 * Retrieve PCI and chip revisions.
271 	 */
272 	pcirev = pci_get_revid(dev);
273 	chiprev = (AE_READ_4(sc, AE_MASTER_REG) >> AE_MASTER_REVNUM_SHIFT) &
274 	    AE_MASTER_REVNUM_MASK;
275 	if (bootverbose) {
276 		device_printf(dev, "pci device revision: %#04x\n", pcirev);
277 		device_printf(dev, "chip id: %#02x\n", chiprev);
278 	}
279 	nmsi = pci_msi_count(dev);
280 	if (bootverbose)
281 		device_printf(dev, "MSI count: %d.\n", nmsi);
282 
283 	/*
284 	 * Allocate interrupt resources.
285 	 */
286 	if (msi_disable == 0 && nmsi == 1) {
287 		error = pci_alloc_msi(dev, &nmsi);
288 		if (error == 0) {
289 			device_printf(dev, "Using MSI messages.\n");
290 			sc->spec_irq = ae_res_spec_msi;
291 			error = bus_alloc_resources(dev, sc->spec_irq, sc->irq);
292 			if (error != 0) {
293 				device_printf(dev, "MSI allocation failed.\n");
294 				sc->spec_irq = NULL;
295 				pci_release_msi(dev);
296 			} else {
297 				sc->flags |= AE_FLAG_MSI;
298 			}
299 		}
300 	}
301 	if (sc->spec_irq == NULL) {
302 		sc->spec_irq = ae_res_spec_irq;
303 		error = bus_alloc_resources(dev, sc->spec_irq, sc->irq);
304 		if (error != 0) {
305 			device_printf(dev, "could not allocate IRQ resources.\n");
306 			sc->spec_irq = NULL;
307 			goto fail;
308 		}
309 	}
310 
311 	ae_init_tunables(sc);
312 
313 	ae_phy_reset(sc);		/* Reset PHY. */
314 	error = ae_reset(sc);		/* Reset the controller itself. */
315 	if (error != 0)
316 		goto fail;
317 
318 	ae_pcie_init(sc);
319 
320 	ae_retrieve_address(sc);	/* Load MAC address. */
321 
322 	error = ae_alloc_rings(sc);	/* Allocate ring buffers. */
323 	if (error != 0)
324 		goto fail;
325 
326 	ifp = sc->ifp = if_alloc(IFT_ETHER);
327 	if (ifp == NULL) {
328 		device_printf(dev, "could not allocate ifnet structure.\n");
329 		error = ENXIO;
330 		goto fail;
331 	}
332 
333 	ifp->if_softc = sc;
334 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
335 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
336 	ifp->if_ioctl = ae_ioctl;
337 	ifp->if_start = ae_start;
338 	ifp->if_init = ae_init;
339 	ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
340 	ifp->if_hwassist = 0;
341 	ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
342 	IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
343 	IFQ_SET_READY(&ifp->if_snd);
344 	if (pci_find_cap(dev, PCIY_PMG, &pmc) == 0) {
345 		ifp->if_capabilities |= IFCAP_WOL_MAGIC;
346 		sc->flags |= AE_FLAG_PMG;
347 	}
348 	ifp->if_capenable = ifp->if_capabilities;
349 
350 	/*
351 	 * Configure and attach MII bus.
352 	 */
353 	error = mii_attach(dev, &sc->miibus, ifp, ae_mediachange,
354 	    ae_mediastatus, BMSR_DEFCAPMASK, AE_PHYADDR_DEFAULT,
355 	    MII_OFFSET_ANY, 0);
356 	if (error != 0) {
357 		device_printf(dev, "attaching PHYs failed\n");
358 		goto fail;
359 	}
360 
361 	ether_ifattach(ifp, sc->eaddr);
362 	/* Tell the upper layer(s) we support long frames. */
363 	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
364 
365 	/*
366 	 * Create and run all helper tasks.
367 	 */
368 	sc->tq = taskqueue_create_fast("ae_taskq", M_WAITOK,
369             taskqueue_thread_enqueue, &sc->tq);
370 	if (sc->tq == NULL) {
371 		device_printf(dev, "could not create taskqueue.\n");
372 		ether_ifdetach(ifp);
373 		error = ENXIO;
374 		goto fail;
375 	}
376 	taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
377 	    device_get_nameunit(sc->dev));
378 
379 	/*
380 	 * Configure interrupt handlers.
381 	 */
382 	error = bus_setup_intr(dev, sc->irq[0], INTR_TYPE_NET | INTR_MPSAFE,
383 	    ae_intr, NULL, sc, &sc->intrhand);
384 	if (error != 0) {
385 		device_printf(dev, "could not set up interrupt handler.\n");
386 		taskqueue_free(sc->tq);
387 		sc->tq = NULL;
388 		ether_ifdetach(ifp);
389 		goto fail;
390 	}
391 
392 fail:
393 	if (error != 0)
394 		ae_detach(dev);
395 
396 	return (error);
397 }
398 
399 #define	AE_SYSCTL(stx, parent, name, desc, ptr)	\
400 	SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, name, CTLFLAG_RD, ptr, 0, desc)
401 
402 static void
403 ae_init_tunables(ae_softc_t *sc)
404 {
405 	struct sysctl_ctx_list *ctx;
406 	struct sysctl_oid *root, *stats, *stats_rx, *stats_tx;
407 	struct ae_stats *ae_stats;
408 
409 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
410 	ae_stats = &sc->stats;
411 
412 	ctx = device_get_sysctl_ctx(sc->dev);
413 	root = device_get_sysctl_tree(sc->dev);
414 	stats = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(root), OID_AUTO, "stats",
415 	    CTLFLAG_RD, NULL, "ae statistics");
416 
417 	/*
418 	 * Receiver statistcics.
419 	 */
420 	stats_rx = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(stats), OID_AUTO, "rx",
421 	    CTLFLAG_RD, NULL, "Rx MAC statistics");
422 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "bcast",
423 	    "broadcast frames", &ae_stats->rx_bcast);
424 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "mcast",
425 	    "multicast frames", &ae_stats->rx_mcast);
426 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "pause",
427 	    "PAUSE frames", &ae_stats->rx_pause);
428 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "control",
429 	    "control frames", &ae_stats->rx_ctrl);
430 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "crc_errors",
431 	    "frames with CRC errors", &ae_stats->rx_crcerr);
432 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "code_errors",
433 	    "frames with invalid opcode", &ae_stats->rx_codeerr);
434 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "runt",
435 	    "runt frames", &ae_stats->rx_runt);
436 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "frag",
437 	    "fragmented frames", &ae_stats->rx_frag);
438 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "align_errors",
439 	    "frames with alignment errors", &ae_stats->rx_align);
440 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_rx), "truncated",
441 	    "frames truncated due to Rx FIFO inderrun", &ae_stats->rx_trunc);
442 
443 	/*
444 	 * Receiver statistcics.
445 	 */
446 	stats_tx = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(stats), OID_AUTO, "tx",
447 	    CTLFLAG_RD, NULL, "Tx MAC statistics");
448 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "bcast",
449 	    "broadcast frames", &ae_stats->tx_bcast);
450 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "mcast",
451 	    "multicast frames", &ae_stats->tx_mcast);
452 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "pause",
453 	    "PAUSE frames", &ae_stats->tx_pause);
454 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "control",
455 	    "control frames", &ae_stats->tx_ctrl);
456 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "defers",
457 	    "deferrals occuried", &ae_stats->tx_defer);
458 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "exc_defers",
459 	    "excessive deferrals occuried", &ae_stats->tx_excdefer);
460 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "singlecols",
461 	    "single collisions occuried", &ae_stats->tx_singlecol);
462 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "multicols",
463 	    "multiple collisions occuried", &ae_stats->tx_multicol);
464 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "latecols",
465 	    "late collisions occuried", &ae_stats->tx_latecol);
466 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "aborts",
467 	    "transmit aborts due collisions", &ae_stats->tx_abortcol);
468 	AE_SYSCTL(ctx, SYSCTL_CHILDREN(stats_tx), "underruns",
469 	    "Tx FIFO underruns", &ae_stats->tx_underrun);
470 }
471 
472 static void
473 ae_pcie_init(ae_softc_t *sc)
474 {
475 
476 	AE_WRITE_4(sc, AE_PCIE_LTSSM_TESTMODE_REG, AE_PCIE_LTSSM_TESTMODE_DEFAULT);
477 	AE_WRITE_4(sc, AE_PCIE_DLL_TX_CTRL_REG, AE_PCIE_DLL_TX_CTRL_DEFAULT);
478 }
479 
480 static void
481 ae_phy_reset(ae_softc_t *sc)
482 {
483 
484 	AE_WRITE_4(sc, AE_PHY_ENABLE_REG, AE_PHY_ENABLE);
485 	DELAY(1000);	/* XXX: pause(9) ? */
486 }
487 
488 static int
489 ae_reset(ae_softc_t *sc)
490 {
491 	int i;
492 
493 	/*
494 	 * Issue a soft reset.
495 	 */
496 	AE_WRITE_4(sc, AE_MASTER_REG, AE_MASTER_SOFT_RESET);
497 	bus_barrier(sc->mem[0], AE_MASTER_REG, 4,
498 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
499 
500 	/*
501 	 * Wait for reset to complete.
502 	 */
503 	for (i = 0; i < AE_RESET_TIMEOUT; i++) {
504 		if ((AE_READ_4(sc, AE_MASTER_REG) & AE_MASTER_SOFT_RESET) == 0)
505 			break;
506 		DELAY(10);
507 	}
508 	if (i == AE_RESET_TIMEOUT) {
509 		device_printf(sc->dev, "reset timeout.\n");
510 		return (ENXIO);
511 	}
512 
513 	/*
514 	 * Wait for everything to enter idle state.
515 	 */
516 	for (i = 0; i < AE_IDLE_TIMEOUT; i++) {
517 		if (AE_READ_4(sc, AE_IDLE_REG) == 0)
518 			break;
519 		DELAY(100);
520 	}
521 	if (i == AE_IDLE_TIMEOUT) {
522 		device_printf(sc->dev, "could not enter idle state.\n");
523 		return (ENXIO);
524 	}
525 	return (0);
526 }
527 
528 static void
529 ae_init(void *arg)
530 {
531 	ae_softc_t *sc;
532 
533 	sc = (ae_softc_t *)arg;
534 	AE_LOCK(sc);
535 	ae_init_locked(sc);
536 	AE_UNLOCK(sc);
537 }
538 
539 static void
540 ae_phy_init(ae_softc_t *sc)
541 {
542 
543 	/*
544 	 * Enable link status change interrupt.
545 	 * XXX magic numbers.
546 	 */
547 #ifdef notyet
548 	AE_PHY_WRITE(sc, 18, 0xc00);
549 #endif
550 }
551 
552 static int
553 ae_init_locked(ae_softc_t *sc)
554 {
555 	struct ifnet *ifp;
556 	struct mii_data *mii;
557 	uint8_t eaddr[ETHER_ADDR_LEN];
558 	uint32_t val;
559 	bus_addr_t addr;
560 
561 	AE_LOCK_ASSERT(sc);
562 
563 	ifp = sc->ifp;
564 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
565 		return (0);
566 	mii = device_get_softc(sc->miibus);
567 
568 	ae_stop(sc);
569 	ae_reset(sc);
570 	ae_pcie_init(sc);		/* Initialize PCIE stuff. */
571 	ae_phy_init(sc);
572 	ae_powersave_disable(sc);
573 
574 	/*
575 	 * Clear and disable interrupts.
576 	 */
577 	AE_WRITE_4(sc, AE_ISR_REG, 0xffffffff);
578 
579 	/*
580 	 * Set the MAC address.
581 	 */
582 	bcopy(IF_LLADDR(ifp), eaddr, ETHER_ADDR_LEN);
583 	val = eaddr[2] << 24 | eaddr[3] << 16 | eaddr[4] << 8 | eaddr[5];
584 	AE_WRITE_4(sc, AE_EADDR0_REG, val);
585 	val = eaddr[0] << 8 | eaddr[1];
586 	AE_WRITE_4(sc, AE_EADDR1_REG, val);
587 
588 	/*
589 	 * Set ring buffers base addresses.
590 	 */
591 	addr = sc->dma_rxd_busaddr;
592 	AE_WRITE_4(sc, AE_DESC_ADDR_HI_REG, BUS_ADDR_HI(addr));
593 	AE_WRITE_4(sc, AE_RXD_ADDR_LO_REG, BUS_ADDR_LO(addr));
594 	addr = sc->dma_txd_busaddr;
595 	AE_WRITE_4(sc, AE_TXD_ADDR_LO_REG, BUS_ADDR_LO(addr));
596 	addr = sc->dma_txs_busaddr;
597 	AE_WRITE_4(sc, AE_TXS_ADDR_LO_REG, BUS_ADDR_LO(addr));
598 
599 	/*
600 	 * Configure ring buffers sizes.
601 	 */
602 	AE_WRITE_2(sc, AE_RXD_COUNT_REG, AE_RXD_COUNT_DEFAULT);
603 	AE_WRITE_2(sc, AE_TXD_BUFSIZE_REG, AE_TXD_BUFSIZE_DEFAULT / 4);
604 	AE_WRITE_2(sc, AE_TXS_COUNT_REG, AE_TXS_COUNT_DEFAULT);
605 
606 	/*
607 	 * Configure interframe gap parameters.
608 	 */
609 	val = ((AE_IFG_TXIPG_DEFAULT << AE_IFG_TXIPG_SHIFT) &
610 	    AE_IFG_TXIPG_MASK) |
611 	    ((AE_IFG_RXIPG_DEFAULT << AE_IFG_RXIPG_SHIFT) &
612 	    AE_IFG_RXIPG_MASK) |
613 	    ((AE_IFG_IPGR1_DEFAULT << AE_IFG_IPGR1_SHIFT) &
614 	    AE_IFG_IPGR1_MASK) |
615 	    ((AE_IFG_IPGR2_DEFAULT << AE_IFG_IPGR2_SHIFT) &
616 	    AE_IFG_IPGR2_MASK);
617 	AE_WRITE_4(sc, AE_IFG_REG, val);
618 
619 	/*
620 	 * Configure half-duplex operation.
621 	 */
622 	val = ((AE_HDPX_LCOL_DEFAULT << AE_HDPX_LCOL_SHIFT) &
623 	    AE_HDPX_LCOL_MASK) |
624 	    ((AE_HDPX_RETRY_DEFAULT << AE_HDPX_RETRY_SHIFT) &
625 	    AE_HDPX_RETRY_MASK) |
626 	    ((AE_HDPX_ABEBT_DEFAULT << AE_HDPX_ABEBT_SHIFT) &
627 	    AE_HDPX_ABEBT_MASK) |
628 	    ((AE_HDPX_JAMIPG_DEFAULT << AE_HDPX_JAMIPG_SHIFT) &
629 	    AE_HDPX_JAMIPG_MASK) | AE_HDPX_EXC_EN;
630 	AE_WRITE_4(sc, AE_HDPX_REG, val);
631 
632 	/*
633 	 * Configure interrupt moderate timer.
634 	 */
635 	AE_WRITE_2(sc, AE_IMT_REG, AE_IMT_DEFAULT);
636 	val = AE_READ_4(sc, AE_MASTER_REG);
637 	val |= AE_MASTER_IMT_EN;
638 	AE_WRITE_4(sc, AE_MASTER_REG, val);
639 
640 	/*
641 	 * Configure interrupt clearing timer.
642 	 */
643 	AE_WRITE_2(sc, AE_ICT_REG, AE_ICT_DEFAULT);
644 
645 	/*
646 	 * Configure MTU.
647 	 */
648 	val = ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN +
649 	    ETHER_CRC_LEN;
650 	AE_WRITE_2(sc, AE_MTU_REG, val);
651 
652 	/*
653 	 * Configure cut-through threshold.
654 	 */
655 	AE_WRITE_4(sc, AE_CUT_THRESH_REG, AE_CUT_THRESH_DEFAULT);
656 
657 	/*
658 	 * Configure flow control.
659 	 */
660 	AE_WRITE_2(sc, AE_FLOW_THRESH_HI_REG, (AE_RXD_COUNT_DEFAULT / 8) * 7);
661 	AE_WRITE_2(sc, AE_FLOW_THRESH_LO_REG, (AE_RXD_COUNT_MIN / 8) >
662 	    (AE_RXD_COUNT_DEFAULT / 12) ? (AE_RXD_COUNT_MIN / 8) :
663 	    (AE_RXD_COUNT_DEFAULT / 12));
664 
665 	/*
666 	 * Init mailboxes.
667 	 */
668 	sc->txd_cur = sc->rxd_cur = 0;
669 	sc->txs_ack = sc->txd_ack = 0;
670 	sc->rxd_cur = 0;
671 	AE_WRITE_2(sc, AE_MB_TXD_IDX_REG, sc->txd_cur);
672 	AE_WRITE_2(sc, AE_MB_RXD_IDX_REG, sc->rxd_cur);
673 
674 	sc->tx_inproc = 0;	/* Number of packets the chip processes now. */
675 	sc->flags |= AE_FLAG_TXAVAIL;	/* Free Tx's available. */
676 
677 	/*
678 	 * Enable DMA.
679 	 */
680 	AE_WRITE_1(sc, AE_DMAREAD_REG, AE_DMAREAD_EN);
681 	AE_WRITE_1(sc, AE_DMAWRITE_REG, AE_DMAWRITE_EN);
682 
683 	/*
684 	 * Check if everything is OK.
685 	 */
686 	val = AE_READ_4(sc, AE_ISR_REG);
687 	if ((val & AE_ISR_PHY_LINKDOWN) != 0) {
688 		device_printf(sc->dev, "Initialization failed.\n");
689 		return (ENXIO);
690 	}
691 
692 	/*
693 	 * Clear interrupt status.
694 	 */
695 	AE_WRITE_4(sc, AE_ISR_REG, 0x3fffffff);
696 	AE_WRITE_4(sc, AE_ISR_REG, 0x0);
697 
698 	/*
699 	 * Enable interrupts.
700 	 */
701 	val = AE_READ_4(sc, AE_MASTER_REG);
702 	AE_WRITE_4(sc, AE_MASTER_REG, val | AE_MASTER_MANUAL_INT);
703 	AE_WRITE_4(sc, AE_IMR_REG, AE_IMR_DEFAULT);
704 
705 	/*
706 	 * Disable WOL.
707 	 */
708 	AE_WRITE_4(sc, AE_WOL_REG, 0);
709 
710 	/*
711 	 * Configure MAC.
712 	 */
713 	val = AE_MAC_TX_CRC_EN | AE_MAC_TX_AUTOPAD |
714 	    AE_MAC_FULL_DUPLEX | AE_MAC_CLK_PHY |
715 	    AE_MAC_TX_FLOW_EN | AE_MAC_RX_FLOW_EN |
716 	    ((AE_HALFBUF_DEFAULT << AE_HALFBUF_SHIFT) & AE_HALFBUF_MASK) |
717 	    ((AE_MAC_PREAMBLE_DEFAULT << AE_MAC_PREAMBLE_SHIFT) &
718 	    AE_MAC_PREAMBLE_MASK);
719 	AE_WRITE_4(sc, AE_MAC_REG, val);
720 
721 	/*
722 	 * Configure Rx MAC.
723 	 */
724 	ae_rxfilter(sc);
725 	ae_rxvlan(sc);
726 
727 	/*
728 	 * Enable Tx/Rx.
729 	 */
730 	val = AE_READ_4(sc, AE_MAC_REG);
731 	AE_WRITE_4(sc, AE_MAC_REG, val | AE_MAC_TX_EN | AE_MAC_RX_EN);
732 
733 	sc->flags &= ~AE_FLAG_LINK;
734 	mii_mediachg(mii);	/* Switch to the current media. */
735 
736 	callout_reset(&sc->tick_ch, hz, ae_tick, sc);
737 
738 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
739 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
740 
741 #ifdef AE_DEBUG
742 	device_printf(sc->dev, "Initialization complete.\n");
743 #endif
744 
745 	return (0);
746 }
747 
748 static int
749 ae_detach(device_t dev)
750 {
751 	struct ae_softc *sc;
752 	struct ifnet *ifp;
753 
754 	sc = device_get_softc(dev);
755 	KASSERT(sc != NULL, ("[ae: %d]: sc is NULL", __LINE__));
756 	ifp = sc->ifp;
757 	if (device_is_attached(dev)) {
758 		AE_LOCK(sc);
759 		sc->flags |= AE_FLAG_DETACH;
760 		ae_stop(sc);
761 		AE_UNLOCK(sc);
762 		callout_drain(&sc->tick_ch);
763 		taskqueue_drain(sc->tq, &sc->int_task);
764 		taskqueue_drain(taskqueue_swi, &sc->link_task);
765 		ether_ifdetach(ifp);
766 	}
767 	if (sc->tq != NULL) {
768 		taskqueue_drain(sc->tq, &sc->int_task);
769 		taskqueue_free(sc->tq);
770 		sc->tq = NULL;
771 	}
772 	if (sc->miibus != NULL) {
773 		device_delete_child(dev, sc->miibus);
774 		sc->miibus = NULL;
775 	}
776 	bus_generic_detach(sc->dev);
777 	ae_dma_free(sc);
778 	if (sc->intrhand != NULL) {
779 		bus_teardown_intr(dev, sc->irq[0], sc->intrhand);
780 		sc->intrhand = NULL;
781 	}
782 	if (ifp != NULL) {
783 		if_free(ifp);
784 		sc->ifp = NULL;
785 	}
786 	if (sc->spec_irq != NULL)
787 		bus_release_resources(dev, sc->spec_irq, sc->irq);
788 	if (sc->spec_mem != NULL)
789 		bus_release_resources(dev, sc->spec_mem, sc->mem);
790 	if ((sc->flags & AE_FLAG_MSI) != 0)
791 		pci_release_msi(dev);
792 	mtx_destroy(&sc->mtx);
793 
794 	return (0);
795 }
796 
797 static int
798 ae_miibus_readreg(device_t dev, int phy, int reg)
799 {
800 	ae_softc_t *sc;
801 	uint32_t val;
802 	int i;
803 
804 	sc = device_get_softc(dev);
805 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
806 
807 	/*
808 	 * Locking is done in upper layers.
809 	 */
810 
811 	val = ((reg << AE_MDIO_REGADDR_SHIFT) & AE_MDIO_REGADDR_MASK) |
812 	    AE_MDIO_START | AE_MDIO_READ | AE_MDIO_SUP_PREAMBLE |
813 	    ((AE_MDIO_CLK_25_4 << AE_MDIO_CLK_SHIFT) & AE_MDIO_CLK_MASK);
814 	AE_WRITE_4(sc, AE_MDIO_REG, val);
815 
816 	/*
817 	 * Wait for operation to complete.
818 	 */
819 	for (i = 0; i < AE_MDIO_TIMEOUT; i++) {
820 		DELAY(2);
821 		val = AE_READ_4(sc, AE_MDIO_REG);
822 		if ((val & (AE_MDIO_START | AE_MDIO_BUSY)) == 0)
823 			break;
824 	}
825 	if (i == AE_MDIO_TIMEOUT) {
826 		device_printf(sc->dev, "phy read timeout: %d.\n", reg);
827 		return (0);
828 	}
829 	return ((val << AE_MDIO_DATA_SHIFT) & AE_MDIO_DATA_MASK);
830 }
831 
832 static int
833 ae_miibus_writereg(device_t dev, int phy, int reg, int val)
834 {
835 	ae_softc_t *sc;
836 	uint32_t aereg;
837 	int i;
838 
839 	sc = device_get_softc(dev);
840 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
841 
842 	/*
843 	 * Locking is done in upper layers.
844 	 */
845 
846 	aereg = ((reg << AE_MDIO_REGADDR_SHIFT) & AE_MDIO_REGADDR_MASK) |
847 	    AE_MDIO_START | AE_MDIO_SUP_PREAMBLE |
848 	    ((AE_MDIO_CLK_25_4 << AE_MDIO_CLK_SHIFT) & AE_MDIO_CLK_MASK) |
849 	    ((val << AE_MDIO_DATA_SHIFT) & AE_MDIO_DATA_MASK);
850 	AE_WRITE_4(sc, AE_MDIO_REG, aereg);
851 
852 	/*
853 	 * Wait for operation to complete.
854 	 */
855 	for (i = 0; i < AE_MDIO_TIMEOUT; i++) {
856 		DELAY(2);
857 		aereg = AE_READ_4(sc, AE_MDIO_REG);
858 		if ((aereg & (AE_MDIO_START | AE_MDIO_BUSY)) == 0)
859 			break;
860 	}
861 	if (i == AE_MDIO_TIMEOUT) {
862 		device_printf(sc->dev, "phy write timeout: %d.\n", reg);
863 	}
864 	return (0);
865 }
866 
867 static void
868 ae_miibus_statchg(device_t dev)
869 {
870 	ae_softc_t *sc;
871 
872 	sc = device_get_softc(dev);
873 	taskqueue_enqueue(taskqueue_swi, &sc->link_task);
874 }
875 
876 static void
877 ae_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
878 {
879 	ae_softc_t *sc;
880 	struct mii_data *mii;
881 
882 	sc = ifp->if_softc;
883 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
884 
885 	AE_LOCK(sc);
886 	mii = device_get_softc(sc->miibus);
887 	mii_pollstat(mii);
888 	ifmr->ifm_status = mii->mii_media_status;
889 	ifmr->ifm_active = mii->mii_media_active;
890 	AE_UNLOCK(sc);
891 }
892 
893 static int
894 ae_mediachange(struct ifnet *ifp)
895 {
896 	ae_softc_t *sc;
897 	struct mii_data *mii;
898 	struct mii_softc *mii_sc;
899 	int error;
900 
901 	/* XXX: check IFF_UP ?? */
902 	sc = ifp->if_softc;
903 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
904 	AE_LOCK(sc);
905 	mii = device_get_softc(sc->miibus);
906 	LIST_FOREACH(mii_sc, &mii->mii_phys, mii_list)
907 		PHY_RESET(mii_sc);
908 	error = mii_mediachg(mii);
909 	AE_UNLOCK(sc);
910 
911 	return (error);
912 }
913 
914 static int
915 ae_check_eeprom_present(ae_softc_t *sc, int *vpdc)
916 {
917 	int error;
918 	uint32_t val;
919 
920 	KASSERT(vpdc != NULL, ("[ae, %d]: vpdc is NULL!\n", __LINE__));
921 
922 	/*
923 	 * Not sure why, but Linux does this.
924 	 */
925 	val = AE_READ_4(sc, AE_SPICTL_REG);
926 	if ((val & AE_SPICTL_VPD_EN) != 0) {
927 		val &= ~AE_SPICTL_VPD_EN;
928 		AE_WRITE_4(sc, AE_SPICTL_REG, val);
929 	}
930 	error = pci_find_cap(sc->dev, PCIY_VPD, vpdc);
931 	return (error);
932 }
933 
934 static int
935 ae_vpd_read_word(ae_softc_t *sc, int reg, uint32_t *word)
936 {
937 	uint32_t val;
938 	int i;
939 
940 	AE_WRITE_4(sc, AE_VPD_DATA_REG, 0);	/* Clear register value. */
941 
942 	/*
943 	 * VPD registers start at offset 0x100. Read them.
944 	 */
945 	val = 0x100 + reg * 4;
946 	AE_WRITE_4(sc, AE_VPD_CAP_REG, (val << AE_VPD_CAP_ADDR_SHIFT) &
947 	    AE_VPD_CAP_ADDR_MASK);
948 	for (i = 0; i < AE_VPD_TIMEOUT; i++) {
949 		DELAY(2000);
950 		val = AE_READ_4(sc, AE_VPD_CAP_REG);
951 		if ((val & AE_VPD_CAP_DONE) != 0)
952 			break;
953 	}
954 	if (i == AE_VPD_TIMEOUT) {
955 		device_printf(sc->dev, "timeout reading VPD register %d.\n",
956 		    reg);
957 		return (ETIMEDOUT);
958 	}
959 	*word = AE_READ_4(sc, AE_VPD_DATA_REG);
960 	return (0);
961 }
962 
963 static int
964 ae_get_vpd_eaddr(ae_softc_t *sc, uint32_t *eaddr)
965 {
966 	uint32_t word, reg, val;
967 	int error;
968 	int found;
969 	int vpdc;
970 	int i;
971 
972 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
973 	KASSERT(eaddr != NULL, ("[ae, %d]: eaddr is NULL", __LINE__));
974 
975 	/*
976 	 * Check for EEPROM.
977 	 */
978 	error = ae_check_eeprom_present(sc, &vpdc);
979 	if (error != 0)
980 		return (error);
981 
982 	/*
983 	 * Read the VPD configuration space.
984 	 * Each register is prefixed with signature,
985 	 * so we can check if it is valid.
986 	 */
987 	for (i = 0, found = 0; i < AE_VPD_NREGS; i++) {
988 		error = ae_vpd_read_word(sc, i, &word);
989 		if (error != 0)
990 			break;
991 
992 		/*
993 		 * Check signature.
994 		 */
995 		if ((word & AE_VPD_SIG_MASK) != AE_VPD_SIG)
996 			break;
997 		reg = word >> AE_VPD_REG_SHIFT;
998 		i++;	/* Move to the next word. */
999 
1000 		if (reg != AE_EADDR0_REG && reg != AE_EADDR1_REG)
1001 			continue;
1002 
1003 		error = ae_vpd_read_word(sc, i, &val);
1004 		if (error != 0)
1005 			break;
1006 		if (reg == AE_EADDR0_REG)
1007 			eaddr[0] = val;
1008 		else
1009 			eaddr[1] = val;
1010 		found++;
1011 	}
1012 
1013 	if (found < 2)
1014 		return (ENOENT);
1015 
1016 	eaddr[1] &= 0xffff;	/* Only last 2 bytes are used. */
1017 	if (AE_CHECK_EADDR_VALID(eaddr) != 0) {
1018 		if (bootverbose)
1019 			device_printf(sc->dev,
1020 			    "VPD ethernet address registers are invalid.\n");
1021 		return (EINVAL);
1022 	}
1023 	return (0);
1024 }
1025 
1026 static int
1027 ae_get_reg_eaddr(ae_softc_t *sc, uint32_t *eaddr)
1028 {
1029 
1030 	/*
1031 	 * BIOS is supposed to set this.
1032 	 */
1033 	eaddr[0] = AE_READ_4(sc, AE_EADDR0_REG);
1034 	eaddr[1] = AE_READ_4(sc, AE_EADDR1_REG);
1035 	eaddr[1] &= 0xffff;	/* Only last 2 bytes are used. */
1036 
1037 	if (AE_CHECK_EADDR_VALID(eaddr) != 0) {
1038 		if (bootverbose)
1039 			device_printf(sc->dev,
1040 			    "Ethernet address registers are invalid.\n");
1041 		return (EINVAL);
1042 	}
1043 	return (0);
1044 }
1045 
1046 static void
1047 ae_retrieve_address(ae_softc_t *sc)
1048 {
1049 	uint32_t eaddr[2] = {0, 0};
1050 	int error;
1051 
1052 	/*
1053 	 *Check for EEPROM.
1054 	 */
1055 	error = ae_get_vpd_eaddr(sc, eaddr);
1056 	if (error != 0)
1057 		error = ae_get_reg_eaddr(sc, eaddr);
1058 	if (error != 0) {
1059 		if (bootverbose)
1060 			device_printf(sc->dev,
1061 			    "Generating random ethernet address.\n");
1062 		eaddr[0] = arc4random();
1063 
1064 		/*
1065 		 * Set OUI to ASUSTek COMPUTER INC.
1066 		 */
1067 		sc->eaddr[0] = 0x02;	/* U/L bit set. */
1068 		sc->eaddr[1] = 0x1f;
1069 		sc->eaddr[2] = 0xc6;
1070 		sc->eaddr[3] = (eaddr[0] >> 16) & 0xff;
1071 		sc->eaddr[4] = (eaddr[0] >> 8) & 0xff;
1072 		sc->eaddr[5] = (eaddr[0] >> 0) & 0xff;
1073 	} else {
1074 		sc->eaddr[0] = (eaddr[1] >> 8) & 0xff;
1075 		sc->eaddr[1] = (eaddr[1] >> 0) & 0xff;
1076 		sc->eaddr[2] = (eaddr[0] >> 24) & 0xff;
1077 		sc->eaddr[3] = (eaddr[0] >> 16) & 0xff;
1078 		sc->eaddr[4] = (eaddr[0] >> 8) & 0xff;
1079 		sc->eaddr[5] = (eaddr[0] >> 0) & 0xff;
1080 	}
1081 }
1082 
1083 static void
1084 ae_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1085 {
1086 	bus_addr_t *addr = arg;
1087 
1088 	if (error != 0)
1089 		return;
1090 	KASSERT(nsegs == 1, ("[ae, %d]: %d segments instead of 1!", __LINE__,
1091 	    nsegs));
1092 	*addr = segs[0].ds_addr;
1093 }
1094 
1095 static int
1096 ae_alloc_rings(ae_softc_t *sc)
1097 {
1098 	bus_addr_t busaddr;
1099 	int error;
1100 
1101 	/*
1102 	 * Create parent DMA tag.
1103 	 */
1104 	error = bus_dma_tag_create(bus_get_dma_tag(sc->dev),
1105 	    1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
1106 	    NULL, NULL, BUS_SPACE_MAXSIZE_32BIT, 0,
1107 	    BUS_SPACE_MAXSIZE_32BIT, 0, NULL, NULL,
1108 	    &sc->dma_parent_tag);
1109 	if (error != 0) {
1110 		device_printf(sc->dev, "could not creare parent DMA tag.\n");
1111 		return (error);
1112 	}
1113 
1114 	/*
1115 	 * Create DMA tag for TxD.
1116 	 */
1117 	error = bus_dma_tag_create(sc->dma_parent_tag,
1118 	    4, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
1119 	    NULL, NULL, AE_TXD_BUFSIZE_DEFAULT, 1,
1120 	    AE_TXD_BUFSIZE_DEFAULT, 0, NULL, NULL,
1121 	    &sc->dma_txd_tag);
1122 	if (error != 0) {
1123 		device_printf(sc->dev, "could not creare TxD DMA tag.\n");
1124 		return (error);
1125 	}
1126 
1127 	/*
1128 	 * Create DMA tag for TxS.
1129 	 */
1130 	error = bus_dma_tag_create(sc->dma_parent_tag,
1131 	    4, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
1132 	    NULL, NULL, AE_TXS_COUNT_DEFAULT * 4, 1,
1133 	    AE_TXS_COUNT_DEFAULT * 4, 0, NULL, NULL,
1134 	    &sc->dma_txs_tag);
1135 	if (error != 0) {
1136 		device_printf(sc->dev, "could not creare TxS DMA tag.\n");
1137 		return (error);
1138 	}
1139 
1140 	/*
1141 	 * Create DMA tag for RxD.
1142 	 */
1143 	error = bus_dma_tag_create(sc->dma_parent_tag,
1144 	    128, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
1145 	    NULL, NULL, AE_RXD_COUNT_DEFAULT * 1536 + 120, 1,
1146 	    AE_RXD_COUNT_DEFAULT * 1536 + 120, 0, NULL, NULL,
1147 	    &sc->dma_rxd_tag);
1148 	if (error != 0) {
1149 		device_printf(sc->dev, "could not creare TxS DMA tag.\n");
1150 		return (error);
1151 	}
1152 
1153 	/*
1154 	 * Allocate TxD DMA memory.
1155 	 */
1156 	error = bus_dmamem_alloc(sc->dma_txd_tag, (void **)&sc->txd_base,
1157 	    BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
1158 	    &sc->dma_txd_map);
1159 	if (error != 0) {
1160 		device_printf(sc->dev,
1161 		    "could not allocate DMA memory for TxD ring.\n");
1162 		return (error);
1163 	}
1164 	error = bus_dmamap_load(sc->dma_txd_tag, sc->dma_txd_map, sc->txd_base,
1165 	    AE_TXD_BUFSIZE_DEFAULT, ae_dmamap_cb, &busaddr, BUS_DMA_NOWAIT);
1166 	if (error != 0 || busaddr == 0) {
1167 		device_printf(sc->dev,
1168 		    "could not load DMA map for TxD ring.\n");
1169 		return (error);
1170 	}
1171 	sc->dma_txd_busaddr = busaddr;
1172 
1173 	/*
1174 	 * Allocate TxS DMA memory.
1175 	 */
1176 	error = bus_dmamem_alloc(sc->dma_txs_tag, (void **)&sc->txs_base,
1177 	    BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
1178 	    &sc->dma_txs_map);
1179 	if (error != 0) {
1180 		device_printf(sc->dev,
1181 		    "could not allocate DMA memory for TxS ring.\n");
1182 		return (error);
1183 	}
1184 	error = bus_dmamap_load(sc->dma_txs_tag, sc->dma_txs_map, sc->txs_base,
1185 	    AE_TXS_COUNT_DEFAULT * 4, ae_dmamap_cb, &busaddr, BUS_DMA_NOWAIT);
1186 	if (error != 0 || busaddr == 0) {
1187 		device_printf(sc->dev,
1188 		    "could not load DMA map for TxS ring.\n");
1189 		return (error);
1190 	}
1191 	sc->dma_txs_busaddr = busaddr;
1192 
1193 	/*
1194 	 * Allocate RxD DMA memory.
1195 	 */
1196 	error = bus_dmamem_alloc(sc->dma_rxd_tag, (void **)&sc->rxd_base_dma,
1197 	    BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
1198 	    &sc->dma_rxd_map);
1199 	if (error != 0) {
1200 		device_printf(sc->dev,
1201 		    "could not allocate DMA memory for RxD ring.\n");
1202 		return (error);
1203 	}
1204 	error = bus_dmamap_load(sc->dma_rxd_tag, sc->dma_rxd_map,
1205 	    sc->rxd_base_dma, AE_RXD_COUNT_DEFAULT * 1536 + 120, ae_dmamap_cb,
1206 	    &busaddr, BUS_DMA_NOWAIT);
1207 	if (error != 0 || busaddr == 0) {
1208 		device_printf(sc->dev,
1209 		    "could not load DMA map for RxD ring.\n");
1210 		return (error);
1211 	}
1212 	sc->dma_rxd_busaddr = busaddr + 120;
1213 	sc->rxd_base = (ae_rxd_t *)(sc->rxd_base_dma + 120);
1214 
1215 	return (0);
1216 }
1217 
1218 static void
1219 ae_dma_free(ae_softc_t *sc)
1220 {
1221 
1222 	if (sc->dma_txd_tag != NULL) {
1223 		if (sc->dma_txd_map != NULL) {
1224 			bus_dmamap_unload(sc->dma_txd_tag, sc->dma_txd_map);
1225 			if (sc->txd_base != NULL)
1226 				bus_dmamem_free(sc->dma_txd_tag, sc->txd_base,
1227 				    sc->dma_txd_map);
1228 
1229 		}
1230 		bus_dma_tag_destroy(sc->dma_txd_tag);
1231 		sc->dma_txd_map = NULL;
1232 		sc->dma_txd_tag = NULL;
1233 		sc->txd_base = NULL;
1234 	}
1235 	if (sc->dma_txs_tag != NULL) {
1236 		if (sc->dma_txs_map != NULL) {
1237 			bus_dmamap_unload(sc->dma_txs_tag, sc->dma_txs_map);
1238 			if (sc->txs_base != NULL)
1239 				bus_dmamem_free(sc->dma_txs_tag, sc->txs_base,
1240 				    sc->dma_txs_map);
1241 
1242 		}
1243 		bus_dma_tag_destroy(sc->dma_txs_tag);
1244 		sc->dma_txs_map = NULL;
1245 		sc->dma_txs_tag = NULL;
1246 		sc->txs_base = NULL;
1247 	}
1248 	if (sc->dma_rxd_tag != NULL) {
1249 		if (sc->dma_rxd_map != NULL) {
1250 			bus_dmamap_unload(sc->dma_rxd_tag, sc->dma_rxd_map);
1251 			if (sc->rxd_base_dma != NULL)
1252 				bus_dmamem_free(sc->dma_rxd_tag,
1253 				    sc->rxd_base_dma, sc->dma_rxd_map);
1254 
1255 		}
1256 		bus_dma_tag_destroy(sc->dma_rxd_tag);
1257 		sc->dma_rxd_map = NULL;
1258 		sc->dma_rxd_tag = NULL;
1259 		sc->rxd_base_dma = NULL;
1260 	}
1261 	if (sc->dma_parent_tag != NULL) {
1262 		bus_dma_tag_destroy(sc->dma_parent_tag);
1263 		sc->dma_parent_tag = NULL;
1264 	}
1265 }
1266 
1267 static int
1268 ae_shutdown(device_t dev)
1269 {
1270 	ae_softc_t *sc;
1271 	int error;
1272 
1273 	sc = device_get_softc(dev);
1274 	KASSERT(sc != NULL, ("[ae: %d]: sc is NULL", __LINE__));
1275 
1276 	error = ae_suspend(dev);
1277 	AE_LOCK(sc);
1278 	ae_powersave_enable(sc);
1279 	AE_UNLOCK(sc);
1280 	return (error);
1281 }
1282 
1283 static void
1284 ae_powersave_disable(ae_softc_t *sc)
1285 {
1286 	uint32_t val;
1287 
1288 	AE_LOCK_ASSERT(sc);
1289 
1290 	AE_PHY_WRITE(sc, AE_PHY_DBG_ADDR, 0);
1291 	val = AE_PHY_READ(sc, AE_PHY_DBG_DATA);
1292 	if (val & AE_PHY_DBG_POWERSAVE) {
1293 		val &= ~AE_PHY_DBG_POWERSAVE;
1294 		AE_PHY_WRITE(sc, AE_PHY_DBG_DATA, val);
1295 		DELAY(1000);
1296 	}
1297 }
1298 
1299 static void
1300 ae_powersave_enable(ae_softc_t *sc)
1301 {
1302 	uint32_t val;
1303 
1304 	AE_LOCK_ASSERT(sc);
1305 
1306 	/*
1307 	 * XXX magic numbers.
1308 	 */
1309 	AE_PHY_WRITE(sc, AE_PHY_DBG_ADDR, 0);
1310 	val = AE_PHY_READ(sc, AE_PHY_DBG_DATA);
1311 	AE_PHY_WRITE(sc, AE_PHY_DBG_ADDR, val | 0x1000);
1312 	AE_PHY_WRITE(sc, AE_PHY_DBG_ADDR, 2);
1313 	AE_PHY_WRITE(sc, AE_PHY_DBG_DATA, 0x3000);
1314 	AE_PHY_WRITE(sc, AE_PHY_DBG_ADDR, 3);
1315 	AE_PHY_WRITE(sc, AE_PHY_DBG_DATA, 0);
1316 }
1317 
1318 static void
1319 ae_pm_init(ae_softc_t *sc)
1320 {
1321 	struct ifnet *ifp;
1322 	uint32_t val;
1323 	uint16_t pmstat;
1324 	struct mii_data *mii;
1325 	int pmc;
1326 
1327 	AE_LOCK_ASSERT(sc);
1328 
1329 	ifp = sc->ifp;
1330 	if ((sc->flags & AE_FLAG_PMG) == 0) {
1331 		/* Disable WOL entirely. */
1332 		AE_WRITE_4(sc, AE_WOL_REG, 0);
1333 		return;
1334 	}
1335 
1336 	/*
1337 	 * Configure WOL if enabled.
1338 	 */
1339 	if ((ifp->if_capenable & IFCAP_WOL) != 0) {
1340 		mii = device_get_softc(sc->miibus);
1341 		mii_pollstat(mii);
1342 		if ((mii->mii_media_status & IFM_AVALID) != 0 &&
1343 		    (mii->mii_media_status & IFM_ACTIVE) != 0) {
1344 			AE_WRITE_4(sc, AE_WOL_REG, AE_WOL_MAGIC | \
1345 			    AE_WOL_MAGIC_PME);
1346 
1347 			/*
1348 			 * Configure MAC.
1349 			 */
1350 			val = AE_MAC_RX_EN | AE_MAC_CLK_PHY | \
1351 			    AE_MAC_TX_CRC_EN | AE_MAC_TX_AUTOPAD | \
1352 			    ((AE_HALFBUF_DEFAULT << AE_HALFBUF_SHIFT) & \
1353 			    AE_HALFBUF_MASK) | \
1354 			    ((AE_MAC_PREAMBLE_DEFAULT << \
1355 			    AE_MAC_PREAMBLE_SHIFT) & AE_MAC_PREAMBLE_MASK) | \
1356 			    AE_MAC_BCAST_EN | AE_MAC_MCAST_EN;
1357 			if ((IFM_OPTIONS(mii->mii_media_active) & \
1358 			    IFM_FDX) != 0)
1359 				val |= AE_MAC_FULL_DUPLEX;
1360 			AE_WRITE_4(sc, AE_MAC_REG, val);
1361 
1362 		} else {	/* No link. */
1363 			AE_WRITE_4(sc, AE_WOL_REG, AE_WOL_LNKCHG | \
1364 			    AE_WOL_LNKCHG_PME);
1365 			AE_WRITE_4(sc, AE_MAC_REG, 0);
1366 		}
1367 	} else {
1368 		ae_powersave_enable(sc);
1369 	}
1370 
1371 	/*
1372 	 * PCIE hacks. Magic numbers.
1373 	 */
1374 	val = AE_READ_4(sc, AE_PCIE_PHYMISC_REG);
1375 	val |= AE_PCIE_PHYMISC_FORCE_RCV_DET;
1376 	AE_WRITE_4(sc, AE_PCIE_PHYMISC_REG, val);
1377 	val = AE_READ_4(sc, AE_PCIE_DLL_TX_CTRL_REG);
1378 	val |= AE_PCIE_DLL_TX_CTRL_SEL_NOR_CLK;
1379 	AE_WRITE_4(sc, AE_PCIE_DLL_TX_CTRL_REG, val);
1380 
1381 	/*
1382 	 * Configure PME.
1383 	 */
1384 	if (pci_find_cap(sc->dev, PCIY_PMG, &pmc) == 0) {
1385 		pmstat = pci_read_config(sc->dev, pmc + PCIR_POWER_STATUS, 2);
1386 		pmstat &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
1387 		if ((ifp->if_capenable & IFCAP_WOL) != 0)
1388 			pmstat |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
1389 		pci_write_config(sc->dev, pmc + PCIR_POWER_STATUS, pmstat, 2);
1390 	}
1391 }
1392 
1393 static int
1394 ae_suspend(device_t dev)
1395 {
1396 	ae_softc_t *sc;
1397 
1398 	sc = device_get_softc(dev);
1399 
1400 	AE_LOCK(sc);
1401 	ae_stop(sc);
1402 	ae_pm_init(sc);
1403 	AE_UNLOCK(sc);
1404 
1405 	return (0);
1406 }
1407 
1408 static int
1409 ae_resume(device_t dev)
1410 {
1411 	ae_softc_t *sc;
1412 
1413 	sc = device_get_softc(dev);
1414 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
1415 
1416 	AE_LOCK(sc);
1417 	AE_READ_4(sc, AE_WOL_REG);	/* Clear WOL status. */
1418 	if ((sc->ifp->if_flags & IFF_UP) != 0)
1419 		ae_init_locked(sc);
1420 	AE_UNLOCK(sc);
1421 
1422 	return (0);
1423 }
1424 
1425 static unsigned int
1426 ae_tx_avail_size(ae_softc_t *sc)
1427 {
1428 	unsigned int avail;
1429 
1430 	if (sc->txd_cur >= sc->txd_ack)
1431 		avail = AE_TXD_BUFSIZE_DEFAULT - (sc->txd_cur - sc->txd_ack);
1432 	else
1433 		avail = sc->txd_ack - sc->txd_cur;
1434 
1435 	return (avail);
1436 }
1437 
1438 static int
1439 ae_encap(ae_softc_t *sc, struct mbuf **m_head)
1440 {
1441 	struct mbuf *m0;
1442 	ae_txd_t *hdr;
1443 	unsigned int to_end;
1444 	uint16_t len;
1445 
1446 	AE_LOCK_ASSERT(sc);
1447 
1448 	m0 = *m_head;
1449 	len = m0->m_pkthdr.len;
1450 
1451 	if ((sc->flags & AE_FLAG_TXAVAIL) == 0 ||
1452 	    len + sizeof(ae_txd_t) + 3 > ae_tx_avail_size(sc)) {
1453 #ifdef AE_DEBUG
1454 		if_printf(sc->ifp, "No free Tx available.\n");
1455 #endif
1456 		return ENOBUFS;
1457 	}
1458 
1459 	hdr = (ae_txd_t *)(sc->txd_base + sc->txd_cur);
1460 	bzero(hdr, sizeof(*hdr));
1461 	/* Skip header size. */
1462 	sc->txd_cur = (sc->txd_cur + sizeof(ae_txd_t)) % AE_TXD_BUFSIZE_DEFAULT;
1463 	/* Space available to the end of the ring */
1464 	to_end = AE_TXD_BUFSIZE_DEFAULT - sc->txd_cur;
1465 	if (to_end >= len) {
1466 		m_copydata(m0, 0, len, (caddr_t)(sc->txd_base + sc->txd_cur));
1467 	} else {
1468 		m_copydata(m0, 0, to_end, (caddr_t)(sc->txd_base +
1469 		    sc->txd_cur));
1470 		m_copydata(m0, to_end, len - to_end, (caddr_t)sc->txd_base);
1471 	}
1472 
1473 	/*
1474 	 * Set TxD flags and parameters.
1475 	 */
1476 	if ((m0->m_flags & M_VLANTAG) != 0) {
1477 		hdr->vlan = htole16(AE_TXD_VLAN(m0->m_pkthdr.ether_vtag));
1478 		hdr->len = htole16(len | AE_TXD_INSERT_VTAG);
1479 	} else {
1480 		hdr->len = htole16(len);
1481 	}
1482 
1483 	/*
1484 	 * Set current TxD position and round up to a 4-byte boundary.
1485 	 */
1486 	sc->txd_cur = ((sc->txd_cur + len + 3) & ~3) % AE_TXD_BUFSIZE_DEFAULT;
1487 	if (sc->txd_cur == sc->txd_ack)
1488 		sc->flags &= ~AE_FLAG_TXAVAIL;
1489 #ifdef AE_DEBUG
1490 	if_printf(sc->ifp, "New txd_cur = %d.\n", sc->txd_cur);
1491 #endif
1492 
1493 	/*
1494 	 * Update TxS position and check if there are empty TxS available.
1495 	 */
1496 	sc->txs_base[sc->txs_cur].flags &= ~htole16(AE_TXS_UPDATE);
1497 	sc->txs_cur = (sc->txs_cur + 1) % AE_TXS_COUNT_DEFAULT;
1498 	if (sc->txs_cur == sc->txs_ack)
1499 		sc->flags &= ~AE_FLAG_TXAVAIL;
1500 
1501 	/*
1502 	 * Synchronize DMA memory.
1503 	 */
1504 	bus_dmamap_sync(sc->dma_txd_tag, sc->dma_txd_map, BUS_DMASYNC_PREREAD |
1505 	    BUS_DMASYNC_PREWRITE);
1506 	bus_dmamap_sync(sc->dma_txs_tag, sc->dma_txs_map,
1507 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1508 
1509 	return (0);
1510 }
1511 
1512 static void
1513 ae_start(struct ifnet *ifp)
1514 {
1515 	ae_softc_t *sc;
1516 
1517 	sc = ifp->if_softc;
1518 	AE_LOCK(sc);
1519 	ae_start_locked(ifp);
1520 	AE_UNLOCK(sc);
1521 }
1522 
1523 static void
1524 ae_start_locked(struct ifnet *ifp)
1525 {
1526 	ae_softc_t *sc;
1527 	unsigned int count;
1528 	struct mbuf *m0;
1529 	int error;
1530 
1531 	sc = ifp->if_softc;
1532 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
1533 	AE_LOCK_ASSERT(sc);
1534 
1535 #ifdef AE_DEBUG
1536 	if_printf(ifp, "Start called.\n");
1537 #endif
1538 
1539 	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1540 	    IFF_DRV_RUNNING || (sc->flags & AE_FLAG_LINK) == 0)
1541 		return;
1542 
1543 	count = 0;
1544 	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
1545 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
1546 		if (m0 == NULL)
1547 			break;	/* Nothing to do. */
1548 
1549 		error = ae_encap(sc, &m0);
1550 		if (error != 0) {
1551 			if (m0 != NULL) {
1552 				IFQ_DRV_PREPEND(&ifp->if_snd, m0);
1553 				ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1554 #ifdef AE_DEBUG
1555 				if_printf(ifp, "Setting OACTIVE.\n");
1556 #endif
1557 			}
1558 			break;
1559 		}
1560 		count++;
1561 		sc->tx_inproc++;
1562 
1563 		/* Bounce a copy of the frame to BPF. */
1564 		ETHER_BPF_MTAP(ifp, m0);
1565 
1566 		m_freem(m0);
1567 	}
1568 
1569 	if (count > 0) {	/* Something was dequeued. */
1570 		AE_WRITE_2(sc, AE_MB_TXD_IDX_REG, sc->txd_cur / 4);
1571 		sc->wd_timer = AE_TX_TIMEOUT;	/* Load watchdog. */
1572 #ifdef AE_DEBUG
1573 		if_printf(ifp, "%d packets dequeued.\n", count);
1574 		if_printf(ifp, "Tx pos now is %d.\n", sc->txd_cur);
1575 #endif
1576 	}
1577 }
1578 
1579 static void
1580 ae_link_task(void *arg, int pending)
1581 {
1582 	ae_softc_t *sc;
1583 	struct mii_data *mii;
1584 	struct ifnet *ifp;
1585 	uint32_t val;
1586 
1587 	sc = (ae_softc_t *)arg;
1588 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
1589 	AE_LOCK(sc);
1590 
1591 	ifp = sc->ifp;
1592 	mii = device_get_softc(sc->miibus);
1593 	if (mii == NULL || ifp == NULL ||
1594 	    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1595 		AE_UNLOCK(sc);	/* XXX: could happen? */
1596 		return;
1597 	}
1598 
1599 	sc->flags &= ~AE_FLAG_LINK;
1600 	if ((mii->mii_media_status & (IFM_AVALID | IFM_ACTIVE)) ==
1601 	    (IFM_AVALID | IFM_ACTIVE)) {
1602 		switch(IFM_SUBTYPE(mii->mii_media_active)) {
1603 		case IFM_10_T:
1604 		case IFM_100_TX:
1605 			sc->flags |= AE_FLAG_LINK;
1606 			break;
1607 		default:
1608 			break;
1609 		}
1610 	}
1611 
1612 	/*
1613 	 * Stop Rx/Tx MACs.
1614 	 */
1615 	ae_stop_rxmac(sc);
1616 	ae_stop_txmac(sc);
1617 
1618 	if ((sc->flags & AE_FLAG_LINK) != 0) {
1619 		ae_mac_config(sc);
1620 
1621 		/*
1622 		 * Restart DMA engines.
1623 		 */
1624 		AE_WRITE_1(sc, AE_DMAREAD_REG, AE_DMAREAD_EN);
1625 		AE_WRITE_1(sc, AE_DMAWRITE_REG, AE_DMAWRITE_EN);
1626 
1627 		/*
1628 		 * Enable Rx and Tx MACs.
1629 		 */
1630 		val = AE_READ_4(sc, AE_MAC_REG);
1631 		val |= AE_MAC_TX_EN | AE_MAC_RX_EN;
1632 		AE_WRITE_4(sc, AE_MAC_REG, val);
1633 	}
1634 	AE_UNLOCK(sc);
1635 }
1636 
1637 static void
1638 ae_stop_rxmac(ae_softc_t *sc)
1639 {
1640 	uint32_t val;
1641 	int i;
1642 
1643 	AE_LOCK_ASSERT(sc);
1644 
1645 	/*
1646 	 * Stop Rx MAC engine.
1647 	 */
1648 	val = AE_READ_4(sc, AE_MAC_REG);
1649 	if ((val & AE_MAC_RX_EN) != 0) {
1650 		val &= ~AE_MAC_RX_EN;
1651 		AE_WRITE_4(sc, AE_MAC_REG, val);
1652 	}
1653 
1654 	/*
1655 	 * Stop Rx DMA engine.
1656 	 */
1657 	if (AE_READ_1(sc, AE_DMAWRITE_REG) == AE_DMAWRITE_EN)
1658 		AE_WRITE_1(sc, AE_DMAWRITE_REG, 0);
1659 
1660 	/*
1661 	 * Wait for IDLE state.
1662 	 */
1663 	for (i = 0; i < AE_IDLE_TIMEOUT; i--) {
1664 		val = AE_READ_4(sc, AE_IDLE_REG);
1665 		if ((val & (AE_IDLE_RXMAC | AE_IDLE_DMAWRITE)) == 0)
1666 			break;
1667 		DELAY(100);
1668 	}
1669 	if (i == AE_IDLE_TIMEOUT)
1670 		device_printf(sc->dev, "timed out while stopping Rx MAC.\n");
1671 }
1672 
1673 static void
1674 ae_stop_txmac(ae_softc_t *sc)
1675 {
1676 	uint32_t val;
1677 	int i;
1678 
1679 	AE_LOCK_ASSERT(sc);
1680 
1681 	/*
1682 	 * Stop Tx MAC engine.
1683 	 */
1684 	val = AE_READ_4(sc, AE_MAC_REG);
1685 	if ((val & AE_MAC_TX_EN) != 0) {
1686 		val &= ~AE_MAC_TX_EN;
1687 		AE_WRITE_4(sc, AE_MAC_REG, val);
1688 	}
1689 
1690 	/*
1691 	 * Stop Tx DMA engine.
1692 	 */
1693 	if (AE_READ_1(sc, AE_DMAREAD_REG) == AE_DMAREAD_EN)
1694 		AE_WRITE_1(sc, AE_DMAREAD_REG, 0);
1695 
1696 	/*
1697 	 * Wait for IDLE state.
1698 	 */
1699 	for (i = 0; i < AE_IDLE_TIMEOUT; i--) {
1700 		val = AE_READ_4(sc, AE_IDLE_REG);
1701 		if ((val & (AE_IDLE_TXMAC | AE_IDLE_DMAREAD)) == 0)
1702 			break;
1703 		DELAY(100);
1704 	}
1705 	if (i == AE_IDLE_TIMEOUT)
1706 		device_printf(sc->dev, "timed out while stopping Tx MAC.\n");
1707 }
1708 
1709 static void
1710 ae_mac_config(ae_softc_t *sc)
1711 {
1712 	struct mii_data *mii;
1713 	uint32_t val;
1714 
1715 	AE_LOCK_ASSERT(sc);
1716 
1717 	mii = device_get_softc(sc->miibus);
1718 	val = AE_READ_4(sc, AE_MAC_REG);
1719 	val &= ~AE_MAC_FULL_DUPLEX;
1720 	/* XXX disable AE_MAC_TX_FLOW_EN? */
1721 
1722 	if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0)
1723 		val |= AE_MAC_FULL_DUPLEX;
1724 
1725 	AE_WRITE_4(sc, AE_MAC_REG, val);
1726 }
1727 
1728 static int
1729 ae_intr(void *arg)
1730 {
1731 	ae_softc_t *sc;
1732 	uint32_t val;
1733 
1734 	sc = (ae_softc_t *)arg;
1735 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL", __LINE__));
1736 
1737 	val = AE_READ_4(sc, AE_ISR_REG);
1738 	if (val == 0 || (val & AE_IMR_DEFAULT) == 0)
1739 		return (FILTER_STRAY);
1740 
1741 	/* Disable interrupts. */
1742 	AE_WRITE_4(sc, AE_ISR_REG, AE_ISR_DISABLE);
1743 
1744 	/* Schedule interrupt processing. */
1745 	taskqueue_enqueue(sc->tq, &sc->int_task);
1746 
1747 	return (FILTER_HANDLED);
1748 }
1749 
1750 static void
1751 ae_int_task(void *arg, int pending)
1752 {
1753 	ae_softc_t *sc;
1754 	struct ifnet *ifp;
1755 	uint32_t val;
1756 
1757 	sc = (ae_softc_t *)arg;
1758 
1759 	AE_LOCK(sc);
1760 
1761 	ifp = sc->ifp;
1762 
1763 	val = AE_READ_4(sc, AE_ISR_REG);	/* Read interrupt status. */
1764 
1765 	/*
1766 	 * Clear interrupts and disable them.
1767 	 */
1768 	AE_WRITE_4(sc, AE_ISR_REG, val | AE_ISR_DISABLE);
1769 
1770 #ifdef AE_DEBUG
1771 	if_printf(ifp, "Interrupt received: 0x%08x\n", val);
1772 #endif
1773 
1774 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
1775 		if ((val & (AE_ISR_DMAR_TIMEOUT | AE_ISR_DMAW_TIMEOUT |
1776 		    AE_ISR_PHY_LINKDOWN)) != 0) {
1777 			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1778 			ae_init_locked(sc);
1779 			AE_UNLOCK(sc);
1780 			return;
1781 		}
1782 		if ((val & AE_ISR_TX_EVENT) != 0)
1783 			ae_tx_intr(sc);
1784 		if ((val & AE_ISR_RX_EVENT) != 0)
1785 			ae_rx_intr(sc);
1786 	}
1787 
1788 	/*
1789 	 * Re-enable interrupts.
1790 	 */
1791 	AE_WRITE_4(sc, AE_ISR_REG, 0);
1792 
1793 	AE_UNLOCK(sc);
1794 }
1795 
1796 static void
1797 ae_tx_intr(ae_softc_t *sc)
1798 {
1799 	struct ifnet *ifp;
1800 	ae_txd_t *txd;
1801 	ae_txs_t *txs;
1802 	uint16_t flags;
1803 
1804 	AE_LOCK_ASSERT(sc);
1805 
1806 	ifp = sc->ifp;
1807 
1808 #ifdef AE_DEBUG
1809 	if_printf(ifp, "Tx interrupt occuried.\n");
1810 #endif
1811 
1812 	/*
1813 	 * Syncronize DMA buffers.
1814 	 */
1815 	bus_dmamap_sync(sc->dma_txd_tag, sc->dma_txd_map,
1816 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1817 	bus_dmamap_sync(sc->dma_txs_tag, sc->dma_txs_map,
1818 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1819 
1820 	for (;;) {
1821 		txs = sc->txs_base + sc->txs_ack;
1822 		flags = le16toh(txs->flags);
1823 		if ((flags & AE_TXS_UPDATE) == 0)
1824 			break;
1825 		txs->flags = htole16(flags & ~AE_TXS_UPDATE);
1826 		/* Update stats. */
1827 		ae_update_stats_tx(flags, &sc->stats);
1828 
1829 		/*
1830 		 * Update TxS position.
1831 		 */
1832 		sc->txs_ack = (sc->txs_ack + 1) % AE_TXS_COUNT_DEFAULT;
1833 		sc->flags |= AE_FLAG_TXAVAIL;
1834 
1835 		txd = (ae_txd_t *)(sc->txd_base + sc->txd_ack);
1836 		if (txs->len != txd->len)
1837 			device_printf(sc->dev, "Size mismatch: TxS:%d TxD:%d\n",
1838 			    le16toh(txs->len), le16toh(txd->len));
1839 
1840 		/*
1841 		 * Move txd ack and align on 4-byte boundary.
1842 		 */
1843 		sc->txd_ack = ((sc->txd_ack + le16toh(txd->len) +
1844 		    sizeof(ae_txs_t) + 3) & ~3) % AE_TXD_BUFSIZE_DEFAULT;
1845 
1846 		if ((flags & AE_TXS_SUCCESS) != 0)
1847 			ifp->if_opackets++;
1848 		else
1849 			ifp->if_oerrors++;
1850 
1851 		sc->tx_inproc--;
1852 
1853 		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1854 	}
1855 
1856 	if (sc->tx_inproc < 0) {
1857 		if_printf(ifp, "Received stray Tx interrupt(s).\n");
1858 		sc->tx_inproc = 0;
1859 	}
1860 
1861 	if (sc->tx_inproc == 0)
1862 		sc->wd_timer = 0;	/* Unarm watchdog. */
1863 
1864 	if ((sc->flags & AE_FLAG_TXAVAIL) != 0) {
1865 		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1866 			ae_start_locked(ifp);
1867 	}
1868 
1869 	/*
1870 	 * Syncronize DMA buffers.
1871 	 */
1872 	bus_dmamap_sync(sc->dma_txd_tag, sc->dma_txd_map,
1873 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1874 	bus_dmamap_sync(sc->dma_txs_tag, sc->dma_txs_map,
1875 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1876 }
1877 
1878 static int
1879 ae_rxeof(ae_softc_t *sc, ae_rxd_t *rxd)
1880 {
1881 	struct ifnet *ifp;
1882 	struct mbuf *m;
1883 	unsigned int size;
1884 	uint16_t flags;
1885 
1886 	AE_LOCK_ASSERT(sc);
1887 
1888 	ifp = sc->ifp;
1889 	flags = le16toh(rxd->flags);
1890 
1891 #ifdef AE_DEBUG
1892 	if_printf(ifp, "Rx interrupt occuried.\n");
1893 #endif
1894 	size = le16toh(rxd->len) - ETHER_CRC_LEN;
1895 	if (size < (ETHER_MIN_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN)) {
1896 		if_printf(ifp, "Runt frame received.");
1897 		return (EIO);
1898 	}
1899 
1900 	m = m_devget(&rxd->data[0], size, ETHER_ALIGN, ifp, NULL);
1901 	if (m == NULL)
1902 		return (ENOBUFS);
1903 
1904 	if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
1905 	    (flags & AE_RXD_HAS_VLAN) != 0) {
1906 		m->m_pkthdr.ether_vtag = AE_RXD_VLAN(le16toh(rxd->vlan));
1907 		m->m_flags |= M_VLANTAG;
1908 	}
1909 
1910 	/*
1911 	 * Pass it through.
1912 	 */
1913 	AE_UNLOCK(sc);
1914 	(*ifp->if_input)(ifp, m);
1915 	AE_LOCK(sc);
1916 
1917 	return (0);
1918 }
1919 
1920 static void
1921 ae_rx_intr(ae_softc_t *sc)
1922 {
1923 	ae_rxd_t *rxd;
1924 	struct ifnet *ifp;
1925 	uint16_t flags;
1926 	int error;
1927 
1928 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL!", __LINE__));
1929 
1930 	AE_LOCK_ASSERT(sc);
1931 
1932 	ifp = sc->ifp;
1933 
1934 	/*
1935 	 * Syncronize DMA buffers.
1936 	 */
1937 	bus_dmamap_sync(sc->dma_rxd_tag, sc->dma_rxd_map,
1938 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1939 
1940 	for (;;) {
1941 		rxd = (ae_rxd_t *)(sc->rxd_base + sc->rxd_cur);
1942 		flags = le16toh(rxd->flags);
1943 		if ((flags & AE_RXD_UPDATE) == 0)
1944 			break;
1945 		rxd->flags = htole16(flags & ~AE_RXD_UPDATE);
1946 		/* Update stats. */
1947 		ae_update_stats_rx(flags, &sc->stats);
1948 
1949 		/*
1950 		 * Update position index.
1951 		 */
1952 		sc->rxd_cur = (sc->rxd_cur + 1) % AE_RXD_COUNT_DEFAULT;
1953 
1954 		if ((flags & AE_RXD_SUCCESS) == 0) {
1955 			ifp->if_ierrors++;
1956 			continue;
1957 		}
1958 		error = ae_rxeof(sc, rxd);
1959 		if (error != 0) {
1960 			ifp->if_ierrors++;
1961 			continue;
1962 		} else {
1963 			ifp->if_ipackets++;
1964 		}
1965 	}
1966 
1967 	/*
1968 	 * Update Rx index.
1969 	 */
1970 	AE_WRITE_2(sc, AE_MB_RXD_IDX_REG, sc->rxd_cur);
1971 }
1972 
1973 static void
1974 ae_watchdog(ae_softc_t *sc)
1975 {
1976 	struct ifnet *ifp;
1977 
1978 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL!", __LINE__));
1979 	AE_LOCK_ASSERT(sc);
1980 	ifp = sc->ifp;
1981 
1982 	if (sc->wd_timer == 0 || --sc->wd_timer != 0)
1983 		return;		/* Noting to do. */
1984 
1985 	if ((sc->flags & AE_FLAG_LINK) == 0)
1986 		if_printf(ifp, "watchdog timeout (missed link).\n");
1987 	else
1988 		if_printf(ifp, "watchdog timeout - resetting.\n");
1989 
1990 	ifp->if_oerrors++;
1991 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1992 	ae_init_locked(sc);
1993 	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1994 		ae_start_locked(ifp);
1995 }
1996 
1997 static void
1998 ae_tick(void *arg)
1999 {
2000 	ae_softc_t *sc;
2001 	struct mii_data *mii;
2002 
2003 	sc = (ae_softc_t *)arg;
2004 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL!", __LINE__));
2005 	AE_LOCK_ASSERT(sc);
2006 
2007 	mii = device_get_softc(sc->miibus);
2008 	mii_tick(mii);
2009 	ae_watchdog(sc);	/* Watchdog check. */
2010 	callout_reset(&sc->tick_ch, hz, ae_tick, sc);
2011 }
2012 
2013 static void
2014 ae_rxvlan(ae_softc_t *sc)
2015 {
2016 	struct ifnet *ifp;
2017 	uint32_t val;
2018 
2019 	AE_LOCK_ASSERT(sc);
2020 	ifp = sc->ifp;
2021 	val = AE_READ_4(sc, AE_MAC_REG);
2022 	val &= ~AE_MAC_RMVLAN_EN;
2023 	if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0)
2024 		val |= AE_MAC_RMVLAN_EN;
2025 	AE_WRITE_4(sc, AE_MAC_REG, val);
2026 }
2027 
2028 static void
2029 ae_rxfilter(ae_softc_t *sc)
2030 {
2031 	struct ifnet *ifp;
2032 	struct ifmultiaddr *ifma;
2033 	uint32_t crc;
2034 	uint32_t mchash[2];
2035 	uint32_t rxcfg;
2036 
2037 	KASSERT(sc != NULL, ("[ae, %d]: sc is NULL!", __LINE__));
2038 
2039 	AE_LOCK_ASSERT(sc);
2040 
2041 	ifp = sc->ifp;
2042 
2043 	rxcfg = AE_READ_4(sc, AE_MAC_REG);
2044 	rxcfg &= ~(AE_MAC_MCAST_EN | AE_MAC_BCAST_EN | AE_MAC_PROMISC_EN);
2045 
2046 	if ((ifp->if_flags & IFF_BROADCAST) != 0)
2047 		rxcfg |= AE_MAC_BCAST_EN;
2048 	if ((ifp->if_flags & IFF_PROMISC) != 0)
2049 		rxcfg |= AE_MAC_PROMISC_EN;
2050 	if ((ifp->if_flags & IFF_ALLMULTI) != 0)
2051 		rxcfg |= AE_MAC_MCAST_EN;
2052 
2053 	/*
2054 	 * Wipe old settings.
2055 	 */
2056 	AE_WRITE_4(sc, AE_REG_MHT0, 0);
2057 	AE_WRITE_4(sc, AE_REG_MHT1, 0);
2058 	if ((ifp->if_flags & (IFF_PROMISC | IFF_ALLMULTI)) != 0) {
2059 		AE_WRITE_4(sc, AE_REG_MHT0, 0xffffffff);
2060 		AE_WRITE_4(sc, AE_REG_MHT1, 0xffffffff);
2061 		AE_WRITE_4(sc, AE_MAC_REG, rxcfg);
2062 		return;
2063 	}
2064 
2065 	/*
2066 	 * Load multicast tables.
2067 	 */
2068 	bzero(mchash, sizeof(mchash));
2069 	if_maddr_rlock(ifp);
2070 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2071 		if (ifma->ifma_addr->sa_family != AF_LINK)
2072 			continue;
2073 		crc = ether_crc32_be(LLADDR((struct sockaddr_dl *)
2074 			ifma->ifma_addr), ETHER_ADDR_LEN);
2075 		mchash[crc >> 31] |= 1 << ((crc >> 26) & 0x1f);
2076 	}
2077 	if_maddr_runlock(ifp);
2078 	AE_WRITE_4(sc, AE_REG_MHT0, mchash[0]);
2079 	AE_WRITE_4(sc, AE_REG_MHT1, mchash[1]);
2080 	AE_WRITE_4(sc, AE_MAC_REG, rxcfg);
2081 }
2082 
2083 static int
2084 ae_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
2085 {
2086 	struct ae_softc *sc;
2087 	struct ifreq *ifr;
2088 	struct mii_data *mii;
2089 	int error, mask;
2090 
2091 	sc = ifp->if_softc;
2092 	ifr = (struct ifreq *)data;
2093 	error = 0;
2094 
2095 	switch (cmd) {
2096 	case SIOCSIFMTU:
2097 		if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > ETHERMTU)
2098 			error = EINVAL;
2099 		else if (ifp->if_mtu != ifr->ifr_mtu) {
2100 			AE_LOCK(sc);
2101 			ifp->if_mtu = ifr->ifr_mtu;
2102 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2103 				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2104 				ae_init_locked(sc);
2105 			}
2106 			AE_UNLOCK(sc);
2107 		}
2108 		break;
2109 	case SIOCSIFFLAGS:
2110 		AE_LOCK(sc);
2111 		if ((ifp->if_flags & IFF_UP) != 0) {
2112 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2113 				if (((ifp->if_flags ^ sc->if_flags)
2114 				    & (IFF_PROMISC | IFF_ALLMULTI)) != 0)
2115 					ae_rxfilter(sc);
2116 			} else {
2117 				if ((sc->flags & AE_FLAG_DETACH) == 0)
2118 					ae_init_locked(sc);
2119 			}
2120 		} else {
2121 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
2122 				ae_stop(sc);
2123 		}
2124 		sc->if_flags = ifp->if_flags;
2125 		AE_UNLOCK(sc);
2126 		break;
2127 	case SIOCADDMULTI:
2128 	case SIOCDELMULTI:
2129 		AE_LOCK(sc);
2130 		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
2131 			ae_rxfilter(sc);
2132 		AE_UNLOCK(sc);
2133 		break;
2134 	case SIOCSIFMEDIA:
2135 	case SIOCGIFMEDIA:
2136 		mii = device_get_softc(sc->miibus);
2137 		error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
2138 		break;
2139 	case SIOCSIFCAP:
2140 		AE_LOCK(sc);
2141 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2142 		if ((mask & IFCAP_VLAN_HWTAGGING) != 0 &&
2143 		    (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) != 0) {
2144 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2145 			ae_rxvlan(sc);
2146 		}
2147 		VLAN_CAPABILITIES(ifp);
2148 		AE_UNLOCK(sc);
2149 		break;
2150 	default:
2151 		error = ether_ioctl(ifp, cmd, data);
2152 		break;
2153 	}
2154 	return (error);
2155 }
2156 
2157 static void
2158 ae_stop(ae_softc_t *sc)
2159 {
2160 	struct ifnet *ifp;
2161 	int i;
2162 
2163 	AE_LOCK_ASSERT(sc);
2164 
2165 	ifp = sc->ifp;
2166 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2167 	sc->flags &= ~AE_FLAG_LINK;
2168 	sc->wd_timer = 0;	/* Cancel watchdog. */
2169 	callout_stop(&sc->tick_ch);
2170 
2171 	/*
2172 	 * Clear and disable interrupts.
2173 	 */
2174 	AE_WRITE_4(sc, AE_IMR_REG, 0);
2175 	AE_WRITE_4(sc, AE_ISR_REG, 0xffffffff);
2176 
2177 	/*
2178 	 * Stop Rx/Tx MACs.
2179 	 */
2180 	ae_stop_txmac(sc);
2181 	ae_stop_rxmac(sc);
2182 
2183 	/*
2184 	 * Stop DMA engines.
2185 	 */
2186 	AE_WRITE_1(sc, AE_DMAREAD_REG, ~AE_DMAREAD_EN);
2187 	AE_WRITE_1(sc, AE_DMAWRITE_REG, ~AE_DMAWRITE_EN);
2188 
2189 	/*
2190 	 * Wait for everything to enter idle state.
2191 	 */
2192 	for (i = 0; i < AE_IDLE_TIMEOUT; i++) {
2193 		if (AE_READ_4(sc, AE_IDLE_REG) == 0)
2194 			break;
2195 		DELAY(100);
2196 	}
2197 	if (i == AE_IDLE_TIMEOUT)
2198 		device_printf(sc->dev, "could not enter idle state in stop.\n");
2199 }
2200 
2201 static void
2202 ae_update_stats_tx(uint16_t flags, ae_stats_t *stats)
2203 {
2204 
2205 	if ((flags & AE_TXS_BCAST) != 0)
2206 		stats->tx_bcast++;
2207 	if ((flags & AE_TXS_MCAST) != 0)
2208 		stats->tx_mcast++;
2209 	if ((flags & AE_TXS_PAUSE) != 0)
2210 		stats->tx_pause++;
2211 	if ((flags & AE_TXS_CTRL) != 0)
2212 		stats->tx_ctrl++;
2213 	if ((flags & AE_TXS_DEFER) != 0)
2214 		stats->tx_defer++;
2215 	if ((flags & AE_TXS_EXCDEFER) != 0)
2216 		stats->tx_excdefer++;
2217 	if ((flags & AE_TXS_SINGLECOL) != 0)
2218 		stats->tx_singlecol++;
2219 	if ((flags & AE_TXS_MULTICOL) != 0)
2220 		stats->tx_multicol++;
2221 	if ((flags & AE_TXS_LATECOL) != 0)
2222 		stats->tx_latecol++;
2223 	if ((flags & AE_TXS_ABORTCOL) != 0)
2224 		stats->tx_abortcol++;
2225 	if ((flags & AE_TXS_UNDERRUN) != 0)
2226 		stats->tx_underrun++;
2227 }
2228 
2229 static void
2230 ae_update_stats_rx(uint16_t flags, ae_stats_t *stats)
2231 {
2232 
2233 	if ((flags & AE_RXD_BCAST) != 0)
2234 		stats->rx_bcast++;
2235 	if ((flags & AE_RXD_MCAST) != 0)
2236 		stats->rx_mcast++;
2237 	if ((flags & AE_RXD_PAUSE) != 0)
2238 		stats->rx_pause++;
2239 	if ((flags & AE_RXD_CTRL) != 0)
2240 		stats->rx_ctrl++;
2241 	if ((flags & AE_RXD_CRCERR) != 0)
2242 		stats->rx_crcerr++;
2243 	if ((flags & AE_RXD_CODEERR) != 0)
2244 		stats->rx_codeerr++;
2245 	if ((flags & AE_RXD_RUNT) != 0)
2246 		stats->rx_runt++;
2247 	if ((flags & AE_RXD_FRAG) != 0)
2248 		stats->rx_frag++;
2249 	if ((flags & AE_RXD_TRUNC) != 0)
2250 		stats->rx_trunc++;
2251 	if ((flags & AE_RXD_ALIGN) != 0)
2252 		stats->rx_align++;
2253 }
2254