xref: /freebsd/sys/dev/et/if_et.c (revision cc3f4b99653c34ae64f8a1fddea370abefef680e)
1 /*-
2  * Copyright (c) 2007 Sepherosa Ziehau.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Sepherosa Ziehau <sepherosa@gmail.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * $DragonFly: src/sys/dev/netif/et/if_et.c,v 1.10 2008/05/18 07:47:14 sephe Exp $
35  */
36 
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/endian.h>
43 #include <sys/kernel.h>
44 #include <sys/bus.h>
45 #include <sys/malloc.h>
46 #include <sys/mbuf.h>
47 #include <sys/proc.h>
48 #include <sys/rman.h>
49 #include <sys/module.h>
50 #include <sys/socket.h>
51 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 
54 #include <net/ethernet.h>
55 #include <net/if.h>
56 #include <net/if_var.h>
57 #include <net/if_dl.h>
58 #include <net/if_types.h>
59 #include <net/bpf.h>
60 #include <net/if_arp.h>
61 #include <net/if_media.h>
62 #include <net/if_vlan_var.h>
63 
64 #include <machine/bus.h>
65 
66 #include <dev/mii/mii.h>
67 #include <dev/mii/miivar.h>
68 
69 #include <dev/pci/pcireg.h>
70 #include <dev/pci/pcivar.h>
71 
72 #include <dev/et/if_etreg.h>
73 #include <dev/et/if_etvar.h>
74 
75 #include "miibus_if.h"
76 
77 MODULE_DEPEND(et, pci, 1, 1, 1);
78 MODULE_DEPEND(et, ether, 1, 1, 1);
79 MODULE_DEPEND(et, miibus, 1, 1, 1);
80 
81 /* Tunables. */
82 static int msi_disable = 0;
83 TUNABLE_INT("hw.et.msi_disable", &msi_disable);
84 
85 #define	ET_CSUM_FEATURES	(CSUM_IP | CSUM_TCP | CSUM_UDP)
86 
87 static int	et_probe(device_t);
88 static int	et_attach(device_t);
89 static int	et_detach(device_t);
90 static int	et_shutdown(device_t);
91 static int	et_suspend(device_t);
92 static int	et_resume(device_t);
93 
94 static int	et_miibus_readreg(device_t, int, int);
95 static int	et_miibus_writereg(device_t, int, int, int);
96 static void	et_miibus_statchg(device_t);
97 
98 static void	et_init_locked(struct et_softc *);
99 static void	et_init(void *);
100 static int	et_ioctl(struct ifnet *, u_long, caddr_t);
101 static void	et_start_locked(struct ifnet *);
102 static void	et_start(struct ifnet *);
103 static int	et_watchdog(struct et_softc *);
104 static int	et_ifmedia_upd_locked(struct ifnet *);
105 static int	et_ifmedia_upd(struct ifnet *);
106 static void	et_ifmedia_sts(struct ifnet *, struct ifmediareq *);
107 
108 static void	et_add_sysctls(struct et_softc *);
109 static int	et_sysctl_rx_intr_npkts(SYSCTL_HANDLER_ARGS);
110 static int	et_sysctl_rx_intr_delay(SYSCTL_HANDLER_ARGS);
111 
112 static void	et_intr(void *);
113 static void	et_rxeof(struct et_softc *);
114 static void	et_txeof(struct et_softc *);
115 
116 static int	et_dma_alloc(struct et_softc *);
117 static void	et_dma_free(struct et_softc *);
118 static void	et_dma_map_addr(void *, bus_dma_segment_t *, int, int);
119 static int	et_dma_ring_alloc(struct et_softc *, bus_size_t, bus_size_t,
120 		    bus_dma_tag_t *, uint8_t **, bus_dmamap_t *, bus_addr_t *,
121 		    const char *);
122 static void	et_dma_ring_free(struct et_softc *, bus_dma_tag_t *, uint8_t **,
123 		    bus_dmamap_t *);
124 static void	et_init_tx_ring(struct et_softc *);
125 static int	et_init_rx_ring(struct et_softc *);
126 static void	et_free_tx_ring(struct et_softc *);
127 static void	et_free_rx_ring(struct et_softc *);
128 static int	et_encap(struct et_softc *, struct mbuf **);
129 static int	et_newbuf_cluster(struct et_rxbuf_data *, int);
130 static int	et_newbuf_hdr(struct et_rxbuf_data *, int);
131 static void	et_rxbuf_discard(struct et_rxbuf_data *, int);
132 
133 static void	et_stop(struct et_softc *);
134 static int	et_chip_init(struct et_softc *);
135 static void	et_chip_attach(struct et_softc *);
136 static void	et_init_mac(struct et_softc *);
137 static void	et_init_rxmac(struct et_softc *);
138 static void	et_init_txmac(struct et_softc *);
139 static int	et_init_rxdma(struct et_softc *);
140 static int	et_init_txdma(struct et_softc *);
141 static int	et_start_rxdma(struct et_softc *);
142 static int	et_start_txdma(struct et_softc *);
143 static int	et_stop_rxdma(struct et_softc *);
144 static int	et_stop_txdma(struct et_softc *);
145 static void	et_reset(struct et_softc *);
146 static int	et_bus_config(struct et_softc *);
147 static void	et_get_eaddr(device_t, uint8_t[]);
148 static void	et_setmulti(struct et_softc *);
149 static void	et_tick(void *);
150 static void	et_stats_update(struct et_softc *);
151 
152 static const struct et_dev {
153 	uint16_t	vid;
154 	uint16_t	did;
155 	const char	*desc;
156 } et_devices[] = {
157 	{ PCI_VENDOR_LUCENT, PCI_PRODUCT_LUCENT_ET1310,
158 	  "Agere ET1310 Gigabit Ethernet" },
159 	{ PCI_VENDOR_LUCENT, PCI_PRODUCT_LUCENT_ET1310_FAST,
160 	  "Agere ET1310 Fast Ethernet" },
161 	{ 0, 0, NULL }
162 };
163 
164 static device_method_t et_methods[] = {
165 	DEVMETHOD(device_probe,		et_probe),
166 	DEVMETHOD(device_attach,	et_attach),
167 	DEVMETHOD(device_detach,	et_detach),
168 	DEVMETHOD(device_shutdown,	et_shutdown),
169 	DEVMETHOD(device_suspend,	et_suspend),
170 	DEVMETHOD(device_resume,	et_resume),
171 
172 	DEVMETHOD(miibus_readreg,	et_miibus_readreg),
173 	DEVMETHOD(miibus_writereg,	et_miibus_writereg),
174 	DEVMETHOD(miibus_statchg,	et_miibus_statchg),
175 
176 	DEVMETHOD_END
177 };
178 
179 static driver_t et_driver = {
180 	"et",
181 	et_methods,
182 	sizeof(struct et_softc)
183 };
184 
185 static devclass_t et_devclass;
186 
187 DRIVER_MODULE(et, pci, et_driver, et_devclass, 0, 0);
188 DRIVER_MODULE(miibus, et, miibus_driver, miibus_devclass, 0, 0);
189 
190 static int	et_rx_intr_npkts = 32;
191 static int	et_rx_intr_delay = 20;		/* x10 usec */
192 static int	et_tx_intr_nsegs = 126;
193 static uint32_t	et_timer = 1000 * 1000 * 1000;	/* nanosec */
194 
195 TUNABLE_INT("hw.et.timer", &et_timer);
196 TUNABLE_INT("hw.et.rx_intr_npkts", &et_rx_intr_npkts);
197 TUNABLE_INT("hw.et.rx_intr_delay", &et_rx_intr_delay);
198 TUNABLE_INT("hw.et.tx_intr_nsegs", &et_tx_intr_nsegs);
199 
200 static int
201 et_probe(device_t dev)
202 {
203 	const struct et_dev *d;
204 	uint16_t did, vid;
205 
206 	vid = pci_get_vendor(dev);
207 	did = pci_get_device(dev);
208 
209 	for (d = et_devices; d->desc != NULL; ++d) {
210 		if (vid == d->vid && did == d->did) {
211 			device_set_desc(dev, d->desc);
212 			return (BUS_PROBE_DEFAULT);
213 		}
214 	}
215 	return (ENXIO);
216 }
217 
218 static int
219 et_attach(device_t dev)
220 {
221 	struct et_softc *sc;
222 	struct ifnet *ifp;
223 	uint8_t eaddr[ETHER_ADDR_LEN];
224 	uint32_t pmcfg;
225 	int cap, error, msic;
226 
227 	sc = device_get_softc(dev);
228 	sc->dev = dev;
229 	mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
230 	    MTX_DEF);
231 	callout_init_mtx(&sc->sc_tick, &sc->sc_mtx, 0);
232 
233 	ifp = sc->ifp = if_alloc(IFT_ETHER);
234 	if (ifp == NULL) {
235 		device_printf(dev, "can not if_alloc()\n");
236 		error = ENOSPC;
237 		goto fail;
238 	}
239 
240 	/*
241 	 * Initialize tunables
242 	 */
243 	sc->sc_rx_intr_npkts = et_rx_intr_npkts;
244 	sc->sc_rx_intr_delay = et_rx_intr_delay;
245 	sc->sc_tx_intr_nsegs = et_tx_intr_nsegs;
246 	sc->sc_timer = et_timer;
247 
248 	/* Enable bus mastering */
249 	pci_enable_busmaster(dev);
250 
251 	/*
252 	 * Allocate IO memory
253 	 */
254 	sc->sc_mem_rid = PCIR_BAR(0);
255 	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
256 	    &sc->sc_mem_rid, RF_ACTIVE);
257 	if (sc->sc_mem_res == NULL) {
258 		device_printf(dev, "can't allocate IO memory\n");
259 		return (ENXIO);
260 	}
261 
262 	msic = 0;
263 	if (pci_find_cap(dev, PCIY_EXPRESS, &cap) == 0) {
264 		sc->sc_expcap = cap;
265 		sc->sc_flags |= ET_FLAG_PCIE;
266 		msic = pci_msi_count(dev);
267 		if (bootverbose)
268 			device_printf(dev, "MSI count: %d\n", msic);
269 	}
270 	if (msic > 0 && msi_disable == 0) {
271 		msic = 1;
272 		if (pci_alloc_msi(dev, &msic) == 0) {
273 			if (msic == 1) {
274 				device_printf(dev, "Using %d MSI message\n",
275 				    msic);
276 				sc->sc_flags |= ET_FLAG_MSI;
277 			} else
278 				pci_release_msi(dev);
279 		}
280 	}
281 
282 	/*
283 	 * Allocate IRQ
284 	 */
285 	if ((sc->sc_flags & ET_FLAG_MSI) == 0) {
286 		sc->sc_irq_rid = 0;
287 		sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
288 		    &sc->sc_irq_rid, RF_SHAREABLE | RF_ACTIVE);
289 	} else {
290 		sc->sc_irq_rid = 1;
291 		sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
292 		    &sc->sc_irq_rid, RF_ACTIVE);
293 	}
294 	if (sc->sc_irq_res == NULL) {
295 		device_printf(dev, "can't allocate irq\n");
296 		error = ENXIO;
297 		goto fail;
298 	}
299 
300 	if (pci_get_device(dev) == PCI_PRODUCT_LUCENT_ET1310_FAST)
301 		sc->sc_flags |= ET_FLAG_FASTETHER;
302 
303 	error = et_bus_config(sc);
304 	if (error)
305 		goto fail;
306 
307 	et_get_eaddr(dev, eaddr);
308 
309 	/* Take PHY out of COMA and enable clocks. */
310 	pmcfg = ET_PM_SYSCLK_GATE | ET_PM_TXCLK_GATE | ET_PM_RXCLK_GATE;
311 	if ((sc->sc_flags & ET_FLAG_FASTETHER) == 0)
312 		pmcfg |= EM_PM_GIGEPHY_ENB;
313 	CSR_WRITE_4(sc, ET_PM, pmcfg);
314 
315 	et_reset(sc);
316 
317 	error = et_dma_alloc(sc);
318 	if (error)
319 		goto fail;
320 
321 	ifp->if_softc = sc;
322 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
323 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
324 	ifp->if_init = et_init;
325 	ifp->if_ioctl = et_ioctl;
326 	ifp->if_start = et_start;
327 	ifp->if_capabilities = IFCAP_TXCSUM | IFCAP_VLAN_MTU;
328 	ifp->if_capenable = ifp->if_capabilities;
329 	ifp->if_snd.ifq_drv_maxlen = ET_TX_NDESC - 1;
330 	IFQ_SET_MAXLEN(&ifp->if_snd, ET_TX_NDESC - 1);
331 	IFQ_SET_READY(&ifp->if_snd);
332 
333 	et_chip_attach(sc);
334 
335 	error = mii_attach(dev, &sc->sc_miibus, ifp, et_ifmedia_upd,
336 	    et_ifmedia_sts, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY,
337 	    MIIF_DOPAUSE);
338 	if (error) {
339 		device_printf(dev, "attaching PHYs failed\n");
340 		goto fail;
341 	}
342 
343 	ether_ifattach(ifp, eaddr);
344 
345 	/* Tell the upper layer(s) we support long frames. */
346 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
347 
348 	error = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_NET | INTR_MPSAFE,
349 	    NULL, et_intr, sc, &sc->sc_irq_handle);
350 	if (error) {
351 		ether_ifdetach(ifp);
352 		device_printf(dev, "can't setup intr\n");
353 		goto fail;
354 	}
355 
356 	et_add_sysctls(sc);
357 
358 	return (0);
359 fail:
360 	et_detach(dev);
361 	return (error);
362 }
363 
364 static int
365 et_detach(device_t dev)
366 {
367 	struct et_softc *sc;
368 
369 	sc = device_get_softc(dev);
370 	if (device_is_attached(dev)) {
371 		ether_ifdetach(sc->ifp);
372 		ET_LOCK(sc);
373 		et_stop(sc);
374 		ET_UNLOCK(sc);
375 		callout_drain(&sc->sc_tick);
376 	}
377 
378 	if (sc->sc_miibus != NULL)
379 		device_delete_child(dev, sc->sc_miibus);
380 	bus_generic_detach(dev);
381 
382 	if (sc->sc_irq_handle != NULL)
383 		bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_irq_handle);
384 	if (sc->sc_irq_res != NULL)
385 		bus_release_resource(dev, SYS_RES_IRQ,
386 		    rman_get_rid(sc->sc_irq_res), sc->sc_irq_res);
387 	if ((sc->sc_flags & ET_FLAG_MSI) != 0)
388 		pci_release_msi(dev);
389 	if (sc->sc_mem_res != NULL)
390 		bus_release_resource(dev, SYS_RES_MEMORY,
391 		    rman_get_rid(sc->sc_mem_res), sc->sc_mem_res);
392 
393 	if (sc->ifp != NULL)
394 		if_free(sc->ifp);
395 
396 	et_dma_free(sc);
397 
398 	mtx_destroy(&sc->sc_mtx);
399 
400 	return (0);
401 }
402 
403 static int
404 et_shutdown(device_t dev)
405 {
406 	struct et_softc *sc;
407 
408 	sc = device_get_softc(dev);
409 	ET_LOCK(sc);
410 	et_stop(sc);
411 	ET_UNLOCK(sc);
412 	return (0);
413 }
414 
415 static int
416 et_miibus_readreg(device_t dev, int phy, int reg)
417 {
418 	struct et_softc *sc;
419 	uint32_t val;
420 	int i, ret;
421 
422 	sc = device_get_softc(dev);
423 	/* Stop any pending operations */
424 	CSR_WRITE_4(sc, ET_MII_CMD, 0);
425 
426 	val = (phy << ET_MII_ADDR_PHY_SHIFT) & ET_MII_ADDR_PHY_MASK;
427 	val |= (reg << ET_MII_ADDR_REG_SHIFT) & ET_MII_ADDR_REG_MASK;
428 	CSR_WRITE_4(sc, ET_MII_ADDR, val);
429 
430 	/* Start reading */
431 	CSR_WRITE_4(sc, ET_MII_CMD, ET_MII_CMD_READ);
432 
433 #define NRETRY	50
434 
435 	for (i = 0; i < NRETRY; ++i) {
436 		val = CSR_READ_4(sc, ET_MII_IND);
437 		if ((val & (ET_MII_IND_BUSY | ET_MII_IND_INVALID)) == 0)
438 			break;
439 		DELAY(50);
440 	}
441 	if (i == NRETRY) {
442 		if_printf(sc->ifp,
443 			  "read phy %d, reg %d timed out\n", phy, reg);
444 		ret = 0;
445 		goto back;
446 	}
447 
448 #undef NRETRY
449 
450 	val = CSR_READ_4(sc, ET_MII_STAT);
451 	ret = val & ET_MII_STAT_VALUE_MASK;
452 
453 back:
454 	/* Make sure that the current operation is stopped */
455 	CSR_WRITE_4(sc, ET_MII_CMD, 0);
456 	return (ret);
457 }
458 
459 static int
460 et_miibus_writereg(device_t dev, int phy, int reg, int val0)
461 {
462 	struct et_softc *sc;
463 	uint32_t val;
464 	int i;
465 
466 	sc = device_get_softc(dev);
467 	/* Stop any pending operations */
468 	CSR_WRITE_4(sc, ET_MII_CMD, 0);
469 
470 	val = (phy << ET_MII_ADDR_PHY_SHIFT) & ET_MII_ADDR_PHY_MASK;
471 	val |= (reg << ET_MII_ADDR_REG_SHIFT) & ET_MII_ADDR_REG_MASK;
472 	CSR_WRITE_4(sc, ET_MII_ADDR, val);
473 
474 	/* Start writing */
475 	CSR_WRITE_4(sc, ET_MII_CTRL,
476 	    (val0 << ET_MII_CTRL_VALUE_SHIFT) & ET_MII_CTRL_VALUE_MASK);
477 
478 #define NRETRY 100
479 
480 	for (i = 0; i < NRETRY; ++i) {
481 		val = CSR_READ_4(sc, ET_MII_IND);
482 		if ((val & ET_MII_IND_BUSY) == 0)
483 			break;
484 		DELAY(50);
485 	}
486 	if (i == NRETRY) {
487 		if_printf(sc->ifp,
488 			  "write phy %d, reg %d timed out\n", phy, reg);
489 		et_miibus_readreg(dev, phy, reg);
490 	}
491 
492 #undef NRETRY
493 
494 	/* Make sure that the current operation is stopped */
495 	CSR_WRITE_4(sc, ET_MII_CMD, 0);
496 	return (0);
497 }
498 
499 static void
500 et_miibus_statchg(device_t dev)
501 {
502 	struct et_softc *sc;
503 	struct mii_data *mii;
504 	struct ifnet *ifp;
505 	uint32_t cfg1, cfg2, ctrl;
506 	int i;
507 
508 	sc = device_get_softc(dev);
509 
510 	mii = device_get_softc(sc->sc_miibus);
511 	ifp = sc->ifp;
512 	if (mii == NULL || ifp == NULL ||
513 	    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
514 		return;
515 
516 	sc->sc_flags &= ~ET_FLAG_LINK;
517 	if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
518 	    (IFM_ACTIVE | IFM_AVALID)) {
519 		switch (IFM_SUBTYPE(mii->mii_media_active)) {
520 		case IFM_10_T:
521 		case IFM_100_TX:
522 			sc->sc_flags |= ET_FLAG_LINK;
523 			break;
524 		case IFM_1000_T:
525 			if ((sc->sc_flags & ET_FLAG_FASTETHER) == 0)
526 				sc->sc_flags |= ET_FLAG_LINK;
527 			break;
528 		}
529 	}
530 
531 	/* XXX Stop TX/RX MAC? */
532 	if ((sc->sc_flags & ET_FLAG_LINK) == 0)
533 		return;
534 
535 	/* Program MACs with resolved speed/duplex/flow-control. */
536 	ctrl = CSR_READ_4(sc, ET_MAC_CTRL);
537 	ctrl &= ~(ET_MAC_CTRL_GHDX | ET_MAC_CTRL_MODE_MII);
538 	cfg1 = CSR_READ_4(sc, ET_MAC_CFG1);
539 	cfg1 &= ~(ET_MAC_CFG1_TXFLOW | ET_MAC_CFG1_RXFLOW |
540 	    ET_MAC_CFG1_LOOPBACK);
541 	cfg2 = CSR_READ_4(sc, ET_MAC_CFG2);
542 	cfg2 &= ~(ET_MAC_CFG2_MODE_MII | ET_MAC_CFG2_MODE_GMII |
543 	    ET_MAC_CFG2_FDX | ET_MAC_CFG2_BIGFRM);
544 	cfg2 |= ET_MAC_CFG2_LENCHK | ET_MAC_CFG2_CRC | ET_MAC_CFG2_PADCRC |
545 	    ((7 << ET_MAC_CFG2_PREAMBLE_LEN_SHIFT) &
546 	    ET_MAC_CFG2_PREAMBLE_LEN_MASK);
547 
548 	if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T)
549 		cfg2 |= ET_MAC_CFG2_MODE_GMII;
550 	else {
551 		cfg2 |= ET_MAC_CFG2_MODE_MII;
552 		ctrl |= ET_MAC_CTRL_MODE_MII;
553 	}
554 
555 	if (IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) {
556 		cfg2 |= ET_MAC_CFG2_FDX;
557 		/*
558 		 * Controller lacks automatic TX pause frame
559 		 * generation so it should be handled by driver.
560 		 * Even though driver can send pause frame with
561 		 * arbitrary pause time, controller does not
562 		 * provide a way that tells how many free RX
563 		 * buffers are available in controller.  This
564 		 * limitation makes it hard to generate XON frame
565 		 * in time on driver side so don't enable TX flow
566 		 * control.
567 		 */
568 #ifdef notyet
569 		if (IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_TXPAUSE)
570 			cfg1 |= ET_MAC_CFG1_TXFLOW;
571 #endif
572 		if (IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_RXPAUSE)
573 			cfg1 |= ET_MAC_CFG1_RXFLOW;
574 	} else
575 		ctrl |= ET_MAC_CTRL_GHDX;
576 
577 	CSR_WRITE_4(sc, ET_MAC_CTRL, ctrl);
578 	CSR_WRITE_4(sc, ET_MAC_CFG2, cfg2);
579 	cfg1 |= ET_MAC_CFG1_TXEN | ET_MAC_CFG1_RXEN;
580 	CSR_WRITE_4(sc, ET_MAC_CFG1, cfg1);
581 
582 #define NRETRY	50
583 
584 	for (i = 0; i < NRETRY; ++i) {
585 		cfg1 = CSR_READ_4(sc, ET_MAC_CFG1);
586 		if ((cfg1 & (ET_MAC_CFG1_SYNC_TXEN | ET_MAC_CFG1_SYNC_RXEN)) ==
587 		    (ET_MAC_CFG1_SYNC_TXEN | ET_MAC_CFG1_SYNC_RXEN))
588 			break;
589 		DELAY(100);
590 	}
591 	if (i == NRETRY)
592 		if_printf(ifp, "can't enable RX/TX\n");
593 	sc->sc_flags |= ET_FLAG_TXRX_ENABLED;
594 
595 #undef NRETRY
596 }
597 
598 static int
599 et_ifmedia_upd_locked(struct ifnet *ifp)
600 {
601 	struct et_softc *sc;
602 	struct mii_data *mii;
603 	struct mii_softc *miisc;
604 
605 	sc = ifp->if_softc;
606 	mii = device_get_softc(sc->sc_miibus);
607 	LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
608 		PHY_RESET(miisc);
609 	return (mii_mediachg(mii));
610 }
611 
612 static int
613 et_ifmedia_upd(struct ifnet *ifp)
614 {
615 	struct et_softc *sc;
616 	int res;
617 
618 	sc = ifp->if_softc;
619 	ET_LOCK(sc);
620 	res = et_ifmedia_upd_locked(ifp);
621 	ET_UNLOCK(sc);
622 
623 	return (res);
624 }
625 
626 static void
627 et_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
628 {
629 	struct et_softc *sc;
630 	struct mii_data *mii;
631 
632 	sc = ifp->if_softc;
633 	ET_LOCK(sc);
634 	if ((ifp->if_flags & IFF_UP) == 0) {
635 		ET_UNLOCK(sc);
636 		return;
637 	}
638 
639 	mii = device_get_softc(sc->sc_miibus);
640 	mii_pollstat(mii);
641 	ifmr->ifm_active = mii->mii_media_active;
642 	ifmr->ifm_status = mii->mii_media_status;
643 	ET_UNLOCK(sc);
644 }
645 
646 static void
647 et_stop(struct et_softc *sc)
648 {
649 	struct ifnet *ifp;
650 
651 	ET_LOCK_ASSERT(sc);
652 
653 	ifp = sc->ifp;
654 	callout_stop(&sc->sc_tick);
655 	/* Disable interrupts. */
656 	CSR_WRITE_4(sc, ET_INTR_MASK, 0xffffffff);
657 
658 	CSR_WRITE_4(sc, ET_MAC_CFG1, CSR_READ_4(sc, ET_MAC_CFG1) & ~(
659 	    ET_MAC_CFG1_TXEN | ET_MAC_CFG1_RXEN));
660 	DELAY(100);
661 
662 	et_stop_rxdma(sc);
663 	et_stop_txdma(sc);
664 	et_stats_update(sc);
665 
666 	et_free_tx_ring(sc);
667 	et_free_rx_ring(sc);
668 
669 	sc->sc_tx = 0;
670 	sc->sc_tx_intr = 0;
671 	sc->sc_flags &= ~ET_FLAG_TXRX_ENABLED;
672 
673 	sc->watchdog_timer = 0;
674 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
675 }
676 
677 static int
678 et_bus_config(struct et_softc *sc)
679 {
680 	uint32_t val, max_plsz;
681 	uint16_t ack_latency, replay_timer;
682 
683 	/*
684 	 * Test whether EEPROM is valid
685 	 * NOTE: Read twice to get the correct value
686 	 */
687 	pci_read_config(sc->dev, ET_PCIR_EEPROM_STATUS, 1);
688 	val = pci_read_config(sc->dev, ET_PCIR_EEPROM_STATUS, 1);
689 	if (val & ET_PCIM_EEPROM_STATUS_ERROR) {
690 		device_printf(sc->dev, "EEPROM status error 0x%02x\n", val);
691 		return (ENXIO);
692 	}
693 
694 	/* TODO: LED */
695 
696 	if ((sc->sc_flags & ET_FLAG_PCIE) == 0)
697 		return (0);
698 
699 	/*
700 	 * Configure ACK latency and replay timer according to
701 	 * max playload size
702 	 */
703 	val = pci_read_config(sc->dev,
704 	    sc->sc_expcap + PCIER_DEVICE_CAP, 4);
705 	max_plsz = val & PCIEM_CAP_MAX_PAYLOAD;
706 
707 	switch (max_plsz) {
708 	case ET_PCIV_DEVICE_CAPS_PLSZ_128:
709 		ack_latency = ET_PCIV_ACK_LATENCY_128;
710 		replay_timer = ET_PCIV_REPLAY_TIMER_128;
711 		break;
712 
713 	case ET_PCIV_DEVICE_CAPS_PLSZ_256:
714 		ack_latency = ET_PCIV_ACK_LATENCY_256;
715 		replay_timer = ET_PCIV_REPLAY_TIMER_256;
716 		break;
717 
718 	default:
719 		ack_latency = pci_read_config(sc->dev, ET_PCIR_ACK_LATENCY, 2);
720 		replay_timer = pci_read_config(sc->dev,
721 		    ET_PCIR_REPLAY_TIMER, 2);
722 		device_printf(sc->dev, "ack latency %u, replay timer %u\n",
723 			      ack_latency, replay_timer);
724 		break;
725 	}
726 	if (ack_latency != 0) {
727 		pci_write_config(sc->dev, ET_PCIR_ACK_LATENCY, ack_latency, 2);
728 		pci_write_config(sc->dev, ET_PCIR_REPLAY_TIMER, replay_timer,
729 		    2);
730 	}
731 
732 	/*
733 	 * Set L0s and L1 latency timer to 2us
734 	 */
735 	val = pci_read_config(sc->dev, ET_PCIR_L0S_L1_LATENCY, 4);
736 	val &= ~(PCIEM_LINK_CAP_L0S_EXIT | PCIEM_LINK_CAP_L1_EXIT);
737 	/* L0s exit latency : 2us */
738 	val |= 0x00005000;
739 	/* L1 exit latency : 2us */
740 	val |= 0x00028000;
741 	pci_write_config(sc->dev, ET_PCIR_L0S_L1_LATENCY, val, 4);
742 
743 	/*
744 	 * Set max read request size to 2048 bytes
745 	 */
746 	pci_set_max_read_req(sc->dev, 2048);
747 
748 	return (0);
749 }
750 
751 static void
752 et_get_eaddr(device_t dev, uint8_t eaddr[])
753 {
754 	uint32_t val;
755 	int i;
756 
757 	val = pci_read_config(dev, ET_PCIR_MAC_ADDR0, 4);
758 	for (i = 0; i < 4; ++i)
759 		eaddr[i] = (val >> (8 * i)) & 0xff;
760 
761 	val = pci_read_config(dev, ET_PCIR_MAC_ADDR1, 2);
762 	for (; i < ETHER_ADDR_LEN; ++i)
763 		eaddr[i] = (val >> (8 * (i - 4))) & 0xff;
764 }
765 
766 static void
767 et_reset(struct et_softc *sc)
768 {
769 
770 	CSR_WRITE_4(sc, ET_MAC_CFG1,
771 		    ET_MAC_CFG1_RST_TXFUNC | ET_MAC_CFG1_RST_RXFUNC |
772 		    ET_MAC_CFG1_RST_TXMC | ET_MAC_CFG1_RST_RXMC |
773 		    ET_MAC_CFG1_SIM_RST | ET_MAC_CFG1_SOFT_RST);
774 
775 	CSR_WRITE_4(sc, ET_SWRST,
776 		    ET_SWRST_TXDMA | ET_SWRST_RXDMA |
777 		    ET_SWRST_TXMAC | ET_SWRST_RXMAC |
778 		    ET_SWRST_MAC | ET_SWRST_MAC_STAT | ET_SWRST_MMC);
779 
780 	CSR_WRITE_4(sc, ET_MAC_CFG1,
781 		    ET_MAC_CFG1_RST_TXFUNC | ET_MAC_CFG1_RST_RXFUNC |
782 		    ET_MAC_CFG1_RST_TXMC | ET_MAC_CFG1_RST_RXMC);
783 	CSR_WRITE_4(sc, ET_MAC_CFG1, 0);
784 	/* Disable interrupts. */
785 	CSR_WRITE_4(sc, ET_INTR_MASK, 0xffffffff);
786 }
787 
788 struct et_dmamap_arg {
789 	bus_addr_t	et_busaddr;
790 };
791 
792 static void
793 et_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
794 {
795 	struct et_dmamap_arg *ctx;
796 
797 	if (error)
798 		return;
799 
800 	KASSERT(nseg == 1, ("%s: %d segments returned!", __func__, nseg));
801 
802 	ctx = arg;
803 	ctx->et_busaddr = segs->ds_addr;
804 }
805 
806 static int
807 et_dma_ring_alloc(struct et_softc *sc, bus_size_t alignment, bus_size_t maxsize,
808     bus_dma_tag_t *tag, uint8_t **ring, bus_dmamap_t *map, bus_addr_t *paddr,
809     const char *msg)
810 {
811 	struct et_dmamap_arg ctx;
812 	int error;
813 
814 	error = bus_dma_tag_create(sc->sc_dtag, alignment, 0, BUS_SPACE_MAXADDR,
815 	    BUS_SPACE_MAXADDR, NULL, NULL, maxsize, 1, maxsize, 0, NULL, NULL,
816 	    tag);
817 	if (error != 0) {
818 		device_printf(sc->dev, "could not create %s dma tag\n", msg);
819 		return (error);
820 	}
821 	/* Allocate DMA'able memory for ring. */
822 	error = bus_dmamem_alloc(*tag, (void **)ring,
823 	    BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, map);
824 	if (error != 0) {
825 		device_printf(sc->dev,
826 		    "could not allocate DMA'able memory for %s\n", msg);
827 		return (error);
828 	}
829 	/* Load the address of the ring. */
830 	ctx.et_busaddr = 0;
831 	error = bus_dmamap_load(*tag, *map, *ring, maxsize, et_dma_map_addr,
832 	    &ctx, BUS_DMA_NOWAIT);
833 	if (error != 0) {
834 		device_printf(sc->dev,
835 		    "could not load DMA'able memory for %s\n", msg);
836 		return (error);
837 	}
838 	*paddr = ctx.et_busaddr;
839 	return (0);
840 }
841 
842 static void
843 et_dma_ring_free(struct et_softc *sc, bus_dma_tag_t *tag, uint8_t **ring,
844     bus_dmamap_t *map)
845 {
846 
847 	if (*map != NULL)
848 		bus_dmamap_unload(*tag, *map);
849 	if (*map != NULL && *ring != NULL) {
850 		bus_dmamem_free(*tag, *ring, *map);
851 		*ring = NULL;
852 		*map = NULL;
853 	}
854 	if (*tag) {
855 		bus_dma_tag_destroy(*tag);
856 		*tag = NULL;
857 	}
858 }
859 
860 static int
861 et_dma_alloc(struct et_softc *sc)
862 {
863 	struct et_txdesc_ring *tx_ring;
864 	struct et_rxdesc_ring *rx_ring;
865 	struct et_rxstat_ring *rxst_ring;
866 	struct et_rxstatus_data *rxsd;
867 	struct et_rxbuf_data *rbd;
868         struct et_txbuf_data *tbd;
869 	struct et_txstatus_data *txsd;
870 	int i, error;
871 
872 	error = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
873 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
874 	    BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0, NULL, NULL,
875 	    &sc->sc_dtag);
876 	if (error != 0) {
877 		device_printf(sc->dev, "could not allocate parent dma tag\n");
878 		return (error);
879 	}
880 
881 	/* TX ring. */
882 	tx_ring = &sc->sc_tx_ring;
883 	error = et_dma_ring_alloc(sc, ET_RING_ALIGN, ET_TX_RING_SIZE,
884 	    &tx_ring->tr_dtag, (uint8_t **)&tx_ring->tr_desc, &tx_ring->tr_dmap,
885 	    &tx_ring->tr_paddr, "TX ring");
886 	if (error)
887 		return (error);
888 
889 	/* TX status block. */
890 	txsd = &sc->sc_tx_status;
891 	error = et_dma_ring_alloc(sc, ET_STATUS_ALIGN, sizeof(uint32_t),
892 	    &txsd->txsd_dtag, (uint8_t **)&txsd->txsd_status, &txsd->txsd_dmap,
893 	    &txsd->txsd_paddr, "TX status block");
894 	if (error)
895 		return (error);
896 
897 	/* RX ring 0, used as to recive small sized frames. */
898 	rx_ring = &sc->sc_rx_ring[0];
899 	error = et_dma_ring_alloc(sc, ET_RING_ALIGN, ET_RX_RING_SIZE,
900 	    &rx_ring->rr_dtag, (uint8_t **)&rx_ring->rr_desc, &rx_ring->rr_dmap,
901 	    &rx_ring->rr_paddr, "RX ring 0");
902 	rx_ring->rr_posreg = ET_RX_RING0_POS;
903 	if (error)
904 		return (error);
905 
906 	/* RX ring 1, used as to store normal sized frames. */
907 	rx_ring = &sc->sc_rx_ring[1];
908 	error = et_dma_ring_alloc(sc, ET_RING_ALIGN, ET_RX_RING_SIZE,
909 	    &rx_ring->rr_dtag, (uint8_t **)&rx_ring->rr_desc, &rx_ring->rr_dmap,
910 	    &rx_ring->rr_paddr, "RX ring 1");
911 	rx_ring->rr_posreg = ET_RX_RING1_POS;
912 	if (error)
913 		return (error);
914 
915 	/* RX stat ring. */
916 	rxst_ring = &sc->sc_rxstat_ring;
917 	error = et_dma_ring_alloc(sc, ET_RING_ALIGN, ET_RXSTAT_RING_SIZE,
918 	    &rxst_ring->rsr_dtag, (uint8_t **)&rxst_ring->rsr_stat,
919 	    &rxst_ring->rsr_dmap, &rxst_ring->rsr_paddr, "RX stat ring");
920 	if (error)
921 		return (error);
922 
923 	/* RX status block. */
924 	rxsd = &sc->sc_rx_status;
925 	error = et_dma_ring_alloc(sc, ET_STATUS_ALIGN,
926 	    sizeof(struct et_rxstatus), &rxsd->rxsd_dtag,
927 	    (uint8_t **)&rxsd->rxsd_status, &rxsd->rxsd_dmap,
928 	    &rxsd->rxsd_paddr, "RX status block");
929 	if (error)
930 		return (error);
931 
932 	/* Create parent DMA tag for mbufs. */
933 	error = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
934 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
935 	    BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0, NULL, NULL,
936 	    &sc->sc_mbuf_dtag);
937 	if (error != 0) {
938 		device_printf(sc->dev,
939 		    "could not allocate parent dma tag for mbuf\n");
940 		return (error);
941 	}
942 
943 	/* Create DMA tag for mini RX mbufs to use RX ring 0. */
944 	error = bus_dma_tag_create(sc->sc_mbuf_dtag, 1, 0,
945 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MHLEN, 1,
946 	    MHLEN, 0, NULL, NULL, &sc->sc_rx_mini_tag);
947 	if (error) {
948 		device_printf(sc->dev, "could not create mini RX dma tag\n");
949 		return (error);
950 	}
951 
952 	/* Create DMA tag for standard RX mbufs to use RX ring 1. */
953 	error = bus_dma_tag_create(sc->sc_mbuf_dtag, 1, 0,
954 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1,
955 	    MCLBYTES, 0, NULL, NULL, &sc->sc_rx_tag);
956 	if (error) {
957 		device_printf(sc->dev, "could not create RX dma tag\n");
958 		return (error);
959 	}
960 
961 	/* Create DMA tag for TX mbufs. */
962 	error = bus_dma_tag_create(sc->sc_mbuf_dtag, 1, 0,
963 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
964 	    MCLBYTES * ET_NSEG_MAX, ET_NSEG_MAX, MCLBYTES, 0, NULL, NULL,
965 	    &sc->sc_tx_tag);
966 	if (error) {
967 		device_printf(sc->dev, "could not create TX dma tag\n");
968 		return (error);
969 	}
970 
971 	/* Initialize RX ring 0. */
972 	rbd = &sc->sc_rx_data[0];
973 	rbd->rbd_bufsize = ET_RXDMA_CTRL_RING0_128;
974 	rbd->rbd_newbuf = et_newbuf_hdr;
975 	rbd->rbd_discard = et_rxbuf_discard;
976 	rbd->rbd_softc = sc;
977 	rbd->rbd_ring = &sc->sc_rx_ring[0];
978 	/* Create DMA maps for mini RX buffers, ring 0. */
979 	for (i = 0; i < ET_RX_NDESC; i++) {
980 		error = bus_dmamap_create(sc->sc_rx_mini_tag, 0,
981 		    &rbd->rbd_buf[i].rb_dmap);
982 		if (error) {
983 			device_printf(sc->dev,
984 			    "could not create DMA map for mini RX mbufs\n");
985 			return (error);
986 		}
987 	}
988 
989 	/* Create a spare DMA map for mini RX buffers, ring 0. */
990 	error = bus_dmamap_create(sc->sc_rx_mini_tag, 0,
991 	    &sc->sc_rx_mini_sparemap);
992 	if (error) {
993 		device_printf(sc->dev,
994 		    "could not create spare DMA map for mini RX mbuf\n");
995 		return (error);
996 	}
997 
998 	/* Initialize RX ring 1. */
999 	rbd = &sc->sc_rx_data[1];
1000 	rbd->rbd_bufsize = ET_RXDMA_CTRL_RING1_2048;
1001 	rbd->rbd_newbuf = et_newbuf_cluster;
1002 	rbd->rbd_discard = et_rxbuf_discard;
1003 	rbd->rbd_softc = sc;
1004 	rbd->rbd_ring = &sc->sc_rx_ring[1];
1005 	/* Create DMA maps for standard RX buffers, ring 1. */
1006 	for (i = 0; i < ET_RX_NDESC; i++) {
1007 		error = bus_dmamap_create(sc->sc_rx_tag, 0,
1008 		    &rbd->rbd_buf[i].rb_dmap);
1009 		if (error) {
1010 			device_printf(sc->dev,
1011 			    "could not create DMA map for mini RX mbufs\n");
1012 			return (error);
1013 		}
1014 	}
1015 
1016 	/* Create a spare DMA map for standard RX buffers, ring 1. */
1017 	error = bus_dmamap_create(sc->sc_rx_tag, 0, &sc->sc_rx_sparemap);
1018 	if (error) {
1019 		device_printf(sc->dev,
1020 		    "could not create spare DMA map for RX mbuf\n");
1021 		return (error);
1022 	}
1023 
1024 	/* Create DMA maps for TX buffers. */
1025 	tbd = &sc->sc_tx_data;
1026 	for (i = 0; i < ET_TX_NDESC; i++) {
1027 		error = bus_dmamap_create(sc->sc_tx_tag, 0,
1028 		    &tbd->tbd_buf[i].tb_dmap);
1029 		if (error) {
1030 			device_printf(sc->dev,
1031 			    "could not create DMA map for TX mbufs\n");
1032 			return (error);
1033 		}
1034 	}
1035 
1036 	return (0);
1037 }
1038 
1039 static void
1040 et_dma_free(struct et_softc *sc)
1041 {
1042 	struct et_txdesc_ring *tx_ring;
1043 	struct et_rxdesc_ring *rx_ring;
1044 	struct et_txstatus_data *txsd;
1045 	struct et_rxstat_ring *rxst_ring;
1046 	struct et_rxstatus_data *rxsd;
1047 	struct et_rxbuf_data *rbd;
1048         struct et_txbuf_data *tbd;
1049 	int i;
1050 
1051 	/* Destroy DMA maps for mini RX buffers, ring 0. */
1052 	rbd = &sc->sc_rx_data[0];
1053 	for (i = 0; i < ET_RX_NDESC; i++) {
1054 		if (rbd->rbd_buf[i].rb_dmap) {
1055 			bus_dmamap_destroy(sc->sc_rx_mini_tag,
1056 			    rbd->rbd_buf[i].rb_dmap);
1057 			rbd->rbd_buf[i].rb_dmap = NULL;
1058 		}
1059 	}
1060 	if (sc->sc_rx_mini_sparemap) {
1061 		bus_dmamap_destroy(sc->sc_rx_mini_tag, sc->sc_rx_mini_sparemap);
1062 		sc->sc_rx_mini_sparemap = NULL;
1063 	}
1064 	if (sc->sc_rx_mini_tag) {
1065 		bus_dma_tag_destroy(sc->sc_rx_mini_tag);
1066 		sc->sc_rx_mini_tag = NULL;
1067 	}
1068 
1069 	/* Destroy DMA maps for standard RX buffers, ring 1. */
1070 	rbd = &sc->sc_rx_data[1];
1071 	for (i = 0; i < ET_RX_NDESC; i++) {
1072 		if (rbd->rbd_buf[i].rb_dmap) {
1073 			bus_dmamap_destroy(sc->sc_rx_tag,
1074 			    rbd->rbd_buf[i].rb_dmap);
1075 			rbd->rbd_buf[i].rb_dmap = NULL;
1076 		}
1077 	}
1078 	if (sc->sc_rx_sparemap) {
1079 		bus_dmamap_destroy(sc->sc_rx_tag, sc->sc_rx_sparemap);
1080 		sc->sc_rx_sparemap = NULL;
1081 	}
1082 	if (sc->sc_rx_tag) {
1083 		bus_dma_tag_destroy(sc->sc_rx_tag);
1084 		sc->sc_rx_tag = NULL;
1085 	}
1086 
1087 	/* Destroy DMA maps for TX buffers. */
1088 	tbd = &sc->sc_tx_data;
1089 	for (i = 0; i < ET_TX_NDESC; i++) {
1090 		if (tbd->tbd_buf[i].tb_dmap) {
1091 			bus_dmamap_destroy(sc->sc_tx_tag,
1092 			    tbd->tbd_buf[i].tb_dmap);
1093 			tbd->tbd_buf[i].tb_dmap = NULL;
1094 		}
1095 	}
1096 	if (sc->sc_tx_tag) {
1097 		bus_dma_tag_destroy(sc->sc_tx_tag);
1098 		sc->sc_tx_tag = NULL;
1099 	}
1100 
1101 	/* Destroy mini RX ring, ring 0. */
1102 	rx_ring = &sc->sc_rx_ring[0];
1103 	et_dma_ring_free(sc, &rx_ring->rr_dtag, (void *)&rx_ring->rr_desc,
1104 	    &rx_ring->rr_dmap);
1105 	/* Destroy standard RX ring, ring 1. */
1106 	rx_ring = &sc->sc_rx_ring[1];
1107 	et_dma_ring_free(sc, &rx_ring->rr_dtag, (void *)&rx_ring->rr_desc,
1108 	    &rx_ring->rr_dmap);
1109 	/* Destroy RX stat ring. */
1110 	rxst_ring = &sc->sc_rxstat_ring;
1111 	et_dma_ring_free(sc, &rxst_ring->rsr_dtag, (void *)&rxst_ring->rsr_stat,
1112 	    &rxst_ring->rsr_dmap);
1113 	/* Destroy RX status block. */
1114 	rxsd = &sc->sc_rx_status;
1115 	et_dma_ring_free(sc, &rxst_ring->rsr_dtag, (void *)&rxst_ring->rsr_stat,
1116 	    &rxst_ring->rsr_dmap);
1117 	/* Destroy TX ring. */
1118 	tx_ring = &sc->sc_tx_ring;
1119 	et_dma_ring_free(sc, &tx_ring->tr_dtag, (void *)&tx_ring->tr_desc,
1120 	    &tx_ring->tr_dmap);
1121 	/* Destroy TX status block. */
1122 	txsd = &sc->sc_tx_status;
1123 	et_dma_ring_free(sc, &txsd->txsd_dtag, (void *)&txsd->txsd_status,
1124 	    &txsd->txsd_dmap);
1125 
1126 	/* Destroy the parent tag. */
1127 	if (sc->sc_dtag) {
1128 		bus_dma_tag_destroy(sc->sc_dtag);
1129 		sc->sc_dtag = NULL;
1130 	}
1131 }
1132 
1133 static void
1134 et_chip_attach(struct et_softc *sc)
1135 {
1136 	uint32_t val;
1137 
1138 	/*
1139 	 * Perform minimal initialization
1140 	 */
1141 
1142 	/* Disable loopback */
1143 	CSR_WRITE_4(sc, ET_LOOPBACK, 0);
1144 
1145 	/* Reset MAC */
1146 	CSR_WRITE_4(sc, ET_MAC_CFG1,
1147 		    ET_MAC_CFG1_RST_TXFUNC | ET_MAC_CFG1_RST_RXFUNC |
1148 		    ET_MAC_CFG1_RST_TXMC | ET_MAC_CFG1_RST_RXMC |
1149 		    ET_MAC_CFG1_SIM_RST | ET_MAC_CFG1_SOFT_RST);
1150 
1151 	/*
1152 	 * Setup half duplex mode
1153 	 */
1154 	val = (10 << ET_MAC_HDX_ALT_BEB_TRUNC_SHIFT) |
1155 	    (15 << ET_MAC_HDX_REXMIT_MAX_SHIFT) |
1156 	    (55 << ET_MAC_HDX_COLLWIN_SHIFT) |
1157 	    ET_MAC_HDX_EXC_DEFER;
1158 	CSR_WRITE_4(sc, ET_MAC_HDX, val);
1159 
1160 	/* Clear MAC control */
1161 	CSR_WRITE_4(sc, ET_MAC_CTRL, 0);
1162 
1163 	/* Reset MII */
1164 	CSR_WRITE_4(sc, ET_MII_CFG, ET_MII_CFG_CLKRST);
1165 
1166 	/* Bring MAC out of reset state */
1167 	CSR_WRITE_4(sc, ET_MAC_CFG1, 0);
1168 
1169 	/* Enable memory controllers */
1170 	CSR_WRITE_4(sc, ET_MMC_CTRL, ET_MMC_CTRL_ENABLE);
1171 }
1172 
1173 static void
1174 et_intr(void *xsc)
1175 {
1176 	struct et_softc *sc;
1177 	struct ifnet *ifp;
1178 	uint32_t status;
1179 
1180 	sc = xsc;
1181 	ET_LOCK(sc);
1182 	ifp = sc->ifp;
1183 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1184 		goto done;
1185 
1186 	status = CSR_READ_4(sc, ET_INTR_STATUS);
1187 	if ((status & ET_INTRS) == 0)
1188 		goto done;
1189 
1190 	/* Disable further interrupts. */
1191 	CSR_WRITE_4(sc, ET_INTR_MASK, 0xffffffff);
1192 
1193 	if (status & (ET_INTR_RXDMA_ERROR | ET_INTR_TXDMA_ERROR)) {
1194 		device_printf(sc->dev, "DMA error(0x%08x) -- resetting\n",
1195 		    status);
1196 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1197 		et_init_locked(sc);
1198 		ET_UNLOCK(sc);
1199 		return;
1200 	}
1201 	if (status & ET_INTR_RXDMA)
1202 		et_rxeof(sc);
1203 	if (status & (ET_INTR_TXDMA | ET_INTR_TIMER))
1204 		et_txeof(sc);
1205 	if (status & ET_INTR_TIMER)
1206 		CSR_WRITE_4(sc, ET_TIMER, sc->sc_timer);
1207 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1208 		CSR_WRITE_4(sc, ET_INTR_MASK, ~ET_INTRS);
1209 		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1210 			et_start_locked(ifp);
1211 	}
1212 done:
1213 	ET_UNLOCK(sc);
1214 }
1215 
1216 static void
1217 et_init_locked(struct et_softc *sc)
1218 {
1219 	struct ifnet *ifp;
1220 	int error;
1221 
1222 	ET_LOCK_ASSERT(sc);
1223 
1224 	ifp = sc->ifp;
1225 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1226 		return;
1227 
1228 	et_stop(sc);
1229 	et_reset(sc);
1230 
1231 	et_init_tx_ring(sc);
1232 	error = et_init_rx_ring(sc);
1233 	if (error)
1234 		return;
1235 
1236 	error = et_chip_init(sc);
1237 	if (error)
1238 		goto fail;
1239 
1240 	/*
1241 	 * Start TX/RX DMA engine
1242 	 */
1243 	error = et_start_rxdma(sc);
1244 	if (error)
1245 		return;
1246 
1247 	error = et_start_txdma(sc);
1248 	if (error)
1249 		return;
1250 
1251 	/* Enable interrupts. */
1252 	CSR_WRITE_4(sc, ET_INTR_MASK, ~ET_INTRS);
1253 
1254 	CSR_WRITE_4(sc, ET_TIMER, sc->sc_timer);
1255 
1256 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1257 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1258 
1259 	sc->sc_flags &= ~ET_FLAG_LINK;
1260 	et_ifmedia_upd_locked(ifp);
1261 
1262 	callout_reset(&sc->sc_tick, hz, et_tick, sc);
1263 
1264 fail:
1265 	if (error)
1266 		et_stop(sc);
1267 }
1268 
1269 static void
1270 et_init(void *xsc)
1271 {
1272 	struct et_softc *sc = xsc;
1273 
1274 	ET_LOCK(sc);
1275 	et_init_locked(sc);
1276 	ET_UNLOCK(sc);
1277 }
1278 
1279 static int
1280 et_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1281 {
1282 	struct et_softc *sc;
1283 	struct mii_data *mii;
1284 	struct ifreq *ifr;
1285 	int error, mask, max_framelen;
1286 
1287 	sc = ifp->if_softc;
1288 	ifr = (struct ifreq *)data;
1289 	error = 0;
1290 
1291 /* XXX LOCKSUSED */
1292 	switch (cmd) {
1293 	case SIOCSIFFLAGS:
1294 		ET_LOCK(sc);
1295 		if (ifp->if_flags & IFF_UP) {
1296 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1297 				if ((ifp->if_flags ^ sc->sc_if_flags) &
1298 				(IFF_ALLMULTI | IFF_PROMISC | IFF_BROADCAST))
1299 					et_setmulti(sc);
1300 			} else {
1301 				et_init_locked(sc);
1302 			}
1303 		} else {
1304 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1305 				et_stop(sc);
1306 		}
1307 		sc->sc_if_flags = ifp->if_flags;
1308 		ET_UNLOCK(sc);
1309 		break;
1310 
1311 	case SIOCSIFMEDIA:
1312 	case SIOCGIFMEDIA:
1313 		mii = device_get_softc(sc->sc_miibus);
1314 		error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
1315 		break;
1316 
1317 	case SIOCADDMULTI:
1318 	case SIOCDELMULTI:
1319 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1320 			ET_LOCK(sc);
1321 			et_setmulti(sc);
1322 			ET_UNLOCK(sc);
1323 		}
1324 		break;
1325 
1326 	case SIOCSIFMTU:
1327 		ET_LOCK(sc);
1328 #if 0
1329 		if (sc->sc_flags & ET_FLAG_JUMBO)
1330 			max_framelen = ET_JUMBO_FRAMELEN;
1331 		else
1332 #endif
1333 			max_framelen = MCLBYTES - 1;
1334 
1335 		if (ET_FRAMELEN(ifr->ifr_mtu) > max_framelen) {
1336 			error = EOPNOTSUPP;
1337 			ET_UNLOCK(sc);
1338 			break;
1339 		}
1340 
1341 		if (ifp->if_mtu != ifr->ifr_mtu) {
1342 			ifp->if_mtu = ifr->ifr_mtu;
1343 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1344 				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1345 				et_init_locked(sc);
1346 			}
1347 		}
1348 		ET_UNLOCK(sc);
1349 		break;
1350 
1351 	case SIOCSIFCAP:
1352 		ET_LOCK(sc);
1353 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1354 		if ((mask & IFCAP_TXCSUM) != 0 &&
1355 		    (IFCAP_TXCSUM & ifp->if_capabilities) != 0) {
1356 			ifp->if_capenable ^= IFCAP_TXCSUM;
1357 			if ((IFCAP_TXCSUM & ifp->if_capenable) != 0)
1358 				ifp->if_hwassist |= ET_CSUM_FEATURES;
1359 			else
1360 				ifp->if_hwassist &= ~ET_CSUM_FEATURES;
1361 		}
1362 		ET_UNLOCK(sc);
1363 		break;
1364 
1365 	default:
1366 		error = ether_ioctl(ifp, cmd, data);
1367 		break;
1368 	}
1369 	return (error);
1370 }
1371 
1372 static void
1373 et_start_locked(struct ifnet *ifp)
1374 {
1375 	struct et_softc *sc;
1376 	struct mbuf *m_head = NULL;
1377 	struct et_txdesc_ring *tx_ring;
1378 	struct et_txbuf_data *tbd;
1379 	uint32_t tx_ready_pos;
1380 	int enq;
1381 
1382 	sc = ifp->if_softc;
1383 	ET_LOCK_ASSERT(sc);
1384 
1385 	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1386 	    IFF_DRV_RUNNING ||
1387 	    (sc->sc_flags & (ET_FLAG_LINK | ET_FLAG_TXRX_ENABLED)) !=
1388 	    (ET_FLAG_LINK | ET_FLAG_TXRX_ENABLED))
1389 		return;
1390 
1391 	/*
1392 	 * Driver does not request TX completion interrupt for every
1393 	 * queued frames to prevent generating excessive interrupts.
1394 	 * This means driver may wait for TX completion interrupt even
1395 	 * though some frames were sucessfully transmitted.  Reclaiming
1396 	 * transmitted frames will ensure driver see all available
1397 	 * descriptors.
1398 	 */
1399 	tbd = &sc->sc_tx_data;
1400 	if (tbd->tbd_used > (ET_TX_NDESC * 2) / 3)
1401 		et_txeof(sc);
1402 
1403 	for (enq = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd); ) {
1404 		if (tbd->tbd_used + ET_NSEG_SPARE >= ET_TX_NDESC) {
1405 			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1406 			break;
1407 		}
1408 
1409 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
1410 		if (m_head == NULL)
1411 			break;
1412 
1413 		if (et_encap(sc, &m_head)) {
1414 			if (m_head == NULL) {
1415 				ifp->if_oerrors++;
1416 				break;
1417 			}
1418 			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1419 			if (tbd->tbd_used > 0)
1420 				ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1421 			break;
1422 		}
1423 		enq++;
1424 		ETHER_BPF_MTAP(ifp, m_head);
1425 	}
1426 
1427 	if (enq > 0) {
1428 		tx_ring = &sc->sc_tx_ring;
1429 		bus_dmamap_sync(tx_ring->tr_dtag, tx_ring->tr_dmap,
1430 		    BUS_DMASYNC_PREWRITE);
1431 		tx_ready_pos = tx_ring->tr_ready_index &
1432 		    ET_TX_READY_POS_INDEX_MASK;
1433 		if (tx_ring->tr_ready_wrap)
1434 			tx_ready_pos |= ET_TX_READY_POS_WRAP;
1435 		CSR_WRITE_4(sc, ET_TX_READY_POS, tx_ready_pos);
1436 		sc->watchdog_timer = 5;
1437 	}
1438 }
1439 
1440 static void
1441 et_start(struct ifnet *ifp)
1442 {
1443 	struct et_softc *sc;
1444 
1445 	sc = ifp->if_softc;
1446 	ET_LOCK(sc);
1447 	et_start_locked(ifp);
1448 	ET_UNLOCK(sc);
1449 }
1450 
1451 static int
1452 et_watchdog(struct et_softc *sc)
1453 {
1454 	uint32_t status;
1455 
1456 	ET_LOCK_ASSERT(sc);
1457 
1458 	if (sc->watchdog_timer == 0 || --sc->watchdog_timer)
1459 		return (0);
1460 
1461 	bus_dmamap_sync(sc->sc_tx_status.txsd_dtag, sc->sc_tx_status.txsd_dmap,
1462 	    BUS_DMASYNC_POSTREAD);
1463 	status = le32toh(*(sc->sc_tx_status.txsd_status));
1464 	if_printf(sc->ifp, "watchdog timed out (0x%08x) -- resetting\n",
1465 	    status);
1466 
1467 	sc->ifp->if_oerrors++;
1468 	sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1469 	et_init_locked(sc);
1470 	return (EJUSTRETURN);
1471 }
1472 
1473 static int
1474 et_stop_rxdma(struct et_softc *sc)
1475 {
1476 
1477 	CSR_WRITE_4(sc, ET_RXDMA_CTRL,
1478 		    ET_RXDMA_CTRL_HALT | ET_RXDMA_CTRL_RING1_ENABLE);
1479 
1480 	DELAY(5);
1481 	if ((CSR_READ_4(sc, ET_RXDMA_CTRL) & ET_RXDMA_CTRL_HALTED) == 0) {
1482 		if_printf(sc->ifp, "can't stop RX DMA engine\n");
1483 		return (ETIMEDOUT);
1484 	}
1485 	return (0);
1486 }
1487 
1488 static int
1489 et_stop_txdma(struct et_softc *sc)
1490 {
1491 
1492 	CSR_WRITE_4(sc, ET_TXDMA_CTRL,
1493 		    ET_TXDMA_CTRL_HALT | ET_TXDMA_CTRL_SINGLE_EPKT);
1494 	return (0);
1495 }
1496 
1497 static void
1498 et_free_tx_ring(struct et_softc *sc)
1499 {
1500 	struct et_txdesc_ring *tx_ring;
1501 	struct et_txbuf_data *tbd;
1502 	struct et_txbuf *tb;
1503 	int i;
1504 
1505 	tbd = &sc->sc_tx_data;
1506 	tx_ring = &sc->sc_tx_ring;
1507 	for (i = 0; i < ET_TX_NDESC; ++i) {
1508 		tb = &tbd->tbd_buf[i];
1509 		if (tb->tb_mbuf != NULL) {
1510 			bus_dmamap_sync(sc->sc_tx_tag, tb->tb_dmap,
1511 			    BUS_DMASYNC_POSTWRITE);
1512 			bus_dmamap_unload(sc->sc_mbuf_dtag, tb->tb_dmap);
1513 			m_freem(tb->tb_mbuf);
1514 			tb->tb_mbuf = NULL;
1515 		}
1516 	}
1517 }
1518 
1519 static void
1520 et_free_rx_ring(struct et_softc *sc)
1521 {
1522 	struct et_rxbuf_data *rbd;
1523 	struct et_rxdesc_ring *rx_ring;
1524 	struct et_rxbuf *rb;
1525 	int i;
1526 
1527 	/* Ring 0 */
1528 	rx_ring = &sc->sc_rx_ring[0];
1529 	rbd = &sc->sc_rx_data[0];
1530 	for (i = 0; i < ET_RX_NDESC; ++i) {
1531 		rb = &rbd->rbd_buf[i];
1532 		if (rb->rb_mbuf != NULL) {
1533 			bus_dmamap_sync(sc->sc_rx_mini_tag, rx_ring->rr_dmap,
1534 			    BUS_DMASYNC_POSTREAD);
1535 			bus_dmamap_unload(sc->sc_rx_mini_tag, rb->rb_dmap);
1536 			m_freem(rb->rb_mbuf);
1537 			rb->rb_mbuf = NULL;
1538 		}
1539 	}
1540 
1541 	/* Ring 1 */
1542 	rx_ring = &sc->sc_rx_ring[1];
1543 	rbd = &sc->sc_rx_data[1];
1544 	for (i = 0; i < ET_RX_NDESC; ++i) {
1545 		rb = &rbd->rbd_buf[i];
1546 		if (rb->rb_mbuf != NULL) {
1547 			bus_dmamap_sync(sc->sc_rx_tag, rx_ring->rr_dmap,
1548 			    BUS_DMASYNC_POSTREAD);
1549 			bus_dmamap_unload(sc->sc_rx_tag, rb->rb_dmap);
1550 			m_freem(rb->rb_mbuf);
1551 			rb->rb_mbuf = NULL;
1552 		}
1553 	}
1554 }
1555 
1556 static void
1557 et_setmulti(struct et_softc *sc)
1558 {
1559 	struct ifnet *ifp;
1560 	uint32_t hash[4] = { 0, 0, 0, 0 };
1561 	uint32_t rxmac_ctrl, pktfilt;
1562 	struct ifmultiaddr *ifma;
1563 	int i, count;
1564 
1565 	ET_LOCK_ASSERT(sc);
1566 	ifp = sc->ifp;
1567 
1568 	pktfilt = CSR_READ_4(sc, ET_PKTFILT);
1569 	rxmac_ctrl = CSR_READ_4(sc, ET_RXMAC_CTRL);
1570 
1571 	pktfilt &= ~(ET_PKTFILT_BCAST | ET_PKTFILT_MCAST | ET_PKTFILT_UCAST);
1572 	if (ifp->if_flags & (IFF_PROMISC | IFF_ALLMULTI)) {
1573 		rxmac_ctrl |= ET_RXMAC_CTRL_NO_PKTFILT;
1574 		goto back;
1575 	}
1576 
1577 	count = 0;
1578 	if_maddr_rlock(ifp);
1579 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1580 		uint32_t *hp, h;
1581 
1582 		if (ifma->ifma_addr->sa_family != AF_LINK)
1583 			continue;
1584 
1585 		h = ether_crc32_be(LLADDR((struct sockaddr_dl *)
1586 				   ifma->ifma_addr), ETHER_ADDR_LEN);
1587 		h = (h & 0x3f800000) >> 23;
1588 
1589 		hp = &hash[0];
1590 		if (h >= 32 && h < 64) {
1591 			h -= 32;
1592 			hp = &hash[1];
1593 		} else if (h >= 64 && h < 96) {
1594 			h -= 64;
1595 			hp = &hash[2];
1596 		} else if (h >= 96) {
1597 			h -= 96;
1598 			hp = &hash[3];
1599 		}
1600 		*hp |= (1 << h);
1601 
1602 		++count;
1603 	}
1604 	if_maddr_runlock(ifp);
1605 
1606 	for (i = 0; i < 4; ++i)
1607 		CSR_WRITE_4(sc, ET_MULTI_HASH + (i * 4), hash[i]);
1608 
1609 	if (count > 0)
1610 		pktfilt |= ET_PKTFILT_MCAST;
1611 	rxmac_ctrl &= ~ET_RXMAC_CTRL_NO_PKTFILT;
1612 back:
1613 	CSR_WRITE_4(sc, ET_PKTFILT, pktfilt);
1614 	CSR_WRITE_4(sc, ET_RXMAC_CTRL, rxmac_ctrl);
1615 }
1616 
1617 static int
1618 et_chip_init(struct et_softc *sc)
1619 {
1620 	struct ifnet *ifp;
1621 	uint32_t rxq_end;
1622 	int error, frame_len, rxmem_size;
1623 
1624 	ifp = sc->ifp;
1625 	/*
1626 	 * Split 16Kbytes internal memory between TX and RX
1627 	 * according to frame length.
1628 	 */
1629 	frame_len = ET_FRAMELEN(ifp->if_mtu);
1630 	if (frame_len < 2048) {
1631 		rxmem_size = ET_MEM_RXSIZE_DEFAULT;
1632 	} else if (frame_len <= ET_RXMAC_CUT_THRU_FRMLEN) {
1633 		rxmem_size = ET_MEM_SIZE / 2;
1634 	} else {
1635 		rxmem_size = ET_MEM_SIZE -
1636 		roundup(frame_len + ET_MEM_TXSIZE_EX, ET_MEM_UNIT);
1637 	}
1638 	rxq_end = ET_QUEUE_ADDR(rxmem_size);
1639 
1640 	CSR_WRITE_4(sc, ET_RXQUEUE_START, ET_QUEUE_ADDR_START);
1641 	CSR_WRITE_4(sc, ET_RXQUEUE_END, rxq_end);
1642 	CSR_WRITE_4(sc, ET_TXQUEUE_START, rxq_end + 1);
1643 	CSR_WRITE_4(sc, ET_TXQUEUE_END, ET_QUEUE_ADDR_END);
1644 
1645 	/* No loopback */
1646 	CSR_WRITE_4(sc, ET_LOOPBACK, 0);
1647 
1648 	/* Clear MSI configure */
1649 	if ((sc->sc_flags & ET_FLAG_MSI) == 0)
1650 		CSR_WRITE_4(sc, ET_MSI_CFG, 0);
1651 
1652 	/* Disable timer */
1653 	CSR_WRITE_4(sc, ET_TIMER, 0);
1654 
1655 	/* Initialize MAC */
1656 	et_init_mac(sc);
1657 
1658 	/* Enable memory controllers */
1659 	CSR_WRITE_4(sc, ET_MMC_CTRL, ET_MMC_CTRL_ENABLE);
1660 
1661 	/* Initialize RX MAC */
1662 	et_init_rxmac(sc);
1663 
1664 	/* Initialize TX MAC */
1665 	et_init_txmac(sc);
1666 
1667 	/* Initialize RX DMA engine */
1668 	error = et_init_rxdma(sc);
1669 	if (error)
1670 		return (error);
1671 
1672 	/* Initialize TX DMA engine */
1673 	error = et_init_txdma(sc);
1674 	if (error)
1675 		return (error);
1676 
1677 	return (0);
1678 }
1679 
1680 static void
1681 et_init_tx_ring(struct et_softc *sc)
1682 {
1683 	struct et_txdesc_ring *tx_ring;
1684 	struct et_txbuf_data *tbd;
1685 	struct et_txstatus_data *txsd;
1686 
1687 	tx_ring = &sc->sc_tx_ring;
1688 	bzero(tx_ring->tr_desc, ET_TX_RING_SIZE);
1689 	bus_dmamap_sync(tx_ring->tr_dtag, tx_ring->tr_dmap,
1690 	    BUS_DMASYNC_PREWRITE);
1691 
1692 	tbd = &sc->sc_tx_data;
1693 	tbd->tbd_start_index = 0;
1694 	tbd->tbd_start_wrap = 0;
1695 	tbd->tbd_used = 0;
1696 
1697 	txsd = &sc->sc_tx_status;
1698 	bzero(txsd->txsd_status, sizeof(uint32_t));
1699 	bus_dmamap_sync(txsd->txsd_dtag, txsd->txsd_dmap,
1700 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1701 }
1702 
1703 static int
1704 et_init_rx_ring(struct et_softc *sc)
1705 {
1706 	struct et_rxstatus_data *rxsd;
1707 	struct et_rxstat_ring *rxst_ring;
1708 	struct et_rxbuf_data *rbd;
1709 	int i, error, n;
1710 
1711 	for (n = 0; n < ET_RX_NRING; ++n) {
1712 		rbd = &sc->sc_rx_data[n];
1713 		for (i = 0; i < ET_RX_NDESC; ++i) {
1714 			error = rbd->rbd_newbuf(rbd, i);
1715 			if (error) {
1716 				if_printf(sc->ifp, "%d ring %d buf, "
1717 					  "newbuf failed: %d\n", n, i, error);
1718 				return (error);
1719 			}
1720 		}
1721 	}
1722 
1723 	rxsd = &sc->sc_rx_status;
1724 	bzero(rxsd->rxsd_status, sizeof(struct et_rxstatus));
1725 	bus_dmamap_sync(rxsd->rxsd_dtag, rxsd->rxsd_dmap,
1726 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1727 
1728 	rxst_ring = &sc->sc_rxstat_ring;
1729 	bzero(rxst_ring->rsr_stat, ET_RXSTAT_RING_SIZE);
1730 	bus_dmamap_sync(rxst_ring->rsr_dtag, rxst_ring->rsr_dmap,
1731 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1732 
1733 	return (0);
1734 }
1735 
1736 static int
1737 et_init_rxdma(struct et_softc *sc)
1738 {
1739 	struct et_rxstatus_data *rxsd;
1740 	struct et_rxstat_ring *rxst_ring;
1741 	struct et_rxdesc_ring *rx_ring;
1742 	int error;
1743 
1744 	error = et_stop_rxdma(sc);
1745 	if (error) {
1746 		if_printf(sc->ifp, "can't init RX DMA engine\n");
1747 		return (error);
1748 	}
1749 
1750 	/*
1751 	 * Install RX status
1752 	 */
1753 	rxsd = &sc->sc_rx_status;
1754 	CSR_WRITE_4(sc, ET_RX_STATUS_HI, ET_ADDR_HI(rxsd->rxsd_paddr));
1755 	CSR_WRITE_4(sc, ET_RX_STATUS_LO, ET_ADDR_LO(rxsd->rxsd_paddr));
1756 
1757 	/*
1758 	 * Install RX stat ring
1759 	 */
1760 	rxst_ring = &sc->sc_rxstat_ring;
1761 	CSR_WRITE_4(sc, ET_RXSTAT_HI, ET_ADDR_HI(rxst_ring->rsr_paddr));
1762 	CSR_WRITE_4(sc, ET_RXSTAT_LO, ET_ADDR_LO(rxst_ring->rsr_paddr));
1763 	CSR_WRITE_4(sc, ET_RXSTAT_CNT, ET_RX_NSTAT - 1);
1764 	CSR_WRITE_4(sc, ET_RXSTAT_POS, 0);
1765 	CSR_WRITE_4(sc, ET_RXSTAT_MINCNT, ((ET_RX_NSTAT * 15) / 100) - 1);
1766 
1767 	/* Match ET_RXSTAT_POS */
1768 	rxst_ring->rsr_index = 0;
1769 	rxst_ring->rsr_wrap = 0;
1770 
1771 	/*
1772 	 * Install the 2nd RX descriptor ring
1773 	 */
1774 	rx_ring = &sc->sc_rx_ring[1];
1775 	CSR_WRITE_4(sc, ET_RX_RING1_HI, ET_ADDR_HI(rx_ring->rr_paddr));
1776 	CSR_WRITE_4(sc, ET_RX_RING1_LO, ET_ADDR_LO(rx_ring->rr_paddr));
1777 	CSR_WRITE_4(sc, ET_RX_RING1_CNT, ET_RX_NDESC - 1);
1778 	CSR_WRITE_4(sc, ET_RX_RING1_POS, ET_RX_RING1_POS_WRAP);
1779 	CSR_WRITE_4(sc, ET_RX_RING1_MINCNT, ((ET_RX_NDESC * 15) / 100) - 1);
1780 
1781 	/* Match ET_RX_RING1_POS */
1782 	rx_ring->rr_index = 0;
1783 	rx_ring->rr_wrap = 1;
1784 
1785 	/*
1786 	 * Install the 1st RX descriptor ring
1787 	 */
1788 	rx_ring = &sc->sc_rx_ring[0];
1789 	CSR_WRITE_4(sc, ET_RX_RING0_HI, ET_ADDR_HI(rx_ring->rr_paddr));
1790 	CSR_WRITE_4(sc, ET_RX_RING0_LO, ET_ADDR_LO(rx_ring->rr_paddr));
1791 	CSR_WRITE_4(sc, ET_RX_RING0_CNT, ET_RX_NDESC - 1);
1792 	CSR_WRITE_4(sc, ET_RX_RING0_POS, ET_RX_RING0_POS_WRAP);
1793 	CSR_WRITE_4(sc, ET_RX_RING0_MINCNT, ((ET_RX_NDESC * 15) / 100) - 1);
1794 
1795 	/* Match ET_RX_RING0_POS */
1796 	rx_ring->rr_index = 0;
1797 	rx_ring->rr_wrap = 1;
1798 
1799 	/*
1800 	 * RX intr moderation
1801 	 */
1802 	CSR_WRITE_4(sc, ET_RX_INTR_NPKTS, sc->sc_rx_intr_npkts);
1803 	CSR_WRITE_4(sc, ET_RX_INTR_DELAY, sc->sc_rx_intr_delay);
1804 
1805 	return (0);
1806 }
1807 
1808 static int
1809 et_init_txdma(struct et_softc *sc)
1810 {
1811 	struct et_txdesc_ring *tx_ring;
1812 	struct et_txstatus_data *txsd;
1813 	int error;
1814 
1815 	error = et_stop_txdma(sc);
1816 	if (error) {
1817 		if_printf(sc->ifp, "can't init TX DMA engine\n");
1818 		return (error);
1819 	}
1820 
1821 	/*
1822 	 * Install TX descriptor ring
1823 	 */
1824 	tx_ring = &sc->sc_tx_ring;
1825 	CSR_WRITE_4(sc, ET_TX_RING_HI, ET_ADDR_HI(tx_ring->tr_paddr));
1826 	CSR_WRITE_4(sc, ET_TX_RING_LO, ET_ADDR_LO(tx_ring->tr_paddr));
1827 	CSR_WRITE_4(sc, ET_TX_RING_CNT, ET_TX_NDESC - 1);
1828 
1829 	/*
1830 	 * Install TX status
1831 	 */
1832 	txsd = &sc->sc_tx_status;
1833 	CSR_WRITE_4(sc, ET_TX_STATUS_HI, ET_ADDR_HI(txsd->txsd_paddr));
1834 	CSR_WRITE_4(sc, ET_TX_STATUS_LO, ET_ADDR_LO(txsd->txsd_paddr));
1835 
1836 	CSR_WRITE_4(sc, ET_TX_READY_POS, 0);
1837 
1838 	/* Match ET_TX_READY_POS */
1839 	tx_ring->tr_ready_index = 0;
1840 	tx_ring->tr_ready_wrap = 0;
1841 
1842 	return (0);
1843 }
1844 
1845 static void
1846 et_init_mac(struct et_softc *sc)
1847 {
1848 	struct ifnet *ifp;
1849 	const uint8_t *eaddr;
1850 	uint32_t val;
1851 
1852 	/* Reset MAC */
1853 	CSR_WRITE_4(sc, ET_MAC_CFG1,
1854 		    ET_MAC_CFG1_RST_TXFUNC | ET_MAC_CFG1_RST_RXFUNC |
1855 		    ET_MAC_CFG1_RST_TXMC | ET_MAC_CFG1_RST_RXMC |
1856 		    ET_MAC_CFG1_SIM_RST | ET_MAC_CFG1_SOFT_RST);
1857 
1858 	/*
1859 	 * Setup inter packet gap
1860 	 */
1861 	val = (56 << ET_IPG_NONB2B_1_SHIFT) |
1862 	    (88 << ET_IPG_NONB2B_2_SHIFT) |
1863 	    (80 << ET_IPG_MINIFG_SHIFT) |
1864 	    (96 << ET_IPG_B2B_SHIFT);
1865 	CSR_WRITE_4(sc, ET_IPG, val);
1866 
1867 	/*
1868 	 * Setup half duplex mode
1869 	 */
1870 	val = (10 << ET_MAC_HDX_ALT_BEB_TRUNC_SHIFT) |
1871 	    (15 << ET_MAC_HDX_REXMIT_MAX_SHIFT) |
1872 	    (55 << ET_MAC_HDX_COLLWIN_SHIFT) |
1873 	    ET_MAC_HDX_EXC_DEFER;
1874 	CSR_WRITE_4(sc, ET_MAC_HDX, val);
1875 
1876 	/* Clear MAC control */
1877 	CSR_WRITE_4(sc, ET_MAC_CTRL, 0);
1878 
1879 	/* Reset MII */
1880 	CSR_WRITE_4(sc, ET_MII_CFG, ET_MII_CFG_CLKRST);
1881 
1882 	/*
1883 	 * Set MAC address
1884 	 */
1885 	ifp = sc->ifp;
1886 	eaddr = IF_LLADDR(ifp);
1887 	val = eaddr[2] | (eaddr[3] << 8) | (eaddr[4] << 16) | (eaddr[5] << 24);
1888 	CSR_WRITE_4(sc, ET_MAC_ADDR1, val);
1889 	val = (eaddr[0] << 16) | (eaddr[1] << 24);
1890 	CSR_WRITE_4(sc, ET_MAC_ADDR2, val);
1891 
1892 	/* Set max frame length */
1893 	CSR_WRITE_4(sc, ET_MAX_FRMLEN, ET_FRAMELEN(ifp->if_mtu));
1894 
1895 	/* Bring MAC out of reset state */
1896 	CSR_WRITE_4(sc, ET_MAC_CFG1, 0);
1897 }
1898 
1899 static void
1900 et_init_rxmac(struct et_softc *sc)
1901 {
1902 	struct ifnet *ifp;
1903 	const uint8_t *eaddr;
1904 	uint32_t val;
1905 	int i;
1906 
1907 	/* Disable RX MAC and WOL */
1908 	CSR_WRITE_4(sc, ET_RXMAC_CTRL, ET_RXMAC_CTRL_WOL_DISABLE);
1909 
1910 	/*
1911 	 * Clear all WOL related registers
1912 	 */
1913 	for (i = 0; i < 3; ++i)
1914 		CSR_WRITE_4(sc, ET_WOL_CRC + (i * 4), 0);
1915 	for (i = 0; i < 20; ++i)
1916 		CSR_WRITE_4(sc, ET_WOL_MASK + (i * 4), 0);
1917 
1918 	/*
1919 	 * Set WOL source address.  XXX is this necessary?
1920 	 */
1921 	ifp = sc->ifp;
1922 	eaddr = IF_LLADDR(ifp);
1923 	val = (eaddr[2] << 24) | (eaddr[3] << 16) | (eaddr[4] << 8) | eaddr[5];
1924 	CSR_WRITE_4(sc, ET_WOL_SA_LO, val);
1925 	val = (eaddr[0] << 8) | eaddr[1];
1926 	CSR_WRITE_4(sc, ET_WOL_SA_HI, val);
1927 
1928 	/* Clear packet filters */
1929 	CSR_WRITE_4(sc, ET_PKTFILT, 0);
1930 
1931 	/* No ucast filtering */
1932 	CSR_WRITE_4(sc, ET_UCAST_FILTADDR1, 0);
1933 	CSR_WRITE_4(sc, ET_UCAST_FILTADDR2, 0);
1934 	CSR_WRITE_4(sc, ET_UCAST_FILTADDR3, 0);
1935 
1936 	if (ET_FRAMELEN(ifp->if_mtu) > ET_RXMAC_CUT_THRU_FRMLEN) {
1937 		/*
1938 		 * In order to transmit jumbo packets greater than
1939 		 * ET_RXMAC_CUT_THRU_FRMLEN bytes, the FIFO between
1940 		 * RX MAC and RX DMA needs to be reduced in size to
1941 		 * (ET_MEM_SIZE - ET_MEM_TXSIZE_EX - framelen).  In
1942 		 * order to implement this, we must use "cut through"
1943 		 * mode in the RX MAC, which chops packets down into
1944 		 * segments.  In this case we selected 256 bytes,
1945 		 * since this is the size of the PCI-Express TLP's
1946 		 * that the ET1310 uses.
1947 		 */
1948 		val = (ET_RXMAC_SEGSZ(256) & ET_RXMAC_MC_SEGSZ_MAX_MASK) |
1949 		      ET_RXMAC_MC_SEGSZ_ENABLE;
1950 	} else {
1951 		val = 0;
1952 	}
1953 	CSR_WRITE_4(sc, ET_RXMAC_MC_SEGSZ, val);
1954 
1955 	CSR_WRITE_4(sc, ET_RXMAC_MC_WATERMARK, 0);
1956 
1957 	/* Initialize RX MAC management register */
1958 	CSR_WRITE_4(sc, ET_RXMAC_MGT, 0);
1959 
1960 	CSR_WRITE_4(sc, ET_RXMAC_SPACE_AVL, 0);
1961 
1962 	CSR_WRITE_4(sc, ET_RXMAC_MGT,
1963 		    ET_RXMAC_MGT_PASS_ECRC |
1964 		    ET_RXMAC_MGT_PASS_ELEN |
1965 		    ET_RXMAC_MGT_PASS_ETRUNC |
1966 		    ET_RXMAC_MGT_CHECK_PKT);
1967 
1968 	/*
1969 	 * Configure runt filtering (may not work on certain chip generation)
1970 	 */
1971 	val = (ETHER_MIN_LEN << ET_PKTFILT_MINLEN_SHIFT) &
1972 	    ET_PKTFILT_MINLEN_MASK;
1973 	val |= ET_PKTFILT_FRAG;
1974 	CSR_WRITE_4(sc, ET_PKTFILT, val);
1975 
1976 	/* Enable RX MAC but leave WOL disabled */
1977 	CSR_WRITE_4(sc, ET_RXMAC_CTRL,
1978 		    ET_RXMAC_CTRL_WOL_DISABLE | ET_RXMAC_CTRL_ENABLE);
1979 
1980 	/*
1981 	 * Setup multicast hash and allmulti/promisc mode
1982 	 */
1983 	et_setmulti(sc);
1984 }
1985 
1986 static void
1987 et_init_txmac(struct et_softc *sc)
1988 {
1989 
1990 	/* Disable TX MAC and FC(?) */
1991 	CSR_WRITE_4(sc, ET_TXMAC_CTRL, ET_TXMAC_CTRL_FC_DISABLE);
1992 
1993 	/*
1994 	 * Initialize pause time.
1995 	 * This register should be set before XON/XOFF frame is
1996 	 * sent by driver.
1997 	 */
1998 	CSR_WRITE_4(sc, ET_TXMAC_FLOWCTRL, 0 << ET_TXMAC_FLOWCTRL_CFPT_SHIFT);
1999 
2000 	/* Enable TX MAC but leave FC(?) diabled */
2001 	CSR_WRITE_4(sc, ET_TXMAC_CTRL,
2002 		    ET_TXMAC_CTRL_ENABLE | ET_TXMAC_CTRL_FC_DISABLE);
2003 }
2004 
2005 static int
2006 et_start_rxdma(struct et_softc *sc)
2007 {
2008 	uint32_t val;
2009 
2010 	val = (sc->sc_rx_data[0].rbd_bufsize & ET_RXDMA_CTRL_RING0_SIZE_MASK) |
2011 	    ET_RXDMA_CTRL_RING0_ENABLE;
2012 	val |= (sc->sc_rx_data[1].rbd_bufsize & ET_RXDMA_CTRL_RING1_SIZE_MASK) |
2013 	    ET_RXDMA_CTRL_RING1_ENABLE;
2014 
2015 	CSR_WRITE_4(sc, ET_RXDMA_CTRL, val);
2016 
2017 	DELAY(5);
2018 
2019 	if (CSR_READ_4(sc, ET_RXDMA_CTRL) & ET_RXDMA_CTRL_HALTED) {
2020 		if_printf(sc->ifp, "can't start RX DMA engine\n");
2021 		return (ETIMEDOUT);
2022 	}
2023 	return (0);
2024 }
2025 
2026 static int
2027 et_start_txdma(struct et_softc *sc)
2028 {
2029 
2030 	CSR_WRITE_4(sc, ET_TXDMA_CTRL, ET_TXDMA_CTRL_SINGLE_EPKT);
2031 	return (0);
2032 }
2033 
2034 static void
2035 et_rxeof(struct et_softc *sc)
2036 {
2037 	struct et_rxstatus_data *rxsd;
2038 	struct et_rxstat_ring *rxst_ring;
2039 	struct et_rxbuf_data *rbd;
2040 	struct et_rxdesc_ring *rx_ring;
2041 	struct et_rxstat *st;
2042 	struct ifnet *ifp;
2043 	struct mbuf *m;
2044 	uint32_t rxstat_pos, rxring_pos;
2045 	uint32_t rxst_info1, rxst_info2, rxs_stat_ring;
2046 	int buflen, buf_idx, npost[2], ring_idx;
2047 	int rxst_index, rxst_wrap;
2048 
2049 	ET_LOCK_ASSERT(sc);
2050 
2051 	ifp = sc->ifp;
2052 	rxsd = &sc->sc_rx_status;
2053 	rxst_ring = &sc->sc_rxstat_ring;
2054 
2055 	if ((sc->sc_flags & ET_FLAG_TXRX_ENABLED) == 0)
2056 		return;
2057 
2058 	bus_dmamap_sync(rxsd->rxsd_dtag, rxsd->rxsd_dmap,
2059 	    BUS_DMASYNC_POSTREAD);
2060 	bus_dmamap_sync(rxst_ring->rsr_dtag, rxst_ring->rsr_dmap,
2061 	    BUS_DMASYNC_POSTREAD);
2062 
2063 	npost[0] = npost[1] = 0;
2064 	rxs_stat_ring = le32toh(rxsd->rxsd_status->rxs_stat_ring);
2065 	rxst_wrap = (rxs_stat_ring & ET_RXS_STATRING_WRAP) ? 1 : 0;
2066 	rxst_index = (rxs_stat_ring & ET_RXS_STATRING_INDEX_MASK) >>
2067 	    ET_RXS_STATRING_INDEX_SHIFT;
2068 
2069 	while (rxst_index != rxst_ring->rsr_index ||
2070 	    rxst_wrap != rxst_ring->rsr_wrap) {
2071 		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2072 			break;
2073 
2074 		MPASS(rxst_ring->rsr_index < ET_RX_NSTAT);
2075 		st = &rxst_ring->rsr_stat[rxst_ring->rsr_index];
2076 		rxst_info1 = le32toh(st->rxst_info1);
2077 		rxst_info2 = le32toh(st->rxst_info2);
2078 		buflen = (rxst_info2 & ET_RXST_INFO2_LEN_MASK) >>
2079 		    ET_RXST_INFO2_LEN_SHIFT;
2080 		buf_idx = (rxst_info2 & ET_RXST_INFO2_BUFIDX_MASK) >>
2081 		    ET_RXST_INFO2_BUFIDX_SHIFT;
2082 		ring_idx = (rxst_info2 & ET_RXST_INFO2_RINGIDX_MASK) >>
2083 		    ET_RXST_INFO2_RINGIDX_SHIFT;
2084 
2085 		if (++rxst_ring->rsr_index == ET_RX_NSTAT) {
2086 			rxst_ring->rsr_index = 0;
2087 			rxst_ring->rsr_wrap ^= 1;
2088 		}
2089 		rxstat_pos = rxst_ring->rsr_index & ET_RXSTAT_POS_INDEX_MASK;
2090 		if (rxst_ring->rsr_wrap)
2091 			rxstat_pos |= ET_RXSTAT_POS_WRAP;
2092 		CSR_WRITE_4(sc, ET_RXSTAT_POS, rxstat_pos);
2093 
2094 		if (ring_idx >= ET_RX_NRING) {
2095 			ifp->if_ierrors++;
2096 			if_printf(ifp, "invalid ring index %d\n", ring_idx);
2097 			continue;
2098 		}
2099 		if (buf_idx >= ET_RX_NDESC) {
2100 			ifp->if_ierrors++;
2101 			if_printf(ifp, "invalid buf index %d\n", buf_idx);
2102 			continue;
2103 		}
2104 
2105 		rbd = &sc->sc_rx_data[ring_idx];
2106 		m = rbd->rbd_buf[buf_idx].rb_mbuf;
2107 		if ((rxst_info1 & ET_RXST_INFO1_OK) == 0){
2108 			/* Discard errored frame. */
2109 			rbd->rbd_discard(rbd, buf_idx);
2110 		} else if (rbd->rbd_newbuf(rbd, buf_idx) != 0) {
2111 			/* No available mbufs, discard it. */
2112 			ifp->if_iqdrops++;
2113 			rbd->rbd_discard(rbd, buf_idx);
2114 		} else {
2115 			buflen -= ETHER_CRC_LEN;
2116 			if (buflen < ETHER_HDR_LEN) {
2117 				m_freem(m);
2118 				ifp->if_ierrors++;
2119 			} else {
2120 				m->m_pkthdr.len = m->m_len = buflen;
2121 				m->m_pkthdr.rcvif = ifp;
2122 				ET_UNLOCK(sc);
2123 				ifp->if_input(ifp, m);
2124 				ET_LOCK(sc);
2125 			}
2126 		}
2127 
2128 		rx_ring = &sc->sc_rx_ring[ring_idx];
2129 		if (buf_idx != rx_ring->rr_index) {
2130 			if_printf(ifp,
2131 			    "WARNING!! ring %d, buf_idx %d, rr_idx %d\n",
2132 			    ring_idx, buf_idx, rx_ring->rr_index);
2133 		}
2134 
2135 		MPASS(rx_ring->rr_index < ET_RX_NDESC);
2136 		if (++rx_ring->rr_index == ET_RX_NDESC) {
2137 			rx_ring->rr_index = 0;
2138 			rx_ring->rr_wrap ^= 1;
2139 		}
2140 		rxring_pos = rx_ring->rr_index & ET_RX_RING_POS_INDEX_MASK;
2141 		if (rx_ring->rr_wrap)
2142 			rxring_pos |= ET_RX_RING_POS_WRAP;
2143 		CSR_WRITE_4(sc, rx_ring->rr_posreg, rxring_pos);
2144 	}
2145 
2146 	bus_dmamap_sync(rxsd->rxsd_dtag, rxsd->rxsd_dmap,
2147 	    BUS_DMASYNC_PREREAD);
2148 	bus_dmamap_sync(rxst_ring->rsr_dtag, rxst_ring->rsr_dmap,
2149 	    BUS_DMASYNC_PREREAD);
2150 }
2151 
2152 static int
2153 et_encap(struct et_softc *sc, struct mbuf **m0)
2154 {
2155 	struct et_txdesc_ring *tx_ring;
2156 	struct et_txbuf_data *tbd;
2157 	struct et_txdesc *td;
2158 	struct mbuf *m;
2159 	bus_dma_segment_t segs[ET_NSEG_MAX];
2160 	bus_dmamap_t map;
2161 	uint32_t csum_flags, last_td_ctrl2;
2162 	int error, i, idx, first_idx, last_idx, nsegs;
2163 
2164 	tx_ring = &sc->sc_tx_ring;
2165 	MPASS(tx_ring->tr_ready_index < ET_TX_NDESC);
2166 	tbd = &sc->sc_tx_data;
2167 	first_idx = tx_ring->tr_ready_index;
2168 	map = tbd->tbd_buf[first_idx].tb_dmap;
2169 
2170 	error = bus_dmamap_load_mbuf_sg(sc->sc_tx_tag, map, *m0, segs, &nsegs,
2171 	    0);
2172 	if (error == EFBIG) {
2173 		m = m_collapse(*m0, M_NOWAIT, ET_NSEG_MAX);
2174 		if (m == NULL) {
2175 			m_freem(*m0);
2176 			*m0 = NULL;
2177 			return (ENOMEM);
2178 		}
2179 		*m0 = m;
2180 		error = bus_dmamap_load_mbuf_sg(sc->sc_tx_tag, map, *m0, segs,
2181 		    &nsegs, 0);
2182 		if (error != 0) {
2183 			m_freem(*m0);
2184                         *m0 = NULL;
2185 			return (error);
2186 		}
2187 	} else if (error != 0)
2188 		return (error);
2189 
2190 	/* Check for descriptor overruns. */
2191 	if (tbd->tbd_used + nsegs > ET_TX_NDESC - 1) {
2192 		bus_dmamap_unload(sc->sc_tx_tag, map);
2193 		return (ENOBUFS);
2194 	}
2195 	bus_dmamap_sync(sc->sc_tx_tag, map, BUS_DMASYNC_PREWRITE);
2196 
2197 	last_td_ctrl2 = ET_TDCTRL2_LAST_FRAG;
2198 	sc->sc_tx += nsegs;
2199 	if (sc->sc_tx / sc->sc_tx_intr_nsegs != sc->sc_tx_intr) {
2200 		sc->sc_tx_intr = sc->sc_tx / sc->sc_tx_intr_nsegs;
2201 		last_td_ctrl2 |= ET_TDCTRL2_INTR;
2202 	}
2203 
2204 	m = *m0;
2205 	csum_flags = 0;
2206 	if ((m->m_pkthdr.csum_flags & ET_CSUM_FEATURES) != 0) {
2207 		if ((m->m_pkthdr.csum_flags & CSUM_IP) != 0)
2208 			csum_flags |= ET_TDCTRL2_CSUM_IP;
2209 		if ((m->m_pkthdr.csum_flags & CSUM_UDP) != 0)
2210 			csum_flags |= ET_TDCTRL2_CSUM_UDP;
2211 		else if ((m->m_pkthdr.csum_flags & CSUM_TCP) != 0)
2212 			csum_flags |= ET_TDCTRL2_CSUM_TCP;
2213 	}
2214 	last_idx = -1;
2215 	for (i = 0; i < nsegs; ++i) {
2216 		idx = (first_idx + i) % ET_TX_NDESC;
2217 		td = &tx_ring->tr_desc[idx];
2218 		td->td_addr_hi = htole32(ET_ADDR_HI(segs[i].ds_addr));
2219 		td->td_addr_lo = htole32(ET_ADDR_LO(segs[i].ds_addr));
2220 		td->td_ctrl1 =  htole32(segs[i].ds_len & ET_TDCTRL1_LEN_MASK);
2221 		if (i == nsegs - 1) {
2222 			/* Last frag */
2223 			td->td_ctrl2 = htole32(last_td_ctrl2 | csum_flags);
2224 			last_idx = idx;
2225 		} else
2226 			td->td_ctrl2 = htole32(csum_flags);
2227 
2228 		MPASS(tx_ring->tr_ready_index < ET_TX_NDESC);
2229 		if (++tx_ring->tr_ready_index == ET_TX_NDESC) {
2230 			tx_ring->tr_ready_index = 0;
2231 			tx_ring->tr_ready_wrap ^= 1;
2232 		}
2233 	}
2234 	td = &tx_ring->tr_desc[first_idx];
2235 	/* First frag */
2236 	td->td_ctrl2 |= htole32(ET_TDCTRL2_FIRST_FRAG);
2237 
2238 	MPASS(last_idx >= 0);
2239 	tbd->tbd_buf[first_idx].tb_dmap = tbd->tbd_buf[last_idx].tb_dmap;
2240 	tbd->tbd_buf[last_idx].tb_dmap = map;
2241 	tbd->tbd_buf[last_idx].tb_mbuf = m;
2242 
2243 	tbd->tbd_used += nsegs;
2244 	MPASS(tbd->tbd_used <= ET_TX_NDESC);
2245 
2246 	return (0);
2247 }
2248 
2249 static void
2250 et_txeof(struct et_softc *sc)
2251 {
2252 	struct et_txdesc_ring *tx_ring;
2253 	struct et_txbuf_data *tbd;
2254 	struct et_txbuf *tb;
2255 	struct ifnet *ifp;
2256 	uint32_t tx_done;
2257 	int end, wrap;
2258 
2259 	ET_LOCK_ASSERT(sc);
2260 
2261 	ifp = sc->ifp;
2262 	tx_ring = &sc->sc_tx_ring;
2263 	tbd = &sc->sc_tx_data;
2264 
2265 	if ((sc->sc_flags & ET_FLAG_TXRX_ENABLED) == 0)
2266 		return;
2267 
2268 	if (tbd->tbd_used == 0)
2269 		return;
2270 
2271 	bus_dmamap_sync(tx_ring->tr_dtag, tx_ring->tr_dmap,
2272 	    BUS_DMASYNC_POSTWRITE);
2273 
2274 	tx_done = CSR_READ_4(sc, ET_TX_DONE_POS);
2275 	end = tx_done & ET_TX_DONE_POS_INDEX_MASK;
2276 	wrap = (tx_done & ET_TX_DONE_POS_WRAP) ? 1 : 0;
2277 
2278 	while (tbd->tbd_start_index != end || tbd->tbd_start_wrap != wrap) {
2279 		MPASS(tbd->tbd_start_index < ET_TX_NDESC);
2280 		tb = &tbd->tbd_buf[tbd->tbd_start_index];
2281 		if (tb->tb_mbuf != NULL) {
2282 			bus_dmamap_sync(sc->sc_tx_tag, tb->tb_dmap,
2283 			    BUS_DMASYNC_POSTWRITE);
2284 			bus_dmamap_unload(sc->sc_tx_tag, tb->tb_dmap);
2285 			m_freem(tb->tb_mbuf);
2286 			tb->tb_mbuf = NULL;
2287 		}
2288 
2289 		if (++tbd->tbd_start_index == ET_TX_NDESC) {
2290 			tbd->tbd_start_index = 0;
2291 			tbd->tbd_start_wrap ^= 1;
2292 		}
2293 
2294 		MPASS(tbd->tbd_used > 0);
2295 		tbd->tbd_used--;
2296 	}
2297 
2298 	if (tbd->tbd_used == 0)
2299 		sc->watchdog_timer = 0;
2300 	if (tbd->tbd_used + ET_NSEG_SPARE < ET_TX_NDESC)
2301 		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2302 }
2303 
2304 static void
2305 et_tick(void *xsc)
2306 {
2307 	struct et_softc *sc;
2308 	struct ifnet *ifp;
2309 	struct mii_data *mii;
2310 
2311 	sc = xsc;
2312 	ET_LOCK_ASSERT(sc);
2313 	ifp = sc->ifp;
2314 	mii = device_get_softc(sc->sc_miibus);
2315 
2316 	mii_tick(mii);
2317 	et_stats_update(sc);
2318 	if (et_watchdog(sc) == EJUSTRETURN)
2319 		return;
2320 	callout_reset(&sc->sc_tick, hz, et_tick, sc);
2321 }
2322 
2323 static int
2324 et_newbuf_cluster(struct et_rxbuf_data *rbd, int buf_idx)
2325 {
2326 	struct et_softc *sc;
2327 	struct et_rxdesc *desc;
2328 	struct et_rxbuf *rb;
2329 	struct mbuf *m;
2330 	bus_dma_segment_t segs[1];
2331 	bus_dmamap_t dmap;
2332 	int nsegs;
2333 
2334 	MPASS(buf_idx < ET_RX_NDESC);
2335 	m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
2336 	if (m == NULL)
2337 		return (ENOBUFS);
2338 	m->m_len = m->m_pkthdr.len = MCLBYTES;
2339 	m_adj(m, ETHER_ALIGN);
2340 
2341 	sc = rbd->rbd_softc;
2342 	rb = &rbd->rbd_buf[buf_idx];
2343 
2344 	if (bus_dmamap_load_mbuf_sg(sc->sc_rx_tag, sc->sc_rx_sparemap, m,
2345 	    segs, &nsegs, 0) != 0) {
2346 		m_freem(m);
2347 		return (ENOBUFS);
2348 	}
2349 	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
2350 
2351 	if (rb->rb_mbuf != NULL) {
2352 		bus_dmamap_sync(sc->sc_rx_tag, rb->rb_dmap,
2353 		    BUS_DMASYNC_POSTREAD);
2354 		bus_dmamap_unload(sc->sc_rx_tag, rb->rb_dmap);
2355 	}
2356 	dmap = rb->rb_dmap;
2357 	rb->rb_dmap = sc->sc_rx_sparemap;
2358 	sc->sc_rx_sparemap = dmap;
2359 	bus_dmamap_sync(sc->sc_rx_tag, rb->rb_dmap, BUS_DMASYNC_PREREAD);
2360 
2361 	rb->rb_mbuf = m;
2362 	desc = &rbd->rbd_ring->rr_desc[buf_idx];
2363 	desc->rd_addr_hi = htole32(ET_ADDR_HI(segs[0].ds_addr));
2364 	desc->rd_addr_lo = htole32(ET_ADDR_LO(segs[0].ds_addr));
2365 	desc->rd_ctrl = htole32(buf_idx & ET_RDCTRL_BUFIDX_MASK);
2366 	bus_dmamap_sync(rbd->rbd_ring->rr_dtag, rbd->rbd_ring->rr_dmap,
2367 	    BUS_DMASYNC_PREWRITE);
2368 	return (0);
2369 }
2370 
2371 static void
2372 et_rxbuf_discard(struct et_rxbuf_data *rbd, int buf_idx)
2373 {
2374 	struct et_rxdesc *desc;
2375 
2376 	desc = &rbd->rbd_ring->rr_desc[buf_idx];
2377 	desc->rd_ctrl = htole32(buf_idx & ET_RDCTRL_BUFIDX_MASK);
2378 	bus_dmamap_sync(rbd->rbd_ring->rr_dtag, rbd->rbd_ring->rr_dmap,
2379 	    BUS_DMASYNC_PREWRITE);
2380 }
2381 
2382 static int
2383 et_newbuf_hdr(struct et_rxbuf_data *rbd, int buf_idx)
2384 {
2385 	struct et_softc *sc;
2386 	struct et_rxdesc *desc;
2387 	struct et_rxbuf *rb;
2388 	struct mbuf *m;
2389 	bus_dma_segment_t segs[1];
2390 	bus_dmamap_t dmap;
2391 	int nsegs;
2392 
2393 	MPASS(buf_idx < ET_RX_NDESC);
2394 	MGETHDR(m, M_NOWAIT, MT_DATA);
2395 	if (m == NULL)
2396 		return (ENOBUFS);
2397 	m->m_len = m->m_pkthdr.len = MHLEN;
2398 	m_adj(m, ETHER_ALIGN);
2399 
2400 	sc = rbd->rbd_softc;
2401 	rb = &rbd->rbd_buf[buf_idx];
2402 
2403 	if (bus_dmamap_load_mbuf_sg(sc->sc_rx_mini_tag, sc->sc_rx_mini_sparemap,
2404 	    m, segs, &nsegs, 0) != 0) {
2405 		m_freem(m);
2406 		return (ENOBUFS);
2407 	}
2408 	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
2409 
2410 	if (rb->rb_mbuf != NULL) {
2411 		bus_dmamap_sync(sc->sc_rx_mini_tag, rb->rb_dmap,
2412 		    BUS_DMASYNC_POSTREAD);
2413 		bus_dmamap_unload(sc->sc_rx_mini_tag, rb->rb_dmap);
2414 	}
2415 	dmap = rb->rb_dmap;
2416 	rb->rb_dmap = sc->sc_rx_mini_sparemap;
2417 	sc->sc_rx_mini_sparemap = dmap;
2418 	bus_dmamap_sync(sc->sc_rx_mini_tag, rb->rb_dmap, BUS_DMASYNC_PREREAD);
2419 
2420 	rb->rb_mbuf = m;
2421 	desc = &rbd->rbd_ring->rr_desc[buf_idx];
2422 	desc->rd_addr_hi = htole32(ET_ADDR_HI(segs[0].ds_addr));
2423 	desc->rd_addr_lo = htole32(ET_ADDR_LO(segs[0].ds_addr));
2424 	desc->rd_ctrl = htole32(buf_idx & ET_RDCTRL_BUFIDX_MASK);
2425 	bus_dmamap_sync(rbd->rbd_ring->rr_dtag, rbd->rbd_ring->rr_dmap,
2426 	    BUS_DMASYNC_PREWRITE);
2427 	return (0);
2428 }
2429 
2430 #define	ET_SYSCTL_STAT_ADD32(c, h, n, p, d)	\
2431 	    SYSCTL_ADD_UINT(c, h, OID_AUTO, n, CTLFLAG_RD, p, 0, d)
2432 #define	ET_SYSCTL_STAT_ADD64(c, h, n, p, d)	\
2433 	    SYSCTL_ADD_UQUAD(c, h, OID_AUTO, n, CTLFLAG_RD, p, d)
2434 
2435 /*
2436  * Create sysctl tree
2437  */
2438 static void
2439 et_add_sysctls(struct et_softc * sc)
2440 {
2441 	struct sysctl_ctx_list *ctx;
2442 	struct sysctl_oid_list *children, *parent;
2443 	struct sysctl_oid *tree;
2444 	struct et_hw_stats *stats;
2445 
2446 	ctx = device_get_sysctl_ctx(sc->dev);
2447 	children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
2448 
2449 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rx_intr_npkts",
2450 	    CTLTYPE_INT | CTLFLAG_RW, sc, 0, et_sysctl_rx_intr_npkts, "I",
2451 	    "RX IM, # packets per RX interrupt");
2452 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rx_intr_delay",
2453 	    CTLTYPE_INT | CTLFLAG_RW, sc, 0, et_sysctl_rx_intr_delay, "I",
2454 	    "RX IM, RX interrupt delay (x10 usec)");
2455 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_intr_nsegs",
2456 	    CTLFLAG_RW, &sc->sc_tx_intr_nsegs, 0,
2457 	    "TX IM, # segments per TX interrupt");
2458 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "timer",
2459 	    CTLFLAG_RW, &sc->sc_timer, 0, "TX timer");
2460 
2461 	tree = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
2462 	    NULL, "ET statistics");
2463         parent = SYSCTL_CHILDREN(tree);
2464 
2465 	/* TX/RX statistics. */
2466 	stats = &sc->sc_stats;
2467 	ET_SYSCTL_STAT_ADD64(ctx, parent, "frames_64", &stats->pkts_64,
2468 	    "0 to 64 bytes frames");
2469 	ET_SYSCTL_STAT_ADD64(ctx, parent, "frames_65_127", &stats->pkts_65,
2470 	    "65 to 127 bytes frames");
2471 	ET_SYSCTL_STAT_ADD64(ctx, parent, "frames_128_255", &stats->pkts_128,
2472 	    "128 to 255 bytes frames");
2473 	ET_SYSCTL_STAT_ADD64(ctx, parent, "frames_256_511", &stats->pkts_256,
2474 	    "256 to 511 bytes frames");
2475 	ET_SYSCTL_STAT_ADD64(ctx, parent, "frames_512_1023", &stats->pkts_512,
2476 	    "512 to 1023 bytes frames");
2477 	ET_SYSCTL_STAT_ADD64(ctx, parent, "frames_1024_1518", &stats->pkts_1024,
2478 	    "1024 to 1518 bytes frames");
2479 	ET_SYSCTL_STAT_ADD64(ctx, parent, "frames_1519_1522", &stats->pkts_1519,
2480 	    "1519 to 1522 bytes frames");
2481 
2482 	/* RX statistics. */
2483 	tree = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "rx", CTLFLAG_RD,
2484 	    NULL, "RX MAC statistics");
2485 	children = SYSCTL_CHILDREN(tree);
2486 	ET_SYSCTL_STAT_ADD64(ctx, children, "bytes",
2487 	    &stats->rx_bytes, "Good bytes");
2488 	ET_SYSCTL_STAT_ADD64(ctx, children, "frames",
2489 	    &stats->rx_frames, "Good frames");
2490 	ET_SYSCTL_STAT_ADD32(ctx, children, "crc_errs",
2491 	    &stats->rx_crcerrs, "CRC errors");
2492 	ET_SYSCTL_STAT_ADD64(ctx, children, "mcast_frames",
2493 	    &stats->rx_mcast, "Multicast frames");
2494 	ET_SYSCTL_STAT_ADD64(ctx, children, "bcast_frames",
2495 	    &stats->rx_bcast, "Broadcast frames");
2496 	ET_SYSCTL_STAT_ADD32(ctx, children, "control",
2497 	    &stats->rx_control, "Control frames");
2498 	ET_SYSCTL_STAT_ADD32(ctx, children, "pause",
2499 	    &stats->rx_pause, "Pause frames");
2500 	ET_SYSCTL_STAT_ADD32(ctx, children, "unknown_control",
2501 	    &stats->rx_unknown_control, "Unknown control frames");
2502 	ET_SYSCTL_STAT_ADD32(ctx, children, "align_errs",
2503 	    &stats->rx_alignerrs, "Alignment errors");
2504 	ET_SYSCTL_STAT_ADD32(ctx, children, "len_errs",
2505 	    &stats->rx_lenerrs, "Frames with length mismatched");
2506 	ET_SYSCTL_STAT_ADD32(ctx, children, "code_errs",
2507 	    &stats->rx_codeerrs, "Frames with code error");
2508 	ET_SYSCTL_STAT_ADD32(ctx, children, "cs_errs",
2509 	    &stats->rx_cserrs, "Frames with carrier sense error");
2510 	ET_SYSCTL_STAT_ADD32(ctx, children, "runts",
2511 	    &stats->rx_runts, "Too short frames");
2512 	ET_SYSCTL_STAT_ADD64(ctx, children, "oversize",
2513 	    &stats->rx_oversize, "Oversized frames");
2514 	ET_SYSCTL_STAT_ADD32(ctx, children, "fragments",
2515 	    &stats->rx_fragments, "Fragmented frames");
2516 	ET_SYSCTL_STAT_ADD32(ctx, children, "jabbers",
2517 	    &stats->rx_jabbers, "Frames with jabber error");
2518 	ET_SYSCTL_STAT_ADD32(ctx, children, "drop",
2519 	    &stats->rx_drop, "Dropped frames");
2520 
2521 	/* TX statistics. */
2522 	tree = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "tx", CTLFLAG_RD,
2523 	    NULL, "TX MAC statistics");
2524 	children = SYSCTL_CHILDREN(tree);
2525 	ET_SYSCTL_STAT_ADD64(ctx, children, "bytes",
2526 	    &stats->tx_bytes, "Good bytes");
2527 	ET_SYSCTL_STAT_ADD64(ctx, children, "frames",
2528 	    &stats->tx_frames, "Good frames");
2529 	ET_SYSCTL_STAT_ADD64(ctx, children, "mcast_frames",
2530 	    &stats->tx_mcast, "Multicast frames");
2531 	ET_SYSCTL_STAT_ADD64(ctx, children, "bcast_frames",
2532 	    &stats->tx_bcast, "Broadcast frames");
2533 	ET_SYSCTL_STAT_ADD32(ctx, children, "pause",
2534 	    &stats->tx_pause, "Pause frames");
2535 	ET_SYSCTL_STAT_ADD32(ctx, children, "deferred",
2536 	    &stats->tx_deferred, "Deferred frames");
2537 	ET_SYSCTL_STAT_ADD32(ctx, children, "excess_deferred",
2538 	    &stats->tx_excess_deferred, "Excessively deferred frames");
2539 	ET_SYSCTL_STAT_ADD32(ctx, children, "single_colls",
2540 	    &stats->tx_single_colls, "Single collisions");
2541 	ET_SYSCTL_STAT_ADD32(ctx, children, "multi_colls",
2542 	    &stats->tx_multi_colls, "Multiple collisions");
2543 	ET_SYSCTL_STAT_ADD32(ctx, children, "late_colls",
2544 	    &stats->tx_late_colls, "Late collisions");
2545 	ET_SYSCTL_STAT_ADD32(ctx, children, "excess_colls",
2546 	    &stats->tx_excess_colls, "Excess collisions");
2547 	ET_SYSCTL_STAT_ADD32(ctx, children, "total_colls",
2548 	    &stats->tx_total_colls, "Total collisions");
2549 	ET_SYSCTL_STAT_ADD32(ctx, children, "pause_honored",
2550 	    &stats->tx_pause_honored, "Honored pause frames");
2551 	ET_SYSCTL_STAT_ADD32(ctx, children, "drop",
2552 	    &stats->tx_drop, "Dropped frames");
2553 	ET_SYSCTL_STAT_ADD32(ctx, children, "jabbers",
2554 	    &stats->tx_jabbers, "Frames with jabber errors");
2555 	ET_SYSCTL_STAT_ADD32(ctx, children, "crc_errs",
2556 	    &stats->tx_crcerrs, "Frames with CRC errors");
2557 	ET_SYSCTL_STAT_ADD32(ctx, children, "control",
2558 	    &stats->tx_control, "Control frames");
2559 	ET_SYSCTL_STAT_ADD64(ctx, children, "oversize",
2560 	    &stats->tx_oversize, "Oversized frames");
2561 	ET_SYSCTL_STAT_ADD32(ctx, children, "undersize",
2562 	    &stats->tx_undersize, "Undersized frames");
2563 	ET_SYSCTL_STAT_ADD32(ctx, children, "fragments",
2564 	    &stats->tx_fragments, "Fragmented frames");
2565 }
2566 
2567 #undef	ET_SYSCTL_STAT_ADD32
2568 #undef	ET_SYSCTL_STAT_ADD64
2569 
2570 static int
2571 et_sysctl_rx_intr_npkts(SYSCTL_HANDLER_ARGS)
2572 {
2573 	struct et_softc *sc;
2574 	struct ifnet *ifp;
2575 	int error, v;
2576 
2577 	sc = arg1;
2578 	ifp = sc->ifp;
2579 	v = sc->sc_rx_intr_npkts;
2580 	error = sysctl_handle_int(oidp, &v, 0, req);
2581 	if (error || req->newptr == NULL)
2582 		goto back;
2583 	if (v <= 0) {
2584 		error = EINVAL;
2585 		goto back;
2586 	}
2587 
2588 	if (sc->sc_rx_intr_npkts != v) {
2589 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2590 			CSR_WRITE_4(sc, ET_RX_INTR_NPKTS, v);
2591 		sc->sc_rx_intr_npkts = v;
2592 	}
2593 back:
2594 	return (error);
2595 }
2596 
2597 static int
2598 et_sysctl_rx_intr_delay(SYSCTL_HANDLER_ARGS)
2599 {
2600 	struct et_softc *sc;
2601 	struct ifnet *ifp;
2602 	int error, v;
2603 
2604 	sc = arg1;
2605 	ifp = sc->ifp;
2606 	v = sc->sc_rx_intr_delay;
2607 	error = sysctl_handle_int(oidp, &v, 0, req);
2608 	if (error || req->newptr == NULL)
2609 		goto back;
2610 	if (v <= 0) {
2611 		error = EINVAL;
2612 		goto back;
2613 	}
2614 
2615 	if (sc->sc_rx_intr_delay != v) {
2616 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2617 			CSR_WRITE_4(sc, ET_RX_INTR_DELAY, v);
2618 		sc->sc_rx_intr_delay = v;
2619 	}
2620 back:
2621 	return (error);
2622 }
2623 
2624 static void
2625 et_stats_update(struct et_softc *sc)
2626 {
2627 	struct ifnet *ifp;
2628 	struct et_hw_stats *stats;
2629 
2630 	stats = &sc->sc_stats;
2631 	stats->pkts_64 += CSR_READ_4(sc, ET_STAT_PKTS_64);
2632 	stats->pkts_65 += CSR_READ_4(sc, ET_STAT_PKTS_65_127);
2633 	stats->pkts_128 += CSR_READ_4(sc, ET_STAT_PKTS_128_255);
2634 	stats->pkts_256 += CSR_READ_4(sc, ET_STAT_PKTS_256_511);
2635 	stats->pkts_512 += CSR_READ_4(sc, ET_STAT_PKTS_512_1023);
2636 	stats->pkts_1024 += CSR_READ_4(sc, ET_STAT_PKTS_1024_1518);
2637 	stats->pkts_1519 += CSR_READ_4(sc, ET_STAT_PKTS_1519_1522);
2638 
2639 	stats->rx_bytes += CSR_READ_4(sc, ET_STAT_RX_BYTES);
2640 	stats->rx_frames += CSR_READ_4(sc, ET_STAT_RX_FRAMES);
2641 	stats->rx_crcerrs += CSR_READ_4(sc, ET_STAT_RX_CRC_ERR);
2642 	stats->rx_mcast += CSR_READ_4(sc, ET_STAT_RX_MCAST);
2643 	stats->rx_bcast += CSR_READ_4(sc, ET_STAT_RX_BCAST);
2644 	stats->rx_control += CSR_READ_4(sc, ET_STAT_RX_CTL);
2645 	stats->rx_pause += CSR_READ_4(sc, ET_STAT_RX_PAUSE);
2646 	stats->rx_unknown_control += CSR_READ_4(sc, ET_STAT_RX_UNKNOWN_CTL);
2647 	stats->rx_alignerrs += CSR_READ_4(sc, ET_STAT_RX_ALIGN_ERR);
2648 	stats->rx_lenerrs += CSR_READ_4(sc, ET_STAT_RX_LEN_ERR);
2649 	stats->rx_codeerrs += CSR_READ_4(sc, ET_STAT_RX_CODE_ERR);
2650 	stats->rx_cserrs += CSR_READ_4(sc, ET_STAT_RX_CS_ERR);
2651 	stats->rx_runts += CSR_READ_4(sc, ET_STAT_RX_RUNT);
2652 	stats->rx_oversize += CSR_READ_4(sc, ET_STAT_RX_OVERSIZE);
2653 	stats->rx_fragments += CSR_READ_4(sc, ET_STAT_RX_FRAG);
2654 	stats->rx_jabbers += CSR_READ_4(sc, ET_STAT_RX_JABBER);
2655 	stats->rx_drop += CSR_READ_4(sc, ET_STAT_RX_DROP);
2656 
2657 	stats->tx_bytes += CSR_READ_4(sc, ET_STAT_TX_BYTES);
2658 	stats->tx_frames += CSR_READ_4(sc, ET_STAT_TX_FRAMES);
2659 	stats->tx_mcast += CSR_READ_4(sc, ET_STAT_TX_MCAST);
2660 	stats->tx_bcast += CSR_READ_4(sc, ET_STAT_TX_BCAST);
2661 	stats->tx_pause += CSR_READ_4(sc, ET_STAT_TX_PAUSE);
2662 	stats->tx_deferred += CSR_READ_4(sc, ET_STAT_TX_DEFER);
2663 	stats->tx_excess_deferred += CSR_READ_4(sc, ET_STAT_TX_EXCESS_DEFER);
2664 	stats->tx_single_colls += CSR_READ_4(sc, ET_STAT_TX_SINGLE_COL);
2665 	stats->tx_multi_colls += CSR_READ_4(sc, ET_STAT_TX_MULTI_COL);
2666 	stats->tx_late_colls += CSR_READ_4(sc, ET_STAT_TX_LATE_COL);
2667 	stats->tx_excess_colls += CSR_READ_4(sc, ET_STAT_TX_EXCESS_COL);
2668 	stats->tx_total_colls += CSR_READ_4(sc, ET_STAT_TX_TOTAL_COL);
2669 	stats->tx_pause_honored += CSR_READ_4(sc, ET_STAT_TX_PAUSE_HONOR);
2670 	stats->tx_drop += CSR_READ_4(sc, ET_STAT_TX_DROP);
2671 	stats->tx_jabbers += CSR_READ_4(sc, ET_STAT_TX_JABBER);
2672 	stats->tx_crcerrs += CSR_READ_4(sc, ET_STAT_TX_CRC_ERR);
2673 	stats->tx_control += CSR_READ_4(sc, ET_STAT_TX_CTL);
2674 	stats->tx_oversize += CSR_READ_4(sc, ET_STAT_TX_OVERSIZE);
2675 	stats->tx_undersize += CSR_READ_4(sc, ET_STAT_TX_UNDERSIZE);
2676 	stats->tx_fragments += CSR_READ_4(sc, ET_STAT_TX_FRAG);
2677 
2678 	/* Update ifnet counters. */
2679 	ifp = sc->ifp;
2680 	ifp->if_opackets = (u_long)stats->tx_frames;
2681 	ifp->if_collisions = stats->tx_total_colls;
2682 	ifp->if_oerrors = stats->tx_drop + stats->tx_jabbers +
2683 	    stats->tx_crcerrs + stats->tx_excess_deferred +
2684 	    stats->tx_late_colls;
2685 	ifp->if_ipackets = (u_long)stats->rx_frames;
2686 	ifp->if_ierrors = stats->rx_crcerrs + stats->rx_alignerrs +
2687 	    stats->rx_lenerrs + stats->rx_codeerrs + stats->rx_cserrs +
2688 	    stats->rx_runts + stats->rx_jabbers + stats->rx_drop;
2689 }
2690 
2691 static int
2692 et_suspend(device_t dev)
2693 {
2694 	struct et_softc *sc;
2695 	uint32_t pmcfg;
2696 
2697 	sc = device_get_softc(dev);
2698 	ET_LOCK(sc);
2699 	if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
2700 		et_stop(sc);
2701 	/* Diable all clocks and put PHY into COMA. */
2702 	pmcfg = CSR_READ_4(sc, ET_PM);
2703 	pmcfg &= ~(EM_PM_GIGEPHY_ENB | ET_PM_SYSCLK_GATE | ET_PM_TXCLK_GATE |
2704 	    ET_PM_RXCLK_GATE);
2705 	pmcfg |= ET_PM_PHY_SW_COMA;
2706 	CSR_WRITE_4(sc, ET_PM, pmcfg);
2707 	ET_UNLOCK(sc);
2708 	return (0);
2709 }
2710 
2711 static int
2712 et_resume(device_t dev)
2713 {
2714 	struct et_softc *sc;
2715 	uint32_t pmcfg;
2716 
2717 	sc = device_get_softc(dev);
2718 	ET_LOCK(sc);
2719 	/* Take PHY out of COMA and enable clocks. */
2720 	pmcfg = ET_PM_SYSCLK_GATE | ET_PM_TXCLK_GATE | ET_PM_RXCLK_GATE;
2721 	if ((sc->sc_flags & ET_FLAG_FASTETHER) == 0)
2722 		pmcfg |= EM_PM_GIGEPHY_ENB;
2723 	CSR_WRITE_4(sc, ET_PM, pmcfg);
2724 	if ((sc->ifp->if_flags & IFF_UP) != 0)
2725 		et_init_locked(sc);
2726 	ET_UNLOCK(sc);
2727 	return (0);
2728 }
2729