xref: /freebsd/sys/dev/cadence/if_cgem.c (revision dd41de95a84d979615a2ef11df6850622bf6184e)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012-2014 Thomas Skibo <thomasskibo@yahoo.com>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 /*
30  * A network interface driver for Cadence GEM Gigabit Ethernet
31  * interface such as the one used in Xilinx Zynq-7000 SoC.
32  *
33  * Reference: Zynq-7000 All Programmable SoC Technical Reference Manual.
34  * (v1.4) November 16, 2012.  Xilinx doc UG585.  GEM is covered in Ch. 16
35  * and register definitions are in appendix B.18.
36  */
37 
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40 
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/bus.h>
44 #include <sys/kernel.h>
45 #include <sys/malloc.h>
46 #include <sys/mbuf.h>
47 #include <sys/module.h>
48 #include <sys/rman.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
51 #include <sys/sysctl.h>
52 
53 #include <machine/bus.h>
54 
55 #include <net/ethernet.h>
56 #include <net/if.h>
57 #include <net/if_arp.h>
58 #include <net/if_dl.h>
59 #include <net/if_media.h>
60 #include <net/if_mib.h>
61 #include <net/if_types.h>
62 
63 #ifdef INET
64 #include <netinet/in.h>
65 #include <netinet/in_systm.h>
66 #include <netinet/in_var.h>
67 #include <netinet/ip.h>
68 #endif
69 
70 #include <net/bpf.h>
71 #include <net/bpfdesc.h>
72 
73 #include <dev/fdt/fdt_common.h>
74 #include <dev/ofw/ofw_bus.h>
75 #include <dev/ofw/ofw_bus_subr.h>
76 
77 #include <dev/mii/mii.h>
78 #include <dev/mii/miivar.h>
79 
80 #ifdef EXT_RESOURCES
81 #include <dev/extres/clk/clk.h>
82 #endif
83 
84 #if BUS_SPACE_MAXADDR > BUS_SPACE_MAXADDR_32BIT
85 #define CGEM64
86 #endif
87 
88 #include <dev/cadence/if_cgem_hw.h>
89 
90 #include "miibus_if.h"
91 
92 #define IF_CGEM_NAME "cgem"
93 
94 #define CGEM_NUM_RX_DESCS	512	/* size of receive descriptor ring */
95 #define CGEM_NUM_TX_DESCS	512	/* size of transmit descriptor ring */
96 
97 /* Default for sysctl rxbufs.  Must be < CGEM_NUM_RX_DESCS of course. */
98 #define DEFAULT_NUM_RX_BUFS	256	/* number of receive bufs to queue. */
99 
100 #define TX_MAX_DMA_SEGS		8	/* maximum segs in a tx mbuf dma */
101 
102 #define CGEM_CKSUM_ASSIST	(CSUM_IP | CSUM_TCP | CSUM_UDP | \
103 				 CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
104 
105 #define HWTYPE_GENERIC_GEM	1
106 #define HWTYPE_ZYNQ		2
107 #define HWTYPE_ZYNQMP		3
108 #define HWTYPE_SIFIVE_FU540	4
109 
110 static struct ofw_compat_data compat_data[] = {
111 	{ "cdns,zynq-gem",		HWTYPE_ZYNQ },
112 	{ "cdns,zynqmp-gem",		HWTYPE_ZYNQMP },
113 	{ "sifive,fu540-c000-gem",	HWTYPE_SIFIVE_FU540 },
114 	{ "cdns,gem",			HWTYPE_GENERIC_GEM },
115 	{ "cadence,gem",		HWTYPE_GENERIC_GEM },
116 	{ NULL,				0 }
117 };
118 
119 struct cgem_softc {
120 	if_t			ifp;
121 	struct mtx		sc_mtx;
122 	device_t		dev;
123 	device_t		miibus;
124 	u_int			mii_media_active;	/* last active media */
125 	int			if_old_flags;
126 	struct resource		*mem_res;
127 	struct resource		*irq_res;
128 	void			*intrhand;
129 	struct callout		tick_ch;
130 	uint32_t		net_ctl_shadow;
131 	uint32_t		net_cfg_shadow;
132 #ifdef EXT_RESOURCES
133 	clk_t			ref_clk;
134 #else
135 	int			ref_clk_num;
136 #endif
137 	int			neednullqs;
138 
139 	bus_dma_tag_t		desc_dma_tag;
140 	bus_dma_tag_t		mbuf_dma_tag;
141 
142 	/* receive descriptor ring */
143 	struct cgem_rx_desc	*rxring;
144 	bus_addr_t		rxring_physaddr;
145 	struct mbuf		*rxring_m[CGEM_NUM_RX_DESCS];
146 	bus_dmamap_t		rxring_m_dmamap[CGEM_NUM_RX_DESCS];
147 	int			rxring_hd_ptr;	/* where to put rcv bufs */
148 	int			rxring_tl_ptr;	/* where to get receives */
149 	int			rxring_queued;	/* how many rcv bufs queued */
150 	bus_dmamap_t		rxring_dma_map;
151 	int			rxbufs;		/* tunable number rcv bufs */
152 	int			rxhangwar;	/* rx hang work-around */
153 	u_int			rxoverruns;	/* rx overruns */
154 	u_int			rxnobufs;	/* rx buf ring empty events */
155 	u_int			rxdmamapfails;	/* rx dmamap failures */
156 	uint32_t		rx_frames_prev;
157 
158 	/* transmit descriptor ring */
159 	struct cgem_tx_desc	*txring;
160 	bus_addr_t		txring_physaddr;
161 	struct mbuf		*txring_m[CGEM_NUM_TX_DESCS];
162 	bus_dmamap_t		txring_m_dmamap[CGEM_NUM_TX_DESCS];
163 	int			txring_hd_ptr;	/* where to put next xmits */
164 	int			txring_tl_ptr;	/* next xmit mbuf to free */
165 	int			txring_queued;	/* num xmits segs queued */
166 	u_int			txfull;		/* tx ring full events */
167 	u_int			txdefrags;	/* tx calls to m_defrag() */
168 	u_int			txdefragfails;	/* tx m_defrag() failures */
169 	u_int			txdmamapfails;	/* tx dmamap failures */
170 
171 	/* null descriptor rings */
172 	void			*null_qs;
173 	bus_addr_t		null_qs_physaddr;
174 
175 	/* hardware provided statistics */
176 	struct cgem_hw_stats {
177 		uint64_t		tx_bytes;
178 		uint32_t		tx_frames;
179 		uint32_t		tx_frames_bcast;
180 		uint32_t		tx_frames_multi;
181 		uint32_t		tx_frames_pause;
182 		uint32_t		tx_frames_64b;
183 		uint32_t		tx_frames_65to127b;
184 		uint32_t		tx_frames_128to255b;
185 		uint32_t		tx_frames_256to511b;
186 		uint32_t		tx_frames_512to1023b;
187 		uint32_t		tx_frames_1024to1536b;
188 		uint32_t		tx_under_runs;
189 		uint32_t		tx_single_collisn;
190 		uint32_t		tx_multi_collisn;
191 		uint32_t		tx_excsv_collisn;
192 		uint32_t		tx_late_collisn;
193 		uint32_t		tx_deferred_frames;
194 		uint32_t		tx_carrier_sense_errs;
195 
196 		uint64_t		rx_bytes;
197 		uint32_t		rx_frames;
198 		uint32_t		rx_frames_bcast;
199 		uint32_t		rx_frames_multi;
200 		uint32_t		rx_frames_pause;
201 		uint32_t		rx_frames_64b;
202 		uint32_t		rx_frames_65to127b;
203 		uint32_t		rx_frames_128to255b;
204 		uint32_t		rx_frames_256to511b;
205 		uint32_t		rx_frames_512to1023b;
206 		uint32_t		rx_frames_1024to1536b;
207 		uint32_t		rx_frames_undersize;
208 		uint32_t		rx_frames_oversize;
209 		uint32_t		rx_frames_jabber;
210 		uint32_t		rx_frames_fcs_errs;
211 		uint32_t		rx_frames_length_errs;
212 		uint32_t		rx_symbol_errs;
213 		uint32_t		rx_align_errs;
214 		uint32_t		rx_resource_errs;
215 		uint32_t		rx_overrun_errs;
216 		uint32_t		rx_ip_hdr_csum_errs;
217 		uint32_t		rx_tcp_csum_errs;
218 		uint32_t		rx_udp_csum_errs;
219 	} stats;
220 };
221 
222 #define RD4(sc, off)		(bus_read_4((sc)->mem_res, (off)))
223 #define WR4(sc, off, val)	(bus_write_4((sc)->mem_res, (off), (val)))
224 #define BARRIER(sc, off, len, flags) \
225 	(bus_barrier((sc)->mem_res, (off), (len), (flags))
226 
227 #define CGEM_LOCK(sc)		mtx_lock(&(sc)->sc_mtx)
228 #define CGEM_UNLOCK(sc)		mtx_unlock(&(sc)->sc_mtx)
229 #define CGEM_LOCK_INIT(sc)	mtx_init(&(sc)->sc_mtx, \
230 	    device_get_nameunit((sc)->dev), MTX_NETWORK_LOCK, MTX_DEF)
231 #define CGEM_LOCK_DESTROY(sc)	mtx_destroy(&(sc)->sc_mtx)
232 #define CGEM_ASSERT_LOCKED(sc)	mtx_assert(&(sc)->sc_mtx, MA_OWNED)
233 
234 /* Allow platforms to optionally provide a way to set the reference clock. */
235 int cgem_set_ref_clk(int unit, int frequency);
236 
237 static devclass_t cgem_devclass;
238 
239 static int cgem_probe(device_t dev);
240 static int cgem_attach(device_t dev);
241 static int cgem_detach(device_t dev);
242 static void cgem_tick(void *);
243 static void cgem_intr(void *);
244 
245 static void cgem_mediachange(struct cgem_softc *, struct mii_data *);
246 
247 static void
248 cgem_get_mac(struct cgem_softc *sc, u_char eaddr[])
249 {
250 	int i;
251 	uint32_t rnd;
252 
253 	/* See if boot loader gave us a MAC address already. */
254 	for (i = 0; i < 4; i++) {
255 		uint32_t low = RD4(sc, CGEM_SPEC_ADDR_LOW(i));
256 		uint32_t high = RD4(sc, CGEM_SPEC_ADDR_HI(i)) & 0xffff;
257 		if (low != 0 || high != 0) {
258 			eaddr[0] = low & 0xff;
259 			eaddr[1] = (low >> 8) & 0xff;
260 			eaddr[2] = (low >> 16) & 0xff;
261 			eaddr[3] = (low >> 24) & 0xff;
262 			eaddr[4] = high & 0xff;
263 			eaddr[5] = (high >> 8) & 0xff;
264 			break;
265 		}
266 	}
267 
268 	/* No MAC from boot loader?  Assign a random one. */
269 	if (i == 4) {
270 		rnd = arc4random();
271 
272 		eaddr[0] = 'b';
273 		eaddr[1] = 's';
274 		eaddr[2] = 'd';
275 		eaddr[3] = (rnd >> 16) & 0xff;
276 		eaddr[4] = (rnd >> 8) & 0xff;
277 		eaddr[5] = rnd & 0xff;
278 
279 		device_printf(sc->dev, "no mac address found, assigning "
280 		    "random: %02x:%02x:%02x:%02x:%02x:%02x\n", eaddr[0],
281 		    eaddr[1], eaddr[2], eaddr[3], eaddr[4], eaddr[5]);
282 	}
283 
284 	/* Move address to first slot and zero out the rest. */
285 	WR4(sc, CGEM_SPEC_ADDR_LOW(0), (eaddr[3] << 24) |
286 	    (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]);
287 	WR4(sc, CGEM_SPEC_ADDR_HI(0), (eaddr[5] << 8) | eaddr[4]);
288 
289 	for (i = 1; i < 4; i++) {
290 		WR4(sc, CGEM_SPEC_ADDR_LOW(i), 0);
291 		WR4(sc, CGEM_SPEC_ADDR_HI(i), 0);
292 	}
293 }
294 
295 /*
296  * cgem_mac_hash():  map 48-bit address to a 6-bit hash. The 6-bit hash
297  * corresponds to a bit in a 64-bit hash register.  Setting that bit in the
298  * hash register enables reception of all frames with a destination address
299  * that hashes to that 6-bit value.
300  *
301  * The hash function is described in sec. 16.2.3 in the Zynq-7000 Tech
302  * Reference Manual.  Bits 0-5 in the hash are the exclusive-or of
303  * every sixth bit in the destination address.
304  */
305 static int
306 cgem_mac_hash(u_char eaddr[])
307 {
308 	int hash;
309 	int i, j;
310 
311 	hash = 0;
312 	for (i = 0; i < 6; i++)
313 		for (j = i; j < 48; j += 6)
314 			if ((eaddr[j >> 3] & (1 << (j & 7))) != 0)
315 				hash ^= (1 << i);
316 
317 	return hash;
318 }
319 
320 static u_int
321 cgem_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
322 {
323 	uint32_t *hashes = arg;
324 	int index;
325 
326 	index = cgem_mac_hash(LLADDR(sdl));
327 	if (index > 31)
328 		hashes[0] |= (1U << (index - 32));
329 	else
330 		hashes[1] |= (1U << index);
331 
332 	return (1);
333 }
334 
335 /*
336  * After any change in rx flags or multi-cast addresses, set up hash registers
337  * and net config register bits.
338  */
339 static void
340 cgem_rx_filter(struct cgem_softc *sc)
341 {
342 	if_t ifp = sc->ifp;
343 	uint32_t hashes[2] = { 0, 0 };
344 
345 	sc->net_cfg_shadow &= ~(CGEM_NET_CFG_MULTI_HASH_EN |
346 	    CGEM_NET_CFG_NO_BCAST | CGEM_NET_CFG_COPY_ALL);
347 
348 	if ((if_getflags(ifp) & IFF_PROMISC) != 0)
349 		sc->net_cfg_shadow |= CGEM_NET_CFG_COPY_ALL;
350 	else {
351 		if ((if_getflags(ifp) & IFF_BROADCAST) == 0)
352 			sc->net_cfg_shadow |= CGEM_NET_CFG_NO_BCAST;
353 		if ((if_getflags(ifp) & IFF_ALLMULTI) != 0) {
354 			hashes[0] = 0xffffffff;
355 			hashes[1] = 0xffffffff;
356 		} else
357 			if_foreach_llmaddr(ifp, cgem_hash_maddr, hashes);
358 
359 		if (hashes[0] != 0 || hashes[1] != 0)
360 			sc->net_cfg_shadow |= CGEM_NET_CFG_MULTI_HASH_EN;
361 	}
362 
363 	WR4(sc, CGEM_HASH_TOP, hashes[0]);
364 	WR4(sc, CGEM_HASH_BOT, hashes[1]);
365 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
366 }
367 
368 /* For bus_dmamap_load() callback. */
369 static void
370 cgem_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
371 {
372 
373 	if (nsegs != 1 || error != 0)
374 		return;
375 	*(bus_addr_t *)arg = segs[0].ds_addr;
376 }
377 
378 /* Set up null queues for priority queues we actually can't disable. */
379 static void
380 cgem_null_qs(struct cgem_softc *sc)
381 {
382 	struct cgem_rx_desc *rx_desc;
383 	struct cgem_tx_desc *tx_desc;
384 	uint32_t queue_mask;
385 	int n;
386 
387 	/* Read design config register 6 to determine number of queues. */
388 	queue_mask = (RD4(sc, CGEM_DESIGN_CFG6) &
389 	    CGEM_DESIGN_CFG6_DMA_PRIO_Q_MASK) >> 1;
390 	if (queue_mask == 0)
391 		return;
392 
393 	/* Create empty RX queue and empty TX buf queues. */
394 	memset(sc->null_qs, 0, sizeof(struct cgem_rx_desc) +
395 	    sizeof(struct cgem_tx_desc));
396 	rx_desc = sc->null_qs;
397 	rx_desc->addr = CGEM_RXDESC_OWN | CGEM_RXDESC_WRAP;
398 	tx_desc = (struct cgem_tx_desc *)(rx_desc + 1);
399 	tx_desc->ctl = CGEM_TXDESC_USED | CGEM_TXDESC_WRAP;
400 
401 	/* Point all valid ring base pointers to the null queues. */
402 	for (n = 1; (queue_mask & 1) != 0; n++, queue_mask >>= 1) {
403 		WR4(sc, CGEM_RX_QN_BAR(n), sc->null_qs_physaddr);
404 		WR4(sc, CGEM_TX_QN_BAR(n), sc->null_qs_physaddr +
405 		    sizeof(struct cgem_rx_desc));
406 	}
407 }
408 
409 /* Create DMA'able descriptor rings. */
410 static int
411 cgem_setup_descs(struct cgem_softc *sc)
412 {
413 	int i, err;
414 	int desc_rings_size = CGEM_NUM_RX_DESCS * sizeof(struct cgem_rx_desc) +
415 	    CGEM_NUM_TX_DESCS * sizeof(struct cgem_tx_desc);
416 
417 	if (sc->neednullqs)
418 		desc_rings_size += sizeof(struct cgem_rx_desc) +
419 		    sizeof(struct cgem_tx_desc);
420 
421 	sc->txring = NULL;
422 	sc->rxring = NULL;
423 
424 	/* Allocate non-cached DMA space for RX and TX descriptors. */
425 	err = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1,
426 #ifdef CGEM64
427 	    1ULL << 32,	/* Do not cross a 4G boundary. */
428 #else
429 	    0,
430 #endif
431 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
432 	    desc_rings_size, 1, desc_rings_size, 0,
433 	    busdma_lock_mutex, &sc->sc_mtx, &sc->desc_dma_tag);
434 	if (err)
435 		return (err);
436 
437 	/* Set up a bus_dma_tag for mbufs. */
438 	err = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
439 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES,
440 	    TX_MAX_DMA_SEGS, MCLBYTES, 0, busdma_lock_mutex, &sc->sc_mtx,
441 	    &sc->mbuf_dma_tag);
442 	if (err)
443 		return (err);
444 
445 	/*
446 	 * Allocate DMA memory. We allocate transmit, receive and null
447 	 * descriptor queues all at once because the hardware only provides
448 	 * one register for the upper 32 bits of rx and tx descriptor queues
449 	 * hardware addresses.
450 	 */
451 	err = bus_dmamem_alloc(sc->desc_dma_tag, (void **)&sc->rxring,
452 	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO,
453 	    &sc->rxring_dma_map);
454 	if (err)
455 		return (err);
456 
457 	/* Load descriptor DMA memory. */
458 	err = bus_dmamap_load(sc->desc_dma_tag, sc->rxring_dma_map,
459 	    (void *)sc->rxring, desc_rings_size,
460 	    cgem_getaddr, &sc->rxring_physaddr, BUS_DMA_NOWAIT);
461 	if (err)
462 		return (err);
463 
464 	/* Initialize RX descriptors. */
465 	for (i = 0; i < CGEM_NUM_RX_DESCS; i++) {
466 		sc->rxring[i].addr = CGEM_RXDESC_OWN;
467 		sc->rxring[i].ctl = 0;
468 		sc->rxring_m[i] = NULL;
469 		sc->rxring_m_dmamap[i] = NULL;
470 	}
471 	sc->rxring[CGEM_NUM_RX_DESCS - 1].addr |= CGEM_RXDESC_WRAP;
472 
473 	sc->rxring_hd_ptr = 0;
474 	sc->rxring_tl_ptr = 0;
475 	sc->rxring_queued = 0;
476 
477 	sc->txring = (struct cgem_tx_desc *)(sc->rxring + CGEM_NUM_RX_DESCS);
478 	sc->txring_physaddr = sc->rxring_physaddr + CGEM_NUM_RX_DESCS *
479 	    sizeof(struct cgem_rx_desc);
480 
481 	/* Initialize TX descriptor ring. */
482 	for (i = 0; i < CGEM_NUM_TX_DESCS; i++) {
483 		sc->txring[i].addr = 0;
484 		sc->txring[i].ctl = CGEM_TXDESC_USED;
485 		sc->txring_m[i] = NULL;
486 		sc->txring_m_dmamap[i] = NULL;
487 	}
488 	sc->txring[CGEM_NUM_TX_DESCS - 1].ctl |= CGEM_TXDESC_WRAP;
489 
490 	sc->txring_hd_ptr = 0;
491 	sc->txring_tl_ptr = 0;
492 	sc->txring_queued = 0;
493 
494 	if (sc->neednullqs) {
495 		sc->null_qs = (void *)(sc->txring + CGEM_NUM_TX_DESCS);
496 		sc->null_qs_physaddr = sc->txring_physaddr +
497 		    CGEM_NUM_TX_DESCS * sizeof(struct cgem_tx_desc);
498 
499 		cgem_null_qs(sc);
500 	}
501 
502 	return (0);
503 }
504 
505 /* Fill receive descriptor ring with mbufs. */
506 static void
507 cgem_fill_rqueue(struct cgem_softc *sc)
508 {
509 	struct mbuf *m = NULL;
510 	bus_dma_segment_t segs[TX_MAX_DMA_SEGS];
511 	int nsegs;
512 
513 	CGEM_ASSERT_LOCKED(sc);
514 
515 	while (sc->rxring_queued < sc->rxbufs) {
516 		/* Get a cluster mbuf. */
517 		m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
518 		if (m == NULL)
519 			break;
520 
521 		m->m_len = MCLBYTES;
522 		m->m_pkthdr.len = MCLBYTES;
523 		m->m_pkthdr.rcvif = sc->ifp;
524 
525 		/* Load map and plug in physical address. */
526 		if (bus_dmamap_create(sc->mbuf_dma_tag, 0,
527 		    &sc->rxring_m_dmamap[sc->rxring_hd_ptr])) {
528 			sc->rxdmamapfails++;
529 			m_free(m);
530 			break;
531 		}
532 		if (bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
533 		    sc->rxring_m_dmamap[sc->rxring_hd_ptr], m,
534 		    segs, &nsegs, BUS_DMA_NOWAIT)) {
535 			sc->rxdmamapfails++;
536 			bus_dmamap_destroy(sc->mbuf_dma_tag,
537 				   sc->rxring_m_dmamap[sc->rxring_hd_ptr]);
538 			sc->rxring_m_dmamap[sc->rxring_hd_ptr] = NULL;
539 			m_free(m);
540 			break;
541 		}
542 		sc->rxring_m[sc->rxring_hd_ptr] = m;
543 
544 		/* Sync cache with receive buffer. */
545 		bus_dmamap_sync(sc->mbuf_dma_tag,
546 		    sc->rxring_m_dmamap[sc->rxring_hd_ptr],
547 		    BUS_DMASYNC_PREREAD);
548 
549 		/* Write rx descriptor and increment head pointer. */
550 		sc->rxring[sc->rxring_hd_ptr].ctl = 0;
551 #ifdef CGEM64
552 		sc->rxring[sc->rxring_hd_ptr].addrhi = segs[0].ds_addr >> 32;
553 #endif
554 		if (sc->rxring_hd_ptr == CGEM_NUM_RX_DESCS - 1) {
555 			sc->rxring[sc->rxring_hd_ptr].addr = segs[0].ds_addr |
556 			    CGEM_RXDESC_WRAP;
557 			sc->rxring_hd_ptr = 0;
558 		} else
559 			sc->rxring[sc->rxring_hd_ptr++].addr = segs[0].ds_addr;
560 
561 		sc->rxring_queued++;
562 	}
563 }
564 
565 /* Pull received packets off of receive descriptor ring. */
566 static void
567 cgem_recv(struct cgem_softc *sc)
568 {
569 	if_t ifp = sc->ifp;
570 	struct mbuf *m, *m_hd, **m_tl;
571 	uint32_t ctl;
572 
573 	CGEM_ASSERT_LOCKED(sc);
574 
575 	/* Pick up all packets in which the OWN bit is set. */
576 	m_hd = NULL;
577 	m_tl = &m_hd;
578 	while (sc->rxring_queued > 0 &&
579 	    (sc->rxring[sc->rxring_tl_ptr].addr & CGEM_RXDESC_OWN) != 0) {
580 		ctl = sc->rxring[sc->rxring_tl_ptr].ctl;
581 
582 		/* Grab filled mbuf. */
583 		m = sc->rxring_m[sc->rxring_tl_ptr];
584 		sc->rxring_m[sc->rxring_tl_ptr] = NULL;
585 
586 		/* Sync cache with receive buffer. */
587 		bus_dmamap_sync(sc->mbuf_dma_tag,
588 		    sc->rxring_m_dmamap[sc->rxring_tl_ptr],
589 		    BUS_DMASYNC_POSTREAD);
590 
591 		/* Unload and destroy dmamap. */
592 		bus_dmamap_unload(sc->mbuf_dma_tag,
593 		    sc->rxring_m_dmamap[sc->rxring_tl_ptr]);
594 		bus_dmamap_destroy(sc->mbuf_dma_tag,
595 		    sc->rxring_m_dmamap[sc->rxring_tl_ptr]);
596 		sc->rxring_m_dmamap[sc->rxring_tl_ptr] = NULL;
597 
598 		/* Increment tail pointer. */
599 		if (++sc->rxring_tl_ptr == CGEM_NUM_RX_DESCS)
600 			sc->rxring_tl_ptr = 0;
601 		sc->rxring_queued--;
602 
603 		/*
604 		 * Check FCS and make sure entire packet landed in one mbuf
605 		 * cluster (which is much bigger than the largest ethernet
606 		 * packet).
607 		 */
608 		if ((ctl & CGEM_RXDESC_BAD_FCS) != 0 ||
609 		    (ctl & (CGEM_RXDESC_SOF | CGEM_RXDESC_EOF)) !=
610 		    (CGEM_RXDESC_SOF | CGEM_RXDESC_EOF)) {
611 			/* discard. */
612 			m_free(m);
613 			if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
614 			continue;
615 		}
616 
617 		/* Ready it to hand off to upper layers. */
618 		m->m_data += ETHER_ALIGN;
619 		m->m_len = (ctl & CGEM_RXDESC_LENGTH_MASK);
620 		m->m_pkthdr.rcvif = ifp;
621 		m->m_pkthdr.len = m->m_len;
622 
623 		/*
624 		 * Are we using hardware checksumming?  Check the status in the
625 		 * receive descriptor.
626 		 */
627 		if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) {
628 			/* TCP or UDP checks out, IP checks out too. */
629 			if ((ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
630 			    CGEM_RXDESC_CKSUM_STAT_TCP_GOOD ||
631 			    (ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
632 			    CGEM_RXDESC_CKSUM_STAT_UDP_GOOD) {
633 				m->m_pkthdr.csum_flags |=
634 				    CSUM_IP_CHECKED | CSUM_IP_VALID |
635 				    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
636 				m->m_pkthdr.csum_data = 0xffff;
637 			} else if ((ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
638 			    CGEM_RXDESC_CKSUM_STAT_IP_GOOD) {
639 				/* Only IP checks out. */
640 				m->m_pkthdr.csum_flags |=
641 				    CSUM_IP_CHECKED | CSUM_IP_VALID;
642 				m->m_pkthdr.csum_data = 0xffff;
643 			}
644 		}
645 
646 		/* Queue it up for delivery below. */
647 		*m_tl = m;
648 		m_tl = &m->m_next;
649 	}
650 
651 	/* Replenish receive buffers. */
652 	cgem_fill_rqueue(sc);
653 
654 	/* Unlock and send up packets. */
655 	CGEM_UNLOCK(sc);
656 	while (m_hd != NULL) {
657 		m = m_hd;
658 		m_hd = m_hd->m_next;
659 		m->m_next = NULL;
660 		if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
661 		if_input(ifp, m);
662 	}
663 	CGEM_LOCK(sc);
664 }
665 
666 /* Find completed transmits and free their mbufs. */
667 static void
668 cgem_clean_tx(struct cgem_softc *sc)
669 {
670 	struct mbuf *m;
671 	uint32_t ctl;
672 
673 	CGEM_ASSERT_LOCKED(sc);
674 
675 	/* free up finished transmits. */
676 	while (sc->txring_queued > 0 &&
677 	    ((ctl = sc->txring[sc->txring_tl_ptr].ctl) &
678 	    CGEM_TXDESC_USED) != 0) {
679 		/* Sync cache. */
680 		bus_dmamap_sync(sc->mbuf_dma_tag,
681 		    sc->txring_m_dmamap[sc->txring_tl_ptr],
682 		    BUS_DMASYNC_POSTWRITE);
683 
684 		/* Unload and destroy DMA map. */
685 		bus_dmamap_unload(sc->mbuf_dma_tag,
686 		    sc->txring_m_dmamap[sc->txring_tl_ptr]);
687 		bus_dmamap_destroy(sc->mbuf_dma_tag,
688 		    sc->txring_m_dmamap[sc->txring_tl_ptr]);
689 		sc->txring_m_dmamap[sc->txring_tl_ptr] = NULL;
690 
691 		/* Free up the mbuf. */
692 		m = sc->txring_m[sc->txring_tl_ptr];
693 		sc->txring_m[sc->txring_tl_ptr] = NULL;
694 		m_freem(m);
695 
696 		/* Check the status. */
697 		if ((ctl & CGEM_TXDESC_AHB_ERR) != 0) {
698 			/* Serious bus error. log to console. */
699 #ifdef CGEM64
700 			device_printf(sc->dev,
701 			    "cgem_clean_tx: AHB error, addr=0x%x%08x\n",
702 			    sc->txring[sc->txring_tl_ptr].addrhi,
703 			    sc->txring[sc->txring_tl_ptr].addr);
704 #else
705 			device_printf(sc->dev,
706 			    "cgem_clean_tx: AHB error, addr=0x%x\n",
707 			    sc->txring[sc->txring_tl_ptr].addr);
708 #endif
709 		} else if ((ctl & (CGEM_TXDESC_RETRY_ERR |
710 		    CGEM_TXDESC_LATE_COLL)) != 0) {
711 			if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1);
712 		} else
713 			if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
714 
715 		/*
716 		 * If the packet spanned more than one tx descriptor, skip
717 		 * descriptors until we find the end so that only
718 		 * start-of-frame descriptors are processed.
719 		 */
720 		while ((ctl & CGEM_TXDESC_LAST_BUF) == 0) {
721 			if ((ctl & CGEM_TXDESC_WRAP) != 0)
722 				sc->txring_tl_ptr = 0;
723 			else
724 				sc->txring_tl_ptr++;
725 			sc->txring_queued--;
726 
727 			ctl = sc->txring[sc->txring_tl_ptr].ctl;
728 
729 			sc->txring[sc->txring_tl_ptr].ctl =
730 			    ctl | CGEM_TXDESC_USED;
731 		}
732 
733 		/* Next descriptor. */
734 		if ((ctl & CGEM_TXDESC_WRAP) != 0)
735 			sc->txring_tl_ptr = 0;
736 		else
737 			sc->txring_tl_ptr++;
738 		sc->txring_queued--;
739 
740 		if_setdrvflagbits(sc->ifp, 0, IFF_DRV_OACTIVE);
741 	}
742 }
743 
744 /* Start transmits. */
745 static void
746 cgem_start_locked(if_t ifp)
747 {
748 	struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
749 	struct mbuf *m;
750 	bus_dma_segment_t segs[TX_MAX_DMA_SEGS];
751 	uint32_t ctl;
752 	int i, nsegs, wrap, err;
753 
754 	CGEM_ASSERT_LOCKED(sc);
755 
756 	if ((if_getdrvflags(ifp) & IFF_DRV_OACTIVE) != 0)
757 		return;
758 
759 	for (;;) {
760 		/* Check that there is room in the descriptor ring. */
761 		if (sc->txring_queued >=
762 		    CGEM_NUM_TX_DESCS - TX_MAX_DMA_SEGS * 2) {
763 			/* Try to make room. */
764 			cgem_clean_tx(sc);
765 
766 			/* Still no room? */
767 			if (sc->txring_queued >=
768 			    CGEM_NUM_TX_DESCS - TX_MAX_DMA_SEGS * 2) {
769 				if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
770 				sc->txfull++;
771 				break;
772 			}
773 		}
774 
775 		/* Grab next transmit packet. */
776 		m = if_dequeue(ifp);
777 		if (m == NULL)
778 			break;
779 
780 		/* Create and load DMA map. */
781 		if (bus_dmamap_create(sc->mbuf_dma_tag, 0,
782 			&sc->txring_m_dmamap[sc->txring_hd_ptr])) {
783 			m_freem(m);
784 			sc->txdmamapfails++;
785 			continue;
786 		}
787 		err = bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
788 		    sc->txring_m_dmamap[sc->txring_hd_ptr], m, segs, &nsegs,
789 		    BUS_DMA_NOWAIT);
790 		if (err == EFBIG) {
791 			/* Too many segments!  defrag and try again. */
792 			struct mbuf *m2 = m_defrag(m, M_NOWAIT);
793 
794 			if (m2 == NULL) {
795 				sc->txdefragfails++;
796 				m_freem(m);
797 				bus_dmamap_destroy(sc->mbuf_dma_tag,
798 				    sc->txring_m_dmamap[sc->txring_hd_ptr]);
799 				sc->txring_m_dmamap[sc->txring_hd_ptr] = NULL;
800 				continue;
801 			}
802 			m = m2;
803 			err = bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
804 			    sc->txring_m_dmamap[sc->txring_hd_ptr], m, segs,
805 			    &nsegs, BUS_DMA_NOWAIT);
806 			sc->txdefrags++;
807 		}
808 		if (err) {
809 			/* Give up. */
810 			m_freem(m);
811 			bus_dmamap_destroy(sc->mbuf_dma_tag,
812 			    sc->txring_m_dmamap[sc->txring_hd_ptr]);
813 			sc->txring_m_dmamap[sc->txring_hd_ptr] = NULL;
814 			sc->txdmamapfails++;
815 			continue;
816 		}
817 		sc->txring_m[sc->txring_hd_ptr] = m;
818 
819 		/* Sync tx buffer with cache. */
820 		bus_dmamap_sync(sc->mbuf_dma_tag,
821 		    sc->txring_m_dmamap[sc->txring_hd_ptr],
822 		    BUS_DMASYNC_PREWRITE);
823 
824 		/* Set wrap flag if next packet might run off end of ring. */
825 		wrap = sc->txring_hd_ptr + nsegs + TX_MAX_DMA_SEGS >=
826 		    CGEM_NUM_TX_DESCS;
827 
828 		/*
829 		 * Fill in the TX descriptors back to front so that USED bit in
830 		 * first descriptor is cleared last.
831 		 */
832 		for (i = nsegs - 1; i >= 0; i--) {
833 			/* Descriptor address. */
834 			sc->txring[sc->txring_hd_ptr + i].addr =
835 			    segs[i].ds_addr;
836 #ifdef CGEM64
837 			sc->txring[sc->txring_hd_ptr + i].addrhi =
838 			    segs[i].ds_addr >> 32;
839 #endif
840 			/* Descriptor control word. */
841 			ctl = segs[i].ds_len;
842 			if (i == nsegs - 1) {
843 				ctl |= CGEM_TXDESC_LAST_BUF;
844 				if (wrap)
845 					ctl |= CGEM_TXDESC_WRAP;
846 			}
847 			sc->txring[sc->txring_hd_ptr + i].ctl = ctl;
848 
849 			if (i != 0)
850 				sc->txring_m[sc->txring_hd_ptr + i] = NULL;
851 		}
852 
853 		if (wrap)
854 			sc->txring_hd_ptr = 0;
855 		else
856 			sc->txring_hd_ptr += nsegs;
857 		sc->txring_queued += nsegs;
858 
859 		/* Kick the transmitter. */
860 		WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow |
861 		    CGEM_NET_CTRL_START_TX);
862 
863 		/* If there is a BPF listener, bounce a copy to him. */
864 		ETHER_BPF_MTAP(ifp, m);
865 	}
866 }
867 
868 static void
869 cgem_start(if_t ifp)
870 {
871 	struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
872 
873 	CGEM_LOCK(sc);
874 	cgem_start_locked(ifp);
875 	CGEM_UNLOCK(sc);
876 }
877 
878 static void
879 cgem_poll_hw_stats(struct cgem_softc *sc)
880 {
881 	uint32_t n;
882 
883 	CGEM_ASSERT_LOCKED(sc);
884 
885 	sc->stats.tx_bytes += RD4(sc, CGEM_OCTETS_TX_BOT);
886 	sc->stats.tx_bytes += (uint64_t)RD4(sc, CGEM_OCTETS_TX_TOP) << 32;
887 
888 	sc->stats.tx_frames += RD4(sc, CGEM_FRAMES_TX);
889 	sc->stats.tx_frames_bcast += RD4(sc, CGEM_BCAST_FRAMES_TX);
890 	sc->stats.tx_frames_multi += RD4(sc, CGEM_MULTI_FRAMES_TX);
891 	sc->stats.tx_frames_pause += RD4(sc, CGEM_PAUSE_FRAMES_TX);
892 	sc->stats.tx_frames_64b += RD4(sc, CGEM_FRAMES_64B_TX);
893 	sc->stats.tx_frames_65to127b += RD4(sc, CGEM_FRAMES_65_127B_TX);
894 	sc->stats.tx_frames_128to255b += RD4(sc, CGEM_FRAMES_128_255B_TX);
895 	sc->stats.tx_frames_256to511b += RD4(sc, CGEM_FRAMES_256_511B_TX);
896 	sc->stats.tx_frames_512to1023b += RD4(sc, CGEM_FRAMES_512_1023B_TX);
897 	sc->stats.tx_frames_1024to1536b += RD4(sc, CGEM_FRAMES_1024_1518B_TX);
898 	sc->stats.tx_under_runs += RD4(sc, CGEM_TX_UNDERRUNS);
899 
900 	n = RD4(sc, CGEM_SINGLE_COLL_FRAMES);
901 	sc->stats.tx_single_collisn += n;
902 	if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
903 	n = RD4(sc, CGEM_MULTI_COLL_FRAMES);
904 	sc->stats.tx_multi_collisn += n;
905 	if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
906 	n = RD4(sc, CGEM_EXCESSIVE_COLL_FRAMES);
907 	sc->stats.tx_excsv_collisn += n;
908 	if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
909 	n = RD4(sc, CGEM_LATE_COLL);
910 	sc->stats.tx_late_collisn += n;
911 	if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
912 
913 	sc->stats.tx_deferred_frames += RD4(sc, CGEM_DEFERRED_TX_FRAMES);
914 	sc->stats.tx_carrier_sense_errs += RD4(sc, CGEM_CARRIER_SENSE_ERRS);
915 
916 	sc->stats.rx_bytes += RD4(sc, CGEM_OCTETS_RX_BOT);
917 	sc->stats.rx_bytes += (uint64_t)RD4(sc, CGEM_OCTETS_RX_TOP) << 32;
918 
919 	sc->stats.rx_frames += RD4(sc, CGEM_FRAMES_RX);
920 	sc->stats.rx_frames_bcast += RD4(sc, CGEM_BCAST_FRAMES_RX);
921 	sc->stats.rx_frames_multi += RD4(sc, CGEM_MULTI_FRAMES_RX);
922 	sc->stats.rx_frames_pause += RD4(sc, CGEM_PAUSE_FRAMES_RX);
923 	sc->stats.rx_frames_64b += RD4(sc, CGEM_FRAMES_64B_RX);
924 	sc->stats.rx_frames_65to127b += RD4(sc, CGEM_FRAMES_65_127B_RX);
925 	sc->stats.rx_frames_128to255b += RD4(sc, CGEM_FRAMES_128_255B_RX);
926 	sc->stats.rx_frames_256to511b += RD4(sc, CGEM_FRAMES_256_511B_RX);
927 	sc->stats.rx_frames_512to1023b += RD4(sc, CGEM_FRAMES_512_1023B_RX);
928 	sc->stats.rx_frames_1024to1536b += RD4(sc, CGEM_FRAMES_1024_1518B_RX);
929 	sc->stats.rx_frames_undersize += RD4(sc, CGEM_UNDERSZ_RX);
930 	sc->stats.rx_frames_oversize += RD4(sc, CGEM_OVERSZ_RX);
931 	sc->stats.rx_frames_jabber += RD4(sc, CGEM_JABBERS_RX);
932 	sc->stats.rx_frames_fcs_errs += RD4(sc, CGEM_FCS_ERRS);
933 	sc->stats.rx_frames_length_errs += RD4(sc, CGEM_LENGTH_FIELD_ERRS);
934 	sc->stats.rx_symbol_errs += RD4(sc, CGEM_RX_SYMBOL_ERRS);
935 	sc->stats.rx_align_errs += RD4(sc, CGEM_ALIGN_ERRS);
936 	sc->stats.rx_resource_errs += RD4(sc, CGEM_RX_RESOURCE_ERRS);
937 	sc->stats.rx_overrun_errs += RD4(sc, CGEM_RX_OVERRUN_ERRS);
938 	sc->stats.rx_ip_hdr_csum_errs += RD4(sc, CGEM_IP_HDR_CKSUM_ERRS);
939 	sc->stats.rx_tcp_csum_errs += RD4(sc, CGEM_TCP_CKSUM_ERRS);
940 	sc->stats.rx_udp_csum_errs += RD4(sc, CGEM_UDP_CKSUM_ERRS);
941 }
942 
943 static void
944 cgem_tick(void *arg)
945 {
946 	struct cgem_softc *sc = (struct cgem_softc *)arg;
947 	struct mii_data *mii;
948 
949 	CGEM_ASSERT_LOCKED(sc);
950 
951 	/* Poll the phy. */
952 	if (sc->miibus != NULL) {
953 		mii = device_get_softc(sc->miibus);
954 		mii_tick(mii);
955 	}
956 
957 	/* Poll statistics registers. */
958 	cgem_poll_hw_stats(sc);
959 
960 	/* Check for receiver hang. */
961 	if (sc->rxhangwar && sc->rx_frames_prev == sc->stats.rx_frames) {
962 		/*
963 		 * Reset receiver logic by toggling RX_EN bit.  1usec
964 		 * delay is necessary especially when operating at 100mbps
965 		 * and 10mbps speeds.
966 		 */
967 		WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow &
968 		    ~CGEM_NET_CTRL_RX_EN);
969 		DELAY(1);
970 		WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
971 	}
972 	sc->rx_frames_prev = sc->stats.rx_frames;
973 
974 	/* Next callout in one second. */
975 	callout_reset(&sc->tick_ch, hz, cgem_tick, sc);
976 }
977 
978 /* Interrupt handler. */
979 static void
980 cgem_intr(void *arg)
981 {
982 	struct cgem_softc *sc = (struct cgem_softc *)arg;
983 	if_t ifp = sc->ifp;
984 	uint32_t istatus;
985 
986 	CGEM_LOCK(sc);
987 
988 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
989 		CGEM_UNLOCK(sc);
990 		return;
991 	}
992 
993 	/* Read interrupt status and immediately clear the bits. */
994 	istatus = RD4(sc, CGEM_INTR_STAT);
995 	WR4(sc, CGEM_INTR_STAT, istatus);
996 
997 	/* Packets received. */
998 	if ((istatus & CGEM_INTR_RX_COMPLETE) != 0)
999 		cgem_recv(sc);
1000 
1001 	/* Free up any completed transmit buffers. */
1002 	cgem_clean_tx(sc);
1003 
1004 	/* Hresp not ok.  Something is very bad with DMA.  Try to clear. */
1005 	if ((istatus & CGEM_INTR_HRESP_NOT_OK) != 0) {
1006 		device_printf(sc->dev,
1007 		    "cgem_intr: hresp not okay! rx_status=0x%x\n",
1008 		    RD4(sc, CGEM_RX_STAT));
1009 		WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_HRESP_NOT_OK);
1010 	}
1011 
1012 	/* Receiver overrun. */
1013 	if ((istatus & CGEM_INTR_RX_OVERRUN) != 0) {
1014 		/* Clear status bit. */
1015 		WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_OVERRUN);
1016 		sc->rxoverruns++;
1017 	}
1018 
1019 	/* Receiver ran out of bufs. */
1020 	if ((istatus & CGEM_INTR_RX_USED_READ) != 0) {
1021 		WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow |
1022 		    CGEM_NET_CTRL_FLUSH_DPRAM_PKT);
1023 		cgem_fill_rqueue(sc);
1024 		sc->rxnobufs++;
1025 	}
1026 
1027 	/* Restart transmitter if needed. */
1028 	if (!if_sendq_empty(ifp))
1029 		cgem_start_locked(ifp);
1030 
1031 	CGEM_UNLOCK(sc);
1032 }
1033 
1034 /* Reset hardware. */
1035 static void
1036 cgem_reset(struct cgem_softc *sc)
1037 {
1038 
1039 	CGEM_ASSERT_LOCKED(sc);
1040 
1041 	/* Determine data bus width from design configuration register. */
1042 	switch (RD4(sc, CGEM_DESIGN_CFG1) &
1043 	    CGEM_DESIGN_CFG1_DMA_BUS_WIDTH_MASK) {
1044 	case CGEM_DESIGN_CFG1_DMA_BUS_WIDTH_64:
1045 		sc->net_cfg_shadow = CGEM_NET_CFG_DBUS_WIDTH_64;
1046 		break;
1047 	case CGEM_DESIGN_CFG1_DMA_BUS_WIDTH_128:
1048 		sc->net_cfg_shadow = CGEM_NET_CFG_DBUS_WIDTH_128;
1049 		break;
1050 	default:
1051 		sc->net_cfg_shadow = CGEM_NET_CFG_DBUS_WIDTH_32;
1052 	}
1053 
1054 	WR4(sc, CGEM_NET_CTRL, 0);
1055 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1056 	WR4(sc, CGEM_NET_CTRL, CGEM_NET_CTRL_CLR_STAT_REGS);
1057 	WR4(sc, CGEM_TX_STAT, CGEM_TX_STAT_ALL);
1058 	WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_ALL);
1059 	WR4(sc, CGEM_INTR_DIS, CGEM_INTR_ALL);
1060 	WR4(sc, CGEM_HASH_BOT, 0);
1061 	WR4(sc, CGEM_HASH_TOP, 0);
1062 	WR4(sc, CGEM_TX_QBAR, 0);	/* manual says do this. */
1063 	WR4(sc, CGEM_RX_QBAR, 0);
1064 
1065 	/* Get management port running even if interface is down. */
1066 	sc->net_cfg_shadow |= CGEM_NET_CFG_MDC_CLK_DIV_48;
1067 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1068 
1069 	sc->net_ctl_shadow = CGEM_NET_CTRL_MGMT_PORT_EN;
1070 	WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
1071 }
1072 
1073 /* Bring up the hardware. */
1074 static void
1075 cgem_config(struct cgem_softc *sc)
1076 {
1077 	if_t ifp = sc->ifp;
1078 	uint32_t dma_cfg;
1079 	u_char *eaddr = if_getlladdr(ifp);
1080 
1081 	CGEM_ASSERT_LOCKED(sc);
1082 
1083 	/* Program Net Config Register. */
1084 	sc->net_cfg_shadow &= (CGEM_NET_CFG_MDC_CLK_DIV_MASK |
1085 	    CGEM_NET_CFG_DBUS_WIDTH_MASK);
1086 	sc->net_cfg_shadow |= (CGEM_NET_CFG_FCS_REMOVE |
1087 	    CGEM_NET_CFG_RX_BUF_OFFSET(ETHER_ALIGN) |
1088 	    CGEM_NET_CFG_GIGE_EN | CGEM_NET_CFG_1536RXEN |
1089 	    CGEM_NET_CFG_FULL_DUPLEX | CGEM_NET_CFG_SPEED100);
1090 
1091 	/* Enable receive checksum offloading? */
1092 	if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0)
1093 		sc->net_cfg_shadow |=  CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN;
1094 
1095 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1096 
1097 	/* Program DMA Config Register. */
1098 	dma_cfg = CGEM_DMA_CFG_RX_BUF_SIZE(MCLBYTES) |
1099 	    CGEM_DMA_CFG_RX_PKTBUF_MEMSZ_SEL_8K |
1100 	    CGEM_DMA_CFG_TX_PKTBUF_MEMSZ_SEL |
1101 	    CGEM_DMA_CFG_AHB_FIXED_BURST_LEN_16 |
1102 #ifdef CGEM64
1103 	    CGEM_DMA_CFG_ADDR_BUS_64 |
1104 #endif
1105 	    CGEM_DMA_CFG_DISC_WHEN_NO_AHB;
1106 
1107 	/* Enable transmit checksum offloading? */
1108 	if ((if_getcapenable(ifp) & IFCAP_TXCSUM) != 0)
1109 		dma_cfg |= CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN;
1110 
1111 	WR4(sc, CGEM_DMA_CFG, dma_cfg);
1112 
1113 	/* Write the rx and tx descriptor ring addresses to the QBAR regs. */
1114 	WR4(sc, CGEM_RX_QBAR, (uint32_t)sc->rxring_physaddr);
1115 	WR4(sc, CGEM_TX_QBAR, (uint32_t)sc->txring_physaddr);
1116 #ifdef CGEM64
1117 	WR4(sc, CGEM_RX_QBAR_HI, (uint32_t)(sc->rxring_physaddr >> 32));
1118 	WR4(sc, CGEM_TX_QBAR_HI, (uint32_t)(sc->txring_physaddr >> 32));
1119 #endif
1120 
1121 	/* Enable rx and tx. */
1122 	sc->net_ctl_shadow |= (CGEM_NET_CTRL_TX_EN | CGEM_NET_CTRL_RX_EN);
1123 	WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
1124 
1125 	/* Set receive address in case it changed. */
1126 	WR4(sc, CGEM_SPEC_ADDR_LOW(0), (eaddr[3] << 24) |
1127 	    (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]);
1128 	WR4(sc, CGEM_SPEC_ADDR_HI(0), (eaddr[5] << 8) | eaddr[4]);
1129 
1130 	/* Set up interrupts. */
1131 	WR4(sc, CGEM_INTR_EN, CGEM_INTR_RX_COMPLETE | CGEM_INTR_RX_OVERRUN |
1132 	    CGEM_INTR_TX_USED_READ | CGEM_INTR_RX_USED_READ |
1133 	    CGEM_INTR_HRESP_NOT_OK);
1134 }
1135 
1136 /* Turn on interface and load up receive ring with buffers. */
1137 static void
1138 cgem_init_locked(struct cgem_softc *sc)
1139 {
1140 	struct mii_data *mii;
1141 
1142 	CGEM_ASSERT_LOCKED(sc);
1143 
1144 	if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) != 0)
1145 		return;
1146 
1147 	cgem_config(sc);
1148 	cgem_fill_rqueue(sc);
1149 
1150 	if_setdrvflagbits(sc->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1151 
1152 	if (sc->miibus != NULL) {
1153 		mii = device_get_softc(sc->miibus);
1154 		mii_mediachg(mii);
1155 	}
1156 
1157 	callout_reset(&sc->tick_ch, hz, cgem_tick, sc);
1158 }
1159 
1160 static void
1161 cgem_init(void *arg)
1162 {
1163 	struct cgem_softc *sc = (struct cgem_softc *)arg;
1164 
1165 	CGEM_LOCK(sc);
1166 	cgem_init_locked(sc);
1167 	CGEM_UNLOCK(sc);
1168 }
1169 
1170 /* Turn off interface.  Free up any buffers in transmit or receive queues. */
1171 static void
1172 cgem_stop(struct cgem_softc *sc)
1173 {
1174 	int i;
1175 
1176 	CGEM_ASSERT_LOCKED(sc);
1177 
1178 	callout_stop(&sc->tick_ch);
1179 
1180 	/* Shut down hardware. */
1181 	cgem_reset(sc);
1182 
1183 	/* Clear out transmit queue. */
1184 	memset(sc->txring, 0, CGEM_NUM_TX_DESCS * sizeof(struct cgem_tx_desc));
1185 	for (i = 0; i < CGEM_NUM_TX_DESCS; i++) {
1186 		sc->txring[i].ctl = CGEM_TXDESC_USED;
1187 		if (sc->txring_m[i]) {
1188 			/* Unload and destroy dmamap. */
1189 			bus_dmamap_unload(sc->mbuf_dma_tag,
1190 			    sc->txring_m_dmamap[i]);
1191 			bus_dmamap_destroy(sc->mbuf_dma_tag,
1192 			    sc->txring_m_dmamap[i]);
1193 			sc->txring_m_dmamap[i] = NULL;
1194 			m_freem(sc->txring_m[i]);
1195 			sc->txring_m[i] = NULL;
1196 		}
1197 	}
1198 	sc->txring[CGEM_NUM_TX_DESCS - 1].ctl |= CGEM_TXDESC_WRAP;
1199 
1200 	sc->txring_hd_ptr = 0;
1201 	sc->txring_tl_ptr = 0;
1202 	sc->txring_queued = 0;
1203 
1204 	/* Clear out receive queue. */
1205 	memset(sc->rxring, 0, CGEM_NUM_RX_DESCS * sizeof(struct cgem_rx_desc));
1206 	for (i = 0; i < CGEM_NUM_RX_DESCS; i++) {
1207 		sc->rxring[i].addr = CGEM_RXDESC_OWN;
1208 		if (sc->rxring_m[i]) {
1209 			/* Unload and destroy dmamap. */
1210 			bus_dmamap_unload(sc->mbuf_dma_tag,
1211 			    sc->rxring_m_dmamap[i]);
1212 			bus_dmamap_destroy(sc->mbuf_dma_tag,
1213 			    sc->rxring_m_dmamap[i]);
1214 			sc->rxring_m_dmamap[i] = NULL;
1215 
1216 			m_freem(sc->rxring_m[i]);
1217 			sc->rxring_m[i] = NULL;
1218 		}
1219 	}
1220 	sc->rxring[CGEM_NUM_RX_DESCS - 1].addr |= CGEM_RXDESC_WRAP;
1221 
1222 	sc->rxring_hd_ptr = 0;
1223 	sc->rxring_tl_ptr = 0;
1224 	sc->rxring_queued = 0;
1225 
1226 	/* Force next statchg or linkchg to program net config register. */
1227 	sc->mii_media_active = 0;
1228 }
1229 
1230 static int
1231 cgem_ioctl(if_t ifp, u_long cmd, caddr_t data)
1232 {
1233 	struct cgem_softc *sc = if_getsoftc(ifp);
1234 	struct ifreq *ifr = (struct ifreq *)data;
1235 	struct mii_data *mii;
1236 	int error = 0, mask;
1237 
1238 	switch (cmd) {
1239 	case SIOCSIFFLAGS:
1240 		CGEM_LOCK(sc);
1241 		if ((if_getflags(ifp) & IFF_UP) != 0) {
1242 			if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1243 				if (((if_getflags(ifp) ^ sc->if_old_flags) &
1244 				    (IFF_PROMISC | IFF_ALLMULTI)) != 0) {
1245 					cgem_rx_filter(sc);
1246 				}
1247 			} else {
1248 				cgem_init_locked(sc);
1249 			}
1250 		} else if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1251 			if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1252 			cgem_stop(sc);
1253 		}
1254 		sc->if_old_flags = if_getflags(ifp);
1255 		CGEM_UNLOCK(sc);
1256 		break;
1257 
1258 	case SIOCADDMULTI:
1259 	case SIOCDELMULTI:
1260 		/* Set up multi-cast filters. */
1261 		if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1262 			CGEM_LOCK(sc);
1263 			cgem_rx_filter(sc);
1264 			CGEM_UNLOCK(sc);
1265 		}
1266 		break;
1267 
1268 	case SIOCSIFMEDIA:
1269 	case SIOCGIFMEDIA:
1270 		if (sc->miibus == NULL)
1271 			return (ENXIO);
1272 		mii = device_get_softc(sc->miibus);
1273 		error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
1274 		break;
1275 
1276 	case SIOCSIFCAP:
1277 		CGEM_LOCK(sc);
1278 		mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap;
1279 
1280 		if ((mask & IFCAP_TXCSUM) != 0) {
1281 			if ((ifr->ifr_reqcap & IFCAP_TXCSUM) != 0) {
1282 				/* Turn on TX checksumming. */
1283 				if_setcapenablebit(ifp, IFCAP_TXCSUM |
1284 				    IFCAP_TXCSUM_IPV6, 0);
1285 				if_sethwassistbits(ifp, CGEM_CKSUM_ASSIST, 0);
1286 
1287 				WR4(sc, CGEM_DMA_CFG,
1288 				    RD4(sc, CGEM_DMA_CFG) |
1289 				    CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN);
1290 			} else {
1291 				/* Turn off TX checksumming. */
1292 				if_setcapenablebit(ifp, 0, IFCAP_TXCSUM |
1293 				    IFCAP_TXCSUM_IPV6);
1294 				if_sethwassistbits(ifp, 0, CGEM_CKSUM_ASSIST);
1295 
1296 				WR4(sc, CGEM_DMA_CFG,
1297 				    RD4(sc, CGEM_DMA_CFG) &
1298 				    ~CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN);
1299 			}
1300 		}
1301 		if ((mask & IFCAP_RXCSUM) != 0) {
1302 			if ((ifr->ifr_reqcap & IFCAP_RXCSUM) != 0) {
1303 				/* Turn on RX checksumming. */
1304 				if_setcapenablebit(ifp, IFCAP_RXCSUM |
1305 				    IFCAP_RXCSUM_IPV6, 0);
1306 				sc->net_cfg_shadow |=
1307 				    CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN;
1308 				WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1309 			} else {
1310 				/* Turn off RX checksumming. */
1311 				if_setcapenablebit(ifp, 0, IFCAP_RXCSUM |
1312 				    IFCAP_RXCSUM_IPV6);
1313 				sc->net_cfg_shadow &=
1314 				    ~CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN;
1315 				WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1316 			}
1317 		}
1318 		if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_TXCSUM)) ==
1319 		    (IFCAP_RXCSUM | IFCAP_TXCSUM))
1320 			if_setcapenablebit(ifp, IFCAP_VLAN_HWCSUM, 0);
1321 		else
1322 			if_setcapenablebit(ifp, 0, IFCAP_VLAN_HWCSUM);
1323 
1324 		CGEM_UNLOCK(sc);
1325 		break;
1326 	default:
1327 		error = ether_ioctl(ifp, cmd, data);
1328 		break;
1329 	}
1330 
1331 	return (error);
1332 }
1333 
1334 /* MII bus support routines.
1335  */
1336 static int
1337 cgem_ifmedia_upd(if_t ifp)
1338 {
1339 	struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
1340 	struct mii_data *mii;
1341 	struct mii_softc *miisc;
1342 	int error = 0;
1343 
1344 	mii = device_get_softc(sc->miibus);
1345 	CGEM_LOCK(sc);
1346 	if ((if_getflags(ifp) & IFF_UP) != 0) {
1347 		LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
1348 			PHY_RESET(miisc);
1349 		error = mii_mediachg(mii);
1350 	}
1351 	CGEM_UNLOCK(sc);
1352 
1353 	return (error);
1354 }
1355 
1356 static void
1357 cgem_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr)
1358 {
1359 	struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
1360 	struct mii_data *mii;
1361 
1362 	mii = device_get_softc(sc->miibus);
1363 	CGEM_LOCK(sc);
1364 	mii_pollstat(mii);
1365 	ifmr->ifm_active = mii->mii_media_active;
1366 	ifmr->ifm_status = mii->mii_media_status;
1367 	CGEM_UNLOCK(sc);
1368 }
1369 
1370 static int
1371 cgem_miibus_readreg(device_t dev, int phy, int reg)
1372 {
1373 	struct cgem_softc *sc = device_get_softc(dev);
1374 	int tries, val;
1375 
1376 	WR4(sc, CGEM_PHY_MAINT, CGEM_PHY_MAINT_CLAUSE_22 |
1377 	    CGEM_PHY_MAINT_MUST_10 | CGEM_PHY_MAINT_OP_READ |
1378 	    (phy << CGEM_PHY_MAINT_PHY_ADDR_SHIFT) |
1379 	    (reg << CGEM_PHY_MAINT_REG_ADDR_SHIFT));
1380 
1381 	/* Wait for completion. */
1382 	tries=0;
1383 	while ((RD4(sc, CGEM_NET_STAT) & CGEM_NET_STAT_PHY_MGMT_IDLE) == 0) {
1384 		DELAY(5);
1385 		if (++tries > 200) {
1386 			device_printf(dev, "phy read timeout: %d\n", reg);
1387 			return (-1);
1388 		}
1389 	}
1390 
1391 	val = RD4(sc, CGEM_PHY_MAINT) & CGEM_PHY_MAINT_DATA_MASK;
1392 
1393 	if (reg == MII_EXTSR)
1394 		/*
1395 		 * MAC does not support half-duplex at gig speeds.
1396 		 * Let mii(4) exclude the capability.
1397 		 */
1398 		val &= ~(EXTSR_1000XHDX | EXTSR_1000THDX);
1399 
1400 	return (val);
1401 }
1402 
1403 static int
1404 cgem_miibus_writereg(device_t dev, int phy, int reg, int data)
1405 {
1406 	struct cgem_softc *sc = device_get_softc(dev);
1407 	int tries;
1408 
1409 	WR4(sc, CGEM_PHY_MAINT, CGEM_PHY_MAINT_CLAUSE_22 |
1410 	    CGEM_PHY_MAINT_MUST_10 | CGEM_PHY_MAINT_OP_WRITE |
1411 	    (phy << CGEM_PHY_MAINT_PHY_ADDR_SHIFT) |
1412 	    (reg << CGEM_PHY_MAINT_REG_ADDR_SHIFT) |
1413 	    (data & CGEM_PHY_MAINT_DATA_MASK));
1414 
1415 	/* Wait for completion. */
1416 	tries = 0;
1417 	while ((RD4(sc, CGEM_NET_STAT) & CGEM_NET_STAT_PHY_MGMT_IDLE) == 0) {
1418 		DELAY(5);
1419 		if (++tries > 200) {
1420 			device_printf(dev, "phy write timeout: %d\n", reg);
1421 			return (-1);
1422 		}
1423 	}
1424 
1425 	return (0);
1426 }
1427 
1428 static void
1429 cgem_miibus_statchg(device_t dev)
1430 {
1431 	struct cgem_softc *sc  = device_get_softc(dev);
1432 	struct mii_data *mii = device_get_softc(sc->miibus);
1433 
1434 	CGEM_ASSERT_LOCKED(sc);
1435 
1436 	if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1437 	    (IFM_ACTIVE | IFM_AVALID) &&
1438 	    sc->mii_media_active != mii->mii_media_active)
1439 		cgem_mediachange(sc, mii);
1440 }
1441 
1442 static void
1443 cgem_miibus_linkchg(device_t dev)
1444 {
1445 	struct cgem_softc *sc  = device_get_softc(dev);
1446 	struct mii_data *mii = device_get_softc(sc->miibus);
1447 
1448 	CGEM_ASSERT_LOCKED(sc);
1449 
1450 	if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1451 	    (IFM_ACTIVE | IFM_AVALID) &&
1452 	    sc->mii_media_active != mii->mii_media_active)
1453 		cgem_mediachange(sc, mii);
1454 }
1455 
1456 /*
1457  * Overridable weak symbol cgem_set_ref_clk().  This allows platforms to
1458  * provide a function to set the cgem's reference clock.
1459  */
1460 static int __used
1461 cgem_default_set_ref_clk(int unit, int frequency)
1462 {
1463 
1464 	return 0;
1465 }
1466 __weak_reference(cgem_default_set_ref_clk, cgem_set_ref_clk);
1467 
1468 /* Call to set reference clock and network config bits according to media. */
1469 static void
1470 cgem_mediachange(struct cgem_softc *sc,	struct mii_data *mii)
1471 {
1472 	int ref_clk_freq;
1473 
1474 	CGEM_ASSERT_LOCKED(sc);
1475 
1476 	/* Update hardware to reflect media. */
1477 	sc->net_cfg_shadow &= ~(CGEM_NET_CFG_SPEED100 | CGEM_NET_CFG_GIGE_EN |
1478 	    CGEM_NET_CFG_FULL_DUPLEX);
1479 
1480 	switch (IFM_SUBTYPE(mii->mii_media_active)) {
1481 	case IFM_1000_T:
1482 		sc->net_cfg_shadow |= (CGEM_NET_CFG_SPEED100 |
1483 		    CGEM_NET_CFG_GIGE_EN);
1484 		ref_clk_freq = 125000000;
1485 		break;
1486 	case IFM_100_TX:
1487 		sc->net_cfg_shadow |= CGEM_NET_CFG_SPEED100;
1488 		ref_clk_freq = 25000000;
1489 		break;
1490 	default:
1491 		ref_clk_freq = 2500000;
1492 	}
1493 
1494 	if ((mii->mii_media_active & IFM_FDX) != 0)
1495 		sc->net_cfg_shadow |= CGEM_NET_CFG_FULL_DUPLEX;
1496 
1497 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1498 
1499 #ifdef EXT_RESOURCES
1500 	if (sc->ref_clk != NULL) {
1501 		CGEM_UNLOCK(sc);
1502 		if (clk_set_freq(sc->ref_clk, ref_clk_freq, 0))
1503 			device_printf(sc->dev, "could not set ref clk to %d\n",
1504 			    ref_clk_freq);
1505 		CGEM_LOCK(sc);
1506 	}
1507 #else
1508 	/* Set the reference clock if necessary. */
1509 	if (cgem_set_ref_clk(sc->ref_clk_num, ref_clk_freq))
1510 		device_printf(sc->dev,
1511 		    "cgem_mediachange: could not set ref clk%d to %d.\n",
1512 		    sc->ref_clk_num, ref_clk_freq);
1513 #endif
1514 
1515 	sc->mii_media_active = mii->mii_media_active;
1516 }
1517 
1518 static void
1519 cgem_add_sysctls(device_t dev)
1520 {
1521 	struct cgem_softc *sc = device_get_softc(dev);
1522 	struct sysctl_ctx_list *ctx;
1523 	struct sysctl_oid_list *child;
1524 	struct sysctl_oid *tree;
1525 
1526 	ctx = device_get_sysctl_ctx(dev);
1527 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
1528 
1529 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rxbufs", CTLFLAG_RW,
1530 	    &sc->rxbufs, 0, "Number receive buffers to provide");
1531 
1532 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rxhangwar", CTLFLAG_RW,
1533 	    &sc->rxhangwar, 0, "Enable receive hang work-around");
1534 
1535 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxoverruns", CTLFLAG_RD,
1536 	    &sc->rxoverruns, 0, "Receive overrun events");
1537 
1538 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxnobufs", CTLFLAG_RD,
1539 	    &sc->rxnobufs, 0, "Receive buf queue empty events");
1540 
1541 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxdmamapfails", CTLFLAG_RD,
1542 	    &sc->rxdmamapfails, 0, "Receive DMA map failures");
1543 
1544 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txfull", CTLFLAG_RD,
1545 	    &sc->txfull, 0, "Transmit ring full events");
1546 
1547 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdmamapfails", CTLFLAG_RD,
1548 	    &sc->txdmamapfails, 0, "Transmit DMA map failures");
1549 
1550 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdefrags", CTLFLAG_RD,
1551 	    &sc->txdefrags, 0, "Transmit m_defrag() calls");
1552 
1553 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdefragfails", CTLFLAG_RD,
1554 	    &sc->txdefragfails, 0, "Transmit m_defrag() failures");
1555 
1556 	tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "stats",
1557 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "GEM statistics");
1558 	child = SYSCTL_CHILDREN(tree);
1559 
1560 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_bytes", CTLFLAG_RD,
1561 	    &sc->stats.tx_bytes, "Total bytes transmitted");
1562 
1563 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames", CTLFLAG_RD,
1564 	    &sc->stats.tx_frames, 0, "Total frames transmitted");
1565 
1566 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_bcast", CTLFLAG_RD,
1567 	    &sc->stats.tx_frames_bcast, 0,
1568 	    "Number broadcast frames transmitted");
1569 
1570 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_multi", CTLFLAG_RD,
1571 	    &sc->stats.tx_frames_multi, 0,
1572 	    "Number multicast frames transmitted");
1573 
1574 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_pause",
1575 	    CTLFLAG_RD, &sc->stats.tx_frames_pause, 0,
1576 	    "Number pause frames transmitted");
1577 
1578 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_64b", CTLFLAG_RD,
1579 	    &sc->stats.tx_frames_64b, 0,
1580 	    "Number frames transmitted of size 64 bytes or less");
1581 
1582 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_65to127b", CTLFLAG_RD,
1583 	    &sc->stats.tx_frames_65to127b, 0,
1584 	    "Number frames transmitted of size 65-127 bytes");
1585 
1586 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_128to255b",
1587 	    CTLFLAG_RD, &sc->stats.tx_frames_128to255b, 0,
1588 	    "Number frames transmitted of size 128-255 bytes");
1589 
1590 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_256to511b",
1591 	    CTLFLAG_RD, &sc->stats.tx_frames_256to511b, 0,
1592 	    "Number frames transmitted of size 256-511 bytes");
1593 
1594 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_512to1023b",
1595 	    CTLFLAG_RD, &sc->stats.tx_frames_512to1023b, 0,
1596 	    "Number frames transmitted of size 512-1023 bytes");
1597 
1598 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_1024to1536b",
1599 	    CTLFLAG_RD, &sc->stats.tx_frames_1024to1536b, 0,
1600 	    "Number frames transmitted of size 1024-1536 bytes");
1601 
1602 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_under_runs",
1603 	    CTLFLAG_RD, &sc->stats.tx_under_runs, 0,
1604 	    "Number transmit under-run events");
1605 
1606 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_single_collisn",
1607 	    CTLFLAG_RD, &sc->stats.tx_single_collisn, 0,
1608 	    "Number single-collision transmit frames");
1609 
1610 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_multi_collisn",
1611 	    CTLFLAG_RD, &sc->stats.tx_multi_collisn, 0,
1612 	    "Number multi-collision transmit frames");
1613 
1614 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_excsv_collisn",
1615 	    CTLFLAG_RD, &sc->stats.tx_excsv_collisn, 0,
1616 	    "Number excessive collision transmit frames");
1617 
1618 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_late_collisn",
1619 	    CTLFLAG_RD, &sc->stats.tx_late_collisn, 0,
1620 	    "Number late-collision transmit frames");
1621 
1622 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_deferred_frames",
1623 	    CTLFLAG_RD, &sc->stats.tx_deferred_frames, 0,
1624 	    "Number deferred transmit frames");
1625 
1626 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_carrier_sense_errs",
1627 	    CTLFLAG_RD, &sc->stats.tx_carrier_sense_errs, 0,
1628 	    "Number carrier sense errors on transmit");
1629 
1630 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_bytes", CTLFLAG_RD,
1631 	    &sc->stats.rx_bytes, "Total bytes received");
1632 
1633 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames", CTLFLAG_RD,
1634 	    &sc->stats.rx_frames, 0, "Total frames received");
1635 
1636 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_bcast",
1637 	    CTLFLAG_RD, &sc->stats.rx_frames_bcast, 0,
1638 	    "Number broadcast frames received");
1639 
1640 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_multi",
1641 	    CTLFLAG_RD, &sc->stats.rx_frames_multi, 0,
1642 	    "Number multicast frames received");
1643 
1644 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_pause",
1645 	    CTLFLAG_RD, &sc->stats.rx_frames_pause, 0,
1646 	    "Number pause frames received");
1647 
1648 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_64b",
1649 	    CTLFLAG_RD, &sc->stats.rx_frames_64b, 0,
1650 	    "Number frames received of size 64 bytes or less");
1651 
1652 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_65to127b",
1653 	    CTLFLAG_RD, &sc->stats.rx_frames_65to127b, 0,
1654 	    "Number frames received of size 65-127 bytes");
1655 
1656 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_128to255b",
1657 	    CTLFLAG_RD, &sc->stats.rx_frames_128to255b, 0,
1658 	    "Number frames received of size 128-255 bytes");
1659 
1660 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_256to511b",
1661 	    CTLFLAG_RD, &sc->stats.rx_frames_256to511b, 0,
1662 	    "Number frames received of size 256-511 bytes");
1663 
1664 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_512to1023b",
1665 	    CTLFLAG_RD, &sc->stats.rx_frames_512to1023b, 0,
1666 	    "Number frames received of size 512-1023 bytes");
1667 
1668 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_1024to1536b",
1669 	    CTLFLAG_RD, &sc->stats.rx_frames_1024to1536b, 0,
1670 	    "Number frames received of size 1024-1536 bytes");
1671 
1672 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_undersize",
1673 	    CTLFLAG_RD, &sc->stats.rx_frames_undersize, 0,
1674 	    "Number undersize frames received");
1675 
1676 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_oversize",
1677 	    CTLFLAG_RD, &sc->stats.rx_frames_oversize, 0,
1678 	    "Number oversize frames received");
1679 
1680 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_jabber",
1681 	    CTLFLAG_RD, &sc->stats.rx_frames_jabber, 0,
1682 	    "Number jabber frames received");
1683 
1684 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_fcs_errs",
1685 	    CTLFLAG_RD, &sc->stats.rx_frames_fcs_errs, 0,
1686 	    "Number frames received with FCS errors");
1687 
1688 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_length_errs",
1689 	    CTLFLAG_RD, &sc->stats.rx_frames_length_errs, 0,
1690 	    "Number frames received with length errors");
1691 
1692 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_symbol_errs",
1693 	    CTLFLAG_RD, &sc->stats.rx_symbol_errs, 0,
1694 	    "Number receive symbol errors");
1695 
1696 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_align_errs",
1697 	    CTLFLAG_RD, &sc->stats.rx_align_errs, 0,
1698 	    "Number receive alignment errors");
1699 
1700 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_resource_errs",
1701 	    CTLFLAG_RD, &sc->stats.rx_resource_errs, 0,
1702 	    "Number frames received when no rx buffer available");
1703 
1704 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_overrun_errs",
1705 	    CTLFLAG_RD, &sc->stats.rx_overrun_errs, 0,
1706 	    "Number frames received but not copied due to receive overrun");
1707 
1708 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_ip_hdr_csum_errs",
1709 	    CTLFLAG_RD, &sc->stats.rx_ip_hdr_csum_errs, 0,
1710 	    "Number frames received with IP header checksum errors");
1711 
1712 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_tcp_csum_errs",
1713 	    CTLFLAG_RD, &sc->stats.rx_tcp_csum_errs, 0,
1714 	    "Number frames received with TCP checksum errors");
1715 
1716 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_udp_csum_errs",
1717 	    CTLFLAG_RD, &sc->stats.rx_udp_csum_errs, 0,
1718 	    "Number frames received with UDP checksum errors");
1719 }
1720 
1721 static int
1722 cgem_probe(device_t dev)
1723 {
1724 
1725 	if (!ofw_bus_status_okay(dev))
1726 		return (ENXIO);
1727 
1728 	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
1729 		return (ENXIO);
1730 
1731 	device_set_desc(dev, "Cadence CGEM Gigabit Ethernet Interface");
1732 	return (0);
1733 }
1734 
1735 static int
1736 cgem_attach(device_t dev)
1737 {
1738 	struct cgem_softc *sc = device_get_softc(dev);
1739 	if_t ifp = NULL;
1740 	int rid, err;
1741 	u_char eaddr[ETHER_ADDR_LEN];
1742 	int hwtype;
1743 #ifndef EXT_RESOURCES
1744 	phandle_t node;
1745 	pcell_t cell;
1746 #endif
1747 
1748 	sc->dev = dev;
1749 	CGEM_LOCK_INIT(sc);
1750 
1751 	/* Key off of compatible string and set hardware-specific options. */
1752 	hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
1753 	if (hwtype == HWTYPE_ZYNQMP)
1754 		sc->neednullqs = 1;
1755 	if (hwtype == HWTYPE_ZYNQ)
1756 		sc->rxhangwar = 1;
1757 
1758 #ifdef EXT_RESOURCES
1759 	if (hwtype == HWTYPE_ZYNQ || hwtype == HWTYPE_ZYNQMP) {
1760 		if (clk_get_by_ofw_name(dev, 0, "tx_clk", &sc->ref_clk) != 0)
1761 			device_printf(dev,
1762 			    "could not retrieve reference clock.\n");
1763 		else if (clk_enable(sc->ref_clk) != 0)
1764 			device_printf(dev, "could not enable clock.\n");
1765 	}
1766 	else if (hwtype == HWTYPE_SIFIVE_FU540) {
1767 		if (clk_get_by_ofw_name(dev, 0, "pclk", &sc->ref_clk) != 0)
1768 			device_printf(dev,
1769 			    "could not retrieve reference clock.\n");
1770 		else if (clk_enable(sc->ref_clk) != 0)
1771 			device_printf(dev, "could not enable clock.\n");
1772 	}
1773 #else
1774 	/* Get reference clock number and base divider from fdt. */
1775 	node = ofw_bus_get_node(dev);
1776 	sc->ref_clk_num = 0;
1777 	if (OF_getprop(node, "ref-clock-num", &cell, sizeof(cell)) > 0)
1778 		sc->ref_clk_num = fdt32_to_cpu(cell);
1779 #endif
1780 
1781 	/* Get memory resource. */
1782 	rid = 0;
1783 	sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1784 	    RF_ACTIVE);
1785 	if (sc->mem_res == NULL) {
1786 		device_printf(dev, "could not allocate memory resources.\n");
1787 		return (ENOMEM);
1788 	}
1789 
1790 	/* Get IRQ resource. */
1791 	rid = 0;
1792 	sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1793 	    RF_ACTIVE);
1794 	if (sc->irq_res == NULL) {
1795 		device_printf(dev, "could not allocate interrupt resource.\n");
1796 		cgem_detach(dev);
1797 		return (ENOMEM);
1798 	}
1799 
1800 	/* Set up ifnet structure. */
1801 	ifp = sc->ifp = if_alloc(IFT_ETHER);
1802 	if (ifp == NULL) {
1803 		device_printf(dev, "could not allocate ifnet structure\n");
1804 		cgem_detach(dev);
1805 		return (ENOMEM);
1806 	}
1807 	if_setsoftc(ifp, sc);
1808 	if_initname(ifp, IF_CGEM_NAME, device_get_unit(dev));
1809 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
1810 	if_setinitfn(ifp, cgem_init);
1811 	if_setioctlfn(ifp, cgem_ioctl);
1812 	if_setstartfn(ifp, cgem_start);
1813 	if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 |
1814 	    IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM, 0);
1815 	if_setsendqlen(ifp, CGEM_NUM_TX_DESCS);
1816 	if_setsendqready(ifp);
1817 
1818 	/* Disable hardware checksumming by default. */
1819 	if_sethwassist(ifp, 0);
1820 	if_setcapenable(ifp, if_getcapabilities(ifp) &
1821 	    ~(IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_VLAN_HWCSUM));
1822 
1823 	sc->if_old_flags = if_getflags(ifp);
1824 	sc->rxbufs = DEFAULT_NUM_RX_BUFS;
1825 
1826 	/* Reset hardware. */
1827 	CGEM_LOCK(sc);
1828 	cgem_reset(sc);
1829 	CGEM_UNLOCK(sc);
1830 
1831 	/* Attach phy to mii bus. */
1832 	err = mii_attach(dev, &sc->miibus, ifp,
1833 	    cgem_ifmedia_upd, cgem_ifmedia_sts, BMSR_DEFCAPMASK,
1834 	    MII_PHY_ANY, MII_OFFSET_ANY, 0);
1835 	if (err)
1836 		device_printf(dev, "warning: attaching PHYs failed\n");
1837 
1838 	/* Set up TX and RX descriptor area. */
1839 	err = cgem_setup_descs(sc);
1840 	if (err) {
1841 		device_printf(dev, "could not set up dma mem for descs.\n");
1842 		cgem_detach(dev);
1843 		return (ENOMEM);
1844 	}
1845 
1846 	/* Get a MAC address. */
1847 	cgem_get_mac(sc, eaddr);
1848 
1849 	/* Start ticks. */
1850 	callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0);
1851 
1852 	ether_ifattach(ifp, eaddr);
1853 
1854 	err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE |
1855 	    INTR_EXCL, NULL, cgem_intr, sc, &sc->intrhand);
1856 	if (err) {
1857 		device_printf(dev, "could not set interrupt handler.\n");
1858 		ether_ifdetach(ifp);
1859 		cgem_detach(dev);
1860 		return (err);
1861 	}
1862 
1863 	cgem_add_sysctls(dev);
1864 
1865 	return (0);
1866 }
1867 
1868 static int
1869 cgem_detach(device_t dev)
1870 {
1871 	struct cgem_softc *sc = device_get_softc(dev);
1872 	int i;
1873 
1874 	if (sc == NULL)
1875 		return (ENODEV);
1876 
1877 	if (device_is_attached(dev)) {
1878 		CGEM_LOCK(sc);
1879 		cgem_stop(sc);
1880 		CGEM_UNLOCK(sc);
1881 		callout_drain(&sc->tick_ch);
1882 		if_setflagbits(sc->ifp, 0, IFF_UP);
1883 		ether_ifdetach(sc->ifp);
1884 	}
1885 
1886 	if (sc->miibus != NULL) {
1887 		device_delete_child(dev, sc->miibus);
1888 		sc->miibus = NULL;
1889 	}
1890 
1891 	/* Release resources. */
1892 	if (sc->mem_res != NULL) {
1893 		bus_release_resource(dev, SYS_RES_MEMORY,
1894 		    rman_get_rid(sc->mem_res), sc->mem_res);
1895 		sc->mem_res = NULL;
1896 	}
1897 	if (sc->irq_res != NULL) {
1898 		if (sc->intrhand)
1899 			bus_teardown_intr(dev, sc->irq_res, sc->intrhand);
1900 		bus_release_resource(dev, SYS_RES_IRQ,
1901 		    rman_get_rid(sc->irq_res), sc->irq_res);
1902 		sc->irq_res = NULL;
1903 	}
1904 
1905 	/* Release DMA resources. */
1906 	if (sc->rxring != NULL) {
1907 		if (sc->rxring_physaddr != 0) {
1908 			bus_dmamap_unload(sc->desc_dma_tag,
1909 			    sc->rxring_dma_map);
1910 			sc->rxring_physaddr = 0;
1911 			sc->txring_physaddr = 0;
1912 			sc->null_qs_physaddr = 0;
1913 		}
1914 		bus_dmamem_free(sc->desc_dma_tag, sc->rxring,
1915 				sc->rxring_dma_map);
1916 		sc->rxring = NULL;
1917 		sc->txring = NULL;
1918 		sc->null_qs = NULL;
1919 
1920 		for (i = 0; i < CGEM_NUM_RX_DESCS; i++)
1921 			if (sc->rxring_m_dmamap[i] != NULL) {
1922 				bus_dmamap_destroy(sc->mbuf_dma_tag,
1923 				    sc->rxring_m_dmamap[i]);
1924 				sc->rxring_m_dmamap[i] = NULL;
1925 			}
1926 		for (i = 0; i < CGEM_NUM_TX_DESCS; i++)
1927 			if (sc->txring_m_dmamap[i] != NULL) {
1928 				bus_dmamap_destroy(sc->mbuf_dma_tag,
1929 				    sc->txring_m_dmamap[i]);
1930 				sc->txring_m_dmamap[i] = NULL;
1931 			}
1932 	}
1933 	if (sc->desc_dma_tag != NULL) {
1934 		bus_dma_tag_destroy(sc->desc_dma_tag);
1935 		sc->desc_dma_tag = NULL;
1936 	}
1937 	if (sc->mbuf_dma_tag != NULL) {
1938 		bus_dma_tag_destroy(sc->mbuf_dma_tag);
1939 		sc->mbuf_dma_tag = NULL;
1940 	}
1941 
1942 #ifdef EXT_RESOURCES
1943 	if (sc->ref_clk != NULL) {
1944 		clk_release(sc->ref_clk);
1945 		sc->ref_clk = NULL;
1946 	}
1947 #endif
1948 
1949 	bus_generic_detach(dev);
1950 
1951 	CGEM_LOCK_DESTROY(sc);
1952 
1953 	return (0);
1954 }
1955 
1956 static device_method_t cgem_methods[] = {
1957 	/* Device interface */
1958 	DEVMETHOD(device_probe,		cgem_probe),
1959 	DEVMETHOD(device_attach,	cgem_attach),
1960 	DEVMETHOD(device_detach,	cgem_detach),
1961 
1962 	/* MII interface */
1963 	DEVMETHOD(miibus_readreg,	cgem_miibus_readreg),
1964 	DEVMETHOD(miibus_writereg,	cgem_miibus_writereg),
1965 	DEVMETHOD(miibus_statchg,	cgem_miibus_statchg),
1966 	DEVMETHOD(miibus_linkchg,	cgem_miibus_linkchg),
1967 
1968 	DEVMETHOD_END
1969 };
1970 
1971 static driver_t cgem_driver = {
1972 	"cgem",
1973 	cgem_methods,
1974 	sizeof(struct cgem_softc),
1975 };
1976 
1977 DRIVER_MODULE(cgem, simplebus, cgem_driver, cgem_devclass, NULL, NULL);
1978 DRIVER_MODULE(miibus, cgem, miibus_driver, miibus_devclass, NULL, NULL);
1979 MODULE_DEPEND(cgem, miibus, 1, 1, 1);
1980 MODULE_DEPEND(cgem, ether, 1, 1, 1);
1981 SIMPLEBUS_PNP_INFO(compat_data);
1982