xref: /freebsd/sys/dev/cadence/if_cgem.c (revision 34252e89a96c8d498d7537ade16d20602bf2106a)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012-2014 Thomas Skibo <thomasskibo@yahoo.com>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 /*
30  * A network interface driver for Cadence GEM Gigabit Ethernet
31  * interface such as the one used in Xilinx Zynq-7000 SoC.
32  *
33  * Reference: Zynq-7000 All Programmable SoC Technical Reference Manual.
34  * (v1.4) November 16, 2012.  Xilinx doc UG585.  GEM is covered in Ch. 16
35  * and register definitions are in appendix B.18.
36  */
37 
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40 
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/bus.h>
44 #include <sys/kernel.h>
45 #include <sys/malloc.h>
46 #include <sys/mbuf.h>
47 #include <sys/module.h>
48 #include <sys/rman.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
51 #include <sys/sysctl.h>
52 
53 #include <machine/bus.h>
54 
55 #include <net/ethernet.h>
56 #include <net/if.h>
57 #include <net/if_arp.h>
58 #include <net/if_dl.h>
59 #include <net/if_media.h>
60 #include <net/if_mib.h>
61 #include <net/if_types.h>
62 
63 #ifdef INET
64 #include <netinet/in.h>
65 #include <netinet/in_systm.h>
66 #include <netinet/in_var.h>
67 #include <netinet/ip.h>
68 #endif
69 
70 #include <net/bpf.h>
71 #include <net/bpfdesc.h>
72 
73 #include <dev/fdt/fdt_common.h>
74 #include <dev/ofw/ofw_bus.h>
75 #include <dev/ofw/ofw_bus_subr.h>
76 
77 #include <dev/mii/mii.h>
78 #include <dev/mii/miivar.h>
79 #include <dev/mii/mii_fdt.h>
80 
81 #include <dev/extres/clk/clk.h>
82 
83 #if BUS_SPACE_MAXADDR > BUS_SPACE_MAXADDR_32BIT
84 #define CGEM64
85 #endif
86 
87 #include <dev/cadence/if_cgem_hw.h>
88 
89 #include "miibus_if.h"
90 
91 #define IF_CGEM_NAME "cgem"
92 
93 #define CGEM_NUM_RX_DESCS	512	/* size of receive descriptor ring */
94 #define CGEM_NUM_TX_DESCS	512	/* size of transmit descriptor ring */
95 
96 /* Default for sysctl rxbufs.  Must be < CGEM_NUM_RX_DESCS of course. */
97 #define DEFAULT_NUM_RX_BUFS	256	/* number of receive bufs to queue. */
98 
99 #define TX_MAX_DMA_SEGS		8	/* maximum segs in a tx mbuf dma */
100 
101 #define CGEM_CKSUM_ASSIST	(CSUM_IP | CSUM_TCP | CSUM_UDP | \
102 				 CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
103 
104 #define HWQUIRK_NONE		0
105 #define HWQUIRK_NEEDNULLQS	1
106 #define HWQUIRK_RXHANGWAR	2
107 #define HWQUIRK_TXCLK		4
108 #define HWQUIRK_PCLK		8
109 
110 static struct ofw_compat_data compat_data[] = {
111 	{ "cdns,zynq-gem",		HWQUIRK_RXHANGWAR | HWQUIRK_TXCLK },
112 	{ "cdns,zynqmp-gem",		HWQUIRK_NEEDNULLQS | HWQUIRK_TXCLK },
113 	{ "microchip,mpfs-mss-gem",	HWQUIRK_NEEDNULLQS | HWQUIRK_TXCLK },
114 	{ "sifive,fu540-c000-gem",	HWQUIRK_PCLK },
115 	{ "sifive,fu740-c000-gem",	HWQUIRK_PCLK },
116 	{ "cdns,gem",			HWQUIRK_NONE },
117 	{ "cdns,macb",			HWQUIRK_NONE },
118 	{ "cadence,gem",		HWQUIRK_NONE },
119 	{ NULL,				0 }
120 };
121 
122 struct cgem_softc {
123 	if_t			ifp;
124 	struct mtx		sc_mtx;
125 	device_t		dev;
126 	device_t		miibus;
127 	u_int			mii_media_active;	/* last active media */
128 	int			if_old_flags;
129 	struct resource		*mem_res;
130 	struct resource		*irq_res;
131 	void			*intrhand;
132 	struct callout		tick_ch;
133 	uint32_t		net_ctl_shadow;
134 	uint32_t		net_cfg_shadow;
135 	clk_t			ref_clk;
136 	int			neednullqs;
137 	int			phy_contype;
138 
139 	bus_dma_tag_t		desc_dma_tag;
140 	bus_dma_tag_t		mbuf_dma_tag;
141 
142 	/* receive descriptor ring */
143 	struct cgem_rx_desc	*rxring;
144 	bus_addr_t		rxring_physaddr;
145 	struct mbuf		*rxring_m[CGEM_NUM_RX_DESCS];
146 	bus_dmamap_t		rxring_m_dmamap[CGEM_NUM_RX_DESCS];
147 	int			rxring_hd_ptr;	/* where to put rcv bufs */
148 	int			rxring_tl_ptr;	/* where to get receives */
149 	int			rxring_queued;	/* how many rcv bufs queued */
150 	bus_dmamap_t		rxring_dma_map;
151 	int			rxbufs;		/* tunable number rcv bufs */
152 	int			rxhangwar;	/* rx hang work-around */
153 	u_int			rxoverruns;	/* rx overruns */
154 	u_int			rxnobufs;	/* rx buf ring empty events */
155 	u_int			rxdmamapfails;	/* rx dmamap failures */
156 	uint32_t		rx_frames_prev;
157 
158 	/* transmit descriptor ring */
159 	struct cgem_tx_desc	*txring;
160 	bus_addr_t		txring_physaddr;
161 	struct mbuf		*txring_m[CGEM_NUM_TX_DESCS];
162 	bus_dmamap_t		txring_m_dmamap[CGEM_NUM_TX_DESCS];
163 	int			txring_hd_ptr;	/* where to put next xmits */
164 	int			txring_tl_ptr;	/* next xmit mbuf to free */
165 	int			txring_queued;	/* num xmits segs queued */
166 	u_int			txfull;		/* tx ring full events */
167 	u_int			txdefrags;	/* tx calls to m_defrag() */
168 	u_int			txdefragfails;	/* tx m_defrag() failures */
169 	u_int			txdmamapfails;	/* tx dmamap failures */
170 
171 	/* null descriptor rings */
172 	void			*null_qs;
173 	bus_addr_t		null_qs_physaddr;
174 
175 	/* hardware provided statistics */
176 	struct cgem_hw_stats {
177 		uint64_t		tx_bytes;
178 		uint32_t		tx_frames;
179 		uint32_t		tx_frames_bcast;
180 		uint32_t		tx_frames_multi;
181 		uint32_t		tx_frames_pause;
182 		uint32_t		tx_frames_64b;
183 		uint32_t		tx_frames_65to127b;
184 		uint32_t		tx_frames_128to255b;
185 		uint32_t		tx_frames_256to511b;
186 		uint32_t		tx_frames_512to1023b;
187 		uint32_t		tx_frames_1024to1536b;
188 		uint32_t		tx_under_runs;
189 		uint32_t		tx_single_collisn;
190 		uint32_t		tx_multi_collisn;
191 		uint32_t		tx_excsv_collisn;
192 		uint32_t		tx_late_collisn;
193 		uint32_t		tx_deferred_frames;
194 		uint32_t		tx_carrier_sense_errs;
195 
196 		uint64_t		rx_bytes;
197 		uint32_t		rx_frames;
198 		uint32_t		rx_frames_bcast;
199 		uint32_t		rx_frames_multi;
200 		uint32_t		rx_frames_pause;
201 		uint32_t		rx_frames_64b;
202 		uint32_t		rx_frames_65to127b;
203 		uint32_t		rx_frames_128to255b;
204 		uint32_t		rx_frames_256to511b;
205 		uint32_t		rx_frames_512to1023b;
206 		uint32_t		rx_frames_1024to1536b;
207 		uint32_t		rx_frames_undersize;
208 		uint32_t		rx_frames_oversize;
209 		uint32_t		rx_frames_jabber;
210 		uint32_t		rx_frames_fcs_errs;
211 		uint32_t		rx_frames_length_errs;
212 		uint32_t		rx_symbol_errs;
213 		uint32_t		rx_align_errs;
214 		uint32_t		rx_resource_errs;
215 		uint32_t		rx_overrun_errs;
216 		uint32_t		rx_ip_hdr_csum_errs;
217 		uint32_t		rx_tcp_csum_errs;
218 		uint32_t		rx_udp_csum_errs;
219 	} stats;
220 };
221 
222 #define RD4(sc, off)		(bus_read_4((sc)->mem_res, (off)))
223 #define WR4(sc, off, val)	(bus_write_4((sc)->mem_res, (off), (val)))
224 #define BARRIER(sc, off, len, flags) \
225 	(bus_barrier((sc)->mem_res, (off), (len), (flags))
226 
227 #define CGEM_LOCK(sc)		mtx_lock(&(sc)->sc_mtx)
228 #define CGEM_UNLOCK(sc)		mtx_unlock(&(sc)->sc_mtx)
229 #define CGEM_LOCK_INIT(sc)	mtx_init(&(sc)->sc_mtx, \
230 	    device_get_nameunit((sc)->dev), MTX_NETWORK_LOCK, MTX_DEF)
231 #define CGEM_LOCK_DESTROY(sc)	mtx_destroy(&(sc)->sc_mtx)
232 #define CGEM_ASSERT_LOCKED(sc)	mtx_assert(&(sc)->sc_mtx, MA_OWNED)
233 
234 /* Allow platforms to optionally provide a way to set the reference clock. */
235 int cgem_set_ref_clk(int unit, int frequency);
236 
237 static devclass_t cgem_devclass;
238 
239 static int cgem_probe(device_t dev);
240 static int cgem_attach(device_t dev);
241 static int cgem_detach(device_t dev);
242 static void cgem_tick(void *);
243 static void cgem_intr(void *);
244 
245 static void cgem_mediachange(struct cgem_softc *, struct mii_data *);
246 
247 static void
248 cgem_get_mac(struct cgem_softc *sc, u_char eaddr[])
249 {
250 	int i;
251 	uint32_t rnd;
252 
253 	/* See if boot loader gave us a MAC address already. */
254 	for (i = 0; i < 4; i++) {
255 		uint32_t low = RD4(sc, CGEM_SPEC_ADDR_LOW(i));
256 		uint32_t high = RD4(sc, CGEM_SPEC_ADDR_HI(i)) & 0xffff;
257 		if (low != 0 || high != 0) {
258 			eaddr[0] = low & 0xff;
259 			eaddr[1] = (low >> 8) & 0xff;
260 			eaddr[2] = (low >> 16) & 0xff;
261 			eaddr[3] = (low >> 24) & 0xff;
262 			eaddr[4] = high & 0xff;
263 			eaddr[5] = (high >> 8) & 0xff;
264 			break;
265 		}
266 	}
267 
268 	/* No MAC from boot loader?  Assign a random one. */
269 	if (i == 4) {
270 		rnd = arc4random();
271 
272 		eaddr[0] = 'b';
273 		eaddr[1] = 's';
274 		eaddr[2] = 'd';
275 		eaddr[3] = (rnd >> 16) & 0xff;
276 		eaddr[4] = (rnd >> 8) & 0xff;
277 		eaddr[5] = rnd & 0xff;
278 
279 		device_printf(sc->dev, "no mac address found, assigning "
280 		    "random: %02x:%02x:%02x:%02x:%02x:%02x\n", eaddr[0],
281 		    eaddr[1], eaddr[2], eaddr[3], eaddr[4], eaddr[5]);
282 	}
283 
284 	/* Move address to first slot and zero out the rest. */
285 	WR4(sc, CGEM_SPEC_ADDR_LOW(0), (eaddr[3] << 24) |
286 	    (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]);
287 	WR4(sc, CGEM_SPEC_ADDR_HI(0), (eaddr[5] << 8) | eaddr[4]);
288 
289 	for (i = 1; i < 4; i++) {
290 		WR4(sc, CGEM_SPEC_ADDR_LOW(i), 0);
291 		WR4(sc, CGEM_SPEC_ADDR_HI(i), 0);
292 	}
293 }
294 
295 /*
296  * cgem_mac_hash():  map 48-bit address to a 6-bit hash. The 6-bit hash
297  * corresponds to a bit in a 64-bit hash register.  Setting that bit in the
298  * hash register enables reception of all frames with a destination address
299  * that hashes to that 6-bit value.
300  *
301  * The hash function is described in sec. 16.2.3 in the Zynq-7000 Tech
302  * Reference Manual.  Bits 0-5 in the hash are the exclusive-or of
303  * every sixth bit in the destination address.
304  */
305 static int
306 cgem_mac_hash(u_char eaddr[])
307 {
308 	int hash;
309 	int i, j;
310 
311 	hash = 0;
312 	for (i = 0; i < 6; i++)
313 		for (j = i; j < 48; j += 6)
314 			if ((eaddr[j >> 3] & (1 << (j & 7))) != 0)
315 				hash ^= (1 << i);
316 
317 	return hash;
318 }
319 
320 static u_int
321 cgem_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
322 {
323 	uint32_t *hashes = arg;
324 	int index;
325 
326 	index = cgem_mac_hash(LLADDR(sdl));
327 	if (index > 31)
328 		hashes[0] |= (1U << (index - 32));
329 	else
330 		hashes[1] |= (1U << index);
331 
332 	return (1);
333 }
334 
335 /*
336  * After any change in rx flags or multi-cast addresses, set up hash registers
337  * and net config register bits.
338  */
339 static void
340 cgem_rx_filter(struct cgem_softc *sc)
341 {
342 	if_t ifp = sc->ifp;
343 	uint32_t hashes[2] = { 0, 0 };
344 
345 	sc->net_cfg_shadow &= ~(CGEM_NET_CFG_MULTI_HASH_EN |
346 	    CGEM_NET_CFG_NO_BCAST | CGEM_NET_CFG_COPY_ALL);
347 
348 	if ((if_getflags(ifp) & IFF_PROMISC) != 0)
349 		sc->net_cfg_shadow |= CGEM_NET_CFG_COPY_ALL;
350 	else {
351 		if ((if_getflags(ifp) & IFF_BROADCAST) == 0)
352 			sc->net_cfg_shadow |= CGEM_NET_CFG_NO_BCAST;
353 		if ((if_getflags(ifp) & IFF_ALLMULTI) != 0) {
354 			hashes[0] = 0xffffffff;
355 			hashes[1] = 0xffffffff;
356 		} else
357 			if_foreach_llmaddr(ifp, cgem_hash_maddr, hashes);
358 
359 		if (hashes[0] != 0 || hashes[1] != 0)
360 			sc->net_cfg_shadow |= CGEM_NET_CFG_MULTI_HASH_EN;
361 	}
362 
363 	WR4(sc, CGEM_HASH_TOP, hashes[0]);
364 	WR4(sc, CGEM_HASH_BOT, hashes[1]);
365 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
366 }
367 
368 /* For bus_dmamap_load() callback. */
369 static void
370 cgem_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
371 {
372 
373 	if (nsegs != 1 || error != 0)
374 		return;
375 	*(bus_addr_t *)arg = segs[0].ds_addr;
376 }
377 
378 /* Set up null queues for priority queues we actually can't disable. */
379 static void
380 cgem_null_qs(struct cgem_softc *sc)
381 {
382 	struct cgem_rx_desc *rx_desc;
383 	struct cgem_tx_desc *tx_desc;
384 	uint32_t queue_mask;
385 	int n;
386 
387 	/* Read design config register 6 to determine number of queues. */
388 	queue_mask = (RD4(sc, CGEM_DESIGN_CFG6) &
389 	    CGEM_DESIGN_CFG6_DMA_PRIO_Q_MASK) >> 1;
390 	if (queue_mask == 0)
391 		return;
392 
393 	/* Create empty RX queue and empty TX buf queues. */
394 	memset(sc->null_qs, 0, sizeof(struct cgem_rx_desc) +
395 	    sizeof(struct cgem_tx_desc));
396 	rx_desc = sc->null_qs;
397 	rx_desc->addr = CGEM_RXDESC_OWN | CGEM_RXDESC_WRAP;
398 	tx_desc = (struct cgem_tx_desc *)(rx_desc + 1);
399 	tx_desc->ctl = CGEM_TXDESC_USED | CGEM_TXDESC_WRAP;
400 
401 	/* Point all valid ring base pointers to the null queues. */
402 	for (n = 1; (queue_mask & 1) != 0; n++, queue_mask >>= 1) {
403 		WR4(sc, CGEM_RX_QN_BAR(n), sc->null_qs_physaddr);
404 		WR4(sc, CGEM_TX_QN_BAR(n), sc->null_qs_physaddr +
405 		    sizeof(struct cgem_rx_desc));
406 	}
407 }
408 
409 /* Create DMA'able descriptor rings. */
410 static int
411 cgem_setup_descs(struct cgem_softc *sc)
412 {
413 	int i, err;
414 	int desc_rings_size = CGEM_NUM_RX_DESCS * sizeof(struct cgem_rx_desc) +
415 	    CGEM_NUM_TX_DESCS * sizeof(struct cgem_tx_desc);
416 
417 	if (sc->neednullqs)
418 		desc_rings_size += sizeof(struct cgem_rx_desc) +
419 		    sizeof(struct cgem_tx_desc);
420 
421 	sc->txring = NULL;
422 	sc->rxring = NULL;
423 
424 	/* Allocate non-cached DMA space for RX and TX descriptors. */
425 	err = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1,
426 #ifdef CGEM64
427 	    1ULL << 32,	/* Do not cross a 4G boundary. */
428 #else
429 	    0,
430 #endif
431 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
432 	    desc_rings_size, 1, desc_rings_size, 0,
433 	    busdma_lock_mutex, &sc->sc_mtx, &sc->desc_dma_tag);
434 	if (err)
435 		return (err);
436 
437 	/* Set up a bus_dma_tag for mbufs. */
438 	err = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
439 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES,
440 	    TX_MAX_DMA_SEGS, MCLBYTES, 0, busdma_lock_mutex, &sc->sc_mtx,
441 	    &sc->mbuf_dma_tag);
442 	if (err)
443 		return (err);
444 
445 	/*
446 	 * Allocate DMA memory. We allocate transmit, receive and null
447 	 * descriptor queues all at once because the hardware only provides
448 	 * one register for the upper 32 bits of rx and tx descriptor queues
449 	 * hardware addresses.
450 	 */
451 	err = bus_dmamem_alloc(sc->desc_dma_tag, (void **)&sc->rxring,
452 	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO,
453 	    &sc->rxring_dma_map);
454 	if (err)
455 		return (err);
456 
457 	/* Load descriptor DMA memory. */
458 	err = bus_dmamap_load(sc->desc_dma_tag, sc->rxring_dma_map,
459 	    (void *)sc->rxring, desc_rings_size,
460 	    cgem_getaddr, &sc->rxring_physaddr, BUS_DMA_NOWAIT);
461 	if (err)
462 		return (err);
463 
464 	/* Initialize RX descriptors. */
465 	for (i = 0; i < CGEM_NUM_RX_DESCS; i++) {
466 		sc->rxring[i].addr = CGEM_RXDESC_OWN;
467 		sc->rxring[i].ctl = 0;
468 		sc->rxring_m[i] = NULL;
469 		sc->rxring_m_dmamap[i] = NULL;
470 	}
471 	sc->rxring[CGEM_NUM_RX_DESCS - 1].addr |= CGEM_RXDESC_WRAP;
472 
473 	sc->rxring_hd_ptr = 0;
474 	sc->rxring_tl_ptr = 0;
475 	sc->rxring_queued = 0;
476 
477 	sc->txring = (struct cgem_tx_desc *)(sc->rxring + CGEM_NUM_RX_DESCS);
478 	sc->txring_physaddr = sc->rxring_physaddr + CGEM_NUM_RX_DESCS *
479 	    sizeof(struct cgem_rx_desc);
480 
481 	/* Initialize TX descriptor ring. */
482 	for (i = 0; i < CGEM_NUM_TX_DESCS; i++) {
483 		sc->txring[i].addr = 0;
484 		sc->txring[i].ctl = CGEM_TXDESC_USED;
485 		sc->txring_m[i] = NULL;
486 		sc->txring_m_dmamap[i] = NULL;
487 	}
488 	sc->txring[CGEM_NUM_TX_DESCS - 1].ctl |= CGEM_TXDESC_WRAP;
489 
490 	sc->txring_hd_ptr = 0;
491 	sc->txring_tl_ptr = 0;
492 	sc->txring_queued = 0;
493 
494 	if (sc->neednullqs) {
495 		sc->null_qs = (void *)(sc->txring + CGEM_NUM_TX_DESCS);
496 		sc->null_qs_physaddr = sc->txring_physaddr +
497 		    CGEM_NUM_TX_DESCS * sizeof(struct cgem_tx_desc);
498 
499 		cgem_null_qs(sc);
500 	}
501 
502 	return (0);
503 }
504 
505 /* Fill receive descriptor ring with mbufs. */
506 static void
507 cgem_fill_rqueue(struct cgem_softc *sc)
508 {
509 	struct mbuf *m = NULL;
510 	bus_dma_segment_t segs[TX_MAX_DMA_SEGS];
511 	int nsegs;
512 
513 	CGEM_ASSERT_LOCKED(sc);
514 
515 	while (sc->rxring_queued < sc->rxbufs) {
516 		/* Get a cluster mbuf. */
517 		m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
518 		if (m == NULL)
519 			break;
520 
521 		m->m_len = MCLBYTES;
522 		m->m_pkthdr.len = MCLBYTES;
523 		m->m_pkthdr.rcvif = sc->ifp;
524 
525 		/* Load map and plug in physical address. */
526 		if (bus_dmamap_create(sc->mbuf_dma_tag, 0,
527 		    &sc->rxring_m_dmamap[sc->rxring_hd_ptr])) {
528 			sc->rxdmamapfails++;
529 			m_free(m);
530 			break;
531 		}
532 		if (bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
533 		    sc->rxring_m_dmamap[sc->rxring_hd_ptr], m,
534 		    segs, &nsegs, BUS_DMA_NOWAIT)) {
535 			sc->rxdmamapfails++;
536 			bus_dmamap_destroy(sc->mbuf_dma_tag,
537 				   sc->rxring_m_dmamap[sc->rxring_hd_ptr]);
538 			sc->rxring_m_dmamap[sc->rxring_hd_ptr] = NULL;
539 			m_free(m);
540 			break;
541 		}
542 		sc->rxring_m[sc->rxring_hd_ptr] = m;
543 
544 		/* Sync cache with receive buffer. */
545 		bus_dmamap_sync(sc->mbuf_dma_tag,
546 		    sc->rxring_m_dmamap[sc->rxring_hd_ptr],
547 		    BUS_DMASYNC_PREREAD);
548 
549 		/* Write rx descriptor and increment head pointer. */
550 		sc->rxring[sc->rxring_hd_ptr].ctl = 0;
551 #ifdef CGEM64
552 		sc->rxring[sc->rxring_hd_ptr].addrhi = segs[0].ds_addr >> 32;
553 #endif
554 		if (sc->rxring_hd_ptr == CGEM_NUM_RX_DESCS - 1) {
555 			sc->rxring[sc->rxring_hd_ptr].addr = segs[0].ds_addr |
556 			    CGEM_RXDESC_WRAP;
557 			sc->rxring_hd_ptr = 0;
558 		} else
559 			sc->rxring[sc->rxring_hd_ptr++].addr = segs[0].ds_addr;
560 
561 		sc->rxring_queued++;
562 	}
563 }
564 
565 /* Pull received packets off of receive descriptor ring. */
566 static void
567 cgem_recv(struct cgem_softc *sc)
568 {
569 	if_t ifp = sc->ifp;
570 	struct mbuf *m, *m_hd, **m_tl;
571 	uint32_t ctl;
572 
573 	CGEM_ASSERT_LOCKED(sc);
574 
575 	/* Pick up all packets in which the OWN bit is set. */
576 	m_hd = NULL;
577 	m_tl = &m_hd;
578 	while (sc->rxring_queued > 0 &&
579 	    (sc->rxring[sc->rxring_tl_ptr].addr & CGEM_RXDESC_OWN) != 0) {
580 		ctl = sc->rxring[sc->rxring_tl_ptr].ctl;
581 
582 		/* Grab filled mbuf. */
583 		m = sc->rxring_m[sc->rxring_tl_ptr];
584 		sc->rxring_m[sc->rxring_tl_ptr] = NULL;
585 
586 		/* Sync cache with receive buffer. */
587 		bus_dmamap_sync(sc->mbuf_dma_tag,
588 		    sc->rxring_m_dmamap[sc->rxring_tl_ptr],
589 		    BUS_DMASYNC_POSTREAD);
590 
591 		/* Unload and destroy dmamap. */
592 		bus_dmamap_unload(sc->mbuf_dma_tag,
593 		    sc->rxring_m_dmamap[sc->rxring_tl_ptr]);
594 		bus_dmamap_destroy(sc->mbuf_dma_tag,
595 		    sc->rxring_m_dmamap[sc->rxring_tl_ptr]);
596 		sc->rxring_m_dmamap[sc->rxring_tl_ptr] = NULL;
597 
598 		/* Increment tail pointer. */
599 		if (++sc->rxring_tl_ptr == CGEM_NUM_RX_DESCS)
600 			sc->rxring_tl_ptr = 0;
601 		sc->rxring_queued--;
602 
603 		/*
604 		 * Check FCS and make sure entire packet landed in one mbuf
605 		 * cluster (which is much bigger than the largest ethernet
606 		 * packet).
607 		 */
608 		if ((ctl & CGEM_RXDESC_BAD_FCS) != 0 ||
609 		    (ctl & (CGEM_RXDESC_SOF | CGEM_RXDESC_EOF)) !=
610 		    (CGEM_RXDESC_SOF | CGEM_RXDESC_EOF)) {
611 			/* discard. */
612 			m_free(m);
613 			if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
614 			continue;
615 		}
616 
617 		/* Ready it to hand off to upper layers. */
618 		m->m_data += ETHER_ALIGN;
619 		m->m_len = (ctl & CGEM_RXDESC_LENGTH_MASK);
620 		m->m_pkthdr.rcvif = ifp;
621 		m->m_pkthdr.len = m->m_len;
622 
623 		/*
624 		 * Are we using hardware checksumming?  Check the status in the
625 		 * receive descriptor.
626 		 */
627 		if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) {
628 			/* TCP or UDP checks out, IP checks out too. */
629 			if ((ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
630 			    CGEM_RXDESC_CKSUM_STAT_TCP_GOOD ||
631 			    (ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
632 			    CGEM_RXDESC_CKSUM_STAT_UDP_GOOD) {
633 				m->m_pkthdr.csum_flags |=
634 				    CSUM_IP_CHECKED | CSUM_IP_VALID |
635 				    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
636 				m->m_pkthdr.csum_data = 0xffff;
637 			} else if ((ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
638 			    CGEM_RXDESC_CKSUM_STAT_IP_GOOD) {
639 				/* Only IP checks out. */
640 				m->m_pkthdr.csum_flags |=
641 				    CSUM_IP_CHECKED | CSUM_IP_VALID;
642 				m->m_pkthdr.csum_data = 0xffff;
643 			}
644 		}
645 
646 		/* Queue it up for delivery below. */
647 		*m_tl = m;
648 		m_tl = &m->m_next;
649 	}
650 
651 	/* Replenish receive buffers. */
652 	cgem_fill_rqueue(sc);
653 
654 	/* Unlock and send up packets. */
655 	CGEM_UNLOCK(sc);
656 	while (m_hd != NULL) {
657 		m = m_hd;
658 		m_hd = m_hd->m_next;
659 		m->m_next = NULL;
660 		if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
661 		if_input(ifp, m);
662 	}
663 	CGEM_LOCK(sc);
664 }
665 
666 /* Find completed transmits and free their mbufs. */
667 static void
668 cgem_clean_tx(struct cgem_softc *sc)
669 {
670 	struct mbuf *m;
671 	uint32_t ctl;
672 
673 	CGEM_ASSERT_LOCKED(sc);
674 
675 	/* free up finished transmits. */
676 	while (sc->txring_queued > 0 &&
677 	    ((ctl = sc->txring[sc->txring_tl_ptr].ctl) &
678 	    CGEM_TXDESC_USED) != 0) {
679 		/* Sync cache. */
680 		bus_dmamap_sync(sc->mbuf_dma_tag,
681 		    sc->txring_m_dmamap[sc->txring_tl_ptr],
682 		    BUS_DMASYNC_POSTWRITE);
683 
684 		/* Unload and destroy DMA map. */
685 		bus_dmamap_unload(sc->mbuf_dma_tag,
686 		    sc->txring_m_dmamap[sc->txring_tl_ptr]);
687 		bus_dmamap_destroy(sc->mbuf_dma_tag,
688 		    sc->txring_m_dmamap[sc->txring_tl_ptr]);
689 		sc->txring_m_dmamap[sc->txring_tl_ptr] = NULL;
690 
691 		/* Free up the mbuf. */
692 		m = sc->txring_m[sc->txring_tl_ptr];
693 		sc->txring_m[sc->txring_tl_ptr] = NULL;
694 		m_freem(m);
695 
696 		/* Check the status. */
697 		if ((ctl & CGEM_TXDESC_AHB_ERR) != 0) {
698 			/* Serious bus error. log to console. */
699 #ifdef CGEM64
700 			device_printf(sc->dev,
701 			    "cgem_clean_tx: AHB error, addr=0x%x%08x\n",
702 			    sc->txring[sc->txring_tl_ptr].addrhi,
703 			    sc->txring[sc->txring_tl_ptr].addr);
704 #else
705 			device_printf(sc->dev,
706 			    "cgem_clean_tx: AHB error, addr=0x%x\n",
707 			    sc->txring[sc->txring_tl_ptr].addr);
708 #endif
709 		} else if ((ctl & (CGEM_TXDESC_RETRY_ERR |
710 		    CGEM_TXDESC_LATE_COLL)) != 0) {
711 			if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1);
712 		} else
713 			if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
714 
715 		/*
716 		 * If the packet spanned more than one tx descriptor, skip
717 		 * descriptors until we find the end so that only
718 		 * start-of-frame descriptors are processed.
719 		 */
720 		while ((ctl & CGEM_TXDESC_LAST_BUF) == 0) {
721 			if ((ctl & CGEM_TXDESC_WRAP) != 0)
722 				sc->txring_tl_ptr = 0;
723 			else
724 				sc->txring_tl_ptr++;
725 			sc->txring_queued--;
726 
727 			ctl = sc->txring[sc->txring_tl_ptr].ctl;
728 
729 			sc->txring[sc->txring_tl_ptr].ctl =
730 			    ctl | CGEM_TXDESC_USED;
731 		}
732 
733 		/* Next descriptor. */
734 		if ((ctl & CGEM_TXDESC_WRAP) != 0)
735 			sc->txring_tl_ptr = 0;
736 		else
737 			sc->txring_tl_ptr++;
738 		sc->txring_queued--;
739 
740 		if_setdrvflagbits(sc->ifp, 0, IFF_DRV_OACTIVE);
741 	}
742 }
743 
744 /* Start transmits. */
745 static void
746 cgem_start_locked(if_t ifp)
747 {
748 	struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
749 	struct mbuf *m;
750 	bus_dma_segment_t segs[TX_MAX_DMA_SEGS];
751 	uint32_t ctl;
752 	int i, nsegs, wrap, err;
753 
754 	CGEM_ASSERT_LOCKED(sc);
755 
756 	if ((if_getdrvflags(ifp) & IFF_DRV_OACTIVE) != 0)
757 		return;
758 
759 	for (;;) {
760 		/* Check that there is room in the descriptor ring. */
761 		if (sc->txring_queued >=
762 		    CGEM_NUM_TX_DESCS - TX_MAX_DMA_SEGS * 2) {
763 			/* Try to make room. */
764 			cgem_clean_tx(sc);
765 
766 			/* Still no room? */
767 			if (sc->txring_queued >=
768 			    CGEM_NUM_TX_DESCS - TX_MAX_DMA_SEGS * 2) {
769 				if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
770 				sc->txfull++;
771 				break;
772 			}
773 		}
774 
775 		/* Grab next transmit packet. */
776 		m = if_dequeue(ifp);
777 		if (m == NULL)
778 			break;
779 
780 		/* Create and load DMA map. */
781 		if (bus_dmamap_create(sc->mbuf_dma_tag, 0,
782 			&sc->txring_m_dmamap[sc->txring_hd_ptr])) {
783 			m_freem(m);
784 			sc->txdmamapfails++;
785 			continue;
786 		}
787 		err = bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
788 		    sc->txring_m_dmamap[sc->txring_hd_ptr], m, segs, &nsegs,
789 		    BUS_DMA_NOWAIT);
790 		if (err == EFBIG) {
791 			/* Too many segments!  defrag and try again. */
792 			struct mbuf *m2 = m_defrag(m, M_NOWAIT);
793 
794 			if (m2 == NULL) {
795 				sc->txdefragfails++;
796 				m_freem(m);
797 				bus_dmamap_destroy(sc->mbuf_dma_tag,
798 				    sc->txring_m_dmamap[sc->txring_hd_ptr]);
799 				sc->txring_m_dmamap[sc->txring_hd_ptr] = NULL;
800 				continue;
801 			}
802 			m = m2;
803 			err = bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
804 			    sc->txring_m_dmamap[sc->txring_hd_ptr], m, segs,
805 			    &nsegs, BUS_DMA_NOWAIT);
806 			sc->txdefrags++;
807 		}
808 		if (err) {
809 			/* Give up. */
810 			m_freem(m);
811 			bus_dmamap_destroy(sc->mbuf_dma_tag,
812 			    sc->txring_m_dmamap[sc->txring_hd_ptr]);
813 			sc->txring_m_dmamap[sc->txring_hd_ptr] = NULL;
814 			sc->txdmamapfails++;
815 			continue;
816 		}
817 		sc->txring_m[sc->txring_hd_ptr] = m;
818 
819 		/* Sync tx buffer with cache. */
820 		bus_dmamap_sync(sc->mbuf_dma_tag,
821 		    sc->txring_m_dmamap[sc->txring_hd_ptr],
822 		    BUS_DMASYNC_PREWRITE);
823 
824 		/* Set wrap flag if next packet might run off end of ring. */
825 		wrap = sc->txring_hd_ptr + nsegs + TX_MAX_DMA_SEGS >=
826 		    CGEM_NUM_TX_DESCS;
827 
828 		/*
829 		 * Fill in the TX descriptors back to front so that USED bit in
830 		 * first descriptor is cleared last.
831 		 */
832 		for (i = nsegs - 1; i >= 0; i--) {
833 			/* Descriptor address. */
834 			sc->txring[sc->txring_hd_ptr + i].addr =
835 			    segs[i].ds_addr;
836 #ifdef CGEM64
837 			sc->txring[sc->txring_hd_ptr + i].addrhi =
838 			    segs[i].ds_addr >> 32;
839 #endif
840 			/* Descriptor control word. */
841 			ctl = segs[i].ds_len;
842 			if (i == nsegs - 1) {
843 				ctl |= CGEM_TXDESC_LAST_BUF;
844 				if (wrap)
845 					ctl |= CGEM_TXDESC_WRAP;
846 			}
847 			sc->txring[sc->txring_hd_ptr + i].ctl = ctl;
848 
849 			if (i != 0)
850 				sc->txring_m[sc->txring_hd_ptr + i] = NULL;
851 		}
852 
853 		if (wrap)
854 			sc->txring_hd_ptr = 0;
855 		else
856 			sc->txring_hd_ptr += nsegs;
857 		sc->txring_queued += nsegs;
858 
859 		/* Kick the transmitter. */
860 		WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow |
861 		    CGEM_NET_CTRL_START_TX);
862 
863 		/* If there is a BPF listener, bounce a copy to him. */
864 		ETHER_BPF_MTAP(ifp, m);
865 	}
866 }
867 
868 static void
869 cgem_start(if_t ifp)
870 {
871 	struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
872 
873 	CGEM_LOCK(sc);
874 	cgem_start_locked(ifp);
875 	CGEM_UNLOCK(sc);
876 }
877 
878 static void
879 cgem_poll_hw_stats(struct cgem_softc *sc)
880 {
881 	uint32_t n;
882 
883 	CGEM_ASSERT_LOCKED(sc);
884 
885 	sc->stats.tx_bytes += RD4(sc, CGEM_OCTETS_TX_BOT);
886 	sc->stats.tx_bytes += (uint64_t)RD4(sc, CGEM_OCTETS_TX_TOP) << 32;
887 
888 	sc->stats.tx_frames += RD4(sc, CGEM_FRAMES_TX);
889 	sc->stats.tx_frames_bcast += RD4(sc, CGEM_BCAST_FRAMES_TX);
890 	sc->stats.tx_frames_multi += RD4(sc, CGEM_MULTI_FRAMES_TX);
891 	sc->stats.tx_frames_pause += RD4(sc, CGEM_PAUSE_FRAMES_TX);
892 	sc->stats.tx_frames_64b += RD4(sc, CGEM_FRAMES_64B_TX);
893 	sc->stats.tx_frames_65to127b += RD4(sc, CGEM_FRAMES_65_127B_TX);
894 	sc->stats.tx_frames_128to255b += RD4(sc, CGEM_FRAMES_128_255B_TX);
895 	sc->stats.tx_frames_256to511b += RD4(sc, CGEM_FRAMES_256_511B_TX);
896 	sc->stats.tx_frames_512to1023b += RD4(sc, CGEM_FRAMES_512_1023B_TX);
897 	sc->stats.tx_frames_1024to1536b += RD4(sc, CGEM_FRAMES_1024_1518B_TX);
898 	sc->stats.tx_under_runs += RD4(sc, CGEM_TX_UNDERRUNS);
899 
900 	n = RD4(sc, CGEM_SINGLE_COLL_FRAMES);
901 	sc->stats.tx_single_collisn += n;
902 	if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
903 	n = RD4(sc, CGEM_MULTI_COLL_FRAMES);
904 	sc->stats.tx_multi_collisn += n;
905 	if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
906 	n = RD4(sc, CGEM_EXCESSIVE_COLL_FRAMES);
907 	sc->stats.tx_excsv_collisn += n;
908 	if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
909 	n = RD4(sc, CGEM_LATE_COLL);
910 	sc->stats.tx_late_collisn += n;
911 	if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
912 
913 	sc->stats.tx_deferred_frames += RD4(sc, CGEM_DEFERRED_TX_FRAMES);
914 	sc->stats.tx_carrier_sense_errs += RD4(sc, CGEM_CARRIER_SENSE_ERRS);
915 
916 	sc->stats.rx_bytes += RD4(sc, CGEM_OCTETS_RX_BOT);
917 	sc->stats.rx_bytes += (uint64_t)RD4(sc, CGEM_OCTETS_RX_TOP) << 32;
918 
919 	sc->stats.rx_frames += RD4(sc, CGEM_FRAMES_RX);
920 	sc->stats.rx_frames_bcast += RD4(sc, CGEM_BCAST_FRAMES_RX);
921 	sc->stats.rx_frames_multi += RD4(sc, CGEM_MULTI_FRAMES_RX);
922 	sc->stats.rx_frames_pause += RD4(sc, CGEM_PAUSE_FRAMES_RX);
923 	sc->stats.rx_frames_64b += RD4(sc, CGEM_FRAMES_64B_RX);
924 	sc->stats.rx_frames_65to127b += RD4(sc, CGEM_FRAMES_65_127B_RX);
925 	sc->stats.rx_frames_128to255b += RD4(sc, CGEM_FRAMES_128_255B_RX);
926 	sc->stats.rx_frames_256to511b += RD4(sc, CGEM_FRAMES_256_511B_RX);
927 	sc->stats.rx_frames_512to1023b += RD4(sc, CGEM_FRAMES_512_1023B_RX);
928 	sc->stats.rx_frames_1024to1536b += RD4(sc, CGEM_FRAMES_1024_1518B_RX);
929 	sc->stats.rx_frames_undersize += RD4(sc, CGEM_UNDERSZ_RX);
930 	sc->stats.rx_frames_oversize += RD4(sc, CGEM_OVERSZ_RX);
931 	sc->stats.rx_frames_jabber += RD4(sc, CGEM_JABBERS_RX);
932 	sc->stats.rx_frames_fcs_errs += RD4(sc, CGEM_FCS_ERRS);
933 	sc->stats.rx_frames_length_errs += RD4(sc, CGEM_LENGTH_FIELD_ERRS);
934 	sc->stats.rx_symbol_errs += RD4(sc, CGEM_RX_SYMBOL_ERRS);
935 	sc->stats.rx_align_errs += RD4(sc, CGEM_ALIGN_ERRS);
936 	sc->stats.rx_resource_errs += RD4(sc, CGEM_RX_RESOURCE_ERRS);
937 	sc->stats.rx_overrun_errs += RD4(sc, CGEM_RX_OVERRUN_ERRS);
938 	sc->stats.rx_ip_hdr_csum_errs += RD4(sc, CGEM_IP_HDR_CKSUM_ERRS);
939 	sc->stats.rx_tcp_csum_errs += RD4(sc, CGEM_TCP_CKSUM_ERRS);
940 	sc->stats.rx_udp_csum_errs += RD4(sc, CGEM_UDP_CKSUM_ERRS);
941 }
942 
943 static void
944 cgem_tick(void *arg)
945 {
946 	struct cgem_softc *sc = (struct cgem_softc *)arg;
947 	struct mii_data *mii;
948 
949 	CGEM_ASSERT_LOCKED(sc);
950 
951 	/* Poll the phy. */
952 	if (sc->miibus != NULL) {
953 		mii = device_get_softc(sc->miibus);
954 		mii_tick(mii);
955 	}
956 
957 	/* Poll statistics registers. */
958 	cgem_poll_hw_stats(sc);
959 
960 	/* Check for receiver hang. */
961 	if (sc->rxhangwar && sc->rx_frames_prev == sc->stats.rx_frames) {
962 		/*
963 		 * Reset receiver logic by toggling RX_EN bit.  1usec
964 		 * delay is necessary especially when operating at 100mbps
965 		 * and 10mbps speeds.
966 		 */
967 		WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow &
968 		    ~CGEM_NET_CTRL_RX_EN);
969 		DELAY(1);
970 		WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
971 	}
972 	sc->rx_frames_prev = sc->stats.rx_frames;
973 
974 	/* Next callout in one second. */
975 	callout_reset(&sc->tick_ch, hz, cgem_tick, sc);
976 }
977 
978 /* Interrupt handler. */
979 static void
980 cgem_intr(void *arg)
981 {
982 	struct cgem_softc *sc = (struct cgem_softc *)arg;
983 	if_t ifp = sc->ifp;
984 	uint32_t istatus;
985 
986 	CGEM_LOCK(sc);
987 
988 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
989 		CGEM_UNLOCK(sc);
990 		return;
991 	}
992 
993 	/* Read interrupt status and immediately clear the bits. */
994 	istatus = RD4(sc, CGEM_INTR_STAT);
995 	WR4(sc, CGEM_INTR_STAT, istatus);
996 
997 	/* Packets received. */
998 	if ((istatus & CGEM_INTR_RX_COMPLETE) != 0)
999 		cgem_recv(sc);
1000 
1001 	/* Free up any completed transmit buffers. */
1002 	cgem_clean_tx(sc);
1003 
1004 	/* Hresp not ok.  Something is very bad with DMA.  Try to clear. */
1005 	if ((istatus & CGEM_INTR_HRESP_NOT_OK) != 0) {
1006 		device_printf(sc->dev,
1007 		    "cgem_intr: hresp not okay! rx_status=0x%x\n",
1008 		    RD4(sc, CGEM_RX_STAT));
1009 		WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_HRESP_NOT_OK);
1010 	}
1011 
1012 	/* Receiver overrun. */
1013 	if ((istatus & CGEM_INTR_RX_OVERRUN) != 0) {
1014 		/* Clear status bit. */
1015 		WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_OVERRUN);
1016 		sc->rxoverruns++;
1017 	}
1018 
1019 	/* Receiver ran out of bufs. */
1020 	if ((istatus & CGEM_INTR_RX_USED_READ) != 0) {
1021 		WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow |
1022 		    CGEM_NET_CTRL_FLUSH_DPRAM_PKT);
1023 		cgem_fill_rqueue(sc);
1024 		sc->rxnobufs++;
1025 	}
1026 
1027 	/* Restart transmitter if needed. */
1028 	if (!if_sendq_empty(ifp))
1029 		cgem_start_locked(ifp);
1030 
1031 	CGEM_UNLOCK(sc);
1032 }
1033 
1034 /* Reset hardware. */
1035 static void
1036 cgem_reset(struct cgem_softc *sc)
1037 {
1038 
1039 	CGEM_ASSERT_LOCKED(sc);
1040 
1041 	/* Determine data bus width from design configuration register. */
1042 	switch (RD4(sc, CGEM_DESIGN_CFG1) &
1043 	    CGEM_DESIGN_CFG1_DMA_BUS_WIDTH_MASK) {
1044 	case CGEM_DESIGN_CFG1_DMA_BUS_WIDTH_64:
1045 		sc->net_cfg_shadow = CGEM_NET_CFG_DBUS_WIDTH_64;
1046 		break;
1047 	case CGEM_DESIGN_CFG1_DMA_BUS_WIDTH_128:
1048 		sc->net_cfg_shadow = CGEM_NET_CFG_DBUS_WIDTH_128;
1049 		break;
1050 	default:
1051 		sc->net_cfg_shadow = CGEM_NET_CFG_DBUS_WIDTH_32;
1052 	}
1053 
1054 	WR4(sc, CGEM_NET_CTRL, 0);
1055 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1056 	WR4(sc, CGEM_NET_CTRL, CGEM_NET_CTRL_CLR_STAT_REGS);
1057 	WR4(sc, CGEM_TX_STAT, CGEM_TX_STAT_ALL);
1058 	WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_ALL);
1059 	WR4(sc, CGEM_INTR_DIS, CGEM_INTR_ALL);
1060 	WR4(sc, CGEM_HASH_BOT, 0);
1061 	WR4(sc, CGEM_HASH_TOP, 0);
1062 	WR4(sc, CGEM_TX_QBAR, 0);	/* manual says do this. */
1063 	WR4(sc, CGEM_RX_QBAR, 0);
1064 
1065 	/* Get management port running even if interface is down. */
1066 	sc->net_cfg_shadow |= CGEM_NET_CFG_MDC_CLK_DIV_48;
1067 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1068 
1069 	sc->net_ctl_shadow = CGEM_NET_CTRL_MGMT_PORT_EN;
1070 	WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
1071 }
1072 
1073 /* Bring up the hardware. */
1074 static void
1075 cgem_config(struct cgem_softc *sc)
1076 {
1077 	if_t ifp = sc->ifp;
1078 	uint32_t dma_cfg;
1079 	u_char *eaddr = if_getlladdr(ifp);
1080 
1081 	CGEM_ASSERT_LOCKED(sc);
1082 
1083 	/* Program Net Config Register. */
1084 	sc->net_cfg_shadow &= (CGEM_NET_CFG_MDC_CLK_DIV_MASK |
1085 	    CGEM_NET_CFG_DBUS_WIDTH_MASK);
1086 	sc->net_cfg_shadow |= (CGEM_NET_CFG_FCS_REMOVE |
1087 	    CGEM_NET_CFG_RX_BUF_OFFSET(ETHER_ALIGN) |
1088 	    CGEM_NET_CFG_GIGE_EN | CGEM_NET_CFG_1536RXEN |
1089 	    CGEM_NET_CFG_FULL_DUPLEX | CGEM_NET_CFG_SPEED100);
1090 
1091 	/* Check connection type, enable SGMII bits if necessary. */
1092 	if (sc->phy_contype == MII_CONTYPE_SGMII) {
1093 		sc->net_cfg_shadow |= CGEM_NET_CFG_SGMII_EN;
1094 		sc->net_cfg_shadow |= CGEM_NET_CFG_PCS_SEL;
1095 	}
1096 
1097 	/* Enable receive checksum offloading? */
1098 	if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0)
1099 		sc->net_cfg_shadow |=  CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN;
1100 
1101 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1102 
1103 	/* Program DMA Config Register. */
1104 	dma_cfg = CGEM_DMA_CFG_RX_BUF_SIZE(MCLBYTES) |
1105 	    CGEM_DMA_CFG_RX_PKTBUF_MEMSZ_SEL_8K |
1106 	    CGEM_DMA_CFG_TX_PKTBUF_MEMSZ_SEL |
1107 	    CGEM_DMA_CFG_AHB_FIXED_BURST_LEN_16 |
1108 #ifdef CGEM64
1109 	    CGEM_DMA_CFG_ADDR_BUS_64 |
1110 #endif
1111 	    CGEM_DMA_CFG_DISC_WHEN_NO_AHB;
1112 
1113 	/* Enable transmit checksum offloading? */
1114 	if ((if_getcapenable(ifp) & IFCAP_TXCSUM) != 0)
1115 		dma_cfg |= CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN;
1116 
1117 	WR4(sc, CGEM_DMA_CFG, dma_cfg);
1118 
1119 	/* Write the rx and tx descriptor ring addresses to the QBAR regs. */
1120 	WR4(sc, CGEM_RX_QBAR, (uint32_t)sc->rxring_physaddr);
1121 	WR4(sc, CGEM_TX_QBAR, (uint32_t)sc->txring_physaddr);
1122 #ifdef CGEM64
1123 	WR4(sc, CGEM_RX_QBAR_HI, (uint32_t)(sc->rxring_physaddr >> 32));
1124 	WR4(sc, CGEM_TX_QBAR_HI, (uint32_t)(sc->txring_physaddr >> 32));
1125 #endif
1126 
1127 	/* Enable rx and tx. */
1128 	sc->net_ctl_shadow |= (CGEM_NET_CTRL_TX_EN | CGEM_NET_CTRL_RX_EN);
1129 	WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
1130 
1131 	/* Set receive address in case it changed. */
1132 	WR4(sc, CGEM_SPEC_ADDR_LOW(0), (eaddr[3] << 24) |
1133 	    (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]);
1134 	WR4(sc, CGEM_SPEC_ADDR_HI(0), (eaddr[5] << 8) | eaddr[4]);
1135 
1136 	/* Set up interrupts. */
1137 	WR4(sc, CGEM_INTR_EN, CGEM_INTR_RX_COMPLETE | CGEM_INTR_RX_OVERRUN |
1138 	    CGEM_INTR_TX_USED_READ | CGEM_INTR_RX_USED_READ |
1139 	    CGEM_INTR_HRESP_NOT_OK);
1140 }
1141 
1142 /* Turn on interface and load up receive ring with buffers. */
1143 static void
1144 cgem_init_locked(struct cgem_softc *sc)
1145 {
1146 	struct mii_data *mii;
1147 
1148 	CGEM_ASSERT_LOCKED(sc);
1149 
1150 	if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) != 0)
1151 		return;
1152 
1153 	cgem_config(sc);
1154 	cgem_fill_rqueue(sc);
1155 
1156 	if_setdrvflagbits(sc->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1157 
1158 	if (sc->miibus != NULL) {
1159 		mii = device_get_softc(sc->miibus);
1160 		mii_mediachg(mii);
1161 	}
1162 
1163 	callout_reset(&sc->tick_ch, hz, cgem_tick, sc);
1164 }
1165 
1166 static void
1167 cgem_init(void *arg)
1168 {
1169 	struct cgem_softc *sc = (struct cgem_softc *)arg;
1170 
1171 	CGEM_LOCK(sc);
1172 	cgem_init_locked(sc);
1173 	CGEM_UNLOCK(sc);
1174 }
1175 
1176 /* Turn off interface.  Free up any buffers in transmit or receive queues. */
1177 static void
1178 cgem_stop(struct cgem_softc *sc)
1179 {
1180 	int i;
1181 
1182 	CGEM_ASSERT_LOCKED(sc);
1183 
1184 	callout_stop(&sc->tick_ch);
1185 
1186 	/* Shut down hardware. */
1187 	cgem_reset(sc);
1188 
1189 	/* Clear out transmit queue. */
1190 	memset(sc->txring, 0, CGEM_NUM_TX_DESCS * sizeof(struct cgem_tx_desc));
1191 	for (i = 0; i < CGEM_NUM_TX_DESCS; i++) {
1192 		sc->txring[i].ctl = CGEM_TXDESC_USED;
1193 		if (sc->txring_m[i]) {
1194 			/* Unload and destroy dmamap. */
1195 			bus_dmamap_unload(sc->mbuf_dma_tag,
1196 			    sc->txring_m_dmamap[i]);
1197 			bus_dmamap_destroy(sc->mbuf_dma_tag,
1198 			    sc->txring_m_dmamap[i]);
1199 			sc->txring_m_dmamap[i] = NULL;
1200 			m_freem(sc->txring_m[i]);
1201 			sc->txring_m[i] = NULL;
1202 		}
1203 	}
1204 	sc->txring[CGEM_NUM_TX_DESCS - 1].ctl |= CGEM_TXDESC_WRAP;
1205 
1206 	sc->txring_hd_ptr = 0;
1207 	sc->txring_tl_ptr = 0;
1208 	sc->txring_queued = 0;
1209 
1210 	/* Clear out receive queue. */
1211 	memset(sc->rxring, 0, CGEM_NUM_RX_DESCS * sizeof(struct cgem_rx_desc));
1212 	for (i = 0; i < CGEM_NUM_RX_DESCS; i++) {
1213 		sc->rxring[i].addr = CGEM_RXDESC_OWN;
1214 		if (sc->rxring_m[i]) {
1215 			/* Unload and destroy dmamap. */
1216 			bus_dmamap_unload(sc->mbuf_dma_tag,
1217 			    sc->rxring_m_dmamap[i]);
1218 			bus_dmamap_destroy(sc->mbuf_dma_tag,
1219 			    sc->rxring_m_dmamap[i]);
1220 			sc->rxring_m_dmamap[i] = NULL;
1221 
1222 			m_freem(sc->rxring_m[i]);
1223 			sc->rxring_m[i] = NULL;
1224 		}
1225 	}
1226 	sc->rxring[CGEM_NUM_RX_DESCS - 1].addr |= CGEM_RXDESC_WRAP;
1227 
1228 	sc->rxring_hd_ptr = 0;
1229 	sc->rxring_tl_ptr = 0;
1230 	sc->rxring_queued = 0;
1231 
1232 	/* Force next statchg or linkchg to program net config register. */
1233 	sc->mii_media_active = 0;
1234 }
1235 
1236 static int
1237 cgem_ioctl(if_t ifp, u_long cmd, caddr_t data)
1238 {
1239 	struct cgem_softc *sc = if_getsoftc(ifp);
1240 	struct ifreq *ifr = (struct ifreq *)data;
1241 	struct mii_data *mii;
1242 	int error = 0, mask;
1243 
1244 	switch (cmd) {
1245 	case SIOCSIFFLAGS:
1246 		CGEM_LOCK(sc);
1247 		if ((if_getflags(ifp) & IFF_UP) != 0) {
1248 			if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1249 				if (((if_getflags(ifp) ^ sc->if_old_flags) &
1250 				    (IFF_PROMISC | IFF_ALLMULTI)) != 0) {
1251 					cgem_rx_filter(sc);
1252 				}
1253 			} else {
1254 				cgem_init_locked(sc);
1255 			}
1256 		} else if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1257 			if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1258 			cgem_stop(sc);
1259 		}
1260 		sc->if_old_flags = if_getflags(ifp);
1261 		CGEM_UNLOCK(sc);
1262 		break;
1263 
1264 	case SIOCADDMULTI:
1265 	case SIOCDELMULTI:
1266 		/* Set up multi-cast filters. */
1267 		if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1268 			CGEM_LOCK(sc);
1269 			cgem_rx_filter(sc);
1270 			CGEM_UNLOCK(sc);
1271 		}
1272 		break;
1273 
1274 	case SIOCSIFMEDIA:
1275 	case SIOCGIFMEDIA:
1276 		if (sc->miibus == NULL)
1277 			return (ENXIO);
1278 		mii = device_get_softc(sc->miibus);
1279 		error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
1280 		break;
1281 
1282 	case SIOCSIFCAP:
1283 		CGEM_LOCK(sc);
1284 		mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap;
1285 
1286 		if ((mask & IFCAP_TXCSUM) != 0) {
1287 			if ((ifr->ifr_reqcap & IFCAP_TXCSUM) != 0) {
1288 				/* Turn on TX checksumming. */
1289 				if_setcapenablebit(ifp, IFCAP_TXCSUM |
1290 				    IFCAP_TXCSUM_IPV6, 0);
1291 				if_sethwassistbits(ifp, CGEM_CKSUM_ASSIST, 0);
1292 
1293 				WR4(sc, CGEM_DMA_CFG,
1294 				    RD4(sc, CGEM_DMA_CFG) |
1295 				    CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN);
1296 			} else {
1297 				/* Turn off TX checksumming. */
1298 				if_setcapenablebit(ifp, 0, IFCAP_TXCSUM |
1299 				    IFCAP_TXCSUM_IPV6);
1300 				if_sethwassistbits(ifp, 0, CGEM_CKSUM_ASSIST);
1301 
1302 				WR4(sc, CGEM_DMA_CFG,
1303 				    RD4(sc, CGEM_DMA_CFG) &
1304 				    ~CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN);
1305 			}
1306 		}
1307 		if ((mask & IFCAP_RXCSUM) != 0) {
1308 			if ((ifr->ifr_reqcap & IFCAP_RXCSUM) != 0) {
1309 				/* Turn on RX checksumming. */
1310 				if_setcapenablebit(ifp, IFCAP_RXCSUM |
1311 				    IFCAP_RXCSUM_IPV6, 0);
1312 				sc->net_cfg_shadow |=
1313 				    CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN;
1314 				WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1315 			} else {
1316 				/* Turn off RX checksumming. */
1317 				if_setcapenablebit(ifp, 0, IFCAP_RXCSUM |
1318 				    IFCAP_RXCSUM_IPV6);
1319 				sc->net_cfg_shadow &=
1320 				    ~CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN;
1321 				WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1322 			}
1323 		}
1324 		if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_TXCSUM)) ==
1325 		    (IFCAP_RXCSUM | IFCAP_TXCSUM))
1326 			if_setcapenablebit(ifp, IFCAP_VLAN_HWCSUM, 0);
1327 		else
1328 			if_setcapenablebit(ifp, 0, IFCAP_VLAN_HWCSUM);
1329 
1330 		CGEM_UNLOCK(sc);
1331 		break;
1332 	default:
1333 		error = ether_ioctl(ifp, cmd, data);
1334 		break;
1335 	}
1336 
1337 	return (error);
1338 }
1339 
1340 /* MII bus support routines.
1341  */
1342 static int
1343 cgem_ifmedia_upd(if_t ifp)
1344 {
1345 	struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
1346 	struct mii_data *mii;
1347 	struct mii_softc *miisc;
1348 	int error = 0;
1349 
1350 	mii = device_get_softc(sc->miibus);
1351 	CGEM_LOCK(sc);
1352 	if ((if_getflags(ifp) & IFF_UP) != 0) {
1353 		LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
1354 			PHY_RESET(miisc);
1355 		error = mii_mediachg(mii);
1356 	}
1357 	CGEM_UNLOCK(sc);
1358 
1359 	return (error);
1360 }
1361 
1362 static void
1363 cgem_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr)
1364 {
1365 	struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
1366 	struct mii_data *mii;
1367 
1368 	mii = device_get_softc(sc->miibus);
1369 	CGEM_LOCK(sc);
1370 	mii_pollstat(mii);
1371 	ifmr->ifm_active = mii->mii_media_active;
1372 	ifmr->ifm_status = mii->mii_media_status;
1373 	CGEM_UNLOCK(sc);
1374 }
1375 
1376 static int
1377 cgem_miibus_readreg(device_t dev, int phy, int reg)
1378 {
1379 	struct cgem_softc *sc = device_get_softc(dev);
1380 	int tries, val;
1381 
1382 	WR4(sc, CGEM_PHY_MAINT, CGEM_PHY_MAINT_CLAUSE_22 |
1383 	    CGEM_PHY_MAINT_MUST_10 | CGEM_PHY_MAINT_OP_READ |
1384 	    (phy << CGEM_PHY_MAINT_PHY_ADDR_SHIFT) |
1385 	    (reg << CGEM_PHY_MAINT_REG_ADDR_SHIFT));
1386 
1387 	/* Wait for completion. */
1388 	tries=0;
1389 	while ((RD4(sc, CGEM_NET_STAT) & CGEM_NET_STAT_PHY_MGMT_IDLE) == 0) {
1390 		DELAY(5);
1391 		if (++tries > 200) {
1392 			device_printf(dev, "phy read timeout: %d\n", reg);
1393 			return (-1);
1394 		}
1395 	}
1396 
1397 	val = RD4(sc, CGEM_PHY_MAINT) & CGEM_PHY_MAINT_DATA_MASK;
1398 
1399 	if (reg == MII_EXTSR)
1400 		/*
1401 		 * MAC does not support half-duplex at gig speeds.
1402 		 * Let mii(4) exclude the capability.
1403 		 */
1404 		val &= ~(EXTSR_1000XHDX | EXTSR_1000THDX);
1405 
1406 	return (val);
1407 }
1408 
1409 static int
1410 cgem_miibus_writereg(device_t dev, int phy, int reg, int data)
1411 {
1412 	struct cgem_softc *sc = device_get_softc(dev);
1413 	int tries;
1414 
1415 	WR4(sc, CGEM_PHY_MAINT, CGEM_PHY_MAINT_CLAUSE_22 |
1416 	    CGEM_PHY_MAINT_MUST_10 | CGEM_PHY_MAINT_OP_WRITE |
1417 	    (phy << CGEM_PHY_MAINT_PHY_ADDR_SHIFT) |
1418 	    (reg << CGEM_PHY_MAINT_REG_ADDR_SHIFT) |
1419 	    (data & CGEM_PHY_MAINT_DATA_MASK));
1420 
1421 	/* Wait for completion. */
1422 	tries = 0;
1423 	while ((RD4(sc, CGEM_NET_STAT) & CGEM_NET_STAT_PHY_MGMT_IDLE) == 0) {
1424 		DELAY(5);
1425 		if (++tries > 200) {
1426 			device_printf(dev, "phy write timeout: %d\n", reg);
1427 			return (-1);
1428 		}
1429 	}
1430 
1431 	return (0);
1432 }
1433 
1434 static void
1435 cgem_miibus_statchg(device_t dev)
1436 {
1437 	struct cgem_softc *sc  = device_get_softc(dev);
1438 	struct mii_data *mii = device_get_softc(sc->miibus);
1439 
1440 	CGEM_ASSERT_LOCKED(sc);
1441 
1442 	if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1443 	    (IFM_ACTIVE | IFM_AVALID) &&
1444 	    sc->mii_media_active != mii->mii_media_active)
1445 		cgem_mediachange(sc, mii);
1446 }
1447 
1448 static void
1449 cgem_miibus_linkchg(device_t dev)
1450 {
1451 	struct cgem_softc *sc  = device_get_softc(dev);
1452 	struct mii_data *mii = device_get_softc(sc->miibus);
1453 
1454 	CGEM_ASSERT_LOCKED(sc);
1455 
1456 	if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1457 	    (IFM_ACTIVE | IFM_AVALID) &&
1458 	    sc->mii_media_active != mii->mii_media_active)
1459 		cgem_mediachange(sc, mii);
1460 }
1461 
1462 /*
1463  * Overridable weak symbol cgem_set_ref_clk().  This allows platforms to
1464  * provide a function to set the cgem's reference clock.
1465  */
1466 static int __used
1467 cgem_default_set_ref_clk(int unit, int frequency)
1468 {
1469 
1470 	return 0;
1471 }
1472 __weak_reference(cgem_default_set_ref_clk, cgem_set_ref_clk);
1473 
1474 /* Call to set reference clock and network config bits according to media. */
1475 static void
1476 cgem_mediachange(struct cgem_softc *sc,	struct mii_data *mii)
1477 {
1478 	int ref_clk_freq;
1479 
1480 	CGEM_ASSERT_LOCKED(sc);
1481 
1482 	/* Update hardware to reflect media. */
1483 	sc->net_cfg_shadow &= ~(CGEM_NET_CFG_SPEED100 | CGEM_NET_CFG_GIGE_EN |
1484 	    CGEM_NET_CFG_FULL_DUPLEX);
1485 
1486 	switch (IFM_SUBTYPE(mii->mii_media_active)) {
1487 	case IFM_1000_T:
1488 		sc->net_cfg_shadow |= (CGEM_NET_CFG_SPEED100 |
1489 		    CGEM_NET_CFG_GIGE_EN);
1490 		ref_clk_freq = 125000000;
1491 		break;
1492 	case IFM_100_TX:
1493 		sc->net_cfg_shadow |= CGEM_NET_CFG_SPEED100;
1494 		ref_clk_freq = 25000000;
1495 		break;
1496 	default:
1497 		ref_clk_freq = 2500000;
1498 	}
1499 
1500 	if ((mii->mii_media_active & IFM_FDX) != 0)
1501 		sc->net_cfg_shadow |= CGEM_NET_CFG_FULL_DUPLEX;
1502 
1503 	WR4(sc, CGEM_NET_CFG, sc->net_cfg_shadow);
1504 
1505 	if (sc->ref_clk != NULL) {
1506 		CGEM_UNLOCK(sc);
1507 		if (clk_set_freq(sc->ref_clk, ref_clk_freq, 0))
1508 			device_printf(sc->dev, "could not set ref clk to %d\n",
1509 			    ref_clk_freq);
1510 		CGEM_LOCK(sc);
1511 	}
1512 
1513 	sc->mii_media_active = mii->mii_media_active;
1514 }
1515 
1516 static void
1517 cgem_add_sysctls(device_t dev)
1518 {
1519 	struct cgem_softc *sc = device_get_softc(dev);
1520 	struct sysctl_ctx_list *ctx;
1521 	struct sysctl_oid_list *child;
1522 	struct sysctl_oid *tree;
1523 
1524 	ctx = device_get_sysctl_ctx(dev);
1525 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
1526 
1527 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rxbufs", CTLFLAG_RW,
1528 	    &sc->rxbufs, 0, "Number receive buffers to provide");
1529 
1530 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rxhangwar", CTLFLAG_RW,
1531 	    &sc->rxhangwar, 0, "Enable receive hang work-around");
1532 
1533 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxoverruns", CTLFLAG_RD,
1534 	    &sc->rxoverruns, 0, "Receive overrun events");
1535 
1536 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxnobufs", CTLFLAG_RD,
1537 	    &sc->rxnobufs, 0, "Receive buf queue empty events");
1538 
1539 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxdmamapfails", CTLFLAG_RD,
1540 	    &sc->rxdmamapfails, 0, "Receive DMA map failures");
1541 
1542 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txfull", CTLFLAG_RD,
1543 	    &sc->txfull, 0, "Transmit ring full events");
1544 
1545 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdmamapfails", CTLFLAG_RD,
1546 	    &sc->txdmamapfails, 0, "Transmit DMA map failures");
1547 
1548 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdefrags", CTLFLAG_RD,
1549 	    &sc->txdefrags, 0, "Transmit m_defrag() calls");
1550 
1551 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdefragfails", CTLFLAG_RD,
1552 	    &sc->txdefragfails, 0, "Transmit m_defrag() failures");
1553 
1554 	tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "stats",
1555 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "GEM statistics");
1556 	child = SYSCTL_CHILDREN(tree);
1557 
1558 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_bytes", CTLFLAG_RD,
1559 	    &sc->stats.tx_bytes, "Total bytes transmitted");
1560 
1561 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames", CTLFLAG_RD,
1562 	    &sc->stats.tx_frames, 0, "Total frames transmitted");
1563 
1564 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_bcast", CTLFLAG_RD,
1565 	    &sc->stats.tx_frames_bcast, 0,
1566 	    "Number broadcast frames transmitted");
1567 
1568 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_multi", CTLFLAG_RD,
1569 	    &sc->stats.tx_frames_multi, 0,
1570 	    "Number multicast frames transmitted");
1571 
1572 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_pause",
1573 	    CTLFLAG_RD, &sc->stats.tx_frames_pause, 0,
1574 	    "Number pause frames transmitted");
1575 
1576 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_64b", CTLFLAG_RD,
1577 	    &sc->stats.tx_frames_64b, 0,
1578 	    "Number frames transmitted of size 64 bytes or less");
1579 
1580 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_65to127b", CTLFLAG_RD,
1581 	    &sc->stats.tx_frames_65to127b, 0,
1582 	    "Number frames transmitted of size 65-127 bytes");
1583 
1584 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_128to255b",
1585 	    CTLFLAG_RD, &sc->stats.tx_frames_128to255b, 0,
1586 	    "Number frames transmitted of size 128-255 bytes");
1587 
1588 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_256to511b",
1589 	    CTLFLAG_RD, &sc->stats.tx_frames_256to511b, 0,
1590 	    "Number frames transmitted of size 256-511 bytes");
1591 
1592 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_512to1023b",
1593 	    CTLFLAG_RD, &sc->stats.tx_frames_512to1023b, 0,
1594 	    "Number frames transmitted of size 512-1023 bytes");
1595 
1596 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_1024to1536b",
1597 	    CTLFLAG_RD, &sc->stats.tx_frames_1024to1536b, 0,
1598 	    "Number frames transmitted of size 1024-1536 bytes");
1599 
1600 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_under_runs",
1601 	    CTLFLAG_RD, &sc->stats.tx_under_runs, 0,
1602 	    "Number transmit under-run events");
1603 
1604 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_single_collisn",
1605 	    CTLFLAG_RD, &sc->stats.tx_single_collisn, 0,
1606 	    "Number single-collision transmit frames");
1607 
1608 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_multi_collisn",
1609 	    CTLFLAG_RD, &sc->stats.tx_multi_collisn, 0,
1610 	    "Number multi-collision transmit frames");
1611 
1612 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_excsv_collisn",
1613 	    CTLFLAG_RD, &sc->stats.tx_excsv_collisn, 0,
1614 	    "Number excessive collision transmit frames");
1615 
1616 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_late_collisn",
1617 	    CTLFLAG_RD, &sc->stats.tx_late_collisn, 0,
1618 	    "Number late-collision transmit frames");
1619 
1620 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_deferred_frames",
1621 	    CTLFLAG_RD, &sc->stats.tx_deferred_frames, 0,
1622 	    "Number deferred transmit frames");
1623 
1624 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_carrier_sense_errs",
1625 	    CTLFLAG_RD, &sc->stats.tx_carrier_sense_errs, 0,
1626 	    "Number carrier sense errors on transmit");
1627 
1628 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_bytes", CTLFLAG_RD,
1629 	    &sc->stats.rx_bytes, "Total bytes received");
1630 
1631 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames", CTLFLAG_RD,
1632 	    &sc->stats.rx_frames, 0, "Total frames received");
1633 
1634 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_bcast",
1635 	    CTLFLAG_RD, &sc->stats.rx_frames_bcast, 0,
1636 	    "Number broadcast frames received");
1637 
1638 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_multi",
1639 	    CTLFLAG_RD, &sc->stats.rx_frames_multi, 0,
1640 	    "Number multicast frames received");
1641 
1642 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_pause",
1643 	    CTLFLAG_RD, &sc->stats.rx_frames_pause, 0,
1644 	    "Number pause frames received");
1645 
1646 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_64b",
1647 	    CTLFLAG_RD, &sc->stats.rx_frames_64b, 0,
1648 	    "Number frames received of size 64 bytes or less");
1649 
1650 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_65to127b",
1651 	    CTLFLAG_RD, &sc->stats.rx_frames_65to127b, 0,
1652 	    "Number frames received of size 65-127 bytes");
1653 
1654 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_128to255b",
1655 	    CTLFLAG_RD, &sc->stats.rx_frames_128to255b, 0,
1656 	    "Number frames received of size 128-255 bytes");
1657 
1658 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_256to511b",
1659 	    CTLFLAG_RD, &sc->stats.rx_frames_256to511b, 0,
1660 	    "Number frames received of size 256-511 bytes");
1661 
1662 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_512to1023b",
1663 	    CTLFLAG_RD, &sc->stats.rx_frames_512to1023b, 0,
1664 	    "Number frames received of size 512-1023 bytes");
1665 
1666 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_1024to1536b",
1667 	    CTLFLAG_RD, &sc->stats.rx_frames_1024to1536b, 0,
1668 	    "Number frames received of size 1024-1536 bytes");
1669 
1670 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_undersize",
1671 	    CTLFLAG_RD, &sc->stats.rx_frames_undersize, 0,
1672 	    "Number undersize frames received");
1673 
1674 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_oversize",
1675 	    CTLFLAG_RD, &sc->stats.rx_frames_oversize, 0,
1676 	    "Number oversize frames received");
1677 
1678 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_jabber",
1679 	    CTLFLAG_RD, &sc->stats.rx_frames_jabber, 0,
1680 	    "Number jabber frames received");
1681 
1682 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_fcs_errs",
1683 	    CTLFLAG_RD, &sc->stats.rx_frames_fcs_errs, 0,
1684 	    "Number frames received with FCS errors");
1685 
1686 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_length_errs",
1687 	    CTLFLAG_RD, &sc->stats.rx_frames_length_errs, 0,
1688 	    "Number frames received with length errors");
1689 
1690 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_symbol_errs",
1691 	    CTLFLAG_RD, &sc->stats.rx_symbol_errs, 0,
1692 	    "Number receive symbol errors");
1693 
1694 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_align_errs",
1695 	    CTLFLAG_RD, &sc->stats.rx_align_errs, 0,
1696 	    "Number receive alignment errors");
1697 
1698 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_resource_errs",
1699 	    CTLFLAG_RD, &sc->stats.rx_resource_errs, 0,
1700 	    "Number frames received when no rx buffer available");
1701 
1702 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_overrun_errs",
1703 	    CTLFLAG_RD, &sc->stats.rx_overrun_errs, 0,
1704 	    "Number frames received but not copied due to receive overrun");
1705 
1706 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_ip_hdr_csum_errs",
1707 	    CTLFLAG_RD, &sc->stats.rx_ip_hdr_csum_errs, 0,
1708 	    "Number frames received with IP header checksum errors");
1709 
1710 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_tcp_csum_errs",
1711 	    CTLFLAG_RD, &sc->stats.rx_tcp_csum_errs, 0,
1712 	    "Number frames received with TCP checksum errors");
1713 
1714 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_udp_csum_errs",
1715 	    CTLFLAG_RD, &sc->stats.rx_udp_csum_errs, 0,
1716 	    "Number frames received with UDP checksum errors");
1717 }
1718 
1719 static int
1720 cgem_probe(device_t dev)
1721 {
1722 
1723 	if (!ofw_bus_status_okay(dev))
1724 		return (ENXIO);
1725 
1726 	if (ofw_bus_search_compatible(dev, compat_data)->ocd_str == NULL)
1727 		return (ENXIO);
1728 
1729 	device_set_desc(dev, "Cadence CGEM Gigabit Ethernet Interface");
1730 	return (0);
1731 }
1732 
1733 static int
1734 cgem_attach(device_t dev)
1735 {
1736 	struct cgem_softc *sc = device_get_softc(dev);
1737 	if_t ifp = NULL;
1738 	int rid, err;
1739 	u_char eaddr[ETHER_ADDR_LEN];
1740 	int hwquirks;
1741 	phandle_t node;
1742 
1743 	sc->dev = dev;
1744 	CGEM_LOCK_INIT(sc);
1745 
1746 	/* Key off of compatible string and set hardware-specific options. */
1747 	hwquirks = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
1748 	if ((hwquirks & HWQUIRK_NEEDNULLQS) != 0)
1749 		sc->neednullqs = 1;
1750 	if ((hwquirks & HWQUIRK_RXHANGWAR) != 0)
1751 		sc->rxhangwar = 1;
1752 	if ((hwquirks & HWQUIRK_TXCLK) != 0) {
1753 		if (clk_get_by_ofw_name(dev, 0, "tx_clk", &sc->ref_clk) != 0)
1754 			device_printf(dev,
1755 			    "could not retrieve reference clock.\n");
1756 		else if (clk_enable(sc->ref_clk) != 0)
1757 			device_printf(dev, "could not enable clock.\n");
1758 	}
1759 	if ((hwquirks & HWQUIRK_PCLK) != 0) {
1760 		if (clk_get_by_ofw_name(dev, 0, "pclk", &sc->ref_clk) != 0)
1761 			device_printf(dev,
1762 			    "could not retrieve reference clock.\n");
1763 		else if (clk_enable(sc->ref_clk) != 0)
1764 			device_printf(dev, "could not enable clock.\n");
1765 	}
1766 
1767 	node = ofw_bus_get_node(dev);
1768 	sc->phy_contype = mii_fdt_get_contype(node);
1769 
1770 	/* Get memory resource. */
1771 	rid = 0;
1772 	sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1773 	    RF_ACTIVE);
1774 	if (sc->mem_res == NULL) {
1775 		device_printf(dev, "could not allocate memory resources.\n");
1776 		return (ENOMEM);
1777 	}
1778 
1779 	/* Get IRQ resource. */
1780 	rid = 0;
1781 	sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1782 	    RF_ACTIVE);
1783 	if (sc->irq_res == NULL) {
1784 		device_printf(dev, "could not allocate interrupt resource.\n");
1785 		cgem_detach(dev);
1786 		return (ENOMEM);
1787 	}
1788 
1789 	/* Set up ifnet structure. */
1790 	ifp = sc->ifp = if_alloc(IFT_ETHER);
1791 	if (ifp == NULL) {
1792 		device_printf(dev, "could not allocate ifnet structure\n");
1793 		cgem_detach(dev);
1794 		return (ENOMEM);
1795 	}
1796 	if_setsoftc(ifp, sc);
1797 	if_initname(ifp, IF_CGEM_NAME, device_get_unit(dev));
1798 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
1799 	if_setinitfn(ifp, cgem_init);
1800 	if_setioctlfn(ifp, cgem_ioctl);
1801 	if_setstartfn(ifp, cgem_start);
1802 	if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 |
1803 	    IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM, 0);
1804 	if_setsendqlen(ifp, CGEM_NUM_TX_DESCS);
1805 	if_setsendqready(ifp);
1806 
1807 	/* Disable hardware checksumming by default. */
1808 	if_sethwassist(ifp, 0);
1809 	if_setcapenable(ifp, if_getcapabilities(ifp) &
1810 	    ~(IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_VLAN_HWCSUM));
1811 
1812 	sc->if_old_flags = if_getflags(ifp);
1813 	sc->rxbufs = DEFAULT_NUM_RX_BUFS;
1814 
1815 	/* Reset hardware. */
1816 	CGEM_LOCK(sc);
1817 	cgem_reset(sc);
1818 	CGEM_UNLOCK(sc);
1819 
1820 	/* Attach phy to mii bus. */
1821 	err = mii_attach(dev, &sc->miibus, ifp,
1822 	    cgem_ifmedia_upd, cgem_ifmedia_sts, BMSR_DEFCAPMASK,
1823 	    MII_PHY_ANY, MII_OFFSET_ANY, 0);
1824 	if (err)
1825 		device_printf(dev, "warning: attaching PHYs failed\n");
1826 
1827 	/* Set up TX and RX descriptor area. */
1828 	err = cgem_setup_descs(sc);
1829 	if (err) {
1830 		device_printf(dev, "could not set up dma mem for descs.\n");
1831 		cgem_detach(dev);
1832 		return (ENOMEM);
1833 	}
1834 
1835 	/* Get a MAC address. */
1836 	cgem_get_mac(sc, eaddr);
1837 
1838 	/* Start ticks. */
1839 	callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0);
1840 
1841 	ether_ifattach(ifp, eaddr);
1842 
1843 	err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE |
1844 	    INTR_EXCL, NULL, cgem_intr, sc, &sc->intrhand);
1845 	if (err) {
1846 		device_printf(dev, "could not set interrupt handler.\n");
1847 		ether_ifdetach(ifp);
1848 		cgem_detach(dev);
1849 		return (err);
1850 	}
1851 
1852 	cgem_add_sysctls(dev);
1853 
1854 	return (0);
1855 }
1856 
1857 static int
1858 cgem_detach(device_t dev)
1859 {
1860 	struct cgem_softc *sc = device_get_softc(dev);
1861 	int i;
1862 
1863 	if (sc == NULL)
1864 		return (ENODEV);
1865 
1866 	if (device_is_attached(dev)) {
1867 		CGEM_LOCK(sc);
1868 		cgem_stop(sc);
1869 		CGEM_UNLOCK(sc);
1870 		callout_drain(&sc->tick_ch);
1871 		if_setflagbits(sc->ifp, 0, IFF_UP);
1872 		ether_ifdetach(sc->ifp);
1873 	}
1874 
1875 	if (sc->miibus != NULL) {
1876 		device_delete_child(dev, sc->miibus);
1877 		sc->miibus = NULL;
1878 	}
1879 
1880 	/* Release resources. */
1881 	if (sc->mem_res != NULL) {
1882 		bus_release_resource(dev, SYS_RES_MEMORY,
1883 		    rman_get_rid(sc->mem_res), sc->mem_res);
1884 		sc->mem_res = NULL;
1885 	}
1886 	if (sc->irq_res != NULL) {
1887 		if (sc->intrhand)
1888 			bus_teardown_intr(dev, sc->irq_res, sc->intrhand);
1889 		bus_release_resource(dev, SYS_RES_IRQ,
1890 		    rman_get_rid(sc->irq_res), sc->irq_res);
1891 		sc->irq_res = NULL;
1892 	}
1893 
1894 	/* Release DMA resources. */
1895 	if (sc->rxring != NULL) {
1896 		if (sc->rxring_physaddr != 0) {
1897 			bus_dmamap_unload(sc->desc_dma_tag,
1898 			    sc->rxring_dma_map);
1899 			sc->rxring_physaddr = 0;
1900 			sc->txring_physaddr = 0;
1901 			sc->null_qs_physaddr = 0;
1902 		}
1903 		bus_dmamem_free(sc->desc_dma_tag, sc->rxring,
1904 				sc->rxring_dma_map);
1905 		sc->rxring = NULL;
1906 		sc->txring = NULL;
1907 		sc->null_qs = NULL;
1908 
1909 		for (i = 0; i < CGEM_NUM_RX_DESCS; i++)
1910 			if (sc->rxring_m_dmamap[i] != NULL) {
1911 				bus_dmamap_destroy(sc->mbuf_dma_tag,
1912 				    sc->rxring_m_dmamap[i]);
1913 				sc->rxring_m_dmamap[i] = NULL;
1914 			}
1915 		for (i = 0; i < CGEM_NUM_TX_DESCS; i++)
1916 			if (sc->txring_m_dmamap[i] != NULL) {
1917 				bus_dmamap_destroy(sc->mbuf_dma_tag,
1918 				    sc->txring_m_dmamap[i]);
1919 				sc->txring_m_dmamap[i] = NULL;
1920 			}
1921 	}
1922 	if (sc->desc_dma_tag != NULL) {
1923 		bus_dma_tag_destroy(sc->desc_dma_tag);
1924 		sc->desc_dma_tag = NULL;
1925 	}
1926 	if (sc->mbuf_dma_tag != NULL) {
1927 		bus_dma_tag_destroy(sc->mbuf_dma_tag);
1928 		sc->mbuf_dma_tag = NULL;
1929 	}
1930 
1931 	if (sc->ref_clk != NULL) {
1932 		clk_release(sc->ref_clk);
1933 		sc->ref_clk = NULL;
1934 	}
1935 
1936 	bus_generic_detach(dev);
1937 
1938 	CGEM_LOCK_DESTROY(sc);
1939 
1940 	return (0);
1941 }
1942 
1943 static device_method_t cgem_methods[] = {
1944 	/* Device interface */
1945 	DEVMETHOD(device_probe,		cgem_probe),
1946 	DEVMETHOD(device_attach,	cgem_attach),
1947 	DEVMETHOD(device_detach,	cgem_detach),
1948 
1949 	/* MII interface */
1950 	DEVMETHOD(miibus_readreg,	cgem_miibus_readreg),
1951 	DEVMETHOD(miibus_writereg,	cgem_miibus_writereg),
1952 	DEVMETHOD(miibus_statchg,	cgem_miibus_statchg),
1953 	DEVMETHOD(miibus_linkchg,	cgem_miibus_linkchg),
1954 
1955 	DEVMETHOD_END
1956 };
1957 
1958 static driver_t cgem_driver = {
1959 	"cgem",
1960 	cgem_methods,
1961 	sizeof(struct cgem_softc),
1962 };
1963 
1964 DRIVER_MODULE(cgem, simplebus, cgem_driver, cgem_devclass, NULL, NULL);
1965 DRIVER_MODULE(miibus, cgem, miibus_driver, miibus_devclass, NULL, NULL);
1966 MODULE_DEPEND(cgem, miibus, 1, 1, 1);
1967 MODULE_DEPEND(cgem, ether, 1, 1, 1);
1968 SIMPLEBUS_PNP_INFO(compat_data);
1969