1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019, 2020, 2023-2025 Kevin Lo <kevlo@openbsd.org>
5 * Copyright (c) 2025 Adrian Chadd <adrian@FreeBSD.org>
6 *
7 * Hardware programming portions from Realtek Semiconductor.
8 *
9 * Permission to use, copy, modify, and distribute this software for any
10 * purpose with or without fee is hereby granted, provided that the above
11 * copyright notice and this permission notice appear in all copies.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
14 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
15 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
16 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
17 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
18 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
19 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
20 */
21
22 /* $OpenBSD: if_rge.c,v 1.38 2025/09/19 00:41:14 kevlo Exp $ */
23
24 #include <sys/param.h>
25 #include <sys/systm.h>
26 #include <sys/sockio.h>
27 #include <sys/mbuf.h>
28 #include <sys/malloc.h>
29 #include <sys/endian.h>
30 #include <sys/socket.h>
31 #include <net/if.h>
32 #include <net/if_media.h>
33 #include <sys/queue.h>
34 #include <sys/taskqueue.h>
35 #include <sys/bus.h>
36 #include <sys/module.h>
37 #include <sys/rman.h>
38 #include <sys/kernel.h>
39
40 #include <netinet/in.h>
41 #include <netinet/if_ether.h>
42
43 #include <net/bpf.h>
44 #include <net/ethernet.h>
45 #include <net/if.h>
46 #include <net/if_var.h>
47 #include <net/if_arp.h>
48 #include <net/if_dl.h>
49 #include <net/if_media.h>
50 #include <net/if_types.h>
51 #include <net/if_vlan_var.h>
52
53 #include <machine/bus.h>
54 #include <machine/resource.h>
55
56 #include <dev/mii/mii.h>
57
58 #include <dev/pci/pcivar.h>
59 #include <dev/pci/pcireg.h>
60
61 #include "if_rge_vendor.h"
62 #include "if_rgereg.h"
63 #include "if_rgevar.h"
64 #include "if_rge_hw.h"
65 #include "if_rge_microcode.h"
66 #include "if_rge_debug.h"
67 #include "if_rge_sysctl.h"
68 #include "if_rge_stats.h"
69
70 #define RGE_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP)
71
72 static int rge_attach(device_t);
73 static int rge_detach(device_t);
74
75 #if 0
76 int rge_activate(struct device *, int);
77 #endif
78 static void rge_intr_msi(void *);
79 static int rge_ioctl(struct ifnet *, u_long, caddr_t);
80 static int rge_transmit_if(if_t, struct mbuf *);
81 static void rge_qflush_if(if_t);
82 static void rge_init_if(void *);
83 static void rge_init_locked(struct rge_softc *);
84 static void rge_stop_locked(struct rge_softc *);
85 static int rge_ifmedia_upd(if_t);
86 static void rge_ifmedia_sts(if_t, struct ifmediareq *);
87 static int rge_allocmem(struct rge_softc *);
88 static int rge_alloc_stats_mem(struct rge_softc *);
89 static int rge_freemem(struct rge_softc *);
90 static int rge_free_stats_mem(struct rge_softc *);
91 static int rge_newbuf(struct rge_queues *);
92 static void rge_rx_list_init(struct rge_queues *);
93 static void rge_tx_list_init(struct rge_queues *);
94 static void rge_fill_rx_ring(struct rge_queues *);
95 static int rge_rxeof(struct rge_queues *, struct mbufq *);
96 static int rge_txeof(struct rge_queues *);
97 static void rge_iff_locked(struct rge_softc *);
98 static void rge_add_media_types(struct rge_softc *);
99 static void rge_tx_task(void *, int);
100 static void rge_txq_flush_mbufs(struct rge_softc *sc);
101 static void rge_tick(void *);
102 static void rge_link_state(struct rge_softc *);
103 #if 0
104 #ifndef SMALL_KERNEL
105 int rge_wol(struct ifnet *, int);
106 void rge_wol_power(struct rge_softc *);
107 #endif
108 #endif
109
110 struct rge_matchid {
111 uint16_t vendor;
112 uint16_t device;
113 const char *name;
114 };
115
116 const struct rge_matchid rge_devices[] = {
117 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_E3000, "Killer E3000" },
118 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RTL8125, "RTL8125" },
119 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RTL8126, "RTL8126", },
120 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RTL8127, "RTL8127" },
121 { 0, 0, NULL }
122 };
123
124 static int
rge_probe(device_t dev)125 rge_probe(device_t dev)
126 {
127 uint16_t vendor, device;
128 const struct rge_matchid *ri;
129
130 vendor = pci_get_vendor(dev);
131 device = pci_get_device(dev);
132
133 for (ri = rge_devices; ri->name != NULL; ri++) {
134 if ((vendor == ri->vendor) && (device == ri->device)) {
135 device_set_desc(dev, ri->name);
136 return (BUS_PROBE_DEFAULT);
137 }
138 }
139
140 return (ENXIO);
141 }
142
143 static void
rge_attach_if(struct rge_softc * sc,const char * eaddr)144 rge_attach_if(struct rge_softc *sc, const char *eaddr)
145 {
146 if_initname(sc->sc_ifp, device_get_name(sc->sc_dev),
147 device_get_unit(sc->sc_dev));
148 if_setdev(sc->sc_ifp, sc->sc_dev);
149 if_setinitfn(sc->sc_ifp, rge_init_if);
150 if_setsoftc(sc->sc_ifp, sc);
151 if_setflags(sc->sc_ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
152 if_setioctlfn(sc->sc_ifp, rge_ioctl);
153 if_settransmitfn(sc->sc_ifp, rge_transmit_if);
154 if_setqflushfn(sc->sc_ifp, rge_qflush_if);
155
156 /* Set offload as appropriate */
157 if_sethwassist(sc->sc_ifp, CSUM_IP | CSUM_TCP | CSUM_UDP);
158 if_setcapabilities(sc->sc_ifp, IFCAP_HWCSUM);
159 if_setcapenable(sc->sc_ifp, if_getcapabilities(sc->sc_ifp));
160
161 /* TODO: set WOL */
162
163 /* Attach interface */
164 ether_ifattach(sc->sc_ifp, eaddr);
165 sc->sc_ether_attached = true;
166
167 /* post ether_ifattach() bits */
168
169 /* VLAN capabilities */
170 if_setcapabilitiesbit(sc->sc_ifp, IFCAP_VLAN_MTU |
171 IFCAP_VLAN_HWTAGGING, 0);
172 if_setcapabilitiesbit(sc->sc_ifp, IFCAP_VLAN_HWCSUM, 0);
173 if_setcapenable(sc->sc_ifp, if_getcapabilities(sc->sc_ifp));
174
175 if_setifheaderlen(sc->sc_ifp, sizeof(struct ether_vlan_header));
176
177 /* TODO: is this needed for iftransmit? */
178 if_setsendqlen(sc->sc_ifp, RGE_TX_LIST_CNT - 1);
179 if_setsendqready(sc->sc_ifp);
180 }
181
182 static int
rge_attach(device_t dev)183 rge_attach(device_t dev)
184 {
185 uint8_t eaddr[ETHER_ADDR_LEN];
186 struct rge_softc *sc;
187 struct rge_queues *q;
188 uint32_t hwrev, reg;
189 int i, rid;
190 int error;
191 int msic;
192
193 sc = device_get_softc(dev);
194 sc->sc_dev = dev;
195 sc->sc_ifp = if_gethandle(IFT_ETHER);
196 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
197 MTX_DEF);
198
199 /* Enable bus mastering */
200 pci_enable_busmaster(dev);
201
202 /*
203 * Map control/status registers.
204 */
205
206 /*
207 * The openbsd driver (and my E3000 NIC) handle registering three
208 * kinds of BARs - a 64 bit MMIO BAR, a 32 bit MMIO BAR, and then
209 * a legacy IO port BAR.
210 *
211 * To simplify bring-up, I'm going to request resources for the first
212 * MMIO BAR (BAR2) which should be a 32 bit BAR.
213 */
214 rid = PCIR_BAR(2);
215 sc->sc_bres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
216 RF_ACTIVE);
217 if (sc->sc_bres == NULL) {
218 RGE_PRINT_ERROR(sc,
219 "Unable to allocate bus resource: memory\n");
220 goto fail;
221 }
222 sc->rge_bhandle = rman_get_bushandle(sc->sc_bres);
223 sc->rge_btag = rman_get_bustag(sc->sc_bres);
224 sc->rge_bsize = rman_get_size(sc->sc_bres);
225
226 q = malloc(sizeof(struct rge_queues), M_DEVBUF, M_NOWAIT | M_ZERO);
227 if (q == NULL) {
228 RGE_PRINT_ERROR(sc, "Unable to malloc rge_queues memory\n");
229 goto fail;
230 }
231 q->q_sc = sc;
232 q->q_index = 0;
233
234 sc->sc_queues = q;
235 sc->sc_nqueues = 1;
236
237 /* Check if PCIe */
238 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) {
239 sc->rge_flags |= RGE_FLAG_PCIE;
240 sc->sc_expcap = reg;
241 }
242
243 /* Allocate MSI */
244 msic = pci_msi_count(dev);
245 if (msic == 0) {
246 RGE_PRINT_ERROR(sc, "%s: only MSI interrupts supported\n",
247 __func__);
248 goto fail;
249 }
250
251 msic = RGE_MSI_MESSAGES;
252 if (pci_alloc_msi(dev, &msic) != 0) {
253 RGE_PRINT_ERROR(sc, "%s: failed to allocate MSI\n",
254 __func__);
255 goto fail;
256 }
257
258 sc->rge_flags |= RGE_FLAG_MSI;
259
260 /* We need at least one MSI */
261 if (msic < RGE_MSI_MESSAGES) {
262 RGE_PRINT_ERROR(sc, "%s: didn't allocate enough MSI\n",
263 __func__);
264 goto fail;
265 }
266
267 /*
268 * Allocate interrupt entries.
269 */
270 for (i = 0, rid = 1; i < RGE_MSI_MESSAGES; i++, rid++) {
271 sc->sc_irq[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ,
272 &rid, RF_ACTIVE);
273 if (sc->sc_irq[i] == NULL) {
274 RGE_PRINT_ERROR(sc, "%s: couldn't allocate MSI %d",
275 __func__, rid);
276 goto fail;
277 }
278 }
279
280 /* Hook interrupts */
281 for (i = 0; i < RGE_MSI_MESSAGES; i++) {
282 error = bus_setup_intr(dev, sc->sc_irq[i],
283 INTR_TYPE_NET | INTR_MPSAFE, NULL, rge_intr_msi,
284 sc, &sc->sc_ih[i]);
285 if (error != 0) {
286 RGE_PRINT_ERROR(sc,
287 "%s: couldn't setup intr %d (error %d)", __func__,
288 i, error);
289 goto fail;
290 }
291 }
292
293 /* Allocate top level bus DMA tag */
294 error = bus_dma_tag_create(bus_get_dma_tag(dev),
295 1, /* alignment */
296 0, /* boundary */
297 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
298 NULL, NULL, /* filter (unused) */
299 BUS_SPACE_MAXADDR, /* maxsize */
300 BUS_SPACE_UNRESTRICTED, /* nsegments */
301 BUS_SPACE_MAXADDR, /* maxsegsize */
302 0, /* flags */
303 NULL, NULL, /* lockfunc, lockarg */
304 &sc->sc_dmat);
305 if (error) {
306 RGE_PRINT_ERROR(sc,
307 "couldn't allocate device DMA tag (error %d)\n", error);
308 goto fail;
309 }
310
311 /* Allocate TX/RX descriptor and buffer tags */
312 error = bus_dma_tag_create(sc->sc_dmat,
313 RGE_ALIGN, /* alignment */
314 0, /* boundary */
315 BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
316 NULL, NULL, /* filter (unused) */
317 RGE_TX_LIST_SZ, /* maxsize */
318 1, /* nsegments */
319 RGE_TX_LIST_SZ, /* maxsegsize */
320 0, /* flags */
321 NULL, NULL, /* lockfunc, lockarg */
322 &sc->sc_dmat_tx_desc);
323 if (error) {
324 RGE_PRINT_ERROR(sc,
325 "couldn't allocate device TX descriptor "
326 "DMA tag (error %d)\n", error);
327 goto fail;
328 }
329
330 error = bus_dma_tag_create(sc->sc_dmat,
331 1, /* alignment */
332 0, /* boundary */
333 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
334 NULL, NULL, /* filter (unused) */
335 RGE_JUMBO_FRAMELEN, /* maxsize */
336 RGE_TX_NSEGS, /* nsegments */
337 RGE_JUMBO_FRAMELEN, /* maxsegsize */
338 0, /* flags */
339 NULL, NULL, /* lockfunc, lockarg */
340 &sc->sc_dmat_tx_buf);
341 if (error) {
342 RGE_PRINT_ERROR(sc,
343 "couldn't allocate device TX buffer DMA tag (error %d)\n",
344 error);
345 goto fail;
346 }
347
348 error = bus_dma_tag_create(sc->sc_dmat,
349 RGE_ALIGN, /* alignment */
350 0, /* boundary */
351 BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
352 NULL, NULL, /* filter (unused) */
353 RGE_RX_LIST_SZ, /* maxsize */
354 1, /* nsegments */
355 RGE_RX_LIST_SZ, /* maxsegsize */
356 0, /* flags */
357 NULL, NULL, /* lockfunc, lockarg */
358 &sc->sc_dmat_rx_desc);
359 if (error) {
360 RGE_PRINT_ERROR(sc,
361 "couldn't allocate device RX descriptor "
362 "DMA tag (error %d)\n", error);
363 goto fail;
364 }
365
366 error = bus_dma_tag_create(sc->sc_dmat,
367 1, /* alignment */
368 0, /* boundary */
369 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
370 NULL, NULL, /* filter (unused) */
371 MCLBYTES, /* maxsize */
372 1, /* nsegments */
373 MCLBYTES, /* maxsegsize */
374 0, /* flags */
375 NULL, NULL, /* lockfunc, lockarg */
376 &sc->sc_dmat_rx_buf);
377 if (error) {
378 RGE_PRINT_ERROR(sc,
379 "couldn't allocate device RX buffer DMA tag (error %d)\n",
380 error);
381 goto fail;
382 }
383
384 error = bus_dma_tag_create(sc->sc_dmat,
385 RGE_STATS_ALIGNMENT, /* alignment */
386 0, /* boundary */
387 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
388 NULL, NULL, /* filter (unused) */
389 RGE_STATS_BUF_SIZE, /* maxsize */
390 1, /* nsegments */
391 RGE_STATS_BUF_SIZE, /* maxsegsize */
392 0, /* flags */
393 NULL, NULL, /* lockfunc, lockarg */
394 &sc->sc_dmat_stats_buf);
395 if (error) {
396 RGE_PRINT_ERROR(sc,
397 "couldn't allocate device RX buffer DMA tag (error %d)\n",
398 error);
399 goto fail;
400 }
401
402
403 /* Attach sysctl nodes */
404 rge_sysctl_attach(sc);
405
406 /* Determine hardware revision */
407 hwrev = RGE_READ_4(sc, RGE_TXCFG) & RGE_TXCFG_HWREV;
408 switch (hwrev) {
409 case 0x60900000:
410 sc->rge_type = MAC_R25;
411 // device_printf(dev, "RTL8125\n");
412 break;
413 case 0x64100000:
414 sc->rge_type = MAC_R25B;
415 // device_printf(dev, "RTL8125B\n");
416 break;
417 case 0x64900000:
418 sc->rge_type = MAC_R26;
419 // device_printf(dev, "RTL8126\n");
420 break;
421 case 0x68800000:
422 sc->rge_type = MAC_R25D;
423 // device_printf(dev, "RTL8125D\n");
424 break;
425 case 0x6c900000:
426 sc->rge_type = MAC_R27;
427 // device_printf(dev, "RTL8127\n");
428 break;
429 default:
430 RGE_PRINT_ERROR(sc, "unknown version 0x%08x\n", hwrev);
431 goto fail;
432 }
433
434 rge_config_imtype(sc, RGE_IMTYPE_SIM);
435
436 /* TODO: disable ASPM/ECPM? */
437
438 #if 0
439 /*
440 * PCI Express check.
441 */
442 if (pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_PCIEXPRESS,
443 &offset, NULL)) {
444 /* Disable PCIe ASPM and ECPM. */
445 reg = pci_conf_read(pa->pa_pc, pa->pa_tag,
446 offset + PCI_PCIE_LCSR);
447 reg &= ~(PCI_PCIE_LCSR_ASPM_L0S | PCI_PCIE_LCSR_ASPM_L1 |
448 PCI_PCIE_LCSR_ECPM);
449 pci_conf_write(pa->pa_pc, pa->pa_tag, offset + PCI_PCIE_LCSR,
450 reg);
451 }
452 #endif
453
454 RGE_LOCK(sc);
455 if (rge_chipinit(sc)) {
456 RGE_UNLOCK(sc);
457 goto fail;
458 }
459
460 rge_get_macaddr(sc, eaddr);
461 RGE_UNLOCK(sc);
462
463 if (rge_allocmem(sc))
464 goto fail;
465 if (rge_alloc_stats_mem(sc))
466 goto fail;
467
468 /* Initialize ifmedia structures. */
469 ifmedia_init(&sc->sc_media, IFM_IMASK, rge_ifmedia_upd,
470 rge_ifmedia_sts);
471 rge_add_media_types(sc);
472 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
473 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
474 sc->sc_media.ifm_media = sc->sc_media.ifm_cur->ifm_media;
475
476 rge_attach_if(sc, eaddr);
477
478 /*
479 * TODO: technically should be per txq but we only support
480 * one TXQ at the moment.
481 */
482 mbufq_init(&sc->sc_txq, RGE_TX_LIST_CNT);
483
484 snprintf(sc->sc_tq_name, sizeof(sc->sc_tq_name),
485 "%s taskq", device_get_nameunit(sc->sc_dev));
486 snprintf(sc->sc_tq_thr_name, sizeof(sc->sc_tq_thr_name),
487 "%s taskq thread", device_get_nameunit(sc->sc_dev));
488
489 sc->sc_tq = taskqueue_create(sc->sc_tq_name, M_NOWAIT,
490 taskqueue_thread_enqueue, &sc->sc_tq);
491 taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s",
492 sc->sc_tq_thr_name);
493
494 TASK_INIT(&sc->sc_tx_task, 0, rge_tx_task, sc);
495
496 callout_init_mtx(&sc->sc_timeout, &sc->sc_mtx, 0);
497
498 return (0);
499 fail:
500 rge_detach(dev);
501 return (ENXIO);
502 }
503
504 /**
505 * @brief flush the mbufq queue
506 *
507 * Again this should likely be per-TXQ.
508 *
509 * This should be called with the driver lock held.
510 */
511 static void
rge_txq_flush_mbufs(struct rge_softc * sc)512 rge_txq_flush_mbufs(struct rge_softc *sc)
513 {
514 struct mbuf *m;
515 int ntx = 0;
516
517 RGE_ASSERT_LOCKED(sc);
518
519 while ((m = mbufq_dequeue(&sc->sc_txq)) != NULL) {
520 m_freem(m);
521 ntx++;
522 }
523
524 RGE_DPRINTF(sc, RGE_DEBUG_XMIT, "%s: %d frames flushed\n", __func__,
525 ntx);
526 }
527
528 static int
rge_detach(device_t dev)529 rge_detach(device_t dev)
530 {
531 struct rge_softc *sc = device_get_softc(dev);
532 int i, rid;
533
534 /* global flag, detaching */
535 RGE_LOCK(sc);
536 sc->sc_stopped = true;
537 sc->sc_detaching = true;
538 RGE_UNLOCK(sc);
539
540 /* stop/drain network interface */
541 callout_drain(&sc->sc_timeout);
542
543 /* Make sure TX task isn't running */
544 if (sc->sc_tq != NULL) {
545 while (taskqueue_cancel(sc->sc_tq, &sc->sc_tx_task, NULL) != 0)
546 taskqueue_drain(sc->sc_tq, &sc->sc_tx_task);
547 }
548
549 RGE_LOCK(sc);
550 callout_stop(&sc->sc_timeout);
551
552 /* stop NIC / DMA */
553 rge_stop_locked(sc);
554
555 /* TODO: wait for completion */
556
557 /* Free pending TX mbufs */
558 rge_txq_flush_mbufs(sc);
559
560 RGE_UNLOCK(sc);
561
562 /* Free taskqueue */
563 if (sc->sc_tq != NULL) {
564 taskqueue_free(sc->sc_tq);
565 sc->sc_tq = NULL;
566 }
567
568 /* Free descriptor memory */
569 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: freemem\n", __func__);
570 rge_freemem(sc);
571 rge_free_stats_mem(sc);
572
573 if (sc->sc_ifp) {
574 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: ifdetach/if_free\n",
575 __func__);
576 if (sc->sc_ether_attached)
577 ether_ifdetach(sc->sc_ifp);
578 if_free(sc->sc_ifp);
579 }
580
581 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_tx_desc\n", __func__);
582 if (sc->sc_dmat_tx_desc)
583 bus_dma_tag_destroy(sc->sc_dmat_tx_desc);
584 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_tx_buf\n", __func__);
585 if (sc->sc_dmat_tx_buf)
586 bus_dma_tag_destroy(sc->sc_dmat_tx_buf);
587 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_rx_desc\n", __func__);
588 if (sc->sc_dmat_rx_desc)
589 bus_dma_tag_destroy(sc->sc_dmat_rx_desc);
590 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_rx_buf\n", __func__);
591 if (sc->sc_dmat_rx_buf)
592 bus_dma_tag_destroy(sc->sc_dmat_rx_buf);
593 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_stats_buf\n", __func__);
594 if (sc->sc_dmat_stats_buf)
595 bus_dma_tag_destroy(sc->sc_dmat_stats_buf);
596 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat\n", __func__);
597 if (sc->sc_dmat)
598 bus_dma_tag_destroy(sc->sc_dmat);
599
600 /* Teardown interrupts */
601 for (i = 0; i < RGE_MSI_MESSAGES; i++) {
602 if (sc->sc_ih[i] != NULL) {
603 bus_teardown_intr(sc->sc_dev, sc->sc_irq[i],
604 sc->sc_ih[i]);
605 sc->sc_ih[i] = NULL;
606 }
607 }
608
609 /* Free interrupt resources */
610 for (i = 0, rid = 1; i < RGE_MSI_MESSAGES; i++, rid++) {
611 if (sc->sc_irq[i] != NULL) {
612 bus_release_resource(sc->sc_dev, SYS_RES_IRQ,
613 rid, sc->sc_irq[i]);
614 sc->sc_irq[i] = NULL;
615 }
616 }
617
618 /* Free MSI allocation */
619 if (sc->rge_flags & RGE_FLAG_MSI)
620 pci_release_msi(dev);
621
622 if (sc->sc_bres) {
623 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: release mmio\n",
624 __func__);
625 bus_release_resource(dev, SYS_RES_MEMORY,
626 rman_get_rid(sc->sc_bres), sc->sc_bres);
627 sc->sc_bres = NULL;
628 }
629
630 if (sc->sc_queues) {
631 free(sc->sc_queues, M_DEVBUF);
632 sc->sc_queues = NULL;
633 }
634
635 mtx_destroy(&sc->sc_mtx);
636
637 return (0);
638 }
639
640 #if 0
641
642 int
643 rge_activate(struct device *self, int act)
644 {
645 #ifndef SMALL_KERNEL
646 struct rge_softc *sc = (struct rge_softc *)self;
647 #endif
648
649 switch (act) {
650 case DVACT_POWERDOWN:
651 #ifndef SMALL_KERNEL
652 rge_wol_power(sc);
653 #endif
654 break;
655 }
656 return (0);
657 }
658 #endif
659
660 static void
rge_intr_msi(void * arg)661 rge_intr_msi(void *arg)
662 {
663 struct mbufq rx_mq;
664 struct epoch_tracker et;
665 struct mbuf *m;
666 struct rge_softc *sc = arg;
667 struct rge_queues *q = sc->sc_queues;
668 uint32_t status;
669 int claimed = 0, rv;
670
671 sc->sc_drv_stats.intr_cnt++;
672
673 mbufq_init(&rx_mq, RGE_RX_LIST_CNT);
674
675 if ((if_getdrvflags(sc->sc_ifp) & IFF_DRV_RUNNING) == 0)
676 return;
677
678 RGE_LOCK(sc);
679
680 if (sc->sc_suspended || sc->sc_stopped || sc->sc_detaching) {
681 RGE_UNLOCK(sc);
682 return;
683 }
684
685 /* Disable interrupts. */
686 RGE_WRITE_4(sc, RGE_IMR, 0);
687
688 if (!(sc->rge_flags & RGE_FLAG_MSI)) {
689 if ((RGE_READ_4(sc, RGE_ISR) & sc->rge_intrs) == 0)
690 goto done;
691 }
692
693 status = RGE_READ_4(sc, RGE_ISR);
694 if (status)
695 RGE_WRITE_4(sc, RGE_ISR, status);
696
697 if (status & RGE_ISR_PCS_TIMEOUT)
698 claimed = 1;
699
700 rv = 0;
701 if (status & sc->rge_intrs) {
702
703 (void) q;
704 rv |= rge_rxeof(q, &rx_mq);
705 rv |= rge_txeof(q);
706
707 if (status & RGE_ISR_SYSTEM_ERR) {
708 sc->sc_drv_stats.intr_system_err_cnt++;
709 rge_init_locked(sc);
710 }
711 claimed = 1;
712 }
713
714 if (sc->rge_timerintr) {
715 if (!rv) {
716 /*
717 * Nothing needs to be processed, fallback
718 * to use TX/RX interrupts.
719 */
720 rge_setup_intr(sc, RGE_IMTYPE_NONE);
721
722 /*
723 * Recollect, mainly to avoid the possible
724 * race introduced by changing interrupt
725 * masks.
726 */
727 rge_rxeof(q, &rx_mq);
728 rge_txeof(q);
729 } else
730 RGE_WRITE_4(sc, RGE_TIMERCNT, 1);
731 } else if (rv) {
732 /*
733 * Assume that using simulated interrupt moderation
734 * (hardware timer based) could reduce the interrupt
735 * rate.
736 */
737 rge_setup_intr(sc, RGE_IMTYPE_SIM);
738 }
739
740 RGE_WRITE_4(sc, RGE_IMR, sc->rge_intrs);
741
742 done:
743 RGE_UNLOCK(sc);
744
745 NET_EPOCH_ENTER(et);
746 /* Handle any RX frames, outside of the driver lock */
747 while ((m = mbufq_dequeue(&rx_mq)) != NULL) {
748 sc->sc_drv_stats.recv_input_cnt++;
749 if_input(sc->sc_ifp, m);
750 }
751 NET_EPOCH_EXIT(et);
752
753 (void) claimed;
754 }
755
756 static inline void
rge_tx_list_sync(struct rge_softc * sc,struct rge_queues * q,unsigned int idx,unsigned int len,int ops)757 rge_tx_list_sync(struct rge_softc *sc, struct rge_queues *q,
758 unsigned int idx, unsigned int len, int ops)
759 {
760 bus_dmamap_sync(sc->sc_dmat_tx_desc, q->q_tx.rge_tx_list_map, ops);
761 }
762
763 /**
764 * @brief Queue the given mbuf at the given TX slot index for transmit.
765 *
766 * If the frame couldn't be enqueued then 0 is returned.
767 * The caller needs to handle that and free/re-queue the mbuf as required.
768 *
769 * Note that this doesn't actually kick-start the transmit itself;
770 * see rge_txstart() for the register to poke to start transmit.
771 *
772 * This must be called with the driver lock held.
773 *
774 * @param sc driver softc
775 * @param q TX queue ring
776 * @param m mbuf to enqueue
777 * @returns if the mbuf is enqueued, it's consumed here and the number of
778 * TX descriptors used is returned; if there's no space then 0 is
779 * returned; if the mbuf couldn't be defragged and the caller
780 * should free it then -1 is returned.
781 */
782 static int
rge_encap(struct rge_softc * sc,struct rge_queues * q,struct mbuf * m,int idx)783 rge_encap(struct rge_softc *sc, struct rge_queues *q, struct mbuf *m, int idx)
784 {
785 struct rge_tx_desc *d = NULL;
786 struct rge_txq *txq;
787 bus_dmamap_t txmap;
788 uint32_t cmdsts, cflags = 0;
789 int cur, error, i;
790 bus_dma_segment_t seg[RGE_TX_NSEGS];
791 int nsegs;
792
793 RGE_ASSERT_LOCKED(sc);
794
795 txq = &q->q_tx.rge_txq[idx];
796 txmap = txq->txq_dmamap;
797
798 sc->sc_drv_stats.tx_encap_cnt++;
799
800 nsegs = RGE_TX_NSEGS;
801 error = bus_dmamap_load_mbuf_sg(sc->sc_dmat_tx_buf, txmap, m,
802 seg, &nsegs, BUS_DMA_NOWAIT);
803
804 switch (error) {
805 case 0:
806 break;
807 case EFBIG: /* mbuf chain is too fragmented */
808 sc->sc_drv_stats.tx_encap_refrag_cnt++;
809 nsegs = RGE_TX_NSEGS;
810 if (m_defrag(m, M_NOWAIT) == 0 &&
811 bus_dmamap_load_mbuf_sg(sc->sc_dmat_tx_buf, txmap, m,
812 seg, &nsegs, BUS_DMA_NOWAIT) == 0)
813 break;
814 /* FALLTHROUGH */
815 default:
816 sc->sc_drv_stats.tx_encap_err_toofrag++;
817 return (-1);
818 }
819
820 bus_dmamap_sync(sc->sc_dmat_tx_buf, txmap, BUS_DMASYNC_PREWRITE);
821
822 /*
823 * Set RGE_TDEXTSTS_IPCSUM if any checksum offloading is requested.
824 * Otherwise, RGE_TDEXTSTS_TCPCSUM / RGE_TDEXTSTS_UDPCSUM does not
825 * take affect.
826 */
827 if ((m->m_pkthdr.csum_flags & RGE_CSUM_FEATURES) != 0) {
828 cflags |= RGE_TDEXTSTS_IPCSUM;
829 sc->sc_drv_stats.tx_offload_ip_csum_set++;
830 if (m->m_pkthdr.csum_flags & CSUM_TCP) {
831 sc->sc_drv_stats.tx_offload_tcp_csum_set++;
832 cflags |= RGE_TDEXTSTS_TCPCSUM;
833 }
834 if (m->m_pkthdr.csum_flags & CSUM_UDP) {
835 sc->sc_drv_stats.tx_offload_udp_csum_set++;
836 cflags |= RGE_TDEXTSTS_UDPCSUM;
837 }
838 }
839
840 /* Set up hardware VLAN tagging */
841 if (m->m_flags & M_VLANTAG) {
842 sc->sc_drv_stats.tx_offload_vlan_tag_set++;
843 cflags |= htole16(m->m_pkthdr.ether_vtag) | RGE_TDEXTSTS_VTAG;
844 }
845
846 cur = idx;
847 for (i = 1; i < nsegs; i++) {
848 cur = RGE_NEXT_TX_DESC(cur);
849
850 cmdsts = RGE_TDCMDSTS_OWN;
851 cmdsts |= seg[i].ds_len;
852
853 if (cur == RGE_TX_LIST_CNT - 1)
854 cmdsts |= RGE_TDCMDSTS_EOR;
855 if (i == nsegs - 1)
856 cmdsts |= RGE_TDCMDSTS_EOF;
857
858 /*
859 * Note: vendor driver puts wmb() after opts2/extsts,
860 * before opts1/status.
861 *
862 * See the other place I have this comment for more
863 * information.
864 */
865 d = &q->q_tx.rge_tx_list[cur];
866 d->rge_addr = htole64(seg[i].ds_addr);
867 d->rge_extsts = htole32(cflags);
868 wmb();
869 d->rge_cmdsts = htole32(cmdsts);
870 }
871
872 /* Update info of TX queue and descriptors. */
873 txq->txq_mbuf = m;
874 txq->txq_descidx = cur;
875
876 cmdsts = RGE_TDCMDSTS_SOF;
877 cmdsts |= seg[0].ds_len;
878
879 if (idx == RGE_TX_LIST_CNT - 1)
880 cmdsts |= RGE_TDCMDSTS_EOR;
881 if (nsegs == 1)
882 cmdsts |= RGE_TDCMDSTS_EOF;
883
884 /*
885 * Note: vendor driver puts wmb() after opts2/extsts,
886 * before opts1/status.
887 *
888 * It does this:
889 * - set rge_addr
890 * - set extsts
891 * - wmb
892 * - set status - at this point it's owned by the hardware
893 *
894 */
895 d = &q->q_tx.rge_tx_list[idx];
896 d->rge_addr = htole64(seg[0].ds_addr);
897 d->rge_extsts = htole32(cflags);
898 wmb();
899 d->rge_cmdsts = htole32(cmdsts);
900 wmb();
901
902 if (cur >= idx) {
903 rge_tx_list_sync(sc, q, idx, nsegs,
904 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
905 } else {
906 rge_tx_list_sync(sc, q, idx, RGE_TX_LIST_CNT - idx,
907 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
908 rge_tx_list_sync(sc, q, 0, cur + 1,
909 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
910 }
911
912 /* Transfer ownership of packet to the chip. */
913 cmdsts |= RGE_TDCMDSTS_OWN;
914 rge_tx_list_sync(sc, q, idx, 1, BUS_DMASYNC_POSTWRITE);
915 d->rge_cmdsts = htole32(cmdsts);
916 rge_tx_list_sync(sc, q, idx, 1, BUS_DMASYNC_PREWRITE);
917 wmb();
918
919 return (nsegs);
920 }
921
922 static int
rge_ioctl(struct ifnet * ifp,u_long cmd,caddr_t data)923 rge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
924 {
925 struct rge_softc *sc = if_getsoftc(ifp);
926 struct ifreq *ifr = (struct ifreq *)data;
927 int error = 0;
928
929 switch (cmd) {
930 case SIOCSIFMTU:
931 /* Note: no hardware reinit is required */
932 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > RGE_JUMBO_MTU) {
933 error = EINVAL;
934 break;
935 }
936 if (if_getmtu(ifp) != ifr->ifr_mtu)
937 if_setmtu(ifp, ifr->ifr_mtu);
938
939 VLAN_CAPABILITIES(ifp);
940 break;
941
942 case SIOCSIFFLAGS:
943 RGE_LOCK(sc);
944 if ((if_getflags(ifp) & IFF_UP) != 0) {
945 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
946 /*
947 * TODO: handle promisc/iffmulti changing
948 * without reprogramming everything.
949 */
950 rge_init_locked(sc);
951 } else {
952 /* Reinit promisc/multi just in case */
953 rge_iff_locked(sc);
954 }
955 } else {
956 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
957 rge_stop_locked(sc);
958 }
959 }
960 RGE_UNLOCK(sc);
961 break;
962 case SIOCADDMULTI:
963 case SIOCDELMULTI:
964 RGE_LOCK(sc);
965 if ((if_getflags(ifp) & IFF_DRV_RUNNING) != 0) {
966 rge_iff_locked(sc);
967 }
968 RGE_UNLOCK(sc);
969 break;
970 case SIOCGIFMEDIA:
971 case SIOCSIFMEDIA:
972 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
973 break;
974 case SIOCSIFCAP:
975 {
976 int mask;
977 bool reinit = false;
978
979 /* Get the mask of changed bits */
980 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
981
982 /*
983 * Locked so we don't have a narrow window where frames
984 * are being processed with the updated flags but the
985 * hardware configuration hasn't yet changed.
986 */
987 RGE_LOCK(sc);
988
989 if ((mask & IFCAP_TXCSUM) != 0 &&
990 (if_getcapabilities(ifp) & IFCAP_TXCSUM) != 0) {
991 if_togglecapenable(ifp, IFCAP_TXCSUM);
992 if ((if_getcapenable(ifp) & IFCAP_TXCSUM) != 0)
993 if_sethwassistbits(ifp, RGE_CSUM_FEATURES, 0);
994 else
995 if_sethwassistbits(ifp, 0, RGE_CSUM_FEATURES);
996 reinit = 1;
997 }
998
999 if ((mask & IFCAP_VLAN_HWTAGGING) != 0 &&
1000 (if_getcapabilities(ifp) & IFCAP_VLAN_HWTAGGING) != 0) {
1001 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
1002 reinit = 1;
1003 }
1004
1005 /* TODO: WOL */
1006
1007 if ((mask & IFCAP_RXCSUM) != 0 &&
1008 (if_getcapabilities(ifp) & IFCAP_RXCSUM) != 0) {
1009 if_togglecapenable(ifp, IFCAP_RXCSUM);
1010 reinit = 1;
1011 }
1012
1013 if (reinit && if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1014 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1015 rge_init_locked(sc);
1016 }
1017
1018 RGE_UNLOCK(sc);
1019 VLAN_CAPABILITIES(ifp);
1020 }
1021 break;
1022 default:
1023 error = ether_ioctl(ifp, cmd, data);
1024 break;
1025 }
1026
1027 return (error);
1028 }
1029
1030 static void
rge_qflush_if(if_t ifp)1031 rge_qflush_if(if_t ifp)
1032 {
1033 struct rge_softc *sc = if_getsoftc(ifp);
1034
1035 /* TODO: this should iterate over the TXQs */
1036 RGE_LOCK(sc);
1037 rge_txq_flush_mbufs(sc);
1038 RGE_UNLOCK(sc);
1039 }
1040
1041 /**
1042 * @brief Transmit the given frame to the hardware.
1043 *
1044 * This routine is called by the network stack to send
1045 * a frame to the device.
1046 *
1047 * For now we simply direct dispatch this frame to the
1048 * hardware (and thus avoid maintaining our own internal
1049 * queue)
1050 */
1051 static int
rge_transmit_if(if_t ifp,struct mbuf * m)1052 rge_transmit_if(if_t ifp, struct mbuf *m)
1053 {
1054 struct rge_softc *sc = if_getsoftc(ifp);
1055 int ret;
1056
1057 sc->sc_drv_stats.transmit_call_cnt++;
1058
1059 RGE_LOCK(sc);
1060 if (sc->sc_stopped == true) {
1061 sc->sc_drv_stats.transmit_stopped_cnt++;
1062 RGE_UNLOCK(sc);
1063 return (ENETDOWN); /* TODO: better error? */
1064 }
1065
1066 /* XXX again should be a per-TXQ thing */
1067 ret = mbufq_enqueue(&sc->sc_txq, m);
1068 if (ret != 0) {
1069 sc->sc_drv_stats.transmit_full_cnt++;
1070 RGE_UNLOCK(sc);
1071 return (ret);
1072 }
1073 RGE_UNLOCK(sc);
1074
1075 /* mbuf is owned by the driver, schedule transmit */
1076 taskqueue_enqueue(sc->sc_tq, &sc->sc_tx_task);
1077 sc->sc_drv_stats.transmit_queued_cnt++;
1078
1079 return (0);
1080 }
1081
1082 static void
rge_init_if(void * xsc)1083 rge_init_if(void *xsc)
1084 {
1085 struct rge_softc *sc = xsc;
1086
1087 RGE_LOCK(sc);
1088 rge_init_locked(sc);
1089 RGE_UNLOCK(sc);
1090 }
1091
1092 static void
rge_init_locked(struct rge_softc * sc)1093 rge_init_locked(struct rge_softc *sc)
1094 {
1095 struct rge_queues *q = sc->sc_queues;
1096 uint32_t rxconf, val;
1097 int i, num_miti;
1098
1099 RGE_ASSERT_LOCKED(sc);
1100
1101 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: called!\n", __func__);
1102
1103 /* Don't double-init the hardware */
1104 if ((if_getdrvflags(sc->sc_ifp) & IFF_DRV_RUNNING) != 0) {
1105 /*
1106 * Note: I'm leaving this disabled by default; however
1107 * I'm leaving it in here so I can figure out what's
1108 * causing this to be initialised both from the ioctl
1109 * API and if_init() API.
1110 */
1111 // RGE_PRINT_ERROR(sc, "%s: called whilst running?\n", __func__);
1112 return;
1113 }
1114
1115 /*
1116 * Bring the hardware down so we know it's in a good known
1117 * state before we bring it up in a good known state.
1118 */
1119 rge_stop_locked(sc);
1120
1121 /* Set MAC address. */
1122 rge_set_macaddr(sc, if_getlladdr(sc->sc_ifp));
1123
1124 /* Initialize RX and TX descriptors lists. */
1125 rge_rx_list_init(q);
1126 rge_tx_list_init(q);
1127
1128 if (rge_chipinit(sc)) {
1129 RGE_PRINT_ERROR(sc, "%s: ERROR: chip init fail!\n", __func__);
1130 return;
1131 }
1132
1133 if (rge_phy_config(sc))
1134 return;
1135
1136 RGE_SETBIT_1(sc, RGE_EECMD, RGE_EECMD_WRITECFG);
1137
1138 RGE_CLRBIT_1(sc, 0xf1, 0x80);
1139 rge_disable_aspm_clkreq(sc);
1140 RGE_WRITE_2(sc, RGE_EEE_TXIDLE_TIMER,
1141 RGE_JUMBO_MTU + ETHER_HDR_LEN + 32);
1142
1143 /* Load the addresses of the RX and TX lists into the chip. */
1144 RGE_WRITE_4(sc, RGE_RXDESC_ADDR_LO,
1145 RGE_ADDR_LO(q->q_rx.rge_rx_list_paddr));
1146 RGE_WRITE_4(sc, RGE_RXDESC_ADDR_HI,
1147 RGE_ADDR_HI(q->q_rx.rge_rx_list_paddr));
1148 RGE_WRITE_4(sc, RGE_TXDESC_ADDR_LO,
1149 RGE_ADDR_LO(q->q_tx.rge_tx_list_paddr));
1150 RGE_WRITE_4(sc, RGE_TXDESC_ADDR_HI,
1151 RGE_ADDR_HI(q->q_tx.rge_tx_list_paddr));
1152
1153 /* Set the initial RX and TX configurations. */
1154 if (sc->rge_type == MAC_R25)
1155 rxconf = RGE_RXCFG_CONFIG;
1156 else if (sc->rge_type == MAC_R25B)
1157 rxconf = RGE_RXCFG_CONFIG_8125B;
1158 else if (sc->rge_type == MAC_R25D)
1159 rxconf = RGE_RXCFG_CONFIG_8125D;
1160 else
1161 rxconf = RGE_RXCFG_CONFIG_8126;
1162 RGE_WRITE_4(sc, RGE_RXCFG, rxconf);
1163 RGE_WRITE_4(sc, RGE_TXCFG, RGE_TXCFG_CONFIG);
1164
1165 val = rge_read_csi(sc, 0x70c) & ~0x3f000000;
1166 rge_write_csi(sc, 0x70c, val | 0x27000000);
1167
1168 if (sc->rge_type == MAC_R26 || sc->rge_type == MAC_R27) {
1169 /* Disable L1 timeout. */
1170 val = rge_read_csi(sc, 0x890) & ~0x00000001;
1171 rge_write_csi(sc, 0x890, val);
1172 } else if (sc->rge_type != MAC_R25D)
1173 RGE_WRITE_2(sc, 0x0382, 0x221b);
1174
1175 RGE_WRITE_1(sc, RGE_RSS_CTRL, 0);
1176
1177 val = RGE_READ_2(sc, RGE_RXQUEUE_CTRL) & ~0x001c;
1178 RGE_WRITE_2(sc, RGE_RXQUEUE_CTRL, val | (fls(sc->sc_nqueues) - 1) << 2);
1179
1180 RGE_CLRBIT_1(sc, RGE_CFG1, RGE_CFG1_SPEED_DOWN);
1181
1182 rge_write_mac_ocp(sc, 0xc140, 0xffff);
1183 rge_write_mac_ocp(sc, 0xc142, 0xffff);
1184
1185 RGE_MAC_SETBIT(sc, 0xeb58, 0x0001);
1186
1187 if (sc->rge_type == MAC_R26 || sc->rge_type == MAC_R27) {
1188 RGE_CLRBIT_1(sc, 0xd8, 0x02);
1189 if (sc->rge_type == MAC_R27) {
1190 RGE_CLRBIT_1(sc, 0x20e4, 0x04);
1191 RGE_MAC_CLRBIT(sc, 0xe00c, 0x1000);
1192 RGE_MAC_CLRBIT(sc, 0xc0c2, 0x0040);
1193 }
1194 }
1195
1196 val = rge_read_mac_ocp(sc, 0xe614);
1197 val &= (sc->rge_type == MAC_R27) ? ~0x0f00 : ~0x0700;
1198 if (sc->rge_type == MAC_R25 || sc->rge_type == MAC_R25D)
1199 rge_write_mac_ocp(sc, 0xe614, val | 0x0300);
1200 else if (sc->rge_type == MAC_R25B)
1201 rge_write_mac_ocp(sc, 0xe614, val | 0x0200);
1202 else if (sc->rge_type == MAC_R26)
1203 rge_write_mac_ocp(sc, 0xe614, val | 0x0300);
1204 else
1205 rge_write_mac_ocp(sc, 0xe614, val | 0x0f00);
1206
1207 val = rge_read_mac_ocp(sc, 0xe63e) & ~0x0c00;
1208 rge_write_mac_ocp(sc, 0xe63e, val |
1209 ((fls(sc->sc_nqueues) - 1) & 0x03) << 10);
1210
1211 val = rge_read_mac_ocp(sc, 0xe63e) & ~0x0030;
1212 rge_write_mac_ocp(sc, 0xe63e, val | 0x0020);
1213
1214 RGE_MAC_CLRBIT(sc, 0xc0b4, 0x0001);
1215 RGE_MAC_SETBIT(sc, 0xc0b4, 0x0001);
1216
1217 RGE_MAC_SETBIT(sc, 0xc0b4, 0x000c);
1218
1219 val = rge_read_mac_ocp(sc, 0xeb6a) & ~0x00ff;
1220 rge_write_mac_ocp(sc, 0xeb6a, val | 0x0033);
1221
1222 val = rge_read_mac_ocp(sc, 0xeb50) & ~0x03e0;
1223 rge_write_mac_ocp(sc, 0xeb50, val | 0x0040);
1224
1225 RGE_MAC_CLRBIT(sc, 0xe056, 0x00f0);
1226
1227 RGE_WRITE_1(sc, RGE_TDFNR, 0x10);
1228
1229 RGE_MAC_CLRBIT(sc, 0xe040, 0x1000);
1230
1231 val = rge_read_mac_ocp(sc, 0xea1c) & ~0x0003;
1232 rge_write_mac_ocp(sc, 0xea1c, val | 0x0001);
1233
1234 if (sc->rge_type == MAC_R25D)
1235 rge_write_mac_ocp(sc, 0xe0c0, 0x4403);
1236 else
1237 rge_write_mac_ocp(sc, 0xe0c0, 0x4000);
1238
1239 RGE_MAC_SETBIT(sc, 0xe052, 0x0060);
1240 RGE_MAC_CLRBIT(sc, 0xe052, 0x0088);
1241
1242 val = rge_read_mac_ocp(sc, 0xd430) & ~0x0fff;
1243 rge_write_mac_ocp(sc, 0xd430, val | 0x045f);
1244
1245 RGE_SETBIT_1(sc, RGE_DLLPR, RGE_DLLPR_PFM_EN | RGE_DLLPR_TX_10M_PS_EN);
1246
1247 if (sc->rge_type == MAC_R25)
1248 RGE_SETBIT_1(sc, RGE_MCUCMD, 0x01);
1249
1250 if (sc->rge_type != MAC_R25D) {
1251 /* Disable EEE plus. */
1252 RGE_MAC_CLRBIT(sc, 0xe080, 0x0002);
1253 }
1254
1255 if (sc->rge_type == MAC_R26 || sc->rge_type == MAC_R27)
1256 RGE_MAC_CLRBIT(sc, 0xea1c, 0x0304);
1257 else
1258 RGE_MAC_CLRBIT(sc, 0xea1c, 0x0004);
1259
1260 /* Clear tcam entries. */
1261 RGE_MAC_SETBIT(sc, 0xeb54, 0x0001);
1262 DELAY(1);
1263 RGE_MAC_CLRBIT(sc, 0xeb54, 0x0001);
1264
1265 RGE_CLRBIT_2(sc, 0x1880, 0x0030);
1266
1267 if (sc->rge_type == MAC_R27) {
1268 val = rge_read_mac_ocp(sc, 0xd40c) & ~0xe038;
1269 rge_write_phy_ocp(sc, 0xd40c, val | 0x8020);
1270 }
1271
1272 /* Config interrupt type. */
1273 if (sc->rge_type == MAC_R27)
1274 RGE_CLRBIT_1(sc, RGE_INT_CFG0, RGE_INT_CFG0_AVOID_MISS_INTR);
1275 else if (sc->rge_type != MAC_R25)
1276 RGE_CLRBIT_1(sc, RGE_INT_CFG0, RGE_INT_CFG0_EN);
1277
1278 /* Clear timer interrupts. */
1279 RGE_WRITE_4(sc, RGE_TIMERINT0, 0);
1280 RGE_WRITE_4(sc, RGE_TIMERINT1, 0);
1281 RGE_WRITE_4(sc, RGE_TIMERINT2, 0);
1282 RGE_WRITE_4(sc, RGE_TIMERINT3, 0);
1283
1284 num_miti =
1285 (sc->rge_type == MAC_R25B || sc->rge_type == MAC_R26) ? 32 : 64;
1286 /* Clear interrupt moderation timer. */
1287 for (i = 0; i < num_miti; i++)
1288 RGE_WRITE_4(sc, RGE_INTMITI(i), 0);
1289
1290 if (sc->rge_type == MAC_R26) {
1291 RGE_CLRBIT_1(sc, RGE_INT_CFG0,
1292 RGE_INT_CFG0_TIMEOUT_BYPASS | RGE_INT_CFG0_RDU_BYPASS_8126 |
1293 RGE_INT_CFG0_MITIGATION_BYPASS);
1294 RGE_WRITE_2(sc, RGE_INT_CFG1, 0);
1295 }
1296
1297 RGE_MAC_SETBIT(sc, 0xc0ac, 0x1f80);
1298
1299 rge_write_mac_ocp(sc, 0xe098, 0xc302);
1300
1301 RGE_MAC_CLRBIT(sc, 0xe032, 0x0003);
1302 val = rge_read_csi(sc, 0x98) & ~0x0000ff00;
1303 rge_write_csi(sc, 0x98, val);
1304
1305 if (sc->rge_type == MAC_R25D) {
1306 val = rge_read_mac_ocp(sc, 0xe092) & ~0x00ff;
1307 rge_write_mac_ocp(sc, 0xe092, val | 0x0008);
1308 } else
1309 RGE_MAC_CLRBIT(sc, 0xe092, 0x00ff);
1310
1311 /* Enable/disable HW VLAN tagging based on enabled capability */
1312 if ((if_getcapabilities(sc->sc_ifp) & IFCAP_VLAN_HWTAGGING) != 0)
1313 RGE_SETBIT_4(sc, RGE_RXCFG, RGE_RXCFG_VLANSTRIP);
1314 else
1315 RGE_CLRBIT_4(sc, RGE_RXCFG, RGE_RXCFG_VLANSTRIP);
1316
1317 /* Enable/disable RX checksum based on enabled capability */
1318 if ((if_getcapenable(sc->sc_ifp) & IFCAP_RXCSUM) != 0)
1319 RGE_SETBIT_2(sc, RGE_CPLUSCMD, RGE_CPLUSCMD_RXCSUM);
1320 else
1321 RGE_CLRBIT_2(sc, RGE_CPLUSCMD, RGE_CPLUSCMD_RXCSUM);
1322 RGE_READ_2(sc, RGE_CPLUSCMD);
1323
1324 /* Set Maximum frame size. */
1325 RGE_WRITE_2(sc, RGE_RXMAXSIZE, RGE_JUMBO_FRAMELEN);
1326
1327 /* Disable RXDV gate. */
1328 RGE_CLRBIT_1(sc, RGE_PPSW, 0x08);
1329 DELAY(2000);
1330
1331 /* Program promiscuous mode and multicast filters. */
1332 rge_iff_locked(sc);
1333
1334 if (sc->rge_type == MAC_R27)
1335 RGE_CLRBIT_1(sc, RGE_RADMFIFO_PROTECT, 0x2001);
1336
1337 rge_disable_aspm_clkreq(sc);
1338
1339 RGE_CLRBIT_1(sc, RGE_EECMD, RGE_EECMD_WRITECFG);
1340 DELAY(10);
1341
1342 rge_ifmedia_upd(sc->sc_ifp);
1343
1344 /* Enable transmit and receive. */
1345 RGE_WRITE_1(sc, RGE_CMD, RGE_CMD_TXENB | RGE_CMD_RXENB);
1346
1347 /* Enable interrupts. */
1348 rge_setup_intr(sc, RGE_IMTYPE_SIM);
1349
1350 if_setdrvflagbits(sc->sc_ifp, IFF_DRV_RUNNING, 0);
1351 if_setdrvflagbits(sc->sc_ifp, 0, IFF_DRV_OACTIVE);
1352
1353 callout_reset(&sc->sc_timeout, hz, rge_tick, sc);
1354
1355 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: init completed!\n", __func__);
1356
1357 /* Unblock transmit when we release the lock */
1358 sc->sc_stopped = false;
1359 }
1360
1361 /*
1362 * @brief Stop the adapter and free any mbufs allocated to the RX and TX lists.
1363 *
1364 * Must be called with the driver lock held.
1365 */
1366 void
rge_stop_locked(struct rge_softc * sc)1367 rge_stop_locked(struct rge_softc *sc)
1368 {
1369 struct rge_queues *q = sc->sc_queues;
1370 int i;
1371
1372 RGE_ASSERT_LOCKED(sc);
1373
1374 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: called!\n", __func__);
1375
1376 callout_stop(&sc->sc_timeout);
1377
1378 /* Stop pending TX submissions */
1379 sc->sc_stopped = true;
1380
1381 if_setdrvflagbits(sc->sc_ifp, 0, IFF_DRV_RUNNING);
1382 sc->rge_timerintr = 0;
1383 sc->sc_watchdog = 0;
1384
1385 RGE_CLRBIT_4(sc, RGE_RXCFG, RGE_RXCFG_ALLPHYS | RGE_RXCFG_INDIV |
1386 RGE_RXCFG_MULTI | RGE_RXCFG_BROAD | RGE_RXCFG_RUNT |
1387 RGE_RXCFG_ERRPKT);
1388
1389 rge_hw_reset(sc);
1390
1391 RGE_MAC_CLRBIT(sc, 0xc0ac, 0x1f80);
1392
1393 if_setdrvflagbits(sc->sc_ifp, 0, IFF_DRV_OACTIVE);
1394
1395 if (q->q_rx.rge_head != NULL) {
1396 m_freem(q->q_rx.rge_head);
1397 q->q_rx.rge_head = NULL;
1398 q->q_rx.rge_tail = &q->q_rx.rge_head;
1399 }
1400
1401 /* Free the TX list buffers. */
1402 for (i = 0; i < RGE_TX_LIST_CNT; i++) {
1403 if (q->q_tx.rge_txq[i].txq_mbuf != NULL) {
1404 bus_dmamap_unload(sc->sc_dmat_tx_buf,
1405 q->q_tx.rge_txq[i].txq_dmamap);
1406 m_freem(q->q_tx.rge_txq[i].txq_mbuf);
1407 q->q_tx.rge_txq[i].txq_mbuf = NULL;
1408 }
1409 }
1410
1411 /* Free the RX list buffers. */
1412 for (i = 0; i < RGE_RX_LIST_CNT; i++) {
1413 if (q->q_rx.rge_rxq[i].rxq_mbuf != NULL) {
1414 bus_dmamap_unload(sc->sc_dmat_rx_buf,
1415 q->q_rx.rge_rxq[i].rxq_dmamap);
1416 m_freem(q->q_rx.rge_rxq[i].rxq_mbuf);
1417 q->q_rx.rge_rxq[i].rxq_mbuf = NULL;
1418 }
1419 }
1420
1421 /* Free pending TX frames */
1422 /* TODO: should be per TX queue */
1423 rge_txq_flush_mbufs(sc);
1424 }
1425
1426 /*
1427 * Set media options.
1428 */
1429 static int
rge_ifmedia_upd(if_t ifp)1430 rge_ifmedia_upd(if_t ifp)
1431 {
1432 struct rge_softc *sc = if_getsoftc(ifp);
1433 struct ifmedia *ifm = &sc->sc_media;
1434 int anar, gig, val;
1435
1436 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1437 return (EINVAL);
1438
1439 /* Disable Gigabit Lite. */
1440 RGE_PHY_CLRBIT(sc, 0xa428, 0x0200);
1441 RGE_PHY_CLRBIT(sc, 0xa5ea, 0x0001);
1442 if (sc->rge_type == MAC_R26 || sc->rge_type == MAC_R27)
1443 RGE_PHY_CLRBIT(sc, 0xa5ea, 0x0007);
1444
1445 val = rge_read_phy_ocp(sc, 0xa5d4);
1446 switch (sc->rge_type) {
1447 case MAC_R27:
1448 val &= ~RGE_ADV_10000TFDX;
1449 /* fallthrough */
1450 case MAC_R26:
1451 val &= ~RGE_ADV_5000TFDX;
1452 /* fallthrough */
1453 default:
1454 val &= ~RGE_ADV_2500TFDX;
1455 break;
1456 }
1457
1458 anar = ANAR_TX_FD | ANAR_TX | ANAR_10_FD | ANAR_10;
1459 gig = GTCR_ADV_1000TFDX | GTCR_ADV_1000THDX;
1460
1461 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1462 case IFM_AUTO:
1463 val |= RGE_ADV_2500TFDX;
1464 if (sc->rge_type == MAC_R26)
1465 val |= RGE_ADV_5000TFDX;
1466 else if (sc->rge_type == MAC_R27)
1467 val |= RGE_ADV_5000TFDX | RGE_ADV_10000TFDX;
1468 break;
1469 case IFM_10G_T:
1470 val |= RGE_ADV_10000TFDX;
1471 if_setbaudrate(ifp, IF_Gbps(10));
1472 break;
1473 case IFM_5000_T:
1474 val |= RGE_ADV_5000TFDX;
1475 if_setbaudrate(ifp, IF_Gbps(5));
1476 break;
1477 case IFM_2500_T:
1478 val |= RGE_ADV_2500TFDX;
1479 if_setbaudrate(ifp, IF_Mbps(2500));
1480 break;
1481 case IFM_1000_T:
1482 if_setbaudrate(ifp, IF_Gbps(1));
1483 break;
1484 case IFM_100_TX:
1485 gig = rge_read_phy(sc, 0, MII_100T2CR) &
1486 ~(GTCR_ADV_1000TFDX | GTCR_ADV_1000THDX);
1487 anar = ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) ?
1488 ANAR_TX | ANAR_TX_FD | ANAR_10_FD | ANAR_10 :
1489 ANAR_TX | ANAR_10_FD | ANAR_10;
1490 if_setbaudrate(ifp, IF_Mbps(100));
1491 break;
1492 case IFM_10_T:
1493 gig = rge_read_phy(sc, 0, MII_100T2CR) &
1494 ~(GTCR_ADV_1000TFDX | GTCR_ADV_1000THDX);
1495 anar = ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) ?
1496 ANAR_10_FD | ANAR_10 : ANAR_10;
1497 if_setbaudrate(ifp, IF_Mbps(10));
1498 break;
1499 default:
1500 RGE_PRINT_ERROR(sc, "unsupported media type\n");
1501 return (EINVAL);
1502 }
1503
1504 rge_write_phy(sc, 0, MII_ANAR, anar | ANAR_PAUSE_ASYM | ANAR_FC);
1505 rge_write_phy(sc, 0, MII_100T2CR, gig);
1506 rge_write_phy_ocp(sc, 0xa5d4, val);
1507 rge_write_phy(sc, 0, MII_BMCR, BMCR_RESET | BMCR_AUTOEN |
1508 BMCR_STARTNEG);
1509
1510 return (0);
1511 }
1512
1513 /*
1514 * Report current media status.
1515 */
1516 static void
rge_ifmedia_sts(if_t ifp,struct ifmediareq * ifmr)1517 rge_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr)
1518 {
1519 struct rge_softc *sc = if_getsoftc(ifp);
1520 uint16_t status = 0;
1521
1522 ifmr->ifm_status = IFM_AVALID;
1523 ifmr->ifm_active = IFM_ETHER;
1524
1525 if (rge_get_link_status(sc)) {
1526 ifmr->ifm_status |= IFM_ACTIVE;
1527
1528 status = RGE_READ_2(sc, RGE_PHYSTAT);
1529 if ((status & RGE_PHYSTAT_FDX) ||
1530 (status & (RGE_PHYSTAT_1000MBPS | RGE_PHYSTAT_2500MBPS |
1531 RGE_PHYSTAT_5000MBPS | RGE_PHYSTAT_10000MBPS)))
1532 ifmr->ifm_active |= IFM_FDX;
1533 else
1534 ifmr->ifm_active |= IFM_HDX;
1535
1536 if (status & RGE_PHYSTAT_10MBPS)
1537 ifmr->ifm_active |= IFM_10_T;
1538 else if (status & RGE_PHYSTAT_100MBPS)
1539 ifmr->ifm_active |= IFM_100_TX;
1540 else if (status & RGE_PHYSTAT_1000MBPS)
1541 ifmr->ifm_active |= IFM_1000_T;
1542 else if (status & RGE_PHYSTAT_2500MBPS)
1543 ifmr->ifm_active |= IFM_2500_T;
1544 else if (status & RGE_PHYSTAT_5000MBPS)
1545 ifmr->ifm_active |= IFM_5000_T;
1546 else if (status & RGE_PHYSTAT_5000MBPS)
1547 ifmr->ifm_active |= IFM_5000_T;
1548 else if (status & RGE_PHYSTAT_10000MBPS)
1549 ifmr->ifm_active |= IFM_10G_T;
1550 }
1551 }
1552
1553 /**
1554 * @brief callback to load/populate a single physical address
1555 */
1556 static void
rge_dma_load_cb(void * arg,bus_dma_segment_t * segs,int nsegs,int error)1557 rge_dma_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1558 {
1559 bus_addr_t *paddr = (bus_addr_t *) arg;
1560
1561 *paddr = 0;
1562
1563 if (error) {
1564 printf("%s: error! (%d)\n", __func__, error);
1565 *paddr = 0;
1566 return;
1567 }
1568
1569 if (nsegs != 1) {
1570 printf("%s: too many segs (got %d)\n", __func__, nsegs);
1571 *paddr = 0;
1572 return;
1573 }
1574
1575 *paddr = segs[0].ds_addr;
1576 }
1577
1578 /**
1579 * @brief Allocate memory for RX/TX rings.
1580 *
1581 * Called with the driver lock NOT held.
1582 */
1583 static int
rge_allocmem(struct rge_softc * sc)1584 rge_allocmem(struct rge_softc *sc)
1585 {
1586 struct rge_queues *q = sc->sc_queues;
1587 int error;
1588 int i;
1589
1590 RGE_ASSERT_UNLOCKED(sc);
1591
1592 /* Allocate DMA'able memory for the TX ring. */
1593 error = bus_dmamem_alloc(sc->sc_dmat_tx_desc,
1594 (void **) &q->q_tx.rge_tx_list,
1595 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
1596 &q->q_tx.rge_tx_list_map);
1597 if (error) {
1598 RGE_PRINT_ERROR(sc, "%s: error (alloc tx_list.map) (%d)\n",
1599 __func__, error);
1600 goto error;
1601 }
1602
1603 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: tx_list=%p\n", __func__,
1604 q->q_tx.rge_tx_list);
1605 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: tx_list_map=%p\n", __func__,
1606 q->q_tx.rge_tx_list_map);
1607
1608 /* Load the map for the TX ring. */
1609 error = bus_dmamap_load(sc->sc_dmat_tx_desc,
1610 q->q_tx.rge_tx_list_map,
1611 q->q_tx.rge_tx_list,
1612 RGE_TX_LIST_SZ,
1613 rge_dma_load_cb,
1614 (void *) &q->q_tx.rge_tx_list_paddr,
1615 BUS_DMA_NOWAIT);
1616
1617 if ((error != 0) || (q->q_tx.rge_tx_list_paddr == 0)) {
1618 RGE_PRINT_ERROR(sc, "%s: error (load tx_list.map) (%d)\n",
1619 __func__, error);
1620 goto error;
1621 }
1622
1623 /* Create DMA maps for TX buffers. */
1624 for (i = 0; i < RGE_TX_LIST_CNT; i++) {
1625 error = bus_dmamap_create(sc->sc_dmat_tx_buf,
1626 0, &q->q_tx.rge_txq[i].txq_dmamap);
1627 if (error) {
1628 RGE_PRINT_ERROR(sc,
1629 "can't create DMA map for TX (%d)\n", error);
1630 goto error;
1631 }
1632 }
1633
1634 /* Allocate DMA'able memory for the RX ring. */
1635 error = bus_dmamem_alloc(sc->sc_dmat_rx_desc,
1636 (void **) &q->q_rx.rge_rx_list,
1637 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
1638 &q->q_rx.rge_rx_list_map);
1639 if (error) {
1640 RGE_PRINT_ERROR(sc, "%s: error (alloc rx_list.map) (%d)\n",
1641 __func__, error);
1642 goto error;
1643 }
1644
1645 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: rx_list=%p\n", __func__,
1646 q->q_rx.rge_rx_list);
1647 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: rx_list_map=%p\n", __func__,
1648 q->q_rx.rge_rx_list_map);
1649
1650 /* Load the map for the RX ring. */
1651 error = bus_dmamap_load(sc->sc_dmat_rx_desc,
1652 q->q_rx.rge_rx_list_map,
1653 q->q_rx.rge_rx_list,
1654 RGE_RX_LIST_SZ,
1655 rge_dma_load_cb,
1656 (void *) &q->q_rx.rge_rx_list_paddr,
1657 BUS_DMA_NOWAIT);
1658
1659 if ((error != 0) || (q->q_rx.rge_rx_list_paddr == 0)) {
1660 RGE_PRINT_ERROR(sc, "%s: error (load rx_list.map) (%d)\n",
1661 __func__, error);
1662 goto error;
1663 }
1664
1665 /* Create DMA maps for RX buffers. */
1666 for (i = 0; i < RGE_RX_LIST_CNT; i++) {
1667 error = bus_dmamap_create(sc->sc_dmat_rx_buf,
1668 0, &q->q_rx.rge_rxq[i].rxq_dmamap);
1669 if (error) {
1670 RGE_PRINT_ERROR(sc,
1671 "can't create DMA map for RX (%d)\n", error);
1672 goto error;
1673 }
1674 }
1675
1676 return (0);
1677 error:
1678
1679 rge_freemem(sc);
1680
1681 return (error);
1682 }
1683
1684 /**
1685 * @brief Allocate memory for MAC stats.
1686 *
1687 * Called with the driver lock NOT held.
1688 */
1689 static int
rge_alloc_stats_mem(struct rge_softc * sc)1690 rge_alloc_stats_mem(struct rge_softc *sc)
1691 {
1692 struct rge_mac_stats *ss = &sc->sc_mac_stats;
1693 int error;
1694
1695 RGE_ASSERT_UNLOCKED(sc);
1696
1697 /* Allocate DMA'able memory for the stats buffer. */
1698 error = bus_dmamem_alloc(sc->sc_dmat_stats_buf,
1699 (void **) &ss->stats, BUS_DMA_WAITOK | BUS_DMA_ZERO,
1700 &ss->map);
1701 if (error) {
1702 RGE_PRINT_ERROR(sc, "%s: error (alloc stats) (%d)\n",
1703 __func__, error);
1704 goto error;
1705 }
1706
1707 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: stats=%p\n", __func__, ss->stats);
1708 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: map=%p\n", __func__, ss->map);
1709
1710 /* Load the map for the TX ring. */
1711 error = bus_dmamap_load(sc->sc_dmat_stats_buf,
1712 ss->map,
1713 ss->stats,
1714 RGE_STATS_BUF_SIZE,
1715 rge_dma_load_cb,
1716 (void *) &ss->paddr,
1717 BUS_DMA_NOWAIT);
1718
1719 if ((error != 0) || (ss->paddr == 0)) {
1720 RGE_PRINT_ERROR(sc, "%s: error (load stats.map) (%d)\n",
1721 __func__, error);
1722 if (error == 0)
1723 error = ENXIO;
1724 goto error;
1725 }
1726
1727 return (0);
1728
1729 error:
1730 rge_free_stats_mem(sc);
1731
1732 return (error);
1733 }
1734
1735
1736 /**
1737 * @brief Free the TX/RX DMA buffers and mbufs.
1738 *
1739 * Called with the driver lock NOT held.
1740 */
1741 static int
rge_freemem(struct rge_softc * sc)1742 rge_freemem(struct rge_softc *sc)
1743 {
1744 struct rge_queues *q = sc->sc_queues;
1745 int i;
1746
1747 RGE_ASSERT_UNLOCKED(sc);
1748
1749 /* TX desc */
1750 bus_dmamap_unload(sc->sc_dmat_tx_desc, q->q_tx.rge_tx_list_map);
1751 if (q->q_tx.rge_tx_list != NULL)
1752 bus_dmamem_free(sc->sc_dmat_tx_desc, q->q_tx.rge_tx_list,
1753 q->q_tx.rge_tx_list_map);
1754 memset(&q->q_tx, 0, sizeof(q->q_tx));
1755
1756 /* TX buf */
1757 for (i = 0; i < RGE_TX_LIST_CNT; i++) {
1758 struct rge_txq *tx = &q->q_tx.rge_txq[i];
1759
1760 /* unmap/free mbuf if it's still alloc'ed and mapped */
1761 if (tx->txq_mbuf != NULL) {
1762 static bool do_warning = false;
1763
1764 if (do_warning == false) {
1765 RGE_PRINT_ERROR(sc,
1766 "%s: TX mbuf should've been freed!\n",
1767 __func__);
1768 do_warning = true;
1769 }
1770 if (tx->txq_dmamap != NULL) {
1771 bus_dmamap_sync(sc->sc_dmat_tx_buf,
1772 tx->txq_dmamap, BUS_DMASYNC_POSTREAD);
1773 bus_dmamap_unload(sc->sc_dmat_tx_buf,
1774 tx->txq_dmamap);
1775 }
1776 m_free(tx->txq_mbuf);
1777 tx->txq_mbuf = NULL;
1778 }
1779
1780 /* Destroy the dmamap if it's allocated */
1781 if (tx->txq_dmamap != NULL) {
1782 bus_dmamap_destroy(sc->sc_dmat_tx_buf, tx->txq_dmamap);
1783 tx->txq_dmamap = NULL;
1784 }
1785 }
1786
1787 /* RX desc */
1788 bus_dmamap_unload(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map);
1789 if (q->q_rx.rge_rx_list != 0)
1790 bus_dmamem_free(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list,
1791 q->q_rx.rge_rx_list_map);
1792 memset(&q->q_rx, 0, sizeof(q->q_tx));
1793
1794 /* RX buf */
1795 for (i = 0; i < RGE_RX_LIST_CNT; i++) {
1796 struct rge_rxq *rx = &q->q_rx.rge_rxq[i];
1797
1798 /* unmap/free mbuf if it's still alloc'ed and mapped */
1799 if (rx->rxq_mbuf != NULL) {
1800 if (rx->rxq_dmamap != NULL) {
1801 bus_dmamap_sync(sc->sc_dmat_rx_buf,
1802 rx->rxq_dmamap, BUS_DMASYNC_POSTREAD);
1803 bus_dmamap_unload(sc->sc_dmat_rx_buf,
1804 rx->rxq_dmamap);
1805 }
1806 m_free(rx->rxq_mbuf);
1807 rx->rxq_mbuf = NULL;
1808 }
1809
1810 /* Destroy the dmamap if it's allocated */
1811 if (rx->rxq_dmamap != NULL) {
1812 bus_dmamap_destroy(sc->sc_dmat_rx_buf, rx->rxq_dmamap);
1813 rx->rxq_dmamap = NULL;
1814 }
1815 }
1816
1817 return (0);
1818 }
1819
1820 /**
1821 * @brief Free the stats memory.
1822 *
1823 * Called with the driver lock NOT held.
1824 */
1825 static int
rge_free_stats_mem(struct rge_softc * sc)1826 rge_free_stats_mem(struct rge_softc *sc)
1827 {
1828 struct rge_mac_stats *ss = &sc->sc_mac_stats;
1829
1830 RGE_ASSERT_UNLOCKED(sc);
1831
1832 bus_dmamap_unload(sc->sc_dmat_stats_buf, ss->map);
1833 if (ss->stats != NULL)
1834 bus_dmamem_free(sc->sc_dmat_stats_buf, ss->stats, ss->map);
1835 memset(ss, 0, sizeof(*ss));
1836 return (0);
1837 }
1838
1839 static uint32_t
rx_ring_space(struct rge_queues * q)1840 rx_ring_space(struct rge_queues *q)
1841 {
1842 uint32_t prod, cons;
1843 uint32_t ret;
1844
1845 RGE_ASSERT_LOCKED(q->q_sc);
1846
1847 prod = q->q_rx.rge_rxq_prodidx;
1848 cons = q->q_rx.rge_rxq_considx;
1849
1850 ret = (cons + RGE_RX_LIST_CNT - prod - 1) % RGE_RX_LIST_CNT + 1;
1851
1852 if (ret > RGE_RX_LIST_CNT)
1853 return RGE_RX_LIST_CNT;
1854
1855 return (ret);
1856 }
1857
1858 /*
1859 * Initialize the RX descriptor and attach an mbuf cluster at the given offset.
1860 *
1861 * Note: this relies on the rxr ring buffer abstraction to not
1862 * over-fill the RX ring. For FreeBSD we'll need to use the
1863 * prod/cons RX indexes to know how much RX ring space to
1864 * populate.
1865 *
1866 * This routine will increment the producer index if successful.
1867 *
1868 * This must be called with the driver lock held.
1869 */
1870 static int
rge_newbuf(struct rge_queues * q)1871 rge_newbuf(struct rge_queues *q)
1872 {
1873 struct rge_softc *sc = q->q_sc;
1874 struct mbuf *m;
1875 struct rge_rx_desc *r;
1876 struct rge_rxq *rxq;
1877 bus_dmamap_t rxmap;
1878 bus_dma_segment_t seg[1];
1879 uint32_t cmdsts;
1880 int nsegs;
1881 uint32_t idx;
1882
1883 RGE_ASSERT_LOCKED(q->q_sc);
1884
1885 /*
1886 * Verify we have enough space in the ring; error out
1887 * if we do not.
1888 */
1889 if (rx_ring_space(q) == 0)
1890 return (ENOBUFS);
1891
1892 idx = q->q_rx.rge_rxq_prodidx;
1893 rxq = &q->q_rx.rge_rxq[idx];
1894 rxmap = rxq->rxq_dmamap;
1895
1896 /*
1897 * If we already have an mbuf here then something messed up;
1898 * exit out as the hardware may be DMAing to it.
1899 */
1900 if (rxq->rxq_mbuf != NULL) {
1901 RGE_PRINT_ERROR(sc,
1902 "%s: RX ring slot %d already has an mbuf?\n", __func__,
1903 idx);
1904 return (ENOBUFS);
1905 }
1906
1907 /* Allocate single buffer backed mbuf of MCLBYTES */
1908 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1909 if (m == NULL)
1910 return (ENOBUFS);
1911
1912 m->m_len = m->m_pkthdr.len = MCLBYTES;
1913
1914 nsegs = 1;
1915 if (bus_dmamap_load_mbuf_sg(sc->sc_dmat_rx_buf, rxmap, m, seg, &nsegs,
1916 BUS_DMA_NOWAIT)) {
1917 m_freem(m);
1918 return (ENOBUFS);
1919 }
1920
1921 /*
1922 * Make sure any changes made to the buffer have been flushed to host
1923 * memory.
1924 */
1925 bus_dmamap_sync(sc->sc_dmat_rx_buf, rxmap,
1926 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1927
1928 /*
1929 * Map the segment into RX descriptors. Note that this
1930 * only currently supports a single segment per mbuf;
1931 * the call to load_mbuf_sg above specified a single segment.
1932 */
1933 r = &q->q_rx.rge_rx_list[idx];
1934
1935 rxq->rxq_mbuf = m;
1936
1937 cmdsts = seg[0].ds_len; /* XXX how big is this field in the descriptor? */
1938 if (idx == RGE_RX_LIST_CNT - 1)
1939 cmdsts |= RGE_RDCMDSTS_EOR;
1940
1941 /*
1942 * Configure the DMA pointer and config, but don't hand
1943 * it yet to the hardware.
1944 */
1945 r->hi_qword1.rx_qword4.rge_cmdsts = htole32(cmdsts);
1946 r->hi_qword1.rx_qword4.rge_extsts = htole32(0);
1947 r->hi_qword0.rge_addr = htole64(seg[0].ds_addr);
1948 wmb();
1949
1950 /*
1951 * Mark the specific descriptor slot as "this descriptor is now
1952 * owned by the hardware", which when the hardware next sees
1953 * this, it'll continue RX DMA.
1954 */
1955 cmdsts |= RGE_RDCMDSTS_OWN;
1956 r->hi_qword1.rx_qword4.rge_cmdsts = htole32(cmdsts);
1957 wmb();
1958
1959 /*
1960 * At this point the hope is the whole ring is now updated and
1961 * consistent; if the hardware was waiting for a descriptor to be
1962 * ready to write into then it should be ready here.
1963 */
1964
1965 RGE_DPRINTF(sc, RGE_DEBUG_RECV_DESC,
1966 "%s: [%d]: m=%p, m_data=%p, m_len=%ju, phys=0x%jx len %ju, "
1967 "desc=0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1968 __func__,
1969 idx,
1970 m,
1971 m->m_data,
1972 (uintmax_t) m->m_len,
1973 (uintmax_t) seg[0].ds_addr,
1974 (uintmax_t) seg[0].ds_len,
1975 ((uint32_t *) r)[0],
1976 ((uint32_t *) r)[1],
1977 ((uint32_t *) r)[2],
1978 ((uint32_t *) r)[3],
1979 ((uint32_t *) r)[4],
1980 ((uint32_t *) r)[5],
1981 ((uint32_t *) r)[6],
1982 ((uint32_t *) r)[7]);
1983
1984 q->q_rx.rge_rxq_prodidx = RGE_NEXT_RX_DESC(idx);
1985
1986 return (0);
1987 }
1988
1989 static void
rge_rx_list_init(struct rge_queues * q)1990 rge_rx_list_init(struct rge_queues *q)
1991 {
1992 memset(q->q_rx.rge_rx_list, 0, RGE_RX_LIST_SZ);
1993
1994 RGE_ASSERT_LOCKED(q->q_sc);
1995
1996 q->q_rx.rge_rxq_prodidx = q->q_rx.rge_rxq_considx = 0;
1997 q->q_rx.rge_head = NULL;
1998 q->q_rx.rge_tail = &q->q_rx.rge_head;
1999
2000 RGE_DPRINTF(q->q_sc, RGE_DEBUG_SETUP, "%s: rx_list=%p\n", __func__,
2001 q->q_rx.rge_rx_list);
2002
2003 rge_fill_rx_ring(q);
2004 }
2005
2006 /**
2007 * @brief Fill / refill the RX ring as needed.
2008 *
2009 * Refill the RX ring with one less than the total descriptors needed.
2010 * This makes the check in rge_rxeof() easier - it can just check
2011 * descriptors from cons -> prod and bail once it hits prod.
2012 * If the whole ring is filled then cons == prod, and that shortcut
2013 * fails.
2014 *
2015 * This must be called with the driver lock held.
2016 */
2017 static void
rge_fill_rx_ring(struct rge_queues * q)2018 rge_fill_rx_ring(struct rge_queues *q)
2019 {
2020 struct rge_softc *sc = q->q_sc;
2021 uint32_t count, i, prod, cons;
2022
2023 RGE_ASSERT_LOCKED(q->q_sc);
2024
2025 prod = q->q_rx.rge_rxq_prodidx;
2026 cons = q->q_rx.rge_rxq_considx;
2027 count = rx_ring_space(q);
2028
2029 /* Fill to count-1; bail if we don't have the space */
2030 if (count <= 1)
2031 return;
2032 count--;
2033
2034 RGE_DPRINTF(sc, RGE_DEBUG_RECV_DESC, "%s: prod=%u, cons=%u, space=%u\n",
2035 __func__, prod, cons, count);
2036
2037 /* Make sure device->host changes are visible */
2038 bus_dmamap_sync(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map,
2039 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2040
2041 for (i = 0; i < count; i++) {
2042 if (rge_newbuf(q))
2043 break;
2044 }
2045
2046 /* Make changes visible to the device */
2047 bus_dmamap_sync(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map,
2048 BUS_DMASYNC_PREWRITE);
2049 }
2050
2051 static void
rge_tx_list_init(struct rge_queues * q)2052 rge_tx_list_init(struct rge_queues *q)
2053 {
2054 struct rge_softc *sc = q->q_sc;
2055 struct rge_tx_desc *d;
2056 int i;
2057
2058 RGE_ASSERT_LOCKED(q->q_sc);
2059
2060 memset(q->q_tx.rge_tx_list, 0, RGE_TX_LIST_SZ);
2061
2062 for (i = 0; i < RGE_TX_LIST_CNT; i++)
2063 q->q_tx.rge_txq[i].txq_mbuf = NULL;
2064
2065 d = &q->q_tx.rge_tx_list[RGE_TX_LIST_CNT - 1];
2066 d->rge_cmdsts = htole32(RGE_TDCMDSTS_EOR);
2067
2068 bus_dmamap_sync(sc->sc_dmat_tx_desc, q->q_tx.rge_tx_list_map,
2069 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2070 wmb();
2071
2072 q->q_tx.rge_txq_prodidx = q->q_tx.rge_txq_considx = 0;
2073
2074 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: rx_list=%p\n", __func__,
2075 q->q_tx.rge_tx_list);
2076 }
2077
2078 int
rge_rxeof(struct rge_queues * q,struct mbufq * mq)2079 rge_rxeof(struct rge_queues *q, struct mbufq *mq)
2080 {
2081 struct rge_softc *sc = q->q_sc;
2082 struct mbuf *m;
2083 struct rge_rx_desc *cur_rx;
2084 struct rge_rxq *rxq;
2085 uint32_t rxstat, extsts;
2086 int i, mlen, rx = 0;
2087 int cons, prod;
2088 int maxpkt = 16; /* XXX TODO: make this a tunable */
2089 bool check_hwcsum;
2090
2091 check_hwcsum = ((if_getcapenable(sc->sc_ifp) & IFCAP_RXCSUM) != 0);
2092
2093 RGE_ASSERT_LOCKED(sc);
2094
2095 sc->sc_drv_stats.rxeof_cnt++;
2096
2097 RGE_DPRINTF(sc, RGE_DEBUG_INTR, "%s; called\n", __func__);
2098
2099 /* Note: if_re is POSTREAD/WRITE, rge is only POSTWRITE */
2100 bus_dmamap_sync(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map,
2101 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2102
2103 prod = q->q_rx.rge_rxq_prodidx;
2104
2105 /*
2106 * Loop around until we've run out of active descriptors to check
2107 * or maxpkt has been reached.
2108 */
2109 for (i = cons = q->q_rx.rge_rxq_considx;
2110 maxpkt > 0 && i != prod;
2111 i = RGE_NEXT_RX_DESC(i)) {
2112 /* break out of loop if we're not running */
2113 if ((if_getdrvflags(sc->sc_ifp) & IFF_DRV_RUNNING) == 0)
2114 break;
2115
2116 /* get the current rx descriptor to check descriptor status */
2117 cur_rx = &q->q_rx.rge_rx_list[i];
2118 rxstat = le32toh(cur_rx->hi_qword1.rx_qword4.rge_cmdsts);
2119 if ((rxstat & RGE_RDCMDSTS_OWN) != 0) {
2120 break;
2121 }
2122
2123 /* Ensure everything else has been DMAed */
2124 rmb();
2125
2126 /* Get the current rx buffer, sync */
2127 rxq = &q->q_rx.rge_rxq[i];
2128
2129 /* Ensure any device updates are now visible in host memory */
2130 bus_dmamap_sync(sc->sc_dmat_rx_buf, rxq->rxq_dmamap,
2131 BUS_DMASYNC_POSTREAD);
2132
2133 /* Unload the DMA map, we are done with it here */
2134 bus_dmamap_unload(sc->sc_dmat_rx_buf, rxq->rxq_dmamap);
2135 m = rxq->rxq_mbuf;
2136 rxq->rxq_mbuf = NULL;
2137
2138 rx = 1;
2139
2140 RGE_DPRINTF(sc, RGE_DEBUG_RECV_DESC,
2141 "%s: RX: [%d]: m=%p, m_data=%p, m_len=%ju, "
2142 "desc=0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
2143 __func__,
2144 i,
2145 m,
2146 m->m_data,
2147 (uintmax_t) m->m_len,
2148 ((uint32_t *) cur_rx)[0],
2149 ((uint32_t *) cur_rx)[1],
2150 ((uint32_t *) cur_rx)[2],
2151 ((uint32_t *) cur_rx)[3],
2152 ((uint32_t *) cur_rx)[4],
2153 ((uint32_t *) cur_rx)[5],
2154 ((uint32_t *) cur_rx)[6],
2155 ((uint32_t *) cur_rx)[7]);
2156
2157 if ((rxstat & RGE_RDCMDSTS_SOF) != 0) {
2158 if (q->q_rx.rge_head != NULL) {
2159 sc->sc_drv_stats.rx_desc_err_multidesc++;
2160 if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS,
2161 1);
2162 m_freem(q->q_rx.rge_head);
2163 q->q_rx.rge_tail = &q->q_rx.rge_head;
2164 }
2165
2166 m->m_pkthdr.len = 0;
2167 } else if (q->q_rx.rge_head == NULL) {
2168 m_freem(m);
2169 continue;
2170 } else
2171 m->m_flags &= ~M_PKTHDR;
2172
2173 *q->q_rx.rge_tail = m;
2174 q->q_rx.rge_tail = &m->m_next;
2175
2176 mlen = rxstat & RGE_RDCMDSTS_FRAGLEN;
2177 m->m_len = mlen;
2178
2179 m = q->q_rx.rge_head;
2180 m->m_pkthdr.len += mlen;
2181
2182 /* Ethernet CRC error */
2183 if (rxstat & RGE_RDCMDSTS_RXERRSUM) {
2184 sc->sc_drv_stats.rx_ether_csum_err++;
2185 if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 1);
2186 m_freem(m);
2187 q->q_rx.rge_head = NULL;
2188 q->q_rx.rge_tail = &q->q_rx.rge_head;
2189 continue;
2190 }
2191
2192 /*
2193 * This mbuf is part of a multi-descriptor frame,
2194 * so count it towards that.
2195 *
2196 * Yes, this means we won't be counting the
2197 * final descriptor/mbuf as part of a multi-descriptor
2198 * frame; if someone wishes to do that then it
2199 * shouldn't be too hard to add.
2200 */
2201 if ((rxstat & RGE_RDCMDSTS_EOF) == 0) {
2202 sc->sc_drv_stats.rx_desc_jumbo_frag++;
2203 continue;
2204 }
2205
2206 q->q_rx.rge_head = NULL;
2207 q->q_rx.rge_tail = &q->q_rx.rge_head;
2208
2209 m_adj(m, -ETHER_CRC_LEN);
2210 m->m_pkthdr.rcvif = sc->sc_ifp;
2211 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1);
2212
2213 extsts = le32toh(cur_rx->hi_qword1.rx_qword4.rge_extsts);
2214
2215 /* Check IP header checksum. */
2216 if (check_hwcsum) {
2217 /* Does it exist for IPv4? */
2218 if (extsts & RGE_RDEXTSTS_IPV4) {
2219 sc->sc_drv_stats.rx_offload_csum_ipv4_exists++;
2220 m->m_pkthdr.csum_flags |=
2221 CSUM_IP_CHECKED;
2222 }
2223 /* XXX IPv6 checksum check? */
2224
2225 if (((extsts & RGE_RDEXTSTS_IPCSUMERR) == 0)
2226 && ((extsts & RGE_RDEXTSTS_IPV4) != 0)) {
2227 sc->sc_drv_stats.rx_offload_csum_ipv4_valid++;
2228 m->m_pkthdr.csum_flags |=
2229 CSUM_IP_VALID;
2230 }
2231
2232 /* Check TCP/UDP checksum. */
2233 if ((extsts & (RGE_RDEXTSTS_IPV4 | RGE_RDEXTSTS_IPV6)) &&
2234 (extsts & RGE_RDEXTSTS_TCPPKT)) {
2235 sc->sc_drv_stats.rx_offload_csum_tcp_exists++;
2236 if ((extsts & RGE_RDEXTSTS_TCPCSUMERR) == 0) {
2237 sc->sc_drv_stats.rx_offload_csum_tcp_valid++;
2238 /* TCP checksum OK */
2239 m->m_pkthdr.csum_flags |=
2240 CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
2241 m->m_pkthdr.csum_data = 0xffff;
2242 }
2243 }
2244
2245 if ((extsts & (RGE_RDEXTSTS_IPV4 | RGE_RDEXTSTS_IPV6)) &&
2246 (extsts & RGE_RDEXTSTS_UDPPKT)) {
2247 sc->sc_drv_stats.rx_offload_csum_udp_exists++;
2248 if ((extsts & RGE_RDEXTSTS_UDPCSUMERR) == 0) {
2249 sc->sc_drv_stats.rx_offload_csum_udp_valid++;
2250 /* UDP checksum OK */
2251 m->m_pkthdr.csum_flags |=
2252 CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
2253 m->m_pkthdr.csum_data = 0xffff;
2254 }
2255 }
2256 }
2257
2258 if (extsts & RGE_RDEXTSTS_VTAG) {
2259 sc->sc_drv_stats.rx_offload_vlan_tag++;
2260 m->m_pkthdr.ether_vtag =
2261 ntohs(extsts & RGE_RDEXTSTS_VLAN_MASK);
2262 m->m_flags |= M_VLANTAG;
2263 }
2264
2265 mbufq_enqueue(mq, m);
2266
2267 maxpkt--;
2268 }
2269
2270 if (!rx)
2271 return (0);
2272
2273 /*
2274 * Make sure any device updates to the descriptor ring are
2275 * visible to the host before we continue.
2276 */
2277 bus_dmamap_sync(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map,
2278 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2279 wmb();
2280
2281 /* Update the consumer index, refill the RX ring */
2282 q->q_rx.rge_rxq_considx = i;
2283 rge_fill_rx_ring(q);
2284
2285 return (1);
2286 }
2287
2288 int
rge_txeof(struct rge_queues * q)2289 rge_txeof(struct rge_queues *q)
2290 {
2291 struct rge_softc *sc = q->q_sc;
2292 struct ifnet *ifp = sc->sc_ifp;
2293 struct rge_txq *txq;
2294 uint32_t txstat;
2295 int cons, prod, cur, idx;
2296 int free = 0, ntx = 0;
2297 int pktlen;
2298 bool is_mcast;
2299
2300 RGE_ASSERT_LOCKED(sc);
2301
2302 sc->sc_drv_stats.txeof_cnt++;
2303
2304 prod = q->q_tx.rge_txq_prodidx;
2305 cons = q->q_tx.rge_txq_considx;
2306
2307 idx = cons;
2308 while (idx != prod) {
2309 txq = &q->q_tx.rge_txq[idx];
2310 cur = txq->txq_descidx;
2311
2312 rge_tx_list_sync(sc, q, cur, 1, BUS_DMASYNC_POSTREAD);
2313 txstat = q->q_tx.rge_tx_list[cur].rge_cmdsts;
2314 rge_tx_list_sync(sc, q, cur, 1, BUS_DMASYNC_PREREAD);
2315 if ((txstat & htole32(RGE_TDCMDSTS_OWN)) != 0) {
2316 free = 2;
2317 break;
2318 }
2319
2320 bus_dmamap_sync(sc->sc_dmat_tx_buf, txq->txq_dmamap,
2321 BUS_DMASYNC_POSTWRITE);
2322 bus_dmamap_unload(sc->sc_dmat_tx_buf, txq->txq_dmamap);
2323 pktlen = txq->txq_mbuf->m_pkthdr.len;
2324 is_mcast = ((txq->txq_mbuf->m_flags & M_MCAST) != 0);
2325 m_freem(txq->txq_mbuf);
2326 txq->txq_mbuf = NULL;
2327 ntx++;
2328
2329 if ((txstat &
2330 htole32(RGE_TDCMDSTS_EXCESSCOLL | RGE_TDCMDSTS_COLL)) != 0)
2331 if_inc_counter(ifp, IFCOUNTER_COLLISIONS, 1);
2332 if ((txstat & htole32(RGE_TDCMDSTS_TXERR)) != 0)
2333 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
2334 else {
2335 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
2336 if_inc_counter(ifp, IFCOUNTER_OBYTES, pktlen);
2337 if (is_mcast)
2338 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
2339
2340 }
2341
2342 idx = RGE_NEXT_TX_DESC(cur);
2343 free = 1;
2344 }
2345
2346 /* If we didn't complete any TX descriptors then return 0 */
2347 if (free == 0)
2348 return (0);
2349
2350 if (idx >= cons) {
2351 rge_tx_list_sync(sc, q, cons, idx - cons,
2352 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2353 } else {
2354 rge_tx_list_sync(sc, q, cons, RGE_TX_LIST_CNT - cons,
2355 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2356 rge_tx_list_sync(sc, q, 0, idx,
2357 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2358 }
2359
2360 q->q_tx.rge_txq_considx = idx;
2361
2362 RGE_DPRINTF(sc, RGE_DEBUG_XMIT,
2363 "%s: handled %d frames; prod=%d, cons=%d\n", __func__,
2364 ntx, q->q_tx.rge_txq_prodidx, q->q_tx.rge_txq_considx);
2365
2366 /*
2367 * We processed the ring and hit a descriptor that was still
2368 * owned by the hardware, so there's still pending work.
2369 *
2370 * If we got to the end of the ring and there's no further
2371 * frames owned by the hardware then we can quieten the
2372 * watchdog.
2373 */
2374 if (free == 2)
2375 sc->sc_watchdog = 5;
2376 else
2377 sc->sc_watchdog = 0;
2378
2379 /*
2380 * Kick-start the transmit task just in case we have
2381 * more frames available.
2382 */
2383 taskqueue_enqueue(sc->sc_tq, &sc->sc_tx_task);
2384
2385 return (1);
2386 }
2387
2388 static u_int
rge_hash_maddr(void * arg,struct sockaddr_dl * sdl,u_int cnt)2389 rge_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
2390 {
2391 uint32_t crc, *hashes = arg;
2392
2393 // XXX TODO: validate this does addrlo? */
2394 crc = ether_crc32_be(LLADDR(sdl), ETHER_ADDR_LEN) >> 26;
2395 crc &= 0x3f;
2396
2397 if (crc < 32)
2398 hashes[0] |= (1 << crc);
2399 else
2400 hashes[1] |= (1 << (crc - 32));
2401
2402 return (1);
2403 }
2404
2405 /**
2406 * @brief Configure the RX filter and multicast filter.
2407 *
2408 * This must be called with the driver lock held.
2409 */
2410 static void
rge_iff_locked(struct rge_softc * sc)2411 rge_iff_locked(struct rge_softc *sc)
2412 {
2413 uint32_t hashes[2];
2414 uint32_t rxfilt;
2415
2416 RGE_ASSERT_LOCKED(sc);
2417
2418 rxfilt = RGE_READ_4(sc, RGE_RXCFG);
2419 rxfilt &= ~(RGE_RXCFG_ALLPHYS | RGE_RXCFG_MULTI);
2420
2421 /*
2422 * Always accept frames destined to our station address.
2423 * Always accept broadcast frames.
2424 */
2425 rxfilt |= RGE_RXCFG_INDIV | RGE_RXCFG_BROAD;
2426
2427 if ((if_getflags(sc->sc_ifp) & (IFF_PROMISC | IFF_ALLMULTI)) != 0) {
2428 rxfilt |= RGE_RXCFG_MULTI;
2429 if ((if_getflags(sc->sc_ifp) & IFF_PROMISC) != 0)
2430 rxfilt |= RGE_RXCFG_ALLPHYS;
2431 hashes[0] = hashes[1] = 0xffffffff;
2432 } else {
2433 rxfilt |= RGE_RXCFG_MULTI;
2434 /* Program new filter. */
2435 memset(hashes, 0, sizeof(hashes));
2436 if_foreach_llmaddr(sc->sc_ifp, rge_hash_maddr, &hashes);
2437 }
2438
2439 RGE_WRITE_4(sc, RGE_RXCFG, rxfilt);
2440 RGE_WRITE_4(sc, RGE_MAR0, bswap32(hashes[1]));
2441 RGE_WRITE_4(sc, RGE_MAR4, bswap32(hashes[0]));
2442 }
2443
2444 static void
rge_add_media_types(struct rge_softc * sc)2445 rge_add_media_types(struct rge_softc *sc)
2446 {
2447 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10_T, 0, NULL);
2448 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL);
2449 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_100_TX, 0, NULL);
2450 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL);
2451 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_1000_T, 0, NULL);
2452 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2453 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_2500_T, 0, NULL);
2454 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL);
2455
2456 if (sc->rge_type == MAC_R26) {
2457 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_5000_T, 0, NULL);
2458 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_5000_T | IFM_FDX,
2459 0, NULL);
2460 } else if (sc->rge_type == MAC_R27) {
2461 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10G_T, 0, NULL);
2462 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10G_T | IFM_FDX,
2463 0, NULL);
2464 }
2465 }
2466
2467 /**
2468 * @brief Deferred packet dequeue and submit.
2469 */
2470 static void
rge_tx_task(void * arg,int npending)2471 rge_tx_task(void *arg, int npending)
2472 {
2473 struct rge_softc *sc = (struct rge_softc *) arg;
2474 /* Note: for now, one queue */
2475 struct rge_queues *q = sc->sc_queues;
2476 struct mbuf *m;
2477 int ntx = 0;
2478 int idx, free, used;
2479
2480 RGE_DPRINTF(sc, RGE_DEBUG_XMIT, "%s: running\n", __func__);
2481
2482 RGE_LOCK(sc);
2483 sc->sc_drv_stats.tx_task_cnt++;
2484
2485 if (sc->sc_stopped == true) {
2486 sc->sc_watchdog = 0;
2487 RGE_UNLOCK(sc);
2488 return;
2489 }
2490
2491 /* Calculate free space. */
2492 idx = q->q_tx.rge_txq_prodidx;
2493 free = q->q_tx.rge_txq_considx;
2494 if (free <= idx)
2495 free += RGE_TX_LIST_CNT;
2496 free -= idx;
2497
2498 for (;;) {
2499 if (free < RGE_TX_NSEGS + 2) {
2500 break;
2501 }
2502
2503 /* Dequeue */
2504 m = mbufq_dequeue(&sc->sc_txq);
2505 if (m == NULL)
2506 break;
2507
2508 /* Attempt to encap */
2509 used = rge_encap(sc, q, m, idx);
2510 if (used < 0) {
2511 if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1);
2512 m_freem(m);
2513 continue;
2514 } else if (used == 0) {
2515 mbufq_prepend(&sc->sc_txq, m);
2516 break;
2517 }
2518
2519 /*
2520 * Note: mbuf is now owned by the tx ring, but we hold the
2521 * lock so it's safe to pass it up here to be copied without
2522 * worrying the TX task will run and dequeue/free it before
2523 * we get a shot at it.
2524 */
2525 ETHER_BPF_MTAP(sc->sc_ifp, m);
2526
2527 /* Update free/idx pointers */
2528 free -= used;
2529 idx += used;
2530 if (idx >= RGE_TX_LIST_CNT)
2531 idx -= RGE_TX_LIST_CNT;
2532
2533 ntx++;
2534 }
2535
2536 /* Ok, did we queue anything? If so, poke the hardware */
2537 if (ntx > 0) {
2538 q->q_tx.rge_txq_prodidx = idx;
2539 sc->sc_watchdog = 5;
2540 RGE_WRITE_2(sc, RGE_TXSTART, RGE_TXSTART_START);
2541 }
2542
2543 RGE_DPRINTF(sc, RGE_DEBUG_XMIT,
2544 "%s: handled %d frames; prod=%d, cons=%d\n", __func__,
2545 ntx, q->q_tx.rge_txq_prodidx, q->q_tx.rge_txq_considx);
2546
2547 RGE_UNLOCK(sc);
2548 }
2549
2550 /**
2551 * @brief Called by the sc_timeout callout.
2552 *
2553 * This is called by the callout code with the driver lock held.
2554 */
2555 void
rge_tick(void * arg)2556 rge_tick(void *arg)
2557 {
2558 struct rge_softc *sc = arg;
2559
2560 RGE_ASSERT_LOCKED(sc);
2561
2562 rge_link_state(sc);
2563
2564 /*
2565 * Since we don't have any other place yet to trigger/test this,
2566 * let's do it here every second and just bite the driver
2567 * blocking for a little bit whilst it happens.
2568 */
2569 if ((if_getdrvflags(sc->sc_ifp) & IFF_DRV_RUNNING) != 0)
2570 rge_hw_mac_stats_fetch(sc, &sc->sc_mac_stats.lcl_stats);
2571
2572 /*
2573 * Handle the TX watchdog.
2574 */
2575 if (sc->sc_watchdog > 0) {
2576 sc->sc_watchdog--;
2577 if (sc->sc_watchdog == 0) {
2578 RGE_PRINT_ERROR(sc, "TX timeout (watchdog)\n");
2579 rge_init_locked(sc);
2580 sc->sc_drv_stats.tx_watchdog_timeout_cnt++;
2581 }
2582 }
2583
2584 callout_reset(&sc->sc_timeout, hz, rge_tick, sc);
2585 }
2586
2587 /**
2588 * @brief process a link state change.
2589 *
2590 * Must be called with the driver lock held.
2591 */
2592 void
rge_link_state(struct rge_softc * sc)2593 rge_link_state(struct rge_softc *sc)
2594 {
2595 int link = LINK_STATE_DOWN;
2596
2597 RGE_ASSERT_LOCKED(sc);
2598
2599 if (rge_get_link_status(sc))
2600 link = LINK_STATE_UP;
2601
2602 if (if_getlinkstate(sc->sc_ifp) != link) {
2603 sc->sc_drv_stats.link_state_change_cnt++;
2604 if_link_state_change(sc->sc_ifp, link);
2605 }
2606 }
2607
2608 /**
2609 * @brief Suspend
2610 */
2611 static int
rge_suspend(device_t dev)2612 rge_suspend(device_t dev)
2613 {
2614 struct rge_softc *sc = device_get_softc(dev);
2615
2616 RGE_LOCK(sc);
2617 rge_stop_locked(sc);
2618 /* TODO: wake on lan */
2619 sc->sc_suspended = true;
2620 RGE_UNLOCK(sc);
2621
2622 return (0);
2623 }
2624
2625 /**
2626 * @brief Resume
2627 */
2628 static int
rge_resume(device_t dev)2629 rge_resume(device_t dev)
2630 {
2631 struct rge_softc *sc = device_get_softc(dev);
2632
2633 RGE_LOCK(sc);
2634 /* TODO: wake on lan */
2635
2636 /* reinit if required */
2637 if (if_getflags(sc->sc_ifp) & IFF_UP)
2638 rge_init_locked(sc);
2639
2640 sc->sc_suspended = false;
2641
2642 RGE_UNLOCK(sc);
2643
2644 return (0);
2645 }
2646
2647 /**
2648 * @brief Shutdown the driver during shutdown
2649 */
2650 static int
rge_shutdown(device_t dev)2651 rge_shutdown(device_t dev)
2652 {
2653 struct rge_softc *sc = device_get_softc(dev);
2654
2655 RGE_LOCK(sc);
2656 rge_stop_locked(sc);
2657 RGE_UNLOCK(sc);
2658
2659 return (0);
2660 }
2661
2662 static device_method_t rge_methods[] = {
2663 DEVMETHOD(device_probe, rge_probe),
2664 DEVMETHOD(device_attach, rge_attach),
2665 DEVMETHOD(device_detach, rge_detach),
2666
2667 DEVMETHOD(device_suspend, rge_suspend),
2668 DEVMETHOD(device_resume, rge_resume),
2669 DEVMETHOD(device_shutdown, rge_shutdown),
2670
2671 DEVMETHOD_END
2672 };
2673
2674 static driver_t rge_driver = {
2675 "rge",
2676 rge_methods,
2677 sizeof(struct rge_softc)
2678 };
2679
2680 MODULE_DEPEND(rge, pci, 1, 1, 1);
2681 MODULE_DEPEND(rge, ether, 1, 1, 1);
2682
2683 DRIVER_MODULE_ORDERED(rge, pci, rge_driver, NULL, NULL, SI_ORDER_ANY);
2684 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, rge, rge_devices,
2685 nitems(rge_devices) - 1);
2686