1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019, 2020, 2023-2025 Kevin Lo <kevlo@openbsd.org> 5 * Copyright (c) 2025 Adrian Chadd <adrian@FreeBSD.org> 6 * 7 * Hardware programming portions from Realtek Semiconductor. 8 * 9 * Permission to use, copy, modify, and distribute this software for any 10 * purpose with or without fee is hereby granted, provided that the above 11 * copyright notice and this permission notice appear in all copies. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 14 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 15 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 16 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 17 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 18 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 19 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 20 */ 21 22 /* $OpenBSD: if_rge.c,v 1.38 2025/09/19 00:41:14 kevlo Exp $ */ 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/sockio.h> 27 #include <sys/mbuf.h> 28 #include <sys/malloc.h> 29 #include <sys/endian.h> 30 #include <sys/socket.h> 31 #include <net/if.h> 32 #include <net/if_media.h> 33 #include <sys/queue.h> 34 #include <sys/taskqueue.h> 35 #include <sys/bus.h> 36 #include <sys/module.h> 37 #include <sys/rman.h> 38 #include <sys/kernel.h> 39 40 #include <netinet/in.h> 41 #include <netinet/if_ether.h> 42 43 #include <net/bpf.h> 44 #include <net/ethernet.h> 45 #include <net/if.h> 46 #include <net/if_var.h> 47 #include <net/if_arp.h> 48 #include <net/if_dl.h> 49 #include <net/if_media.h> 50 #include <net/if_types.h> 51 #include <net/if_vlan_var.h> 52 53 #include <machine/bus.h> 54 #include <machine/resource.h> 55 56 #include <dev/mii/mii.h> 57 58 #include <dev/pci/pcivar.h> 59 #include <dev/pci/pcireg.h> 60 61 #include "if_rge_vendor.h" 62 #include "if_rgereg.h" 63 #include "if_rgevar.h" 64 #include "if_rge_hw.h" 65 #include "if_rge_microcode.h" 66 #include "if_rge_debug.h" 67 #include "if_rge_sysctl.h" 68 #include "if_rge_stats.h" 69 70 #define RGE_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP) 71 72 static int rge_attach(device_t); 73 static int rge_detach(device_t); 74 75 #if 0 76 int rge_activate(struct device *, int); 77 #endif 78 static void rge_intr_msi(void *); 79 static int rge_ioctl(struct ifnet *, u_long, caddr_t); 80 static int rge_transmit_if(if_t, struct mbuf *); 81 static void rge_qflush_if(if_t); 82 static void rge_init_if(void *); 83 static void rge_init_locked(struct rge_softc *); 84 static void rge_stop_locked(struct rge_softc *); 85 static int rge_ifmedia_upd(if_t); 86 static void rge_ifmedia_sts(if_t, struct ifmediareq *); 87 static int rge_allocmem(struct rge_softc *); 88 static int rge_alloc_stats_mem(struct rge_softc *); 89 static int rge_freemem(struct rge_softc *); 90 static int rge_free_stats_mem(struct rge_softc *); 91 static int rge_newbuf(struct rge_queues *); 92 static void rge_rx_list_init(struct rge_queues *); 93 static void rge_tx_list_init(struct rge_queues *); 94 static void rge_fill_rx_ring(struct rge_queues *); 95 static int rge_rxeof(struct rge_queues *, struct mbufq *); 96 static int rge_txeof(struct rge_queues *); 97 static void rge_iff_locked(struct rge_softc *); 98 static void rge_add_media_types(struct rge_softc *); 99 static void rge_tx_task(void *, int); 100 static void rge_txq_flush_mbufs(struct rge_softc *sc); 101 static void rge_tick(void *); 102 static void rge_link_state(struct rge_softc *); 103 #if 0 104 #ifndef SMALL_KERNEL 105 int rge_wol(struct ifnet *, int); 106 void rge_wol_power(struct rge_softc *); 107 #endif 108 #endif 109 110 struct rge_matchid { 111 uint16_t vendor; 112 uint16_t device; 113 const char *name; 114 }; 115 116 const struct rge_matchid rge_devices[] = { 117 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_E3000, "Killer E3000" }, 118 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RTL8125, "RTL8125" }, 119 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RTL8126, "RTL8126", }, 120 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RTL8127, "RTL8127" }, 121 { 0, 0, NULL } 122 }; 123 124 static int 125 rge_probe(device_t dev) 126 { 127 uint16_t vendor, device; 128 const struct rge_matchid *ri; 129 130 vendor = pci_get_vendor(dev); 131 device = pci_get_device(dev); 132 133 for (ri = rge_devices; ri->name != NULL; ri++) { 134 if ((vendor == ri->vendor) && (device == ri->device)) { 135 device_set_desc(dev, ri->name); 136 return (BUS_PROBE_DEFAULT); 137 } 138 } 139 140 return (ENXIO); 141 } 142 143 static void 144 rge_attach_if(struct rge_softc *sc, const char *eaddr) 145 { 146 if_initname(sc->sc_ifp, device_get_name(sc->sc_dev), 147 device_get_unit(sc->sc_dev)); 148 if_setdev(sc->sc_ifp, sc->sc_dev); 149 if_setinitfn(sc->sc_ifp, rge_init_if); 150 if_setsoftc(sc->sc_ifp, sc); 151 if_setflags(sc->sc_ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 152 if_setioctlfn(sc->sc_ifp, rge_ioctl); 153 if_settransmitfn(sc->sc_ifp, rge_transmit_if); 154 if_setqflushfn(sc->sc_ifp, rge_qflush_if); 155 156 /* Set offload as appropriate */ 157 if_sethwassist(sc->sc_ifp, CSUM_IP | CSUM_TCP | CSUM_UDP); 158 if_setcapabilities(sc->sc_ifp, IFCAP_HWCSUM); 159 if_setcapenable(sc->sc_ifp, if_getcapabilities(sc->sc_ifp)); 160 161 /* TODO: set WOL */ 162 163 /* Attach interface */ 164 ether_ifattach(sc->sc_ifp, eaddr); 165 sc->sc_ether_attached = true; 166 167 /* post ether_ifattach() bits */ 168 169 /* VLAN capabilities */ 170 if_setcapabilitiesbit(sc->sc_ifp, IFCAP_VLAN_MTU | 171 IFCAP_VLAN_HWTAGGING, 0); 172 if_setcapabilitiesbit(sc->sc_ifp, IFCAP_VLAN_HWCSUM, 0); 173 if_setcapenable(sc->sc_ifp, if_getcapabilities(sc->sc_ifp)); 174 175 if_setifheaderlen(sc->sc_ifp, sizeof(struct ether_vlan_header)); 176 177 /* TODO: is this needed for iftransmit? */ 178 if_setsendqlen(sc->sc_ifp, RGE_TX_LIST_CNT - 1); 179 if_setsendqready(sc->sc_ifp); 180 } 181 182 static int 183 rge_attach(device_t dev) 184 { 185 uint8_t eaddr[ETHER_ADDR_LEN]; 186 struct rge_softc *sc; 187 struct rge_queues *q; 188 uint32_t hwrev, reg; 189 int i, rid; 190 int error; 191 int msic; 192 193 sc = device_get_softc(dev); 194 sc->sc_dev = dev; 195 sc->sc_ifp = if_gethandle(IFT_ETHER); 196 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, 197 MTX_DEF); 198 199 /* Enable bus mastering */ 200 pci_enable_busmaster(dev); 201 202 /* 203 * Map control/status registers. 204 */ 205 206 /* 207 * The openbsd driver (and my E3000 NIC) handle registering three 208 * kinds of BARs - a 64 bit MMIO BAR, a 32 bit MMIO BAR, and then 209 * a legacy IO port BAR. 210 * 211 * To simplify bring-up, I'm going to request resources for the first 212 * MMIO BAR (BAR2) which should be a 32 bit BAR. 213 */ 214 rid = PCIR_BAR(2); 215 sc->sc_bres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 216 RF_ACTIVE); 217 if (sc->sc_bres == NULL) { 218 RGE_PRINT_ERROR(sc, 219 "Unable to allocate bus resource: memory\n"); 220 goto fail; 221 } 222 sc->rge_bhandle = rman_get_bushandle(sc->sc_bres); 223 sc->rge_btag = rman_get_bustag(sc->sc_bres); 224 sc->rge_bsize = rman_get_size(sc->sc_bres); 225 226 q = malloc(sizeof(struct rge_queues), M_DEVBUF, M_NOWAIT | M_ZERO); 227 if (q == NULL) { 228 RGE_PRINT_ERROR(sc, "Unable to malloc rge_queues memory\n"); 229 goto fail; 230 } 231 q->q_sc = sc; 232 q->q_index = 0; 233 234 sc->sc_queues = q; 235 sc->sc_nqueues = 1; 236 237 /* Check if PCIe */ 238 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 239 sc->rge_flags |= RGE_FLAG_PCIE; 240 sc->sc_expcap = reg; 241 } 242 243 /* Allocate MSI */ 244 msic = pci_msi_count(dev); 245 if (msic == 0) { 246 RGE_PRINT_ERROR(sc, "%s: only MSI interrupts supported\n", 247 __func__); 248 goto fail; 249 } 250 251 msic = RGE_MSI_MESSAGES; 252 if (pci_alloc_msi(dev, &msic) != 0) { 253 RGE_PRINT_ERROR(sc, "%s: failed to allocate MSI\n", 254 __func__); 255 goto fail; 256 } 257 258 sc->rge_flags |= RGE_FLAG_MSI; 259 260 /* We need at least one MSI */ 261 if (msic < RGE_MSI_MESSAGES) { 262 RGE_PRINT_ERROR(sc, "%s: didn't allocate enough MSI\n", 263 __func__); 264 goto fail; 265 } 266 267 /* 268 * Allocate interrupt entries. 269 */ 270 for (i = 0, rid = 1; i < RGE_MSI_MESSAGES; i++, rid++) { 271 sc->sc_irq[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, 272 &rid, RF_ACTIVE); 273 if (sc->sc_irq[i] == NULL) { 274 RGE_PRINT_ERROR(sc, "%s: couldn't allocate MSI %d", 275 __func__, rid); 276 goto fail; 277 } 278 } 279 280 /* Hook interrupts */ 281 for (i = 0; i < RGE_MSI_MESSAGES; i++) { 282 error = bus_setup_intr(dev, sc->sc_irq[i], 283 INTR_TYPE_NET | INTR_MPSAFE, NULL, rge_intr_msi, 284 sc, &sc->sc_ih[i]); 285 if (error != 0) { 286 RGE_PRINT_ERROR(sc, 287 "%s: couldn't setup intr %d (error %d)", __func__, 288 i, error); 289 goto fail; 290 } 291 } 292 293 /* Allocate top level bus DMA tag */ 294 error = bus_dma_tag_create(bus_get_dma_tag(dev), 295 1, /* alignment */ 296 0, /* boundary */ 297 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 298 NULL, NULL, /* filter (unused) */ 299 BUS_SPACE_MAXADDR, /* maxsize */ 300 BUS_SPACE_UNRESTRICTED, /* nsegments */ 301 BUS_SPACE_MAXADDR, /* maxsegsize */ 302 0, /* flags */ 303 NULL, NULL, /* lockfunc, lockarg */ 304 &sc->sc_dmat); 305 if (error) { 306 RGE_PRINT_ERROR(sc, 307 "couldn't allocate device DMA tag (error %d)\n", error); 308 goto fail; 309 } 310 311 /* Allocate TX/RX descriptor and buffer tags */ 312 error = bus_dma_tag_create(sc->sc_dmat, 313 RGE_ALIGN, /* alignment */ 314 0, /* boundary */ 315 BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, 316 NULL, NULL, /* filter (unused) */ 317 RGE_TX_LIST_SZ, /* maxsize */ 318 1, /* nsegments */ 319 RGE_TX_LIST_SZ, /* maxsegsize */ 320 0, /* flags */ 321 NULL, NULL, /* lockfunc, lockarg */ 322 &sc->sc_dmat_tx_desc); 323 if (error) { 324 RGE_PRINT_ERROR(sc, 325 "couldn't allocate device TX descriptor " 326 "DMA tag (error %d)\n", error); 327 goto fail; 328 } 329 330 error = bus_dma_tag_create(sc->sc_dmat, 331 1, /* alignment */ 332 0, /* boundary */ 333 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 334 NULL, NULL, /* filter (unused) */ 335 RGE_JUMBO_FRAMELEN, /* maxsize */ 336 RGE_TX_NSEGS, /* nsegments */ 337 RGE_JUMBO_FRAMELEN, /* maxsegsize */ 338 0, /* flags */ 339 NULL, NULL, /* lockfunc, lockarg */ 340 &sc->sc_dmat_tx_buf); 341 if (error) { 342 RGE_PRINT_ERROR(sc, 343 "couldn't allocate device TX buffer DMA tag (error %d)\n", 344 error); 345 goto fail; 346 } 347 348 error = bus_dma_tag_create(sc->sc_dmat, 349 RGE_ALIGN, /* alignment */ 350 0, /* boundary */ 351 BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, 352 NULL, NULL, /* filter (unused) */ 353 RGE_RX_LIST_SZ, /* maxsize */ 354 1, /* nsegments */ 355 RGE_RX_LIST_SZ, /* maxsegsize */ 356 0, /* flags */ 357 NULL, NULL, /* lockfunc, lockarg */ 358 &sc->sc_dmat_rx_desc); 359 if (error) { 360 RGE_PRINT_ERROR(sc, 361 "couldn't allocate device RX descriptor " 362 "DMA tag (error %d)\n", error); 363 goto fail; 364 } 365 366 error = bus_dma_tag_create(sc->sc_dmat, 367 1, /* alignment */ 368 0, /* boundary */ 369 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 370 NULL, NULL, /* filter (unused) */ 371 MCLBYTES, /* maxsize */ 372 1, /* nsegments */ 373 MCLBYTES, /* maxsegsize */ 374 0, /* flags */ 375 NULL, NULL, /* lockfunc, lockarg */ 376 &sc->sc_dmat_rx_buf); 377 if (error) { 378 RGE_PRINT_ERROR(sc, 379 "couldn't allocate device RX buffer DMA tag (error %d)\n", 380 error); 381 goto fail; 382 } 383 384 error = bus_dma_tag_create(sc->sc_dmat, 385 RGE_STATS_ALIGNMENT, /* alignment */ 386 0, /* boundary */ 387 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 388 NULL, NULL, /* filter (unused) */ 389 RGE_STATS_BUF_SIZE, /* maxsize */ 390 1, /* nsegments */ 391 RGE_STATS_BUF_SIZE, /* maxsegsize */ 392 0, /* flags */ 393 NULL, NULL, /* lockfunc, lockarg */ 394 &sc->sc_dmat_stats_buf); 395 if (error) { 396 RGE_PRINT_ERROR(sc, 397 "couldn't allocate device RX buffer DMA tag (error %d)\n", 398 error); 399 goto fail; 400 } 401 402 403 /* Attach sysctl nodes */ 404 rge_sysctl_attach(sc); 405 406 /* Determine hardware revision */ 407 hwrev = RGE_READ_4(sc, RGE_TXCFG) & RGE_TXCFG_HWREV; 408 switch (hwrev) { 409 case 0x60900000: 410 sc->rge_type = MAC_R25; 411 // device_printf(dev, "RTL8125\n"); 412 break; 413 case 0x64100000: 414 sc->rge_type = MAC_R25B; 415 // device_printf(dev, "RTL8125B\n"); 416 break; 417 case 0x64900000: 418 sc->rge_type = MAC_R26; 419 // device_printf(dev, "RTL8126\n"); 420 break; 421 case 0x68800000: 422 sc->rge_type = MAC_R25D; 423 // device_printf(dev, "RTL8125D\n"); 424 break; 425 case 0x6c900000: 426 sc->rge_type = MAC_R27; 427 // device_printf(dev, "RTL8127\n"); 428 break; 429 default: 430 RGE_PRINT_ERROR(sc, "unknown version 0x%08x\n", hwrev); 431 goto fail; 432 } 433 434 rge_config_imtype(sc, RGE_IMTYPE_SIM); 435 436 /* TODO: disable ASPM/ECPM? */ 437 438 #if 0 439 /* 440 * PCI Express check. 441 */ 442 if (pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_PCIEXPRESS, 443 &offset, NULL)) { 444 /* Disable PCIe ASPM and ECPM. */ 445 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, 446 offset + PCI_PCIE_LCSR); 447 reg &= ~(PCI_PCIE_LCSR_ASPM_L0S | PCI_PCIE_LCSR_ASPM_L1 | 448 PCI_PCIE_LCSR_ECPM); 449 pci_conf_write(pa->pa_pc, pa->pa_tag, offset + PCI_PCIE_LCSR, 450 reg); 451 } 452 #endif 453 454 RGE_LOCK(sc); 455 if (rge_chipinit(sc)) { 456 RGE_UNLOCK(sc); 457 goto fail; 458 } 459 460 rge_get_macaddr(sc, eaddr); 461 RGE_UNLOCK(sc); 462 463 if (rge_allocmem(sc)) 464 goto fail; 465 if (rge_alloc_stats_mem(sc)) 466 goto fail; 467 468 /* Initialize ifmedia structures. */ 469 ifmedia_init(&sc->sc_media, IFM_IMASK, rge_ifmedia_upd, 470 rge_ifmedia_sts); 471 rge_add_media_types(sc); 472 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 473 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 474 sc->sc_media.ifm_media = sc->sc_media.ifm_cur->ifm_media; 475 476 rge_attach_if(sc, eaddr); 477 478 /* 479 * TODO: technically should be per txq but we only support 480 * one TXQ at the moment. 481 */ 482 mbufq_init(&sc->sc_txq, RGE_TX_LIST_CNT); 483 484 snprintf(sc->sc_tq_name, sizeof(sc->sc_tq_name), 485 "%s taskq", device_get_nameunit(sc->sc_dev)); 486 snprintf(sc->sc_tq_thr_name, sizeof(sc->sc_tq_thr_name), 487 "%s taskq thread", device_get_nameunit(sc->sc_dev)); 488 489 sc->sc_tq = taskqueue_create(sc->sc_tq_name, M_NOWAIT, 490 taskqueue_thread_enqueue, &sc->sc_tq); 491 taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s", 492 sc->sc_tq_thr_name); 493 494 TASK_INIT(&sc->sc_tx_task, 0, rge_tx_task, sc); 495 496 callout_init_mtx(&sc->sc_timeout, &sc->sc_mtx, 0); 497 498 return (0); 499 fail: 500 rge_detach(dev); 501 return (ENXIO); 502 } 503 504 /** 505 * @brief flush the mbufq queue 506 * 507 * Again this should likely be per-TXQ. 508 * 509 * This should be called with the driver lock held. 510 */ 511 static void 512 rge_txq_flush_mbufs(struct rge_softc *sc) 513 { 514 struct mbuf *m; 515 int ntx = 0; 516 517 RGE_ASSERT_LOCKED(sc); 518 519 while ((m = mbufq_dequeue(&sc->sc_txq)) != NULL) { 520 m_freem(m); 521 ntx++; 522 } 523 524 RGE_DPRINTF(sc, RGE_DEBUG_XMIT, "%s: %d frames flushed\n", __func__, 525 ntx); 526 } 527 528 static int 529 rge_detach(device_t dev) 530 { 531 struct rge_softc *sc = device_get_softc(dev); 532 int i, rid; 533 534 /* global flag, detaching */ 535 RGE_LOCK(sc); 536 sc->sc_stopped = true; 537 sc->sc_detaching = true; 538 RGE_UNLOCK(sc); 539 540 /* stop/drain network interface */ 541 callout_drain(&sc->sc_timeout); 542 543 /* Make sure TX task isn't running */ 544 if (sc->sc_tq != NULL) { 545 while (taskqueue_cancel(sc->sc_tq, &sc->sc_tx_task, NULL) != 0) 546 taskqueue_drain(sc->sc_tq, &sc->sc_tx_task); 547 } 548 549 RGE_LOCK(sc); 550 callout_stop(&sc->sc_timeout); 551 552 /* stop NIC / DMA */ 553 rge_stop_locked(sc); 554 555 /* TODO: wait for completion */ 556 557 /* Free pending TX mbufs */ 558 rge_txq_flush_mbufs(sc); 559 560 RGE_UNLOCK(sc); 561 562 /* Free taskqueue */ 563 if (sc->sc_tq != NULL) { 564 taskqueue_free(sc->sc_tq); 565 sc->sc_tq = NULL; 566 } 567 568 /* Free descriptor memory */ 569 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: freemem\n", __func__); 570 rge_freemem(sc); 571 rge_free_stats_mem(sc); 572 573 if (sc->sc_ifp) { 574 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: ifdetach/if_free\n", 575 __func__); 576 if (sc->sc_ether_attached) 577 ether_ifdetach(sc->sc_ifp); 578 if_free(sc->sc_ifp); 579 } 580 581 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_tx_desc\n", __func__); 582 if (sc->sc_dmat_tx_desc) 583 bus_dma_tag_destroy(sc->sc_dmat_tx_desc); 584 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_tx_buf\n", __func__); 585 if (sc->sc_dmat_tx_buf) 586 bus_dma_tag_destroy(sc->sc_dmat_tx_buf); 587 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_rx_desc\n", __func__); 588 if (sc->sc_dmat_rx_desc) 589 bus_dma_tag_destroy(sc->sc_dmat_rx_desc); 590 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_rx_buf\n", __func__); 591 if (sc->sc_dmat_rx_buf) 592 bus_dma_tag_destroy(sc->sc_dmat_rx_buf); 593 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat_stats_buf\n", __func__); 594 if (sc->sc_dmat_stats_buf) 595 bus_dma_tag_destroy(sc->sc_dmat_stats_buf); 596 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: sc_dmat\n", __func__); 597 if (sc->sc_dmat) 598 bus_dma_tag_destroy(sc->sc_dmat); 599 600 /* Teardown interrupts */ 601 for (i = 0; i < RGE_MSI_MESSAGES; i++) { 602 if (sc->sc_ih[i] != NULL) { 603 bus_teardown_intr(sc->sc_dev, sc->sc_irq[i], 604 sc->sc_ih[i]); 605 sc->sc_ih[i] = NULL; 606 } 607 } 608 609 /* Free interrupt resources */ 610 for (i = 0, rid = 1; i < RGE_MSI_MESSAGES; i++, rid++) { 611 if (sc->sc_irq[i] != NULL) { 612 bus_release_resource(sc->sc_dev, SYS_RES_IRQ, 613 rid, sc->sc_irq[i]); 614 sc->sc_irq[i] = NULL; 615 } 616 } 617 618 /* Free MSI allocation */ 619 if (sc->rge_flags & RGE_FLAG_MSI) 620 pci_release_msi(dev); 621 622 if (sc->sc_bres) { 623 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: release mmio\n", 624 __func__); 625 bus_release_resource(dev, SYS_RES_MEMORY, 626 rman_get_rid(sc->sc_bres), sc->sc_bres); 627 sc->sc_bres = NULL; 628 } 629 630 if (sc->sc_queues) { 631 free(sc->sc_queues, M_DEVBUF); 632 sc->sc_queues = NULL; 633 } 634 635 mtx_destroy(&sc->sc_mtx); 636 637 return (0); 638 } 639 640 #if 0 641 642 int 643 rge_activate(struct device *self, int act) 644 { 645 #ifndef SMALL_KERNEL 646 struct rge_softc *sc = (struct rge_softc *)self; 647 #endif 648 649 switch (act) { 650 case DVACT_POWERDOWN: 651 #ifndef SMALL_KERNEL 652 rge_wol_power(sc); 653 #endif 654 break; 655 } 656 return (0); 657 } 658 #endif 659 660 static void 661 rge_intr_msi(void *arg) 662 { 663 struct mbufq rx_mq; 664 struct epoch_tracker et; 665 struct mbuf *m; 666 struct rge_softc *sc = arg; 667 struct rge_queues *q = sc->sc_queues; 668 uint32_t status; 669 int claimed = 0, rv; 670 671 sc->sc_drv_stats.intr_cnt++; 672 673 mbufq_init(&rx_mq, RGE_RX_LIST_CNT); 674 675 if ((if_getdrvflags(sc->sc_ifp) & IFF_DRV_RUNNING) == 0) 676 return; 677 678 RGE_LOCK(sc); 679 680 if (sc->sc_suspended || sc->sc_stopped || sc->sc_detaching) { 681 RGE_UNLOCK(sc); 682 return; 683 } 684 685 /* Disable interrupts. */ 686 RGE_WRITE_4(sc, RGE_IMR, 0); 687 688 if (!(sc->rge_flags & RGE_FLAG_MSI)) { 689 if ((RGE_READ_4(sc, RGE_ISR) & sc->rge_intrs) == 0) 690 goto done; 691 } 692 693 status = RGE_READ_4(sc, RGE_ISR); 694 if (status) 695 RGE_WRITE_4(sc, RGE_ISR, status); 696 697 if (status & RGE_ISR_PCS_TIMEOUT) 698 claimed = 1; 699 700 rv = 0; 701 if (status & sc->rge_intrs) { 702 703 (void) q; 704 rv |= rge_rxeof(q, &rx_mq); 705 rv |= rge_txeof(q); 706 707 if (status & RGE_ISR_SYSTEM_ERR) { 708 sc->sc_drv_stats.intr_system_err_cnt++; 709 rge_init_locked(sc); 710 } 711 claimed = 1; 712 } 713 714 if (sc->rge_timerintr) { 715 if (!rv) { 716 /* 717 * Nothing needs to be processed, fallback 718 * to use TX/RX interrupts. 719 */ 720 rge_setup_intr(sc, RGE_IMTYPE_NONE); 721 722 /* 723 * Recollect, mainly to avoid the possible 724 * race introduced by changing interrupt 725 * masks. 726 */ 727 rge_rxeof(q, &rx_mq); 728 rge_txeof(q); 729 } else 730 RGE_WRITE_4(sc, RGE_TIMERCNT, 1); 731 } else if (rv) { 732 /* 733 * Assume that using simulated interrupt moderation 734 * (hardware timer based) could reduce the interrupt 735 * rate. 736 */ 737 rge_setup_intr(sc, RGE_IMTYPE_SIM); 738 } 739 740 RGE_WRITE_4(sc, RGE_IMR, sc->rge_intrs); 741 742 done: 743 RGE_UNLOCK(sc); 744 745 NET_EPOCH_ENTER(et); 746 /* Handle any RX frames, outside of the driver lock */ 747 while ((m = mbufq_dequeue(&rx_mq)) != NULL) { 748 sc->sc_drv_stats.recv_input_cnt++; 749 if_input(sc->sc_ifp, m); 750 } 751 NET_EPOCH_EXIT(et); 752 753 (void) claimed; 754 } 755 756 static inline void 757 rge_tx_list_sync(struct rge_softc *sc, struct rge_queues *q, 758 unsigned int idx, unsigned int len, int ops) 759 { 760 bus_dmamap_sync(sc->sc_dmat_tx_desc, q->q_tx.rge_tx_list_map, ops); 761 } 762 763 /** 764 * @brief Queue the given mbuf at the given TX slot index for transmit. 765 * 766 * If the frame couldn't be enqueued then 0 is returned. 767 * The caller needs to handle that and free/re-queue the mbuf as required. 768 * 769 * Note that this doesn't actually kick-start the transmit itself; 770 * see rge_txstart() for the register to poke to start transmit. 771 * 772 * This must be called with the driver lock held. 773 * 774 * @param sc driver softc 775 * @param q TX queue ring 776 * @param m mbuf to enqueue 777 * @returns if the mbuf is enqueued, it's consumed here and the number of 778 * TX descriptors used is returned; if there's no space then 0 is 779 * returned; if the mbuf couldn't be defragged and the caller 780 * should free it then -1 is returned. 781 */ 782 static int 783 rge_encap(struct rge_softc *sc, struct rge_queues *q, struct mbuf *m, int idx) 784 { 785 struct rge_tx_desc *d = NULL; 786 struct rge_txq *txq; 787 bus_dmamap_t txmap; 788 uint32_t cmdsts, cflags = 0; 789 int cur, error, i; 790 bus_dma_segment_t seg[RGE_TX_NSEGS]; 791 int nsegs; 792 793 RGE_ASSERT_LOCKED(sc); 794 795 txq = &q->q_tx.rge_txq[idx]; 796 txmap = txq->txq_dmamap; 797 798 sc->sc_drv_stats.tx_encap_cnt++; 799 800 nsegs = RGE_TX_NSEGS; 801 error = bus_dmamap_load_mbuf_sg(sc->sc_dmat_tx_buf, txmap, m, 802 seg, &nsegs, BUS_DMA_NOWAIT); 803 804 switch (error) { 805 case 0: 806 break; 807 case EFBIG: /* mbuf chain is too fragmented */ 808 sc->sc_drv_stats.tx_encap_refrag_cnt++; 809 nsegs = RGE_TX_NSEGS; 810 if (m_defrag(m, M_NOWAIT) == 0 && 811 bus_dmamap_load_mbuf_sg(sc->sc_dmat_tx_buf, txmap, m, 812 seg, &nsegs, BUS_DMA_NOWAIT) == 0) 813 break; 814 /* FALLTHROUGH */ 815 default: 816 sc->sc_drv_stats.tx_encap_err_toofrag++; 817 return (-1); 818 } 819 820 bus_dmamap_sync(sc->sc_dmat_tx_buf, txmap, BUS_DMASYNC_PREWRITE); 821 822 /* 823 * Set RGE_TDEXTSTS_IPCSUM if any checksum offloading is requested. 824 * Otherwise, RGE_TDEXTSTS_TCPCSUM / RGE_TDEXTSTS_UDPCSUM does not 825 * take affect. 826 */ 827 if ((m->m_pkthdr.csum_flags & RGE_CSUM_FEATURES) != 0) { 828 cflags |= RGE_TDEXTSTS_IPCSUM; 829 sc->sc_drv_stats.tx_offload_ip_csum_set++; 830 if (m->m_pkthdr.csum_flags & CSUM_TCP) { 831 sc->sc_drv_stats.tx_offload_tcp_csum_set++; 832 cflags |= RGE_TDEXTSTS_TCPCSUM; 833 } 834 if (m->m_pkthdr.csum_flags & CSUM_UDP) { 835 sc->sc_drv_stats.tx_offload_udp_csum_set++; 836 cflags |= RGE_TDEXTSTS_UDPCSUM; 837 } 838 } 839 840 /* Set up hardware VLAN tagging */ 841 if (m->m_flags & M_VLANTAG) { 842 sc->sc_drv_stats.tx_offload_vlan_tag_set++; 843 cflags |= htole16(m->m_pkthdr.ether_vtag) | RGE_TDEXTSTS_VTAG; 844 } 845 846 cur = idx; 847 for (i = 1; i < nsegs; i++) { 848 cur = RGE_NEXT_TX_DESC(cur); 849 850 cmdsts = RGE_TDCMDSTS_OWN; 851 cmdsts |= seg[i].ds_len; 852 853 if (cur == RGE_TX_LIST_CNT - 1) 854 cmdsts |= RGE_TDCMDSTS_EOR; 855 if (i == nsegs - 1) 856 cmdsts |= RGE_TDCMDSTS_EOF; 857 858 /* 859 * Note: vendor driver puts wmb() after opts2/extsts, 860 * before opts1/status. 861 * 862 * See the other place I have this comment for more 863 * information. 864 */ 865 d = &q->q_tx.rge_tx_list[cur]; 866 d->rge_addr = htole64(seg[i].ds_addr); 867 d->rge_extsts = htole32(cflags); 868 wmb(); 869 d->rge_cmdsts = htole32(cmdsts); 870 } 871 872 /* Update info of TX queue and descriptors. */ 873 txq->txq_mbuf = m; 874 txq->txq_descidx = cur; 875 876 cmdsts = RGE_TDCMDSTS_SOF; 877 cmdsts |= seg[0].ds_len; 878 879 if (idx == RGE_TX_LIST_CNT - 1) 880 cmdsts |= RGE_TDCMDSTS_EOR; 881 if (nsegs == 1) 882 cmdsts |= RGE_TDCMDSTS_EOF; 883 884 /* 885 * Note: vendor driver puts wmb() after opts2/extsts, 886 * before opts1/status. 887 * 888 * It does this: 889 * - set rge_addr 890 * - set extsts 891 * - wmb 892 * - set status - at this point it's owned by the hardware 893 * 894 */ 895 d = &q->q_tx.rge_tx_list[idx]; 896 d->rge_addr = htole64(seg[0].ds_addr); 897 d->rge_extsts = htole32(cflags); 898 wmb(); 899 d->rge_cmdsts = htole32(cmdsts); 900 wmb(); 901 902 if (cur >= idx) { 903 rge_tx_list_sync(sc, q, idx, nsegs, 904 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 905 } else { 906 rge_tx_list_sync(sc, q, idx, RGE_TX_LIST_CNT - idx, 907 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 908 rge_tx_list_sync(sc, q, 0, cur + 1, 909 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 910 } 911 912 /* Transfer ownership of packet to the chip. */ 913 cmdsts |= RGE_TDCMDSTS_OWN; 914 rge_tx_list_sync(sc, q, idx, 1, BUS_DMASYNC_POSTWRITE); 915 d->rge_cmdsts = htole32(cmdsts); 916 rge_tx_list_sync(sc, q, idx, 1, BUS_DMASYNC_PREWRITE); 917 wmb(); 918 919 return (nsegs); 920 } 921 922 static int 923 rge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 924 { 925 struct rge_softc *sc = if_getsoftc(ifp); 926 struct ifreq *ifr = (struct ifreq *)data; 927 int error = 0; 928 929 switch (cmd) { 930 case SIOCSIFMTU: 931 /* Note: no hardware reinit is required */ 932 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > RGE_JUMBO_MTU) { 933 error = EINVAL; 934 break; 935 } 936 if (if_getmtu(ifp) != ifr->ifr_mtu) 937 if_setmtu(ifp, ifr->ifr_mtu); 938 939 VLAN_CAPABILITIES(ifp); 940 break; 941 942 case SIOCSIFFLAGS: 943 RGE_LOCK(sc); 944 if ((if_getflags(ifp) & IFF_UP) != 0) { 945 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 946 /* 947 * TODO: handle promisc/iffmulti changing 948 * without reprogramming everything. 949 */ 950 rge_init_locked(sc); 951 } else { 952 /* Reinit promisc/multi just in case */ 953 rge_iff_locked(sc); 954 } 955 } else { 956 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 957 rge_stop_locked(sc); 958 } 959 } 960 RGE_UNLOCK(sc); 961 break; 962 case SIOCADDMULTI: 963 case SIOCDELMULTI: 964 RGE_LOCK(sc); 965 if ((if_getflags(ifp) & IFF_DRV_RUNNING) != 0) { 966 rge_iff_locked(sc); 967 } 968 RGE_UNLOCK(sc); 969 break; 970 case SIOCGIFMEDIA: 971 case SIOCSIFMEDIA: 972 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 973 break; 974 case SIOCSIFCAP: 975 { 976 int mask; 977 bool reinit = false; 978 979 /* Get the mask of changed bits */ 980 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); 981 982 /* 983 * Locked so we don't have a narrow window where frames 984 * are being processed with the updated flags but the 985 * hardware configuration hasn't yet changed. 986 */ 987 RGE_LOCK(sc); 988 989 if ((mask & IFCAP_TXCSUM) != 0 && 990 (if_getcapabilities(ifp) & IFCAP_TXCSUM) != 0) { 991 if_togglecapenable(ifp, IFCAP_TXCSUM); 992 if ((if_getcapenable(ifp) & IFCAP_TXCSUM) != 0) 993 if_sethwassistbits(ifp, RGE_CSUM_FEATURES, 0); 994 else 995 if_sethwassistbits(ifp, 0, RGE_CSUM_FEATURES); 996 reinit = 1; 997 } 998 999 if ((mask & IFCAP_VLAN_HWTAGGING) != 0 && 1000 (if_getcapabilities(ifp) & IFCAP_VLAN_HWTAGGING) != 0) { 1001 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); 1002 reinit = 1; 1003 } 1004 1005 /* TODO: WOL */ 1006 1007 if ((mask & IFCAP_RXCSUM) != 0 && 1008 (if_getcapabilities(ifp) & IFCAP_RXCSUM) != 0) { 1009 if_togglecapenable(ifp, IFCAP_RXCSUM); 1010 reinit = 1; 1011 } 1012 1013 if (reinit && if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1014 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 1015 rge_init_locked(sc); 1016 } 1017 1018 RGE_UNLOCK(sc); 1019 VLAN_CAPABILITIES(ifp); 1020 } 1021 break; 1022 default: 1023 error = ether_ioctl(ifp, cmd, data); 1024 break; 1025 } 1026 1027 return (error); 1028 } 1029 1030 static void 1031 rge_qflush_if(if_t ifp) 1032 { 1033 struct rge_softc *sc = if_getsoftc(ifp); 1034 1035 /* TODO: this should iterate over the TXQs */ 1036 RGE_LOCK(sc); 1037 rge_txq_flush_mbufs(sc); 1038 RGE_UNLOCK(sc); 1039 } 1040 1041 /** 1042 * @brief Transmit the given frame to the hardware. 1043 * 1044 * This routine is called by the network stack to send 1045 * a frame to the device. 1046 * 1047 * For now we simply direct dispatch this frame to the 1048 * hardware (and thus avoid maintaining our own internal 1049 * queue) 1050 */ 1051 static int 1052 rge_transmit_if(if_t ifp, struct mbuf *m) 1053 { 1054 struct rge_softc *sc = if_getsoftc(ifp); 1055 int ret; 1056 1057 sc->sc_drv_stats.transmit_call_cnt++; 1058 1059 RGE_LOCK(sc); 1060 if (sc->sc_stopped == true) { 1061 sc->sc_drv_stats.transmit_stopped_cnt++; 1062 RGE_UNLOCK(sc); 1063 return (ENETDOWN); /* TODO: better error? */ 1064 } 1065 1066 /* XXX again should be a per-TXQ thing */ 1067 ret = mbufq_enqueue(&sc->sc_txq, m); 1068 if (ret != 0) { 1069 sc->sc_drv_stats.transmit_full_cnt++; 1070 RGE_UNLOCK(sc); 1071 return (ret); 1072 } 1073 RGE_UNLOCK(sc); 1074 1075 /* mbuf is owned by the driver, schedule transmit */ 1076 taskqueue_enqueue(sc->sc_tq, &sc->sc_tx_task); 1077 sc->sc_drv_stats.transmit_queued_cnt++; 1078 1079 return (0); 1080 } 1081 1082 static void 1083 rge_init_if(void *xsc) 1084 { 1085 struct rge_softc *sc = xsc; 1086 1087 RGE_LOCK(sc); 1088 rge_init_locked(sc); 1089 RGE_UNLOCK(sc); 1090 } 1091 1092 static void 1093 rge_init_locked(struct rge_softc *sc) 1094 { 1095 struct rge_queues *q = sc->sc_queues; 1096 uint32_t rxconf, val; 1097 int i, num_miti; 1098 1099 RGE_ASSERT_LOCKED(sc); 1100 1101 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: called!\n", __func__); 1102 1103 /* Don't double-init the hardware */ 1104 if ((if_getdrvflags(sc->sc_ifp) & IFF_DRV_RUNNING) != 0) { 1105 /* 1106 * Note: I'm leaving this disabled by default; however 1107 * I'm leaving it in here so I can figure out what's 1108 * causing this to be initialised both from the ioctl 1109 * API and if_init() API. 1110 */ 1111 // RGE_PRINT_ERROR(sc, "%s: called whilst running?\n", __func__); 1112 return; 1113 } 1114 1115 /* 1116 * Bring the hardware down so we know it's in a good known 1117 * state before we bring it up in a good known state. 1118 */ 1119 rge_stop_locked(sc); 1120 1121 /* Set MAC address. */ 1122 rge_set_macaddr(sc, if_getlladdr(sc->sc_ifp)); 1123 1124 /* Initialize RX and TX descriptors lists. */ 1125 rge_rx_list_init(q); 1126 rge_tx_list_init(q); 1127 1128 if (rge_chipinit(sc)) { 1129 RGE_PRINT_ERROR(sc, "%s: ERROR: chip init fail!\n", __func__); 1130 return; 1131 } 1132 1133 if (rge_phy_config(sc)) 1134 return; 1135 1136 RGE_SETBIT_1(sc, RGE_EECMD, RGE_EECMD_WRITECFG); 1137 1138 RGE_CLRBIT_1(sc, 0xf1, 0x80); 1139 rge_disable_aspm_clkreq(sc); 1140 RGE_WRITE_2(sc, RGE_EEE_TXIDLE_TIMER, 1141 RGE_JUMBO_MTU + ETHER_HDR_LEN + 32); 1142 1143 /* Load the addresses of the RX and TX lists into the chip. */ 1144 RGE_WRITE_4(sc, RGE_RXDESC_ADDR_LO, 1145 RGE_ADDR_LO(q->q_rx.rge_rx_list_paddr)); 1146 RGE_WRITE_4(sc, RGE_RXDESC_ADDR_HI, 1147 RGE_ADDR_HI(q->q_rx.rge_rx_list_paddr)); 1148 RGE_WRITE_4(sc, RGE_TXDESC_ADDR_LO, 1149 RGE_ADDR_LO(q->q_tx.rge_tx_list_paddr)); 1150 RGE_WRITE_4(sc, RGE_TXDESC_ADDR_HI, 1151 RGE_ADDR_HI(q->q_tx.rge_tx_list_paddr)); 1152 1153 /* Set the initial RX and TX configurations. */ 1154 if (sc->rge_type == MAC_R25) 1155 rxconf = RGE_RXCFG_CONFIG; 1156 else if (sc->rge_type == MAC_R25B) 1157 rxconf = RGE_RXCFG_CONFIG_8125B; 1158 else if (sc->rge_type == MAC_R25D) 1159 rxconf = RGE_RXCFG_CONFIG_8125D; 1160 else 1161 rxconf = RGE_RXCFG_CONFIG_8126; 1162 RGE_WRITE_4(sc, RGE_RXCFG, rxconf); 1163 RGE_WRITE_4(sc, RGE_TXCFG, RGE_TXCFG_CONFIG); 1164 1165 val = rge_read_csi(sc, 0x70c) & ~0x3f000000; 1166 rge_write_csi(sc, 0x70c, val | 0x27000000); 1167 1168 if (sc->rge_type == MAC_R26 || sc->rge_type == MAC_R27) { 1169 /* Disable L1 timeout. */ 1170 val = rge_read_csi(sc, 0x890) & ~0x00000001; 1171 rge_write_csi(sc, 0x890, val); 1172 } else if (sc->rge_type != MAC_R25D) 1173 RGE_WRITE_2(sc, 0x0382, 0x221b); 1174 1175 RGE_WRITE_1(sc, RGE_RSS_CTRL, 0); 1176 1177 val = RGE_READ_2(sc, RGE_RXQUEUE_CTRL) & ~0x001c; 1178 RGE_WRITE_2(sc, RGE_RXQUEUE_CTRL, val | (fls(sc->sc_nqueues) - 1) << 2); 1179 1180 RGE_CLRBIT_1(sc, RGE_CFG1, RGE_CFG1_SPEED_DOWN); 1181 1182 rge_write_mac_ocp(sc, 0xc140, 0xffff); 1183 rge_write_mac_ocp(sc, 0xc142, 0xffff); 1184 1185 RGE_MAC_SETBIT(sc, 0xeb58, 0x0001); 1186 1187 if (sc->rge_type == MAC_R26 || sc->rge_type == MAC_R27) { 1188 RGE_CLRBIT_1(sc, 0xd8, 0x02); 1189 if (sc->rge_type == MAC_R27) { 1190 RGE_CLRBIT_1(sc, 0x20e4, 0x04); 1191 RGE_MAC_CLRBIT(sc, 0xe00c, 0x1000); 1192 RGE_MAC_CLRBIT(sc, 0xc0c2, 0x0040); 1193 } 1194 } 1195 1196 val = rge_read_mac_ocp(sc, 0xe614); 1197 val &= (sc->rge_type == MAC_R27) ? ~0x0f00 : ~0x0700; 1198 if (sc->rge_type == MAC_R25 || sc->rge_type == MAC_R25D) 1199 rge_write_mac_ocp(sc, 0xe614, val | 0x0300); 1200 else if (sc->rge_type == MAC_R25B) 1201 rge_write_mac_ocp(sc, 0xe614, val | 0x0200); 1202 else if (sc->rge_type == MAC_R26) 1203 rge_write_mac_ocp(sc, 0xe614, val | 0x0300); 1204 else 1205 rge_write_mac_ocp(sc, 0xe614, val | 0x0f00); 1206 1207 val = rge_read_mac_ocp(sc, 0xe63e) & ~0x0c00; 1208 rge_write_mac_ocp(sc, 0xe63e, val | 1209 ((fls(sc->sc_nqueues) - 1) & 0x03) << 10); 1210 1211 val = rge_read_mac_ocp(sc, 0xe63e) & ~0x0030; 1212 rge_write_mac_ocp(sc, 0xe63e, val | 0x0020); 1213 1214 RGE_MAC_CLRBIT(sc, 0xc0b4, 0x0001); 1215 RGE_MAC_SETBIT(sc, 0xc0b4, 0x0001); 1216 1217 RGE_MAC_SETBIT(sc, 0xc0b4, 0x000c); 1218 1219 val = rge_read_mac_ocp(sc, 0xeb6a) & ~0x00ff; 1220 rge_write_mac_ocp(sc, 0xeb6a, val | 0x0033); 1221 1222 val = rge_read_mac_ocp(sc, 0xeb50) & ~0x03e0; 1223 rge_write_mac_ocp(sc, 0xeb50, val | 0x0040); 1224 1225 RGE_MAC_CLRBIT(sc, 0xe056, 0x00f0); 1226 1227 RGE_WRITE_1(sc, RGE_TDFNR, 0x10); 1228 1229 RGE_MAC_CLRBIT(sc, 0xe040, 0x1000); 1230 1231 val = rge_read_mac_ocp(sc, 0xea1c) & ~0x0003; 1232 rge_write_mac_ocp(sc, 0xea1c, val | 0x0001); 1233 1234 if (sc->rge_type == MAC_R25D) 1235 rge_write_mac_ocp(sc, 0xe0c0, 0x4403); 1236 else 1237 rge_write_mac_ocp(sc, 0xe0c0, 0x4000); 1238 1239 RGE_MAC_SETBIT(sc, 0xe052, 0x0060); 1240 RGE_MAC_CLRBIT(sc, 0xe052, 0x0088); 1241 1242 val = rge_read_mac_ocp(sc, 0xd430) & ~0x0fff; 1243 rge_write_mac_ocp(sc, 0xd430, val | 0x045f); 1244 1245 RGE_SETBIT_1(sc, RGE_DLLPR, RGE_DLLPR_PFM_EN | RGE_DLLPR_TX_10M_PS_EN); 1246 1247 if (sc->rge_type == MAC_R25) 1248 RGE_SETBIT_1(sc, RGE_MCUCMD, 0x01); 1249 1250 if (sc->rge_type != MAC_R25D) { 1251 /* Disable EEE plus. */ 1252 RGE_MAC_CLRBIT(sc, 0xe080, 0x0002); 1253 } 1254 1255 if (sc->rge_type == MAC_R26 || sc->rge_type == MAC_R27) 1256 RGE_MAC_CLRBIT(sc, 0xea1c, 0x0304); 1257 else 1258 RGE_MAC_CLRBIT(sc, 0xea1c, 0x0004); 1259 1260 /* Clear tcam entries. */ 1261 RGE_MAC_SETBIT(sc, 0xeb54, 0x0001); 1262 DELAY(1); 1263 RGE_MAC_CLRBIT(sc, 0xeb54, 0x0001); 1264 1265 RGE_CLRBIT_2(sc, 0x1880, 0x0030); 1266 1267 if (sc->rge_type == MAC_R27) { 1268 val = rge_read_mac_ocp(sc, 0xd40c) & ~0xe038; 1269 rge_write_phy_ocp(sc, 0xd40c, val | 0x8020); 1270 } 1271 1272 /* Config interrupt type. */ 1273 if (sc->rge_type == MAC_R27) 1274 RGE_CLRBIT_1(sc, RGE_INT_CFG0, RGE_INT_CFG0_AVOID_MISS_INTR); 1275 else if (sc->rge_type != MAC_R25) 1276 RGE_CLRBIT_1(sc, RGE_INT_CFG0, RGE_INT_CFG0_EN); 1277 1278 /* Clear timer interrupts. */ 1279 RGE_WRITE_4(sc, RGE_TIMERINT0, 0); 1280 RGE_WRITE_4(sc, RGE_TIMERINT1, 0); 1281 RGE_WRITE_4(sc, RGE_TIMERINT2, 0); 1282 RGE_WRITE_4(sc, RGE_TIMERINT3, 0); 1283 1284 num_miti = 1285 (sc->rge_type == MAC_R25B || sc->rge_type == MAC_R26) ? 32 : 64; 1286 /* Clear interrupt moderation timer. */ 1287 for (i = 0; i < num_miti; i++) 1288 RGE_WRITE_4(sc, RGE_INTMITI(i), 0); 1289 1290 if (sc->rge_type == MAC_R26) { 1291 RGE_CLRBIT_1(sc, RGE_INT_CFG0, 1292 RGE_INT_CFG0_TIMEOUT_BYPASS | RGE_INT_CFG0_RDU_BYPASS_8126 | 1293 RGE_INT_CFG0_MITIGATION_BYPASS); 1294 RGE_WRITE_2(sc, RGE_INT_CFG1, 0); 1295 } 1296 1297 RGE_MAC_SETBIT(sc, 0xc0ac, 0x1f80); 1298 1299 rge_write_mac_ocp(sc, 0xe098, 0xc302); 1300 1301 RGE_MAC_CLRBIT(sc, 0xe032, 0x0003); 1302 val = rge_read_csi(sc, 0x98) & ~0x0000ff00; 1303 rge_write_csi(sc, 0x98, val); 1304 1305 if (sc->rge_type == MAC_R25D) { 1306 val = rge_read_mac_ocp(sc, 0xe092) & ~0x00ff; 1307 rge_write_mac_ocp(sc, 0xe092, val | 0x0008); 1308 } else 1309 RGE_MAC_CLRBIT(sc, 0xe092, 0x00ff); 1310 1311 /* Enable/disable HW VLAN tagging based on enabled capability */ 1312 if ((if_getcapabilities(sc->sc_ifp) & IFCAP_VLAN_HWTAGGING) != 0) 1313 RGE_SETBIT_4(sc, RGE_RXCFG, RGE_RXCFG_VLANSTRIP); 1314 else 1315 RGE_CLRBIT_4(sc, RGE_RXCFG, RGE_RXCFG_VLANSTRIP); 1316 1317 /* Enable/disable RX checksum based on enabled capability */ 1318 if ((if_getcapenable(sc->sc_ifp) & IFCAP_RXCSUM) != 0) 1319 RGE_SETBIT_2(sc, RGE_CPLUSCMD, RGE_CPLUSCMD_RXCSUM); 1320 else 1321 RGE_CLRBIT_2(sc, RGE_CPLUSCMD, RGE_CPLUSCMD_RXCSUM); 1322 RGE_READ_2(sc, RGE_CPLUSCMD); 1323 1324 /* Set Maximum frame size. */ 1325 RGE_WRITE_2(sc, RGE_RXMAXSIZE, RGE_JUMBO_FRAMELEN); 1326 1327 /* Disable RXDV gate. */ 1328 RGE_CLRBIT_1(sc, RGE_PPSW, 0x08); 1329 DELAY(2000); 1330 1331 /* Program promiscuous mode and multicast filters. */ 1332 rge_iff_locked(sc); 1333 1334 if (sc->rge_type == MAC_R27) 1335 RGE_CLRBIT_1(sc, RGE_RADMFIFO_PROTECT, 0x2001); 1336 1337 rge_disable_aspm_clkreq(sc); 1338 1339 RGE_CLRBIT_1(sc, RGE_EECMD, RGE_EECMD_WRITECFG); 1340 DELAY(10); 1341 1342 rge_ifmedia_upd(sc->sc_ifp); 1343 1344 /* Enable transmit and receive. */ 1345 RGE_WRITE_1(sc, RGE_CMD, RGE_CMD_TXENB | RGE_CMD_RXENB); 1346 1347 /* Enable interrupts. */ 1348 rge_setup_intr(sc, RGE_IMTYPE_SIM); 1349 1350 if_setdrvflagbits(sc->sc_ifp, IFF_DRV_RUNNING, 0); 1351 if_setdrvflagbits(sc->sc_ifp, 0, IFF_DRV_OACTIVE); 1352 1353 callout_reset(&sc->sc_timeout, hz, rge_tick, sc); 1354 1355 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: init completed!\n", __func__); 1356 1357 /* Unblock transmit when we release the lock */ 1358 sc->sc_stopped = false; 1359 } 1360 1361 /* 1362 * @brief Stop the adapter and free any mbufs allocated to the RX and TX lists. 1363 * 1364 * Must be called with the driver lock held. 1365 */ 1366 void 1367 rge_stop_locked(struct rge_softc *sc) 1368 { 1369 struct rge_queues *q = sc->sc_queues; 1370 int i; 1371 1372 RGE_ASSERT_LOCKED(sc); 1373 1374 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: called!\n", __func__); 1375 1376 callout_stop(&sc->sc_timeout); 1377 1378 /* Stop pending TX submissions */ 1379 sc->sc_stopped = true; 1380 1381 if_setdrvflagbits(sc->sc_ifp, 0, IFF_DRV_RUNNING); 1382 sc->rge_timerintr = 0; 1383 sc->sc_watchdog = 0; 1384 1385 RGE_CLRBIT_4(sc, RGE_RXCFG, RGE_RXCFG_ALLPHYS | RGE_RXCFG_INDIV | 1386 RGE_RXCFG_MULTI | RGE_RXCFG_BROAD | RGE_RXCFG_RUNT | 1387 RGE_RXCFG_ERRPKT); 1388 1389 rge_hw_reset(sc); 1390 1391 RGE_MAC_CLRBIT(sc, 0xc0ac, 0x1f80); 1392 1393 if_setdrvflagbits(sc->sc_ifp, 0, IFF_DRV_OACTIVE); 1394 1395 if (q->q_rx.rge_head != NULL) { 1396 m_freem(q->q_rx.rge_head); 1397 q->q_rx.rge_head = NULL; 1398 q->q_rx.rge_tail = &q->q_rx.rge_head; 1399 } 1400 1401 /* Free the TX list buffers. */ 1402 for (i = 0; i < RGE_TX_LIST_CNT; i++) { 1403 if (q->q_tx.rge_txq[i].txq_mbuf != NULL) { 1404 bus_dmamap_unload(sc->sc_dmat_tx_buf, 1405 q->q_tx.rge_txq[i].txq_dmamap); 1406 m_freem(q->q_tx.rge_txq[i].txq_mbuf); 1407 q->q_tx.rge_txq[i].txq_mbuf = NULL; 1408 } 1409 } 1410 1411 /* Free the RX list buffers. */ 1412 for (i = 0; i < RGE_RX_LIST_CNT; i++) { 1413 if (q->q_rx.rge_rxq[i].rxq_mbuf != NULL) { 1414 bus_dmamap_unload(sc->sc_dmat_rx_buf, 1415 q->q_rx.rge_rxq[i].rxq_dmamap); 1416 m_freem(q->q_rx.rge_rxq[i].rxq_mbuf); 1417 q->q_rx.rge_rxq[i].rxq_mbuf = NULL; 1418 } 1419 } 1420 1421 /* Free pending TX frames */ 1422 /* TODO: should be per TX queue */ 1423 rge_txq_flush_mbufs(sc); 1424 } 1425 1426 /* 1427 * Set media options. 1428 */ 1429 static int 1430 rge_ifmedia_upd(if_t ifp) 1431 { 1432 struct rge_softc *sc = if_getsoftc(ifp); 1433 struct ifmedia *ifm = &sc->sc_media; 1434 int anar, gig, val; 1435 1436 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1437 return (EINVAL); 1438 1439 /* Disable Gigabit Lite. */ 1440 RGE_PHY_CLRBIT(sc, 0xa428, 0x0200); 1441 RGE_PHY_CLRBIT(sc, 0xa5ea, 0x0001); 1442 if (sc->rge_type == MAC_R26 || sc->rge_type == MAC_R27) 1443 RGE_PHY_CLRBIT(sc, 0xa5ea, 0x0007); 1444 1445 val = rge_read_phy_ocp(sc, 0xa5d4); 1446 switch (sc->rge_type) { 1447 case MAC_R27: 1448 val &= ~RGE_ADV_10000TFDX; 1449 /* fallthrough */ 1450 case MAC_R26: 1451 val &= ~RGE_ADV_5000TFDX; 1452 /* fallthrough */ 1453 default: 1454 val &= ~RGE_ADV_2500TFDX; 1455 break; 1456 } 1457 1458 anar = ANAR_TX_FD | ANAR_TX | ANAR_10_FD | ANAR_10; 1459 gig = GTCR_ADV_1000TFDX | GTCR_ADV_1000THDX; 1460 1461 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1462 case IFM_AUTO: 1463 val |= RGE_ADV_2500TFDX; 1464 if (sc->rge_type == MAC_R26) 1465 val |= RGE_ADV_5000TFDX; 1466 else if (sc->rge_type == MAC_R27) 1467 val |= RGE_ADV_5000TFDX | RGE_ADV_10000TFDX; 1468 break; 1469 case IFM_10G_T: 1470 val |= RGE_ADV_10000TFDX; 1471 if_setbaudrate(ifp, IF_Gbps(10)); 1472 break; 1473 case IFM_5000_T: 1474 val |= RGE_ADV_5000TFDX; 1475 if_setbaudrate(ifp, IF_Gbps(5)); 1476 break; 1477 case IFM_2500_T: 1478 val |= RGE_ADV_2500TFDX; 1479 if_setbaudrate(ifp, IF_Mbps(2500)); 1480 break; 1481 case IFM_1000_T: 1482 if_setbaudrate(ifp, IF_Gbps(1)); 1483 break; 1484 case IFM_100_TX: 1485 gig = rge_read_phy(sc, 0, MII_100T2CR) & 1486 ~(GTCR_ADV_1000TFDX | GTCR_ADV_1000THDX); 1487 anar = ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) ? 1488 ANAR_TX | ANAR_TX_FD | ANAR_10_FD | ANAR_10 : 1489 ANAR_TX | ANAR_10_FD | ANAR_10; 1490 if_setbaudrate(ifp, IF_Mbps(100)); 1491 break; 1492 case IFM_10_T: 1493 gig = rge_read_phy(sc, 0, MII_100T2CR) & 1494 ~(GTCR_ADV_1000TFDX | GTCR_ADV_1000THDX); 1495 anar = ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) ? 1496 ANAR_10_FD | ANAR_10 : ANAR_10; 1497 if_setbaudrate(ifp, IF_Mbps(10)); 1498 break; 1499 default: 1500 RGE_PRINT_ERROR(sc, "unsupported media type\n"); 1501 return (EINVAL); 1502 } 1503 1504 rge_write_phy(sc, 0, MII_ANAR, anar | ANAR_PAUSE_ASYM | ANAR_FC); 1505 rge_write_phy(sc, 0, MII_100T2CR, gig); 1506 rge_write_phy_ocp(sc, 0xa5d4, val); 1507 rge_write_phy(sc, 0, MII_BMCR, BMCR_RESET | BMCR_AUTOEN | 1508 BMCR_STARTNEG); 1509 1510 return (0); 1511 } 1512 1513 /* 1514 * Report current media status. 1515 */ 1516 static void 1517 rge_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) 1518 { 1519 struct rge_softc *sc = if_getsoftc(ifp); 1520 uint16_t status = 0; 1521 1522 ifmr->ifm_status = IFM_AVALID; 1523 ifmr->ifm_active = IFM_ETHER; 1524 1525 if (rge_get_link_status(sc)) { 1526 ifmr->ifm_status |= IFM_ACTIVE; 1527 1528 status = RGE_READ_2(sc, RGE_PHYSTAT); 1529 if ((status & RGE_PHYSTAT_FDX) || 1530 (status & (RGE_PHYSTAT_1000MBPS | RGE_PHYSTAT_2500MBPS | 1531 RGE_PHYSTAT_5000MBPS | RGE_PHYSTAT_10000MBPS))) 1532 ifmr->ifm_active |= IFM_FDX; 1533 else 1534 ifmr->ifm_active |= IFM_HDX; 1535 1536 if (status & RGE_PHYSTAT_10MBPS) 1537 ifmr->ifm_active |= IFM_10_T; 1538 else if (status & RGE_PHYSTAT_100MBPS) 1539 ifmr->ifm_active |= IFM_100_TX; 1540 else if (status & RGE_PHYSTAT_1000MBPS) 1541 ifmr->ifm_active |= IFM_1000_T; 1542 else if (status & RGE_PHYSTAT_2500MBPS) 1543 ifmr->ifm_active |= IFM_2500_T; 1544 else if (status & RGE_PHYSTAT_5000MBPS) 1545 ifmr->ifm_active |= IFM_5000_T; 1546 else if (status & RGE_PHYSTAT_5000MBPS) 1547 ifmr->ifm_active |= IFM_5000_T; 1548 else if (status & RGE_PHYSTAT_10000MBPS) 1549 ifmr->ifm_active |= IFM_10G_T; 1550 } 1551 } 1552 1553 /** 1554 * @brief callback to load/populate a single physical address 1555 */ 1556 static void 1557 rge_dma_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 1558 { 1559 bus_addr_t *paddr = (bus_addr_t *) arg; 1560 1561 *paddr = 0; 1562 1563 if (error) { 1564 printf("%s: error! (%d)\n", __func__, error); 1565 *paddr = 0; 1566 return; 1567 } 1568 1569 if (nsegs != 1) { 1570 printf("%s: too many segs (got %d)\n", __func__, nsegs); 1571 *paddr = 0; 1572 return; 1573 } 1574 1575 *paddr = segs[0].ds_addr; 1576 } 1577 1578 /** 1579 * @brief Allocate memory for RX/TX rings. 1580 * 1581 * Called with the driver lock NOT held. 1582 */ 1583 static int 1584 rge_allocmem(struct rge_softc *sc) 1585 { 1586 struct rge_queues *q = sc->sc_queues; 1587 int error; 1588 int i; 1589 1590 RGE_ASSERT_UNLOCKED(sc); 1591 1592 /* Allocate DMA'able memory for the TX ring. */ 1593 error = bus_dmamem_alloc(sc->sc_dmat_tx_desc, 1594 (void **) &q->q_tx.rge_tx_list, 1595 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, 1596 &q->q_tx.rge_tx_list_map); 1597 if (error) { 1598 RGE_PRINT_ERROR(sc, "%s: error (alloc tx_list.map) (%d)\n", 1599 __func__, error); 1600 goto error; 1601 } 1602 1603 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: tx_list=%p\n", __func__, 1604 q->q_tx.rge_tx_list); 1605 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: tx_list_map=%p\n", __func__, 1606 q->q_tx.rge_tx_list_map); 1607 1608 /* Load the map for the TX ring. */ 1609 error = bus_dmamap_load(sc->sc_dmat_tx_desc, 1610 q->q_tx.rge_tx_list_map, 1611 q->q_tx.rge_tx_list, 1612 RGE_TX_LIST_SZ, 1613 rge_dma_load_cb, 1614 (void *) &q->q_tx.rge_tx_list_paddr, 1615 BUS_DMA_NOWAIT); 1616 1617 if ((error != 0) || (q->q_tx.rge_tx_list_paddr == 0)) { 1618 RGE_PRINT_ERROR(sc, "%s: error (load tx_list.map) (%d)\n", 1619 __func__, error); 1620 goto error; 1621 } 1622 1623 /* Create DMA maps for TX buffers. */ 1624 for (i = 0; i < RGE_TX_LIST_CNT; i++) { 1625 error = bus_dmamap_create(sc->sc_dmat_tx_buf, 1626 0, &q->q_tx.rge_txq[i].txq_dmamap); 1627 if (error) { 1628 RGE_PRINT_ERROR(sc, 1629 "can't create DMA map for TX (%d)\n", error); 1630 goto error; 1631 } 1632 } 1633 1634 /* Allocate DMA'able memory for the RX ring. */ 1635 error = bus_dmamem_alloc(sc->sc_dmat_rx_desc, 1636 (void **) &q->q_rx.rge_rx_list, 1637 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, 1638 &q->q_rx.rge_rx_list_map); 1639 if (error) { 1640 RGE_PRINT_ERROR(sc, "%s: error (alloc rx_list.map) (%d)\n", 1641 __func__, error); 1642 goto error; 1643 } 1644 1645 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: rx_list=%p\n", __func__, 1646 q->q_rx.rge_rx_list); 1647 RGE_DPRINTF(sc, RGE_DEBUG_INIT, "%s: rx_list_map=%p\n", __func__, 1648 q->q_rx.rge_rx_list_map); 1649 1650 /* Load the map for the RX ring. */ 1651 error = bus_dmamap_load(sc->sc_dmat_rx_desc, 1652 q->q_rx.rge_rx_list_map, 1653 q->q_rx.rge_rx_list, 1654 RGE_RX_LIST_SZ, 1655 rge_dma_load_cb, 1656 (void *) &q->q_rx.rge_rx_list_paddr, 1657 BUS_DMA_NOWAIT); 1658 1659 if ((error != 0) || (q->q_rx.rge_rx_list_paddr == 0)) { 1660 RGE_PRINT_ERROR(sc, "%s: error (load rx_list.map) (%d)\n", 1661 __func__, error); 1662 goto error; 1663 } 1664 1665 /* Create DMA maps for RX buffers. */ 1666 for (i = 0; i < RGE_RX_LIST_CNT; i++) { 1667 error = bus_dmamap_create(sc->sc_dmat_rx_buf, 1668 0, &q->q_rx.rge_rxq[i].rxq_dmamap); 1669 if (error) { 1670 RGE_PRINT_ERROR(sc, 1671 "can't create DMA map for RX (%d)\n", error); 1672 goto error; 1673 } 1674 } 1675 1676 return (0); 1677 error: 1678 1679 rge_freemem(sc); 1680 1681 return (error); 1682 } 1683 1684 /** 1685 * @brief Allocate memory for MAC stats. 1686 * 1687 * Called with the driver lock NOT held. 1688 */ 1689 static int 1690 rge_alloc_stats_mem(struct rge_softc *sc) 1691 { 1692 struct rge_mac_stats *ss = &sc->sc_mac_stats; 1693 int error; 1694 1695 RGE_ASSERT_UNLOCKED(sc); 1696 1697 /* Allocate DMA'able memory for the stats buffer. */ 1698 error = bus_dmamem_alloc(sc->sc_dmat_stats_buf, 1699 (void **) &ss->stats, BUS_DMA_WAITOK | BUS_DMA_ZERO, 1700 &ss->map); 1701 if (error) { 1702 RGE_PRINT_ERROR(sc, "%s: error (alloc stats) (%d)\n", 1703 __func__, error); 1704 goto error; 1705 } 1706 1707 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: stats=%p\n", __func__, ss->stats); 1708 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: map=%p\n", __func__, ss->map); 1709 1710 /* Load the map for the TX ring. */ 1711 error = bus_dmamap_load(sc->sc_dmat_stats_buf, 1712 ss->map, 1713 ss->stats, 1714 RGE_STATS_BUF_SIZE, 1715 rge_dma_load_cb, 1716 (void *) &ss->paddr, 1717 BUS_DMA_NOWAIT); 1718 1719 if ((error != 0) || (ss->paddr == 0)) { 1720 RGE_PRINT_ERROR(sc, "%s: error (load stats.map) (%d)\n", 1721 __func__, error); 1722 if (error == 0) 1723 error = ENXIO; 1724 goto error; 1725 } 1726 1727 return (0); 1728 1729 error: 1730 rge_free_stats_mem(sc); 1731 1732 return (error); 1733 } 1734 1735 1736 /** 1737 * @brief Free the TX/RX DMA buffers and mbufs. 1738 * 1739 * Called with the driver lock NOT held. 1740 */ 1741 static int 1742 rge_freemem(struct rge_softc *sc) 1743 { 1744 struct rge_queues *q = sc->sc_queues; 1745 int i; 1746 1747 RGE_ASSERT_UNLOCKED(sc); 1748 1749 /* TX desc */ 1750 bus_dmamap_unload(sc->sc_dmat_tx_desc, q->q_tx.rge_tx_list_map); 1751 if (q->q_tx.rge_tx_list != NULL) 1752 bus_dmamem_free(sc->sc_dmat_tx_desc, q->q_tx.rge_tx_list, 1753 q->q_tx.rge_tx_list_map); 1754 memset(&q->q_tx, 0, sizeof(q->q_tx)); 1755 1756 /* TX buf */ 1757 for (i = 0; i < RGE_TX_LIST_CNT; i++) { 1758 struct rge_txq *tx = &q->q_tx.rge_txq[i]; 1759 1760 /* unmap/free mbuf if it's still alloc'ed and mapped */ 1761 if (tx->txq_mbuf != NULL) { 1762 static bool do_warning = false; 1763 1764 if (do_warning == false) { 1765 RGE_PRINT_ERROR(sc, 1766 "%s: TX mbuf should've been freed!\n", 1767 __func__); 1768 do_warning = true; 1769 } 1770 if (tx->txq_dmamap != NULL) { 1771 bus_dmamap_sync(sc->sc_dmat_tx_buf, 1772 tx->txq_dmamap, BUS_DMASYNC_POSTREAD); 1773 bus_dmamap_unload(sc->sc_dmat_tx_buf, 1774 tx->txq_dmamap); 1775 } 1776 m_free(tx->txq_mbuf); 1777 tx->txq_mbuf = NULL; 1778 } 1779 1780 /* Destroy the dmamap if it's allocated */ 1781 if (tx->txq_dmamap != NULL) { 1782 bus_dmamap_destroy(sc->sc_dmat_tx_buf, tx->txq_dmamap); 1783 tx->txq_dmamap = NULL; 1784 } 1785 } 1786 1787 /* RX desc */ 1788 bus_dmamap_unload(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map); 1789 if (q->q_rx.rge_rx_list != 0) 1790 bus_dmamem_free(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list, 1791 q->q_rx.rge_rx_list_map); 1792 memset(&q->q_rx, 0, sizeof(q->q_tx)); 1793 1794 /* RX buf */ 1795 for (i = 0; i < RGE_RX_LIST_CNT; i++) { 1796 struct rge_rxq *rx = &q->q_rx.rge_rxq[i]; 1797 1798 /* unmap/free mbuf if it's still alloc'ed and mapped */ 1799 if (rx->rxq_mbuf != NULL) { 1800 if (rx->rxq_dmamap != NULL) { 1801 bus_dmamap_sync(sc->sc_dmat_rx_buf, 1802 rx->rxq_dmamap, BUS_DMASYNC_POSTREAD); 1803 bus_dmamap_unload(sc->sc_dmat_rx_buf, 1804 rx->rxq_dmamap); 1805 } 1806 m_free(rx->rxq_mbuf); 1807 rx->rxq_mbuf = NULL; 1808 } 1809 1810 /* Destroy the dmamap if it's allocated */ 1811 if (rx->rxq_dmamap != NULL) { 1812 bus_dmamap_destroy(sc->sc_dmat_rx_buf, rx->rxq_dmamap); 1813 rx->rxq_dmamap = NULL; 1814 } 1815 } 1816 1817 return (0); 1818 } 1819 1820 /** 1821 * @brief Free the stats memory. 1822 * 1823 * Called with the driver lock NOT held. 1824 */ 1825 static int 1826 rge_free_stats_mem(struct rge_softc *sc) 1827 { 1828 struct rge_mac_stats *ss = &sc->sc_mac_stats; 1829 1830 RGE_ASSERT_UNLOCKED(sc); 1831 1832 bus_dmamap_unload(sc->sc_dmat_stats_buf, ss->map); 1833 if (ss->stats != NULL) 1834 bus_dmamem_free(sc->sc_dmat_stats_buf, ss->stats, ss->map); 1835 memset(ss, 0, sizeof(*ss)); 1836 return (0); 1837 } 1838 1839 static uint32_t 1840 rx_ring_space(struct rge_queues *q) 1841 { 1842 uint32_t prod, cons; 1843 uint32_t ret; 1844 1845 RGE_ASSERT_LOCKED(q->q_sc); 1846 1847 prod = q->q_rx.rge_rxq_prodidx; 1848 cons = q->q_rx.rge_rxq_considx; 1849 1850 ret = (cons + RGE_RX_LIST_CNT - prod - 1) % RGE_RX_LIST_CNT + 1; 1851 1852 if (ret > RGE_RX_LIST_CNT) 1853 return RGE_RX_LIST_CNT; 1854 1855 return (ret); 1856 } 1857 1858 /* 1859 * Initialize the RX descriptor and attach an mbuf cluster at the given offset. 1860 * 1861 * Note: this relies on the rxr ring buffer abstraction to not 1862 * over-fill the RX ring. For FreeBSD we'll need to use the 1863 * prod/cons RX indexes to know how much RX ring space to 1864 * populate. 1865 * 1866 * This routine will increment the producer index if successful. 1867 * 1868 * This must be called with the driver lock held. 1869 */ 1870 static int 1871 rge_newbuf(struct rge_queues *q) 1872 { 1873 struct rge_softc *sc = q->q_sc; 1874 struct mbuf *m; 1875 struct rge_rx_desc *r; 1876 struct rge_rxq *rxq; 1877 bus_dmamap_t rxmap; 1878 bus_dma_segment_t seg[1]; 1879 uint32_t cmdsts; 1880 int nsegs; 1881 uint32_t idx; 1882 1883 RGE_ASSERT_LOCKED(q->q_sc); 1884 1885 /* 1886 * Verify we have enough space in the ring; error out 1887 * if we do not. 1888 */ 1889 if (rx_ring_space(q) == 0) 1890 return (ENOBUFS); 1891 1892 idx = q->q_rx.rge_rxq_prodidx; 1893 rxq = &q->q_rx.rge_rxq[idx]; 1894 rxmap = rxq->rxq_dmamap; 1895 1896 /* 1897 * If we already have an mbuf here then something messed up; 1898 * exit out as the hardware may be DMAing to it. 1899 */ 1900 if (rxq->rxq_mbuf != NULL) { 1901 RGE_PRINT_ERROR(sc, 1902 "%s: RX ring slot %d already has an mbuf?\n", __func__, 1903 idx); 1904 return (ENOBUFS); 1905 } 1906 1907 /* Allocate single buffer backed mbuf of MCLBYTES */ 1908 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1909 if (m == NULL) 1910 return (ENOBUFS); 1911 1912 m->m_len = m->m_pkthdr.len = MCLBYTES; 1913 1914 nsegs = 1; 1915 if (bus_dmamap_load_mbuf_sg(sc->sc_dmat_rx_buf, rxmap, m, seg, &nsegs, 1916 BUS_DMA_NOWAIT)) { 1917 m_freem(m); 1918 return (ENOBUFS); 1919 } 1920 1921 /* 1922 * Make sure any changes made to the buffer have been flushed to host 1923 * memory. 1924 */ 1925 bus_dmamap_sync(sc->sc_dmat_rx_buf, rxmap, 1926 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1927 1928 /* 1929 * Map the segment into RX descriptors. Note that this 1930 * only currently supports a single segment per mbuf; 1931 * the call to load_mbuf_sg above specified a single segment. 1932 */ 1933 r = &q->q_rx.rge_rx_list[idx]; 1934 1935 rxq->rxq_mbuf = m; 1936 1937 cmdsts = seg[0].ds_len; /* XXX how big is this field in the descriptor? */ 1938 if (idx == RGE_RX_LIST_CNT - 1) 1939 cmdsts |= RGE_RDCMDSTS_EOR; 1940 1941 /* 1942 * Configure the DMA pointer and config, but don't hand 1943 * it yet to the hardware. 1944 */ 1945 r->hi_qword1.rx_qword4.rge_cmdsts = htole32(cmdsts); 1946 r->hi_qword1.rx_qword4.rge_extsts = htole32(0); 1947 r->hi_qword0.rge_addr = htole64(seg[0].ds_addr); 1948 wmb(); 1949 1950 /* 1951 * Mark the specific descriptor slot as "this descriptor is now 1952 * owned by the hardware", which when the hardware next sees 1953 * this, it'll continue RX DMA. 1954 */ 1955 cmdsts |= RGE_RDCMDSTS_OWN; 1956 r->hi_qword1.rx_qword4.rge_cmdsts = htole32(cmdsts); 1957 wmb(); 1958 1959 /* 1960 * At this point the hope is the whole ring is now updated and 1961 * consistent; if the hardware was waiting for a descriptor to be 1962 * ready to write into then it should be ready here. 1963 */ 1964 1965 RGE_DPRINTF(sc, RGE_DEBUG_RECV_DESC, 1966 "%s: [%d]: m=%p, m_data=%p, m_len=%ju, phys=0x%jx len %ju, " 1967 "desc=0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 1968 __func__, 1969 idx, 1970 m, 1971 m->m_data, 1972 (uintmax_t) m->m_len, 1973 (uintmax_t) seg[0].ds_addr, 1974 (uintmax_t) seg[0].ds_len, 1975 ((uint32_t *) r)[0], 1976 ((uint32_t *) r)[1], 1977 ((uint32_t *) r)[2], 1978 ((uint32_t *) r)[3], 1979 ((uint32_t *) r)[4], 1980 ((uint32_t *) r)[5], 1981 ((uint32_t *) r)[6], 1982 ((uint32_t *) r)[7]); 1983 1984 q->q_rx.rge_rxq_prodidx = RGE_NEXT_RX_DESC(idx); 1985 1986 return (0); 1987 } 1988 1989 static void 1990 rge_rx_list_init(struct rge_queues *q) 1991 { 1992 memset(q->q_rx.rge_rx_list, 0, RGE_RX_LIST_SZ); 1993 1994 RGE_ASSERT_LOCKED(q->q_sc); 1995 1996 q->q_rx.rge_rxq_prodidx = q->q_rx.rge_rxq_considx = 0; 1997 q->q_rx.rge_head = NULL; 1998 q->q_rx.rge_tail = &q->q_rx.rge_head; 1999 2000 RGE_DPRINTF(q->q_sc, RGE_DEBUG_SETUP, "%s: rx_list=%p\n", __func__, 2001 q->q_rx.rge_rx_list); 2002 2003 rge_fill_rx_ring(q); 2004 } 2005 2006 /** 2007 * @brief Fill / refill the RX ring as needed. 2008 * 2009 * Refill the RX ring with one less than the total descriptors needed. 2010 * This makes the check in rge_rxeof() easier - it can just check 2011 * descriptors from cons -> prod and bail once it hits prod. 2012 * If the whole ring is filled then cons == prod, and that shortcut 2013 * fails. 2014 * 2015 * This must be called with the driver lock held. 2016 */ 2017 static void 2018 rge_fill_rx_ring(struct rge_queues *q) 2019 { 2020 struct rge_softc *sc = q->q_sc; 2021 uint32_t count, i, prod, cons; 2022 2023 RGE_ASSERT_LOCKED(q->q_sc); 2024 2025 prod = q->q_rx.rge_rxq_prodidx; 2026 cons = q->q_rx.rge_rxq_considx; 2027 count = rx_ring_space(q); 2028 2029 /* Fill to count-1; bail if we don't have the space */ 2030 if (count <= 1) 2031 return; 2032 count--; 2033 2034 RGE_DPRINTF(sc, RGE_DEBUG_RECV_DESC, "%s: prod=%u, cons=%u, space=%u\n", 2035 __func__, prod, cons, count); 2036 2037 /* Make sure device->host changes are visible */ 2038 bus_dmamap_sync(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map, 2039 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2040 2041 for (i = 0; i < count; i++) { 2042 if (rge_newbuf(q)) 2043 break; 2044 } 2045 2046 /* Make changes visible to the device */ 2047 bus_dmamap_sync(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map, 2048 BUS_DMASYNC_PREWRITE); 2049 } 2050 2051 static void 2052 rge_tx_list_init(struct rge_queues *q) 2053 { 2054 struct rge_softc *sc = q->q_sc; 2055 struct rge_tx_desc *d; 2056 int i; 2057 2058 RGE_ASSERT_LOCKED(q->q_sc); 2059 2060 memset(q->q_tx.rge_tx_list, 0, RGE_TX_LIST_SZ); 2061 2062 for (i = 0; i < RGE_TX_LIST_CNT; i++) 2063 q->q_tx.rge_txq[i].txq_mbuf = NULL; 2064 2065 d = &q->q_tx.rge_tx_list[RGE_TX_LIST_CNT - 1]; 2066 d->rge_cmdsts = htole32(RGE_TDCMDSTS_EOR); 2067 2068 bus_dmamap_sync(sc->sc_dmat_tx_desc, q->q_tx.rge_tx_list_map, 2069 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2070 wmb(); 2071 2072 q->q_tx.rge_txq_prodidx = q->q_tx.rge_txq_considx = 0; 2073 2074 RGE_DPRINTF(sc, RGE_DEBUG_SETUP, "%s: rx_list=%p\n", __func__, 2075 q->q_tx.rge_tx_list); 2076 } 2077 2078 int 2079 rge_rxeof(struct rge_queues *q, struct mbufq *mq) 2080 { 2081 struct rge_softc *sc = q->q_sc; 2082 struct mbuf *m; 2083 struct rge_rx_desc *cur_rx; 2084 struct rge_rxq *rxq; 2085 uint32_t rxstat, extsts; 2086 int i, mlen, rx = 0; 2087 int cons, prod; 2088 int maxpkt = 16; /* XXX TODO: make this a tunable */ 2089 bool check_hwcsum; 2090 2091 check_hwcsum = ((if_getcapenable(sc->sc_ifp) & IFCAP_RXCSUM) != 0); 2092 2093 RGE_ASSERT_LOCKED(sc); 2094 2095 sc->sc_drv_stats.rxeof_cnt++; 2096 2097 RGE_DPRINTF(sc, RGE_DEBUG_INTR, "%s; called\n", __func__); 2098 2099 /* Note: if_re is POSTREAD/WRITE, rge is only POSTWRITE */ 2100 bus_dmamap_sync(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map, 2101 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2102 2103 prod = q->q_rx.rge_rxq_prodidx; 2104 2105 /* 2106 * Loop around until we've run out of active descriptors to check 2107 * or maxpkt has been reached. 2108 */ 2109 for (i = cons = q->q_rx.rge_rxq_considx; 2110 maxpkt > 0 && i != prod; 2111 i = RGE_NEXT_RX_DESC(i)) { 2112 /* break out of loop if we're not running */ 2113 if ((if_getdrvflags(sc->sc_ifp) & IFF_DRV_RUNNING) == 0) 2114 break; 2115 2116 /* get the current rx descriptor to check descriptor status */ 2117 cur_rx = &q->q_rx.rge_rx_list[i]; 2118 rxstat = le32toh(cur_rx->hi_qword1.rx_qword4.rge_cmdsts); 2119 if ((rxstat & RGE_RDCMDSTS_OWN) != 0) { 2120 break; 2121 } 2122 2123 /* Ensure everything else has been DMAed */ 2124 rmb(); 2125 2126 /* Get the current rx buffer, sync */ 2127 rxq = &q->q_rx.rge_rxq[i]; 2128 2129 /* Ensure any device updates are now visible in host memory */ 2130 bus_dmamap_sync(sc->sc_dmat_rx_buf, rxq->rxq_dmamap, 2131 BUS_DMASYNC_POSTREAD); 2132 2133 /* Unload the DMA map, we are done with it here */ 2134 bus_dmamap_unload(sc->sc_dmat_rx_buf, rxq->rxq_dmamap); 2135 m = rxq->rxq_mbuf; 2136 rxq->rxq_mbuf = NULL; 2137 2138 rx = 1; 2139 2140 RGE_DPRINTF(sc, RGE_DEBUG_RECV_DESC, 2141 "%s: RX: [%d]: m=%p, m_data=%p, m_len=%ju, " 2142 "desc=0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 2143 __func__, 2144 i, 2145 m, 2146 m->m_data, 2147 (uintmax_t) m->m_len, 2148 ((uint32_t *) cur_rx)[0], 2149 ((uint32_t *) cur_rx)[1], 2150 ((uint32_t *) cur_rx)[2], 2151 ((uint32_t *) cur_rx)[3], 2152 ((uint32_t *) cur_rx)[4], 2153 ((uint32_t *) cur_rx)[5], 2154 ((uint32_t *) cur_rx)[6], 2155 ((uint32_t *) cur_rx)[7]); 2156 2157 if ((rxstat & RGE_RDCMDSTS_SOF) != 0) { 2158 if (q->q_rx.rge_head != NULL) { 2159 sc->sc_drv_stats.rx_desc_err_multidesc++; 2160 if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 2161 1); 2162 m_freem(q->q_rx.rge_head); 2163 q->q_rx.rge_tail = &q->q_rx.rge_head; 2164 } 2165 2166 m->m_pkthdr.len = 0; 2167 } else if (q->q_rx.rge_head == NULL) { 2168 m_freem(m); 2169 continue; 2170 } else 2171 m->m_flags &= ~M_PKTHDR; 2172 2173 *q->q_rx.rge_tail = m; 2174 q->q_rx.rge_tail = &m->m_next; 2175 2176 mlen = rxstat & RGE_RDCMDSTS_FRAGLEN; 2177 m->m_len = mlen; 2178 2179 m = q->q_rx.rge_head; 2180 m->m_pkthdr.len += mlen; 2181 2182 /* Ethernet CRC error */ 2183 if (rxstat & RGE_RDCMDSTS_RXERRSUM) { 2184 sc->sc_drv_stats.rx_ether_csum_err++; 2185 if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 1); 2186 m_freem(m); 2187 q->q_rx.rge_head = NULL; 2188 q->q_rx.rge_tail = &q->q_rx.rge_head; 2189 continue; 2190 } 2191 2192 /* 2193 * This mbuf is part of a multi-descriptor frame, 2194 * so count it towards that. 2195 * 2196 * Yes, this means we won't be counting the 2197 * final descriptor/mbuf as part of a multi-descriptor 2198 * frame; if someone wishes to do that then it 2199 * shouldn't be too hard to add. 2200 */ 2201 if ((rxstat & RGE_RDCMDSTS_EOF) == 0) { 2202 sc->sc_drv_stats.rx_desc_jumbo_frag++; 2203 continue; 2204 } 2205 2206 q->q_rx.rge_head = NULL; 2207 q->q_rx.rge_tail = &q->q_rx.rge_head; 2208 2209 m_adj(m, -ETHER_CRC_LEN); 2210 m->m_pkthdr.rcvif = sc->sc_ifp; 2211 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); 2212 2213 extsts = le32toh(cur_rx->hi_qword1.rx_qword4.rge_extsts); 2214 2215 /* Check IP header checksum. */ 2216 if (check_hwcsum) { 2217 /* Does it exist for IPv4? */ 2218 if (extsts & RGE_RDEXTSTS_IPV4) { 2219 sc->sc_drv_stats.rx_offload_csum_ipv4_exists++; 2220 m->m_pkthdr.csum_flags |= 2221 CSUM_IP_CHECKED; 2222 } 2223 /* XXX IPv6 checksum check? */ 2224 2225 if (((extsts & RGE_RDEXTSTS_IPCSUMERR) == 0) 2226 && ((extsts & RGE_RDEXTSTS_IPV4) != 0)) { 2227 sc->sc_drv_stats.rx_offload_csum_ipv4_valid++; 2228 m->m_pkthdr.csum_flags |= 2229 CSUM_IP_VALID; 2230 } 2231 2232 /* Check TCP/UDP checksum. */ 2233 if ((extsts & (RGE_RDEXTSTS_IPV4 | RGE_RDEXTSTS_IPV6)) && 2234 (extsts & RGE_RDEXTSTS_TCPPKT)) { 2235 sc->sc_drv_stats.rx_offload_csum_tcp_exists++; 2236 if ((extsts & RGE_RDEXTSTS_TCPCSUMERR) == 0) { 2237 sc->sc_drv_stats.rx_offload_csum_tcp_valid++; 2238 /* TCP checksum OK */ 2239 m->m_pkthdr.csum_flags |= 2240 CSUM_DATA_VALID|CSUM_PSEUDO_HDR; 2241 m->m_pkthdr.csum_data = 0xffff; 2242 } 2243 } 2244 2245 if ((extsts & (RGE_RDEXTSTS_IPV4 | RGE_RDEXTSTS_IPV6)) && 2246 (extsts & RGE_RDEXTSTS_UDPPKT)) { 2247 sc->sc_drv_stats.rx_offload_csum_udp_exists++; 2248 if ((extsts & RGE_RDEXTSTS_UDPCSUMERR) == 0) { 2249 sc->sc_drv_stats.rx_offload_csum_udp_valid++; 2250 /* UDP checksum OK */ 2251 m->m_pkthdr.csum_flags |= 2252 CSUM_DATA_VALID|CSUM_PSEUDO_HDR; 2253 m->m_pkthdr.csum_data = 0xffff; 2254 } 2255 } 2256 } 2257 2258 if (extsts & RGE_RDEXTSTS_VTAG) { 2259 sc->sc_drv_stats.rx_offload_vlan_tag++; 2260 m->m_pkthdr.ether_vtag = 2261 ntohs(extsts & RGE_RDEXTSTS_VLAN_MASK); 2262 m->m_flags |= M_VLANTAG; 2263 } 2264 2265 mbufq_enqueue(mq, m); 2266 2267 maxpkt--; 2268 } 2269 2270 if (!rx) 2271 return (0); 2272 2273 /* 2274 * Make sure any device updates to the descriptor ring are 2275 * visible to the host before we continue. 2276 */ 2277 bus_dmamap_sync(sc->sc_dmat_rx_desc, q->q_rx.rge_rx_list_map, 2278 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2279 wmb(); 2280 2281 /* Update the consumer index, refill the RX ring */ 2282 q->q_rx.rge_rxq_considx = i; 2283 rge_fill_rx_ring(q); 2284 2285 return (1); 2286 } 2287 2288 int 2289 rge_txeof(struct rge_queues *q) 2290 { 2291 struct rge_softc *sc = q->q_sc; 2292 struct ifnet *ifp = sc->sc_ifp; 2293 struct rge_txq *txq; 2294 uint32_t txstat; 2295 int cons, prod, cur, idx; 2296 int free = 0, ntx = 0; 2297 int pktlen; 2298 bool is_mcast; 2299 2300 RGE_ASSERT_LOCKED(sc); 2301 2302 sc->sc_drv_stats.txeof_cnt++; 2303 2304 prod = q->q_tx.rge_txq_prodidx; 2305 cons = q->q_tx.rge_txq_considx; 2306 2307 idx = cons; 2308 while (idx != prod) { 2309 txq = &q->q_tx.rge_txq[idx]; 2310 cur = txq->txq_descidx; 2311 2312 rge_tx_list_sync(sc, q, cur, 1, BUS_DMASYNC_POSTREAD); 2313 txstat = q->q_tx.rge_tx_list[cur].rge_cmdsts; 2314 rge_tx_list_sync(sc, q, cur, 1, BUS_DMASYNC_PREREAD); 2315 if ((txstat & htole32(RGE_TDCMDSTS_OWN)) != 0) { 2316 free = 2; 2317 break; 2318 } 2319 2320 bus_dmamap_sync(sc->sc_dmat_tx_buf, txq->txq_dmamap, 2321 BUS_DMASYNC_POSTWRITE); 2322 bus_dmamap_unload(sc->sc_dmat_tx_buf, txq->txq_dmamap); 2323 pktlen = txq->txq_mbuf->m_pkthdr.len; 2324 is_mcast = ((txq->txq_mbuf->m_flags & M_MCAST) != 0); 2325 m_freem(txq->txq_mbuf); 2326 txq->txq_mbuf = NULL; 2327 ntx++; 2328 2329 if ((txstat & 2330 htole32(RGE_TDCMDSTS_EXCESSCOLL | RGE_TDCMDSTS_COLL)) != 0) 2331 if_inc_counter(ifp, IFCOUNTER_COLLISIONS, 1); 2332 if ((txstat & htole32(RGE_TDCMDSTS_TXERR)) != 0) 2333 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 2334 else { 2335 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 2336 if_inc_counter(ifp, IFCOUNTER_OBYTES, pktlen); 2337 if (is_mcast) 2338 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 2339 2340 } 2341 2342 idx = RGE_NEXT_TX_DESC(cur); 2343 free = 1; 2344 } 2345 2346 /* If we didn't complete any TX descriptors then return 0 */ 2347 if (free == 0) 2348 return (0); 2349 2350 if (idx >= cons) { 2351 rge_tx_list_sync(sc, q, cons, idx - cons, 2352 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2353 } else { 2354 rge_tx_list_sync(sc, q, cons, RGE_TX_LIST_CNT - cons, 2355 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2356 rge_tx_list_sync(sc, q, 0, idx, 2357 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2358 } 2359 2360 q->q_tx.rge_txq_considx = idx; 2361 2362 RGE_DPRINTF(sc, RGE_DEBUG_XMIT, 2363 "%s: handled %d frames; prod=%d, cons=%d\n", __func__, 2364 ntx, q->q_tx.rge_txq_prodidx, q->q_tx.rge_txq_considx); 2365 2366 /* 2367 * We processed the ring and hit a descriptor that was still 2368 * owned by the hardware, so there's still pending work. 2369 * 2370 * If we got to the end of the ring and there's no further 2371 * frames owned by the hardware then we can quieten the 2372 * watchdog. 2373 */ 2374 if (free == 2) 2375 sc->sc_watchdog = 5; 2376 else 2377 sc->sc_watchdog = 0; 2378 2379 /* 2380 * Kick-start the transmit task just in case we have 2381 * more frames available. 2382 */ 2383 taskqueue_enqueue(sc->sc_tq, &sc->sc_tx_task); 2384 2385 return (1); 2386 } 2387 2388 static u_int 2389 rge_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) 2390 { 2391 uint32_t crc, *hashes = arg; 2392 2393 // XXX TODO: validate this does addrlo? */ 2394 crc = ether_crc32_be(LLADDR(sdl), ETHER_ADDR_LEN) >> 26; 2395 crc &= 0x3f; 2396 2397 if (crc < 32) 2398 hashes[0] |= (1 << crc); 2399 else 2400 hashes[1] |= (1 << (crc - 32)); 2401 2402 return (1); 2403 } 2404 2405 /** 2406 * @brief Configure the RX filter and multicast filter. 2407 * 2408 * This must be called with the driver lock held. 2409 */ 2410 static void 2411 rge_iff_locked(struct rge_softc *sc) 2412 { 2413 uint32_t hashes[2]; 2414 uint32_t rxfilt; 2415 2416 RGE_ASSERT_LOCKED(sc); 2417 2418 rxfilt = RGE_READ_4(sc, RGE_RXCFG); 2419 rxfilt &= ~(RGE_RXCFG_ALLPHYS | RGE_RXCFG_MULTI); 2420 2421 /* 2422 * Always accept frames destined to our station address. 2423 * Always accept broadcast frames. 2424 */ 2425 rxfilt |= RGE_RXCFG_INDIV | RGE_RXCFG_BROAD; 2426 2427 if ((if_getflags(sc->sc_ifp) & (IFF_PROMISC | IFF_ALLMULTI)) != 0) { 2428 rxfilt |= RGE_RXCFG_MULTI; 2429 if ((if_getflags(sc->sc_ifp) & IFF_PROMISC) != 0) 2430 rxfilt |= RGE_RXCFG_ALLPHYS; 2431 hashes[0] = hashes[1] = 0xffffffff; 2432 } else { 2433 rxfilt |= RGE_RXCFG_MULTI; 2434 /* Program new filter. */ 2435 memset(hashes, 0, sizeof(hashes)); 2436 if_foreach_llmaddr(sc->sc_ifp, rge_hash_maddr, &hashes); 2437 } 2438 2439 RGE_WRITE_4(sc, RGE_RXCFG, rxfilt); 2440 RGE_WRITE_4(sc, RGE_MAR0, bswap32(hashes[1])); 2441 RGE_WRITE_4(sc, RGE_MAR4, bswap32(hashes[0])); 2442 } 2443 2444 static void 2445 rge_add_media_types(struct rge_softc *sc) 2446 { 2447 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10_T, 0, NULL); 2448 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 2449 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_100_TX, 0, NULL); 2450 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 2451 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_1000_T, 0, NULL); 2452 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2453 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_2500_T, 0, NULL); 2454 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 2455 2456 if (sc->rge_type == MAC_R26) { 2457 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_5000_T, 0, NULL); 2458 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_5000_T | IFM_FDX, 2459 0, NULL); 2460 } else if (sc->rge_type == MAC_R27) { 2461 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10G_T, 0, NULL); 2462 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10G_T | IFM_FDX, 2463 0, NULL); 2464 } 2465 } 2466 2467 /** 2468 * @brief Deferred packet dequeue and submit. 2469 */ 2470 static void 2471 rge_tx_task(void *arg, int npending) 2472 { 2473 struct rge_softc *sc = (struct rge_softc *) arg; 2474 /* Note: for now, one queue */ 2475 struct rge_queues *q = sc->sc_queues; 2476 struct mbuf *m; 2477 int ntx = 0; 2478 int idx, free, used; 2479 2480 RGE_DPRINTF(sc, RGE_DEBUG_XMIT, "%s: running\n", __func__); 2481 2482 RGE_LOCK(sc); 2483 sc->sc_drv_stats.tx_task_cnt++; 2484 2485 if (sc->sc_stopped == true) { 2486 sc->sc_watchdog = 0; 2487 RGE_UNLOCK(sc); 2488 return; 2489 } 2490 2491 /* Calculate free space. */ 2492 idx = q->q_tx.rge_txq_prodidx; 2493 free = q->q_tx.rge_txq_considx; 2494 if (free <= idx) 2495 free += RGE_TX_LIST_CNT; 2496 free -= idx; 2497 2498 for (;;) { 2499 if (free < RGE_TX_NSEGS + 2) { 2500 break; 2501 } 2502 2503 /* Dequeue */ 2504 m = mbufq_dequeue(&sc->sc_txq); 2505 if (m == NULL) 2506 break; 2507 2508 /* Attempt to encap */ 2509 used = rge_encap(sc, q, m, idx); 2510 if (used < 0) { 2511 if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1); 2512 m_freem(m); 2513 continue; 2514 } else if (used == 0) { 2515 mbufq_prepend(&sc->sc_txq, m); 2516 break; 2517 } 2518 2519 /* 2520 * Note: mbuf is now owned by the tx ring, but we hold the 2521 * lock so it's safe to pass it up here to be copied without 2522 * worrying the TX task will run and dequeue/free it before 2523 * we get a shot at it. 2524 */ 2525 ETHER_BPF_MTAP(sc->sc_ifp, m); 2526 2527 /* Update free/idx pointers */ 2528 free -= used; 2529 idx += used; 2530 if (idx >= RGE_TX_LIST_CNT) 2531 idx -= RGE_TX_LIST_CNT; 2532 2533 ntx++; 2534 } 2535 2536 /* Ok, did we queue anything? If so, poke the hardware */ 2537 if (ntx > 0) { 2538 q->q_tx.rge_txq_prodidx = idx; 2539 sc->sc_watchdog = 5; 2540 RGE_WRITE_2(sc, RGE_TXSTART, RGE_TXSTART_START); 2541 } 2542 2543 RGE_DPRINTF(sc, RGE_DEBUG_XMIT, 2544 "%s: handled %d frames; prod=%d, cons=%d\n", __func__, 2545 ntx, q->q_tx.rge_txq_prodidx, q->q_tx.rge_txq_considx); 2546 2547 RGE_UNLOCK(sc); 2548 } 2549 2550 /** 2551 * @brief Called by the sc_timeout callout. 2552 * 2553 * This is called by the callout code with the driver lock held. 2554 */ 2555 void 2556 rge_tick(void *arg) 2557 { 2558 struct rge_softc *sc = arg; 2559 2560 RGE_ASSERT_LOCKED(sc); 2561 2562 rge_link_state(sc); 2563 2564 /* 2565 * Since we don't have any other place yet to trigger/test this, 2566 * let's do it here every second and just bite the driver 2567 * blocking for a little bit whilst it happens. 2568 */ 2569 if ((if_getdrvflags(sc->sc_ifp) & IFF_DRV_RUNNING) != 0) 2570 rge_hw_mac_stats_fetch(sc, &sc->sc_mac_stats.lcl_stats); 2571 2572 /* 2573 * Handle the TX watchdog. 2574 */ 2575 if (sc->sc_watchdog > 0) { 2576 sc->sc_watchdog--; 2577 if (sc->sc_watchdog == 0) { 2578 RGE_PRINT_ERROR(sc, "TX timeout (watchdog)\n"); 2579 rge_init_locked(sc); 2580 sc->sc_drv_stats.tx_watchdog_timeout_cnt++; 2581 } 2582 } 2583 2584 callout_reset(&sc->sc_timeout, hz, rge_tick, sc); 2585 } 2586 2587 /** 2588 * @brief process a link state change. 2589 * 2590 * Must be called with the driver lock held. 2591 */ 2592 void 2593 rge_link_state(struct rge_softc *sc) 2594 { 2595 int link = LINK_STATE_DOWN; 2596 2597 RGE_ASSERT_LOCKED(sc); 2598 2599 if (rge_get_link_status(sc)) 2600 link = LINK_STATE_UP; 2601 2602 if (if_getlinkstate(sc->sc_ifp) != link) { 2603 sc->sc_drv_stats.link_state_change_cnt++; 2604 if_link_state_change(sc->sc_ifp, link); 2605 } 2606 } 2607 2608 /** 2609 * @brief Suspend 2610 */ 2611 static int 2612 rge_suspend(device_t dev) 2613 { 2614 struct rge_softc *sc = device_get_softc(dev); 2615 2616 RGE_LOCK(sc); 2617 rge_stop_locked(sc); 2618 /* TODO: wake on lan */ 2619 sc->sc_suspended = true; 2620 RGE_UNLOCK(sc); 2621 2622 return (0); 2623 } 2624 2625 /** 2626 * @brief Resume 2627 */ 2628 static int 2629 rge_resume(device_t dev) 2630 { 2631 struct rge_softc *sc = device_get_softc(dev); 2632 2633 RGE_LOCK(sc); 2634 /* TODO: wake on lan */ 2635 2636 /* reinit if required */ 2637 if (if_getflags(sc->sc_ifp) & IFF_UP) 2638 rge_init_locked(sc); 2639 2640 sc->sc_suspended = false; 2641 2642 RGE_UNLOCK(sc); 2643 2644 return (0); 2645 } 2646 2647 /** 2648 * @brief Shutdown the driver during shutdown 2649 */ 2650 static int 2651 rge_shutdown(device_t dev) 2652 { 2653 struct rge_softc *sc = device_get_softc(dev); 2654 2655 RGE_LOCK(sc); 2656 rge_stop_locked(sc); 2657 RGE_UNLOCK(sc); 2658 2659 return (0); 2660 } 2661 2662 static device_method_t rge_methods[] = { 2663 DEVMETHOD(device_probe, rge_probe), 2664 DEVMETHOD(device_attach, rge_attach), 2665 DEVMETHOD(device_detach, rge_detach), 2666 2667 DEVMETHOD(device_suspend, rge_suspend), 2668 DEVMETHOD(device_resume, rge_resume), 2669 DEVMETHOD(device_shutdown, rge_shutdown), 2670 2671 DEVMETHOD_END 2672 }; 2673 2674 static driver_t rge_driver = { 2675 "rge", 2676 rge_methods, 2677 sizeof(struct rge_softc) 2678 }; 2679 2680 MODULE_DEPEND(rge, pci, 1, 1, 1); 2681 MODULE_DEPEND(rge, ether, 1, 1, 1); 2682 2683 DRIVER_MODULE_ORDERED(rge, pci, rge_driver, NULL, NULL, SI_ORDER_ANY); 2684 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, rge, rge_devices, 2685 nitems(rge_devices) - 1); 2686