1 /*- 2 * Copyright (c) 2020 Michael J Karels 3 * Copyright (c) 2016, 2020 Jared McNeill <jmcneill@invisible.ca> 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 21 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 /* 30 * RPi4 (BCM 2711) Gigabit Ethernet ("GENET") controller 31 * 32 * This driver is derived in large part from bcmgenet.c from NetBSD by 33 * Jared McNeill. Parts of the structure and other common code in 34 * this driver have been copied from if_awg.c for the Allwinner EMAC, 35 * also by Jared McNeill. 36 */ 37 38 #include "opt_device_polling.h" 39 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/rman.h> 47 #include <sys/kernel.h> 48 #include <sys/endian.h> 49 #include <sys/mbuf.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/module.h> 53 #include <sys/taskqueue.h> 54 #include <sys/gpio.h> 55 56 #include <net/bpf.h> 57 #include <net/if.h> 58 #include <net/ethernet.h> 59 #include <net/if_dl.h> 60 #include <net/if_media.h> 61 #include <net/if_types.h> 62 #include <net/if_var.h> 63 64 #include <machine/bus.h> 65 66 #include <dev/ofw/ofw_bus.h> 67 #include <dev/ofw/ofw_bus_subr.h> 68 69 #define __BIT(_x) (1 << (_x)) 70 #include "if_genetreg.h" 71 72 #include <dev/mii/mii.h> 73 #include <dev/mii/miivar.h> 74 #include <dev/mii/mii_fdt.h> 75 76 #include <netinet/in.h> 77 #include <netinet/ip.h> 78 #include <netinet/ip6.h> 79 80 #include "syscon_if.h" 81 #include "miibus_if.h" 82 #include "gpio_if.h" 83 84 #define RD4(sc, reg) bus_read_4((sc)->res[_RES_MAC], (reg)) 85 #define WR4(sc, reg, val) bus_write_4((sc)->res[_RES_MAC], (reg), (val)) 86 87 #define GEN_LOCK(sc) mtx_lock(&(sc)->mtx) 88 #define GEN_UNLOCK(sc) mtx_unlock(&(sc)->mtx) 89 #define GEN_ASSERT_LOCKED(sc) mtx_assert(&(sc)->mtx, MA_OWNED) 90 #define GEN_ASSERT_UNLOCKED(sc) mtx_assert(&(sc)->mtx, MA_NOTOWNED) 91 92 #define TX_DESC_COUNT GENET_DMA_DESC_COUNT 93 #define RX_DESC_COUNT GENET_DMA_DESC_COUNT 94 95 #define TX_NEXT(n, count) (((n) + 1) & ((count) - 1)) 96 #define RX_NEXT(n, count) (((n) + 1) & ((count) - 1)) 97 98 99 #define TX_MAX_SEGS 20 100 101 /* Maximum number of mbufs to send to if_input */ 102 static int gen_rx_batch = 16 /* RX_BATCH_DEFAULT */; 103 TUNABLE_INT("hw.gen.rx_batch", &gen_rx_batch); 104 105 static struct ofw_compat_data compat_data[] = { 106 { "brcm,genet-v1", 1 }, 107 { "brcm,genet-v2", 2 }, 108 { "brcm,genet-v3", 3 }, 109 { "brcm,genet-v4", 4 }, 110 { "brcm,genet-v5", 5 }, 111 { NULL, 0 } 112 }; 113 114 enum { 115 _RES_MAC, /* what to call this? */ 116 _RES_IRQ1, 117 _RES_IRQ2, 118 _RES_NITEMS 119 }; 120 121 static struct resource_spec gen_spec[] = { 122 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 123 { SYS_RES_IRQ, 0, RF_ACTIVE }, 124 { SYS_RES_IRQ, 1, RF_ACTIVE }, 125 { -1, 0 } 126 }; 127 128 /* structure per ring entry */ 129 struct gen_ring_ent { 130 bus_dmamap_t map; 131 struct mbuf *mbuf; 132 }; 133 134 struct tx_queue { 135 int hwindex; /* hardware index */ 136 int nentries; 137 u_int queued; /* or avail? */ 138 u_int cur; 139 u_int next; 140 u_int prod_idx; 141 u_int cons_idx; 142 struct gen_ring_ent *entries; 143 }; 144 145 struct rx_queue { 146 int hwindex; /* hardware index */ 147 int nentries; 148 u_int cur; 149 u_int prod_idx; 150 u_int cons_idx; 151 struct gen_ring_ent *entries; 152 }; 153 154 struct gen_softc { 155 struct resource *res[_RES_NITEMS]; 156 struct mtx mtx; 157 if_t ifp; 158 device_t dev; 159 device_t miibus; 160 mii_contype_t phy_mode; 161 162 struct callout stat_ch; 163 struct task link_task; 164 void *ih; 165 void *ih2; 166 int type; 167 int if_flags; 168 int link; 169 bus_dma_tag_t tx_buf_tag; 170 /* 171 * The genet chip has multiple queues for transmit and receive. 172 * This driver uses only one (queue 16, the default), but is cast 173 * with multiple rings. The additional rings are used for different 174 * priorities. 175 */ 176 #define DEF_TXQUEUE 0 177 #define NTXQUEUE 1 178 struct tx_queue tx_queue[NTXQUEUE]; 179 struct gen_ring_ent tx_ring_ent[TX_DESC_COUNT]; /* ring entries */ 180 181 bus_dma_tag_t rx_buf_tag; 182 #define DEF_RXQUEUE 0 183 #define NRXQUEUE 1 184 struct rx_queue rx_queue[NRXQUEUE]; 185 struct gen_ring_ent rx_ring_ent[RX_DESC_COUNT]; /* ring entries */ 186 }; 187 188 static void gen_init(void *softc); 189 static void gen_start(if_t ifp); 190 static void gen_destroy(struct gen_softc *sc); 191 static int gen_encap(struct gen_softc *sc, struct mbuf **mp); 192 static int gen_parse_tx(struct mbuf *m, int csum_flags); 193 static int gen_ioctl(if_t ifp, u_long cmd, caddr_t data); 194 static int gen_get_phy_mode(device_t dev); 195 static bool gen_get_eaddr(device_t dev, struct ether_addr *eaddr); 196 static void gen_set_enaddr(struct gen_softc *sc); 197 static void gen_setup_rxfilter(struct gen_softc *sc); 198 static void gen_reset(struct gen_softc *sc); 199 static void gen_enable(struct gen_softc *sc); 200 static void gen_dma_disable(device_t dev); 201 static int gen_bus_dma_init(struct gen_softc *sc); 202 static void gen_bus_dma_teardown(struct gen_softc *sc); 203 static void gen_enable_intr(struct gen_softc *sc); 204 static void gen_init_txrings(struct gen_softc *sc); 205 static void gen_init_rxrings(struct gen_softc *sc); 206 static void gen_intr(void *softc); 207 static int gen_rxintr(struct gen_softc *sc, struct rx_queue *q); 208 static void gen_txintr(struct gen_softc *sc, struct tx_queue *q); 209 static void gen_intr2(void *softc); 210 static int gen_newbuf_rx(struct gen_softc *sc, struct rx_queue *q, int index); 211 static int gen_mapbuf_rx(struct gen_softc *sc, struct rx_queue *q, int index, 212 struct mbuf *m); 213 static void gen_link_task(void *arg, int pending); 214 static void gen_media_status(if_t ifp, struct ifmediareq *ifmr); 215 static int gen_media_change(if_t ifp); 216 static void gen_tick(void *softc); 217 218 static int 219 gen_probe(device_t dev) 220 { 221 if (!ofw_bus_status_okay(dev)) 222 return (ENXIO); 223 224 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) 225 return (ENXIO); 226 227 device_set_desc(dev, "RPi4 Gigabit Ethernet"); 228 return (BUS_PROBE_DEFAULT); 229 } 230 231 static int 232 gen_attach(device_t dev) 233 { 234 struct ether_addr eaddr; 235 struct gen_softc *sc; 236 int major, minor, error; 237 bool eaddr_found; 238 239 sc = device_get_softc(dev); 240 sc->dev = dev; 241 sc->type = ofw_bus_search_compatible(dev, compat_data)->ocd_data; 242 243 if (bus_alloc_resources(dev, gen_spec, sc->res) != 0) { 244 device_printf(dev, "cannot allocate resources for device\n"); 245 error = ENXIO; 246 goto fail; 247 } 248 249 major = (RD4(sc, GENET_SYS_REV_CTRL) & REV_MAJOR) >> REV_MAJOR_SHIFT; 250 if (major != REV_MAJOR_V5) { 251 device_printf(dev, "version %d is not supported\n", major); 252 error = ENXIO; 253 goto fail; 254 } 255 minor = (RD4(sc, GENET_SYS_REV_CTRL) & REV_MINOR) >> REV_MINOR_SHIFT; 256 device_printf(dev, "GENET version 5.%d phy 0x%04x\n", minor, 257 RD4(sc, GENET_SYS_REV_CTRL) & REV_PHY); 258 259 mtx_init(&sc->mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, MTX_DEF); 260 callout_init_mtx(&sc->stat_ch, &sc->mtx, 0); 261 TASK_INIT(&sc->link_task, 0, gen_link_task, sc); 262 263 error = gen_get_phy_mode(dev); 264 if (error != 0) 265 goto fail; 266 267 bzero(&eaddr, sizeof(eaddr)); 268 eaddr_found = gen_get_eaddr(dev, &eaddr); 269 270 /* reset core */ 271 gen_reset(sc); 272 273 gen_dma_disable(dev); 274 275 /* Setup DMA */ 276 error = gen_bus_dma_init(sc); 277 if (error != 0) { 278 device_printf(dev, "cannot setup bus dma\n"); 279 goto fail; 280 } 281 282 /* Install interrupt handlers */ 283 error = bus_setup_intr(dev, sc->res[_RES_IRQ1], 284 INTR_TYPE_NET | INTR_MPSAFE, NULL, gen_intr, sc, &sc->ih); 285 if (error != 0) { 286 device_printf(dev, "cannot setup interrupt handler1\n"); 287 goto fail; 288 } 289 290 error = bus_setup_intr(dev, sc->res[_RES_IRQ2], 291 INTR_TYPE_NET | INTR_MPSAFE, NULL, gen_intr2, sc, &sc->ih2); 292 if (error != 0) { 293 device_printf(dev, "cannot setup interrupt handler2\n"); 294 goto fail; 295 } 296 297 /* Setup ethernet interface */ 298 sc->ifp = if_alloc(IFT_ETHER); 299 if_setsoftc(sc->ifp, sc); 300 if_initname(sc->ifp, device_get_name(dev), device_get_unit(dev)); 301 if_setflags(sc->ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 302 if_setstartfn(sc->ifp, gen_start); 303 if_setioctlfn(sc->ifp, gen_ioctl); 304 if_setinitfn(sc->ifp, gen_init); 305 if_setsendqlen(sc->ifp, TX_DESC_COUNT - 1); 306 if_setsendqready(sc->ifp); 307 #define GEN_CSUM_FEATURES (CSUM_UDP | CSUM_TCP) 308 if_sethwassist(sc->ifp, GEN_CSUM_FEATURES); 309 if_setcapabilities(sc->ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM | 310 IFCAP_HWCSUM_IPV6); 311 if_setcapenable(sc->ifp, if_getcapabilities(sc->ifp)); 312 313 /* Attach MII driver */ 314 error = mii_attach(dev, &sc->miibus, sc->ifp, gen_media_change, 315 gen_media_status, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 316 MIIF_DOPAUSE); 317 if (error != 0) { 318 device_printf(dev, "cannot attach PHY\n"); 319 goto fail; 320 } 321 322 /* If address was not found, create one based on the hostid and name. */ 323 if (eaddr_found == 0) 324 ether_gen_addr(sc->ifp, &eaddr); 325 /* Attach ethernet interface */ 326 ether_ifattach(sc->ifp, eaddr.octet); 327 328 fail: 329 if (error) 330 gen_destroy(sc); 331 return (error); 332 } 333 334 /* Free resources after failed attach. This is not a complete detach. */ 335 static void 336 gen_destroy(struct gen_softc *sc) 337 { 338 339 if (sc->miibus) { /* can't happen */ 340 device_delete_child(sc->dev, sc->miibus); 341 sc->miibus = NULL; 342 } 343 bus_teardown_intr(sc->dev, sc->res[_RES_IRQ1], sc->ih); 344 bus_teardown_intr(sc->dev, sc->res[_RES_IRQ2], sc->ih2); 345 gen_bus_dma_teardown(sc); 346 callout_drain(&sc->stat_ch); 347 if (mtx_initialized(&sc->mtx)) 348 mtx_destroy(&sc->mtx); 349 bus_release_resources(sc->dev, gen_spec, sc->res); 350 if (sc->ifp != NULL) { 351 if_free(sc->ifp); 352 sc->ifp = NULL; 353 } 354 } 355 356 static int 357 gen_get_phy_mode(device_t dev) 358 { 359 struct gen_softc *sc; 360 phandle_t node; 361 mii_contype_t type; 362 int error = 0; 363 364 sc = device_get_softc(dev); 365 node = ofw_bus_get_node(dev); 366 type = mii_fdt_get_contype(node); 367 368 switch (type) { 369 case MII_CONTYPE_RGMII: 370 case MII_CONTYPE_RGMII_RXID: 371 case MII_CONTYPE_RGMII_TXID: 372 sc->phy_mode = type; 373 break; 374 default: 375 device_printf(dev, "unknown phy-mode '%s'\n", 376 mii_fdt_contype_to_name(type)); 377 error = ENXIO; 378 break; 379 } 380 381 return (error); 382 } 383 384 static bool 385 gen_get_eaddr(device_t dev, struct ether_addr *eaddr) 386 { 387 struct gen_softc *sc; 388 uint32_t maclo, machi, val; 389 phandle_t node; 390 391 sc = device_get_softc(dev); 392 393 node = ofw_bus_get_node(dev); 394 if (OF_getprop(node, "mac-address", eaddr->octet, 395 ETHER_ADDR_LEN) != -1 || 396 OF_getprop(node, "local-mac-address", eaddr->octet, 397 ETHER_ADDR_LEN) != -1 || 398 OF_getprop(node, "address", eaddr->octet, ETHER_ADDR_LEN) != -1) 399 return (true); 400 401 device_printf(dev, "No Ethernet address found in fdt!\n"); 402 maclo = machi = 0; 403 404 val = RD4(sc, GENET_SYS_RBUF_FLUSH_CTRL); 405 if ((val & GENET_SYS_RBUF_FLUSH_RESET) == 0) { 406 maclo = htobe32(RD4(sc, GENET_UMAC_MAC0)); 407 machi = htobe16(RD4(sc, GENET_UMAC_MAC1) & 0xffff); 408 } 409 410 if (maclo == 0 && machi == 0) { 411 if (bootverbose) 412 device_printf(dev, 413 "No Ethernet address found in controller\n"); 414 return (false); 415 } else { 416 eaddr->octet[0] = maclo & 0xff; 417 eaddr->octet[1] = (maclo >> 8) & 0xff; 418 eaddr->octet[2] = (maclo >> 16) & 0xff; 419 eaddr->octet[3] = (maclo >> 24) & 0xff; 420 eaddr->octet[4] = machi & 0xff; 421 eaddr->octet[5] = (machi >> 8) & 0xff; 422 return (true); 423 } 424 } 425 426 static void 427 gen_reset(struct gen_softc *sc) 428 { 429 uint32_t val; 430 431 val = RD4(sc, GENET_SYS_RBUF_FLUSH_CTRL); 432 val |= GENET_SYS_RBUF_FLUSH_RESET; 433 WR4(sc, GENET_SYS_RBUF_FLUSH_CTRL, val); 434 DELAY(10); 435 436 val &= ~GENET_SYS_RBUF_FLUSH_RESET; 437 WR4(sc, GENET_SYS_RBUF_FLUSH_CTRL, val); 438 DELAY(10); 439 440 WR4(sc, GENET_SYS_RBUF_FLUSH_CTRL, 0); 441 DELAY(10); 442 443 WR4(sc, GENET_UMAC_CMD, 0); 444 WR4(sc, GENET_UMAC_CMD, 445 GENET_UMAC_CMD_LCL_LOOP_EN | GENET_UMAC_CMD_SW_RESET); 446 DELAY(10); 447 WR4(sc, GENET_UMAC_CMD, 0); 448 449 WR4(sc, GENET_UMAC_MIB_CTRL, GENET_UMAC_MIB_RESET_RUNT | 450 GENET_UMAC_MIB_RESET_RX | GENET_UMAC_MIB_RESET_TX); 451 WR4(sc, GENET_UMAC_MIB_CTRL, 0); 452 453 WR4(sc, GENET_UMAC_MAX_FRAME_LEN, 1536); 454 455 val = RD4(sc, GENET_RBUF_CTRL); 456 val |= GENET_RBUF_ALIGN_2B; 457 WR4(sc, GENET_RBUF_CTRL, val); 458 459 WR4(sc, GENET_RBUF_TBUF_SIZE_CTRL, 1); 460 } 461 462 static void 463 gen_enable(struct gen_softc *sc) 464 { 465 u_int val; 466 467 /* Enable transmitter and receiver */ 468 val = RD4(sc, GENET_UMAC_CMD); 469 val |= GENET_UMAC_CMD_TXEN; 470 val |= GENET_UMAC_CMD_RXEN; 471 WR4(sc, GENET_UMAC_CMD, val); 472 473 /* Enable interrupts */ 474 gen_enable_intr(sc); 475 WR4(sc, GENET_INTRL2_CPU_CLEAR_MASK, 476 GENET_IRQ_TXDMA_DONE | GENET_IRQ_RXDMA_DONE); 477 } 478 479 static void 480 gen_enable_offload(struct gen_softc *sc) 481 { 482 uint32_t check_ctrl, buf_ctrl; 483 484 check_ctrl = RD4(sc, GENET_RBUF_CHECK_CTRL); 485 buf_ctrl = RD4(sc, GENET_RBUF_CTRL); 486 if ((if_getcapenable(sc->ifp) & IFCAP_RXCSUM) != 0) { 487 check_ctrl |= GENET_RBUF_CHECK_CTRL_EN; 488 buf_ctrl |= GENET_RBUF_64B_EN; 489 } else { 490 check_ctrl &= ~GENET_RBUF_CHECK_CTRL_EN; 491 buf_ctrl &= ~GENET_RBUF_64B_EN; 492 } 493 WR4(sc, GENET_RBUF_CHECK_CTRL, check_ctrl); 494 WR4(sc, GENET_RBUF_CTRL, buf_ctrl); 495 496 buf_ctrl = RD4(sc, GENET_TBUF_CTRL); 497 if ((if_getcapenable(sc->ifp) & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) != 498 0) 499 buf_ctrl |= GENET_RBUF_64B_EN; 500 else 501 buf_ctrl &= ~GENET_RBUF_64B_EN; 502 WR4(sc, GENET_TBUF_CTRL, buf_ctrl); 503 } 504 505 static void 506 gen_dma_disable(device_t dev) 507 { 508 struct gen_softc *sc = device_get_softc(dev); 509 int val; 510 511 val = RD4(sc, GENET_TX_DMA_CTRL); 512 val &= ~GENET_TX_DMA_CTRL_EN; 513 val &= ~GENET_TX_DMA_CTRL_RBUF_EN(GENET_DMA_DEFAULT_QUEUE); 514 WR4(sc, GENET_TX_DMA_CTRL, val); 515 516 val = RD4(sc, GENET_RX_DMA_CTRL); 517 val &= ~GENET_RX_DMA_CTRL_EN; 518 val &= ~GENET_RX_DMA_CTRL_RBUF_EN(GENET_DMA_DEFAULT_QUEUE); 519 WR4(sc, GENET_RX_DMA_CTRL, val); 520 } 521 522 static int 523 gen_bus_dma_init(struct gen_softc *sc) 524 { 525 struct device *dev = sc->dev; 526 int i, error; 527 528 error = bus_dma_tag_create( 529 bus_get_dma_tag(dev), /* Parent tag */ 530 4, 0, /* alignment, boundary */ 531 BUS_SPACE_MAXADDR_40BIT, /* lowaddr */ 532 BUS_SPACE_MAXADDR, /* highaddr */ 533 NULL, NULL, /* filter, filterarg */ 534 MCLBYTES, TX_MAX_SEGS, /* maxsize, nsegs */ 535 MCLBYTES, /* maxsegsize */ 536 0, /* flags */ 537 NULL, NULL, /* lockfunc, lockarg */ 538 &sc->tx_buf_tag); 539 if (error != 0) { 540 device_printf(dev, "cannot create TX buffer tag\n"); 541 return (error); 542 } 543 544 for (i = 0; i < TX_DESC_COUNT; i++) { 545 error = bus_dmamap_create(sc->tx_buf_tag, 0, 546 &sc->tx_ring_ent[i].map); 547 if (error != 0) { 548 device_printf(dev, "cannot create TX buffer map\n"); 549 return (error); 550 } 551 } 552 553 error = bus_dma_tag_create( 554 bus_get_dma_tag(dev), /* Parent tag */ 555 4, 0, /* alignment, boundary */ 556 BUS_SPACE_MAXADDR_40BIT, /* lowaddr */ 557 BUS_SPACE_MAXADDR, /* highaddr */ 558 NULL, NULL, /* filter, filterarg */ 559 MCLBYTES, 1, /* maxsize, nsegs */ 560 MCLBYTES, /* maxsegsize */ 561 0, /* flags */ 562 NULL, NULL, /* lockfunc, lockarg */ 563 &sc->rx_buf_tag); 564 if (error != 0) { 565 device_printf(dev, "cannot create RX buffer tag\n"); 566 return (error); 567 } 568 569 for (i = 0; i < RX_DESC_COUNT; i++) { 570 error = bus_dmamap_create(sc->rx_buf_tag, 0, 571 &sc->rx_ring_ent[i].map); 572 if (error != 0) { 573 device_printf(dev, "cannot create RX buffer map\n"); 574 return (error); 575 } 576 } 577 return (0); 578 } 579 580 static void 581 gen_bus_dma_teardown(struct gen_softc *sc) 582 { 583 int i, error; 584 585 if (sc->tx_buf_tag != NULL) { 586 for (i = 0; i < TX_DESC_COUNT; i++) { 587 error = bus_dmamap_destroy(sc->tx_buf_tag, 588 sc->tx_ring_ent[i].map); 589 sc->tx_ring_ent[i].map = NULL; 590 if (error) 591 device_printf(sc->dev, 592 "%s: bus_dmamap_destroy failed: %d\n", 593 __func__, error); 594 } 595 error = bus_dma_tag_destroy(sc->tx_buf_tag); 596 sc->tx_buf_tag = NULL; 597 if (error) 598 device_printf(sc->dev, 599 "%s: bus_dma_tag_destroy failed: %d\n", __func__, 600 error); 601 } 602 603 if (sc->tx_buf_tag != NULL) { 604 for (i = 0; i < RX_DESC_COUNT; i++) { 605 error = bus_dmamap_destroy(sc->rx_buf_tag, 606 sc->rx_ring_ent[i].map); 607 sc->rx_ring_ent[i].map = NULL; 608 if (error) 609 device_printf(sc->dev, 610 "%s: bus_dmamap_destroy failed: %d\n", 611 __func__, error); 612 } 613 error = bus_dma_tag_destroy(sc->rx_buf_tag); 614 sc->rx_buf_tag = NULL; 615 if (error) 616 device_printf(sc->dev, 617 "%s: bus_dma_tag_destroy failed: %d\n", __func__, 618 error); 619 } 620 } 621 622 static void 623 gen_enable_intr(struct gen_softc *sc) 624 { 625 626 WR4(sc, GENET_INTRL2_CPU_CLEAR_MASK, 627 GENET_IRQ_TXDMA_DONE | GENET_IRQ_RXDMA_DONE); 628 } 629 630 /* 631 * "queue" is the software queue index (0-4); "qid" is the hardware index 632 * (0-16). "base" is the starting index in the ring array. 633 */ 634 static void 635 gen_init_txring(struct gen_softc *sc, int queue, int qid, int base, 636 int nentries) 637 { 638 struct tx_queue *q; 639 uint32_t val; 640 641 q = &sc->tx_queue[queue]; 642 q->entries = &sc->tx_ring_ent[base]; 643 q->hwindex = qid; 644 q->nentries = nentries; 645 646 /* TX ring */ 647 648 q->queued = 0; 649 q->cons_idx = q->prod_idx = 0; 650 651 WR4(sc, GENET_TX_SCB_BURST_SIZE, 0x08); 652 653 WR4(sc, GENET_TX_DMA_READ_PTR_LO(qid), 0); 654 WR4(sc, GENET_TX_DMA_READ_PTR_HI(qid), 0); 655 WR4(sc, GENET_TX_DMA_CONS_INDEX(qid), 0); 656 WR4(sc, GENET_TX_DMA_PROD_INDEX(qid), 0); 657 WR4(sc, GENET_TX_DMA_RING_BUF_SIZE(qid), 658 (nentries << GENET_TX_DMA_RING_BUF_SIZE_DESC_SHIFT) | 659 (MCLBYTES & GENET_TX_DMA_RING_BUF_SIZE_BUF_LEN_MASK)); 660 WR4(sc, GENET_TX_DMA_START_ADDR_LO(qid), 0); 661 WR4(sc, GENET_TX_DMA_START_ADDR_HI(qid), 0); 662 WR4(sc, GENET_TX_DMA_END_ADDR_LO(qid), 663 TX_DESC_COUNT * GENET_DMA_DESC_SIZE / 4 - 1); 664 WR4(sc, GENET_TX_DMA_END_ADDR_HI(qid), 0); 665 WR4(sc, GENET_TX_DMA_MBUF_DONE_THRES(qid), 1); 666 WR4(sc, GENET_TX_DMA_FLOW_PERIOD(qid), 0); 667 WR4(sc, GENET_TX_DMA_WRITE_PTR_LO(qid), 0); 668 WR4(sc, GENET_TX_DMA_WRITE_PTR_HI(qid), 0); 669 670 WR4(sc, GENET_TX_DMA_RING_CFG, __BIT(qid)); /* enable */ 671 672 /* Enable transmit DMA */ 673 val = RD4(sc, GENET_TX_DMA_CTRL); 674 val |= GENET_TX_DMA_CTRL_EN; 675 val |= GENET_TX_DMA_CTRL_RBUF_EN(qid); 676 WR4(sc, GENET_TX_DMA_CTRL, val); 677 } 678 679 /* 680 * "queue" is the software queue index (0-4); "qid" is the hardware index 681 * (0-16). "base" is the starting index in the ring array. 682 */ 683 static void 684 gen_init_rxring(struct gen_softc *sc, int queue, int qid, int base, 685 int nentries) 686 { 687 struct rx_queue *q; 688 uint32_t val; 689 int i; 690 691 q = &sc->rx_queue[queue]; 692 q->entries = &sc->rx_ring_ent[base]; 693 q->hwindex = qid; 694 q->nentries = nentries; 695 q->cons_idx = q->prod_idx = 0; 696 697 WR4(sc, GENET_RX_SCB_BURST_SIZE, 0x08); 698 699 WR4(sc, GENET_RX_DMA_WRITE_PTR_LO(qid), 0); 700 WR4(sc, GENET_RX_DMA_WRITE_PTR_HI(qid), 0); 701 WR4(sc, GENET_RX_DMA_PROD_INDEX(qid), 0); 702 WR4(sc, GENET_RX_DMA_CONS_INDEX(qid), 0); 703 WR4(sc, GENET_RX_DMA_RING_BUF_SIZE(qid), 704 (nentries << GENET_RX_DMA_RING_BUF_SIZE_DESC_SHIFT) | 705 (MCLBYTES & GENET_RX_DMA_RING_BUF_SIZE_BUF_LEN_MASK)); 706 WR4(sc, GENET_RX_DMA_START_ADDR_LO(qid), 0); 707 WR4(sc, GENET_RX_DMA_START_ADDR_HI(qid), 0); 708 WR4(sc, GENET_RX_DMA_END_ADDR_LO(qid), 709 RX_DESC_COUNT * GENET_DMA_DESC_SIZE / 4 - 1); 710 WR4(sc, GENET_RX_DMA_END_ADDR_HI(qid), 0); 711 WR4(sc, GENET_RX_DMA_XON_XOFF_THRES(qid), 712 (5 << GENET_RX_DMA_XON_XOFF_THRES_LO_SHIFT) | (RX_DESC_COUNT >> 4)); 713 WR4(sc, GENET_RX_DMA_READ_PTR_LO(qid), 0); 714 WR4(sc, GENET_RX_DMA_READ_PTR_HI(qid), 0); 715 716 WR4(sc, GENET_RX_DMA_RING_CFG, __BIT(qid)); /* enable */ 717 718 /* fill ring */ 719 for (i = 0; i < RX_DESC_COUNT; i++) 720 gen_newbuf_rx(sc, &sc->rx_queue[DEF_RXQUEUE], i); 721 722 /* Enable receive DMA */ 723 val = RD4(sc, GENET_RX_DMA_CTRL); 724 val |= GENET_RX_DMA_CTRL_EN; 725 val |= GENET_RX_DMA_CTRL_RBUF_EN(qid); 726 WR4(sc, GENET_RX_DMA_CTRL, val); 727 } 728 729 static void 730 gen_init_txrings(struct gen_softc *sc) 731 { 732 int base = 0; 733 #ifdef PRI_RINGS 734 int i; 735 736 /* init priority rings */ 737 for (i = 0; i < PRI_RINGS; i++) { 738 gen_init_txring(sc, i, i, base, TX_DESC_PRICOUNT); 739 sc->tx_queue[i].queue = i; 740 base += TX_DESC_PRICOUNT; 741 dma_ring_conf |= 1 << i; 742 dma_control |= DMA_RENABLE(i); 743 } 744 #endif 745 746 /* init GENET_DMA_DEFAULT_QUEUE (16) */ 747 gen_init_txring(sc, DEF_TXQUEUE, GENET_DMA_DEFAULT_QUEUE, base, 748 TX_DESC_COUNT); 749 sc->tx_queue[DEF_TXQUEUE].hwindex = GENET_DMA_DEFAULT_QUEUE; 750 } 751 752 static void 753 gen_init_rxrings(struct gen_softc *sc) 754 { 755 int base = 0; 756 #ifdef PRI_RINGS 757 int i; 758 759 /* init priority rings */ 760 for (i = 0; i < PRI_RINGS; i++) { 761 gen_init_rxring(sc, i, i, base, TX_DESC_PRICOUNT); 762 sc->rx_queue[i].queue = i; 763 base += TX_DESC_PRICOUNT; 764 dma_ring_conf |= 1 << i; 765 dma_control |= DMA_RENABLE(i); 766 } 767 #endif 768 769 /* init GENET_DMA_DEFAULT_QUEUE (16) */ 770 gen_init_rxring(sc, DEF_RXQUEUE, GENET_DMA_DEFAULT_QUEUE, base, 771 RX_DESC_COUNT); 772 sc->rx_queue[DEF_RXQUEUE].hwindex = GENET_DMA_DEFAULT_QUEUE; 773 774 } 775 776 static void 777 gen_init_locked(struct gen_softc *sc) 778 { 779 struct mii_data *mii; 780 if_t ifp; 781 782 mii = device_get_softc(sc->miibus); 783 ifp = sc->ifp; 784 785 GEN_ASSERT_LOCKED(sc); 786 787 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 788 return; 789 790 if (sc->phy_mode == MII_CONTYPE_RGMII || 791 sc->phy_mode == MII_CONTYPE_RGMII_RXID) 792 WR4(sc, GENET_SYS_PORT_CTRL, 793 GENET_SYS_PORT_MODE_EXT_GPHY); 794 795 gen_set_enaddr(sc); 796 797 /* Setup RX filter */ 798 gen_setup_rxfilter(sc); 799 800 gen_init_txrings(sc); 801 gen_init_rxrings(sc); 802 gen_enable(sc); 803 gen_enable_offload(sc); 804 805 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 806 807 mii_mediachg(mii); 808 callout_reset(&sc->stat_ch, hz, gen_tick, sc); 809 } 810 811 static void 812 gen_init(void *softc) 813 { 814 struct gen_softc *sc; 815 816 sc = softc; 817 GEN_LOCK(sc); 818 gen_init_locked(sc); 819 GEN_UNLOCK(sc); 820 } 821 822 static uint8_t ether_broadcastaddr[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; 823 824 static void 825 gen_setup_rxfilter_mdf(struct gen_softc *sc, u_int n, const uint8_t *ea) 826 { 827 uint32_t addr0 = (ea[0] << 8) | ea[1]; 828 uint32_t addr1 = (ea[2] << 24) | (ea[3] << 16) | (ea[4] << 8) | ea[5]; 829 830 WR4(sc, GENET_UMAC_MDF_ADDR0(n), addr0); 831 WR4(sc, GENET_UMAC_MDF_ADDR1(n), addr1); 832 } 833 834 static u_int 835 gen_setup_multi(void *arg, struct sockaddr_dl *sdl, u_int count) 836 { 837 struct gen_softc *sc = arg; 838 839 /* "count + 2" to account for unicast and broadcast */ 840 gen_setup_rxfilter_mdf(sc, count + 2, LLADDR(sdl)); 841 return (1); /* increment to count */ 842 } 843 844 static void 845 gen_setup_rxfilter(struct gen_softc *sc) 846 { 847 struct ifnet *ifp = sc->ifp; 848 uint32_t cmd, mdf_ctrl; 849 u_int n; 850 851 GEN_ASSERT_LOCKED(sc); 852 853 cmd = RD4(sc, GENET_UMAC_CMD); 854 855 /* 856 * Count the required number of hardware filters. We need one 857 * for each multicast address, plus one for our own address and 858 * the broadcast address. 859 */ 860 n = if_llmaddr_count(ifp) + 2; 861 862 if (n > GENET_MAX_MDF_FILTER) 863 ifp->if_flags |= IFF_ALLMULTI; 864 else 865 ifp->if_flags &= ~IFF_ALLMULTI; 866 867 if ((ifp->if_flags & (IFF_PROMISC|IFF_ALLMULTI)) != 0) { 868 cmd |= GENET_UMAC_CMD_PROMISC; 869 mdf_ctrl = 0; 870 } else { 871 cmd &= ~GENET_UMAC_CMD_PROMISC; 872 gen_setup_rxfilter_mdf(sc, 0, ether_broadcastaddr); 873 gen_setup_rxfilter_mdf(sc, 1, IF_LLADDR(ifp)); 874 (void) if_foreach_llmaddr(ifp, gen_setup_multi, sc); 875 mdf_ctrl = (__BIT(GENET_MAX_MDF_FILTER) - 1) &~ 876 (__BIT(GENET_MAX_MDF_FILTER - n) - 1); 877 } 878 879 WR4(sc, GENET_UMAC_CMD, cmd); 880 WR4(sc, GENET_UMAC_MDF_CTRL, mdf_ctrl); 881 } 882 883 static void 884 gen_set_enaddr(struct gen_softc *sc) 885 { 886 uint8_t *enaddr; 887 uint32_t val; 888 if_t ifp; 889 890 GEN_ASSERT_LOCKED(sc); 891 892 ifp = sc->ifp; 893 894 /* Write our unicast address */ 895 enaddr = IF_LLADDR(ifp); 896 /* Write hardware address */ 897 val = enaddr[3] | (enaddr[2] << 8) | (enaddr[1] << 16) | 898 (enaddr[0] << 24); 899 WR4(sc, GENET_UMAC_MAC0, val); 900 val = enaddr[5] | (enaddr[4] << 8); 901 WR4(sc, GENET_UMAC_MAC1, val); 902 } 903 904 static void 905 gen_start_locked(struct gen_softc *sc) 906 { 907 struct mbuf *m; 908 if_t ifp; 909 int cnt, err; 910 911 GEN_ASSERT_LOCKED(sc); 912 913 if (!sc->link) 914 return; 915 916 ifp = sc->ifp; 917 918 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 919 IFF_DRV_RUNNING) 920 return; 921 922 for (cnt = 0; ; cnt++) { 923 m = if_dequeue(ifp); 924 if (m == NULL) 925 break; 926 927 err = gen_encap(sc, &m); 928 if (err != 0) { 929 if (err == ENOBUFS) 930 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); 931 if (m != NULL) 932 if_sendq_prepend(ifp, m); 933 break; 934 } 935 if_bpfmtap(ifp, m); 936 } 937 } 938 939 static void 940 gen_start(if_t ifp) 941 { 942 struct gen_softc *sc; 943 944 sc = if_getsoftc(ifp); 945 946 GEN_LOCK(sc); 947 gen_start_locked(sc); 948 GEN_UNLOCK(sc); 949 } 950 951 static int 952 gen_encap(struct gen_softc *sc, struct mbuf **mp) 953 { 954 bus_dmamap_t map; 955 bus_dma_segment_t segs[TX_MAX_SEGS]; 956 int error, nsegs, cur, first, i, index, offset; 957 uint32_t csuminfo, length_status, csum_flags = 0, csumdata; 958 struct mbuf *m; 959 struct statusblock *sb = NULL; 960 struct tx_queue *q; 961 struct gen_ring_ent *ent; 962 963 GEN_ASSERT_LOCKED(sc); 964 965 q = &sc->tx_queue[DEF_TXQUEUE]; 966 967 m = *mp; 968 if ((if_getcapenable(sc->ifp) & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) != 969 0) { 970 csum_flags = m->m_pkthdr.csum_flags; 971 csumdata = m->m_pkthdr.csum_data; 972 M_PREPEND(m, sizeof(struct statusblock), M_NOWAIT); 973 if (m == NULL) { 974 if (sc->ifp->if_flags & IFF_DEBUG) 975 device_printf(sc->dev, "prepend fail\n"); 976 *mp = NULL; 977 return (ENOMEM); 978 } 979 offset = gen_parse_tx(m, csum_flags); 980 sb = mtod(m, struct statusblock *); 981 if (csum_flags != 0) { 982 csuminfo = (offset << TXCSUM_OFF_SHIFT) | 983 (offset + csumdata); 984 if (csum_flags & (CSUM_TCP | CSUM_UDP)) 985 csuminfo |= TXCSUM_LEN_VALID; 986 if (csum_flags & CSUM_UDP) 987 csuminfo |= TXCSUM_UDP; 988 sb->txcsuminfo = csuminfo; 989 } else 990 sb->txcsuminfo = 0; 991 } 992 993 *mp = m; 994 995 cur = first = q->cur; 996 ent = &q->entries[cur]; 997 map = ent->map; 998 error = bus_dmamap_load_mbuf_sg(sc->tx_buf_tag, map, m, segs, 999 &nsegs, BUS_DMA_NOWAIT); 1000 if (error == EFBIG) { 1001 m = m_collapse(m, M_NOWAIT, TX_MAX_SEGS); 1002 if (m == NULL) { 1003 device_printf(sc->dev, 1004 "gen_encap: m_collapse failed\n"); 1005 m_freem(*mp); 1006 *mp = NULL; 1007 return (ENOMEM); 1008 } 1009 *mp = m; 1010 error = bus_dmamap_load_mbuf_sg(sc->tx_buf_tag, map, m, 1011 segs, &nsegs, BUS_DMA_NOWAIT); 1012 if (error != 0) { 1013 m_freem(*mp); 1014 *mp = NULL; 1015 } 1016 } 1017 if (error != 0) { 1018 device_printf(sc->dev, 1019 "gen_encap: bus_dmamap_load_mbuf_sg failed\n"); 1020 return (error); 1021 } 1022 if (nsegs == 0) { 1023 m_freem(*mp); 1024 *mp = NULL; 1025 return (EIO); 1026 } 1027 1028 /* Remove statusblock after mapping, before possible requeue or bpf. */ 1029 if (sb != NULL) { 1030 m->m_data += sizeof(struct statusblock); 1031 m->m_len -= sizeof(struct statusblock); 1032 m->m_pkthdr.len -= sizeof(struct statusblock); 1033 } 1034 if (q->queued + nsegs > q->nentries) { 1035 bus_dmamap_unload(sc->tx_buf_tag, map); 1036 return (ENOBUFS); 1037 } 1038 1039 bus_dmamap_sync(sc->tx_buf_tag, map, BUS_DMASYNC_PREWRITE); 1040 1041 index = q->prod_idx & (q->nentries - 1); 1042 for (i = 0; i < nsegs; i++) { 1043 ent = &q->entries[cur]; 1044 length_status = GENET_TX_DESC_STATUS_QTAG_MASK; 1045 if (i == 0) { 1046 length_status |= GENET_TX_DESC_STATUS_SOP | 1047 GENET_TX_DESC_STATUS_CRC; 1048 if (csum_flags != 0) 1049 length_status |= GENET_TX_DESC_STATUS_CKSUM; 1050 } 1051 if (i == nsegs - 1) 1052 length_status |= GENET_TX_DESC_STATUS_EOP; 1053 1054 length_status |= segs[i].ds_len << 1055 GENET_TX_DESC_STATUS_BUFLEN_SHIFT; 1056 1057 WR4(sc, GENET_TX_DESC_ADDRESS_LO(index), 1058 (uint32_t)segs[i].ds_addr); 1059 WR4(sc, GENET_TX_DESC_ADDRESS_HI(index), 1060 (uint32_t)(segs[i].ds_addr >> 32)); 1061 WR4(sc, GENET_TX_DESC_STATUS(index), length_status); 1062 1063 ++q->queued; 1064 cur = TX_NEXT(cur, q->nentries); 1065 index = TX_NEXT(index, q->nentries); 1066 } 1067 1068 q->prod_idx += nsegs; 1069 q->prod_idx &= GENET_TX_DMA_PROD_CONS_MASK; 1070 /* We probably don't need to write the producer index on every iter */ 1071 if (nsegs != 0) 1072 WR4(sc, GENET_TX_DMA_PROD_INDEX(q->hwindex), q->prod_idx); 1073 q->cur = cur; 1074 1075 /* Store mbuf in the last segment */ 1076 q->entries[first].mbuf = m; 1077 1078 return (0); 1079 } 1080 1081 /* 1082 * Parse a packet to find the offset of the transport header for checksum 1083 * offload. Ensure that the link and network headers are contiguous with 1084 * the status block, or transmission fails. 1085 */ 1086 static int 1087 gen_parse_tx(struct mbuf *m, int csum_flags) 1088 { 1089 int offset, off_in_m; 1090 u_char *p, *copy_p = NULL; 1091 struct mbuf *m0 = m; 1092 uint16_t ether_type; 1093 1094 if (m->m_len == sizeof(struct statusblock)) { 1095 /* M_PREPEND placed statusblock at end; move to beginning */ 1096 m->m_data = m->m_pktdat; 1097 copy_p = mtodo(m, sizeof(struct statusblock)); 1098 m = m->m_next; 1099 off_in_m = 0; 1100 p = mtod(m, u_char *); 1101 } else { 1102 p = mtodo(m, sizeof(struct statusblock)); 1103 off_in_m = sizeof(struct statusblock); 1104 } 1105 1106 /* If headers need to be copied contiguous to statusblock, do so. */ 1107 #define COPY(size) { \ 1108 if (copy_p != NULL) { \ 1109 int hsize = size; \ 1110 bcopy(p, copy_p, hsize); \ 1111 m0->m_len += hsize; \ 1112 m0->m_pkthdr.len += hsize; /* unneeded */ \ 1113 copy_p += hsize; \ 1114 m->m_len -= hsize; \ 1115 m->m_data += hsize; \ 1116 } \ 1117 } 1118 1119 KASSERT((sizeof(struct statusblock) + sizeof(struct ether_vlan_header) + 1120 sizeof(struct ip6_hdr) <= MLEN), ("%s: mbuf too small", __func__)); 1121 1122 if (((struct ether_header *)p)->ether_type == htons(ETHERTYPE_VLAN)) { 1123 offset = sizeof(struct ether_vlan_header); 1124 ether_type = ntohs(((struct ether_vlan_header *)p)->evl_proto); 1125 COPY(sizeof(struct ether_vlan_header)); 1126 if (m->m_len == off_in_m + sizeof(struct ether_vlan_header)) { 1127 m = m->m_next; 1128 off_in_m = 0; 1129 p = mtod(m, u_char *); 1130 } else { 1131 off_in_m += sizeof(struct ether_vlan_header); 1132 p += sizeof(struct ether_vlan_header); 1133 } 1134 } else { 1135 offset = sizeof(struct ether_header); 1136 ether_type = ntohs(((struct ether_header *)p)->ether_type); 1137 COPY(sizeof(struct ether_header)); 1138 if (m->m_len == off_in_m + sizeof(struct ether_header)) { 1139 m = m->m_next; 1140 off_in_m = 0; 1141 p = mtod(m, u_char *); 1142 } else { 1143 off_in_m += sizeof(struct ether_header); 1144 p += sizeof(struct ether_header); 1145 } 1146 } 1147 if (ether_type == ETHERTYPE_IP) { 1148 COPY(((struct ip *)p)->ip_hl << 2); 1149 offset += ((struct ip *)p)->ip_hl << 2; 1150 } else if (ether_type == ETHERTYPE_IPV6) { 1151 COPY(sizeof(struct ip6_hdr)); 1152 offset += sizeof(struct ip6_hdr); 1153 } else { 1154 /* 1155 * Unknown whether other cases require moving a header; 1156 * ARP works without. 1157 */ 1158 } 1159 return (offset); 1160 #undef COPY 1161 } 1162 1163 static void 1164 gen_intr(void *arg) 1165 { 1166 struct gen_softc *sc = arg; 1167 uint32_t val; 1168 1169 GEN_LOCK(sc); 1170 1171 val = RD4(sc, GENET_INTRL2_CPU_STAT); 1172 val &= ~RD4(sc, GENET_INTRL2_CPU_STAT_MASK); 1173 WR4(sc, GENET_INTRL2_CPU_CLEAR, val); 1174 1175 if (val & GENET_IRQ_RXDMA_DONE) 1176 gen_rxintr(sc, &sc->rx_queue[DEF_RXQUEUE]); 1177 1178 1179 if (val & GENET_IRQ_TXDMA_DONE) { 1180 gen_txintr(sc, &sc->tx_queue[DEF_TXQUEUE]); 1181 if (!if_sendq_empty(sc->ifp)) 1182 gen_start_locked(sc); 1183 } 1184 1185 GEN_UNLOCK(sc); 1186 } 1187 1188 static int 1189 gen_rxintr(struct gen_softc *sc, struct rx_queue *q) 1190 { 1191 if_t ifp; 1192 struct mbuf *m, *mh, *mt; 1193 struct statusblock *sb = NULL; 1194 int error, index, len, cnt, npkt, n; 1195 uint32_t status, prod_idx, total; 1196 1197 ifp = sc->ifp; 1198 mh = mt = NULL; 1199 cnt = 0; 1200 npkt = 0; 1201 1202 prod_idx = RD4(sc, GENET_RX_DMA_PROD_INDEX(q->hwindex)) & 1203 GENET_RX_DMA_PROD_CONS_MASK; 1204 total = (prod_idx - q->cons_idx) & GENET_RX_DMA_PROD_CONS_MASK; 1205 1206 index = q->cons_idx & (RX_DESC_COUNT - 1); 1207 for (n = 0; n < total; n++) { 1208 bus_dmamap_sync(sc->rx_buf_tag, q->entries[index].map, 1209 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1210 bus_dmamap_unload(sc->rx_buf_tag, q->entries[index].map); 1211 1212 m = q->entries[index].mbuf; 1213 1214 if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) { 1215 sb = mtod(m, struct statusblock *); 1216 status = sb->status_buflen; 1217 } else 1218 status = RD4(sc, GENET_RX_DESC_STATUS(index)); 1219 1220 len = (status & GENET_RX_DESC_STATUS_BUFLEN_MASK) >> 1221 GENET_RX_DESC_STATUS_BUFLEN_SHIFT; 1222 1223 /* check for errors */ 1224 if ((status & 1225 (GENET_RX_DESC_STATUS_SOP | GENET_RX_DESC_STATUS_EOP | 1226 GENET_RX_DESC_STATUS_RX_ERROR)) != 1227 (GENET_RX_DESC_STATUS_SOP | GENET_RX_DESC_STATUS_EOP)) { 1228 if (ifp->if_flags & IFF_DEBUG) 1229 device_printf(sc->dev, 1230 "error/frag %x csum %x\n", status, 1231 sb->rxcsum); 1232 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 1233 continue; 1234 } 1235 1236 error = gen_newbuf_rx(sc, q, index); 1237 if (error != 0) { 1238 if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); 1239 if (ifp->if_flags & IFF_DEBUG) 1240 device_printf(sc->dev, "gen_newbuf_rx %d\n", 1241 error); 1242 /* reuse previous mbuf */ 1243 (void) gen_mapbuf_rx(sc, q, index, m); 1244 continue; 1245 } 1246 1247 if (sb != NULL) { 1248 if (status & GENET_RX_DESC_STATUS_CKSUM_OK) { 1249 /* L4 checksum checked; not sure about L3. */ 1250 m->m_pkthdr.csum_flags = CSUM_DATA_VALID | 1251 CSUM_PSEUDO_HDR; 1252 m->m_pkthdr.csum_data = 0xffff; 1253 } 1254 m->m_data += sizeof(struct statusblock); 1255 m->m_len -= sizeof(struct statusblock); 1256 len -= sizeof(struct statusblock); 1257 } 1258 if (len > ETHER_ALIGN) { 1259 m_adj(m, ETHER_ALIGN); 1260 len -= ETHER_ALIGN; 1261 } 1262 1263 m->m_pkthdr.rcvif = ifp; 1264 m->m_pkthdr.len = len; 1265 m->m_len = len; 1266 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 1267 1268 m->m_nextpkt = NULL; 1269 if (mh == NULL) 1270 mh = m; 1271 else 1272 mt->m_nextpkt = m; 1273 mt = m; 1274 ++cnt; 1275 ++npkt; 1276 1277 index = RX_NEXT(index, q->nentries); 1278 1279 q->cons_idx = (q->cons_idx + 1) & GENET_RX_DMA_PROD_CONS_MASK; 1280 WR4(sc, GENET_RX_DMA_CONS_INDEX(q->hwindex), q->cons_idx); 1281 1282 if (cnt == gen_rx_batch) { 1283 GEN_UNLOCK(sc); 1284 if_input(ifp, mh); 1285 GEN_LOCK(sc); 1286 mh = mt = NULL; 1287 cnt = 0; 1288 } 1289 } 1290 1291 if (mh != NULL) { 1292 GEN_UNLOCK(sc); 1293 if_input(ifp, mh); 1294 GEN_LOCK(sc); 1295 } 1296 1297 return (npkt); 1298 } 1299 1300 static void 1301 gen_txintr(struct gen_softc *sc, struct tx_queue *q) 1302 { 1303 uint32_t cons_idx, total; 1304 struct gen_ring_ent *ent; 1305 if_t ifp; 1306 int i, prog; 1307 1308 GEN_ASSERT_LOCKED(sc); 1309 1310 ifp = sc->ifp; 1311 1312 cons_idx = RD4(sc, GENET_TX_DMA_CONS_INDEX(q->hwindex)) & 1313 GENET_TX_DMA_PROD_CONS_MASK; 1314 total = (cons_idx - q->cons_idx) & GENET_TX_DMA_PROD_CONS_MASK; 1315 1316 prog = 0; 1317 for (i = q->next; q->queued > 0 && total > 0; 1318 i = TX_NEXT(i, q->nentries), total--) { 1319 /* XXX check for errors */ 1320 1321 ent = &q->entries[i]; 1322 if (ent->mbuf != NULL) { 1323 bus_dmamap_sync(sc->tx_buf_tag, ent->map, 1324 BUS_DMASYNC_POSTWRITE); 1325 bus_dmamap_unload(sc->tx_buf_tag, ent->map); 1326 m_freem(ent->mbuf); 1327 ent->mbuf = NULL; 1328 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 1329 } 1330 1331 prog++; 1332 --q->queued; 1333 } 1334 1335 if (prog > 0) { 1336 q->next = i; 1337 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 1338 } 1339 1340 q->cons_idx = cons_idx; 1341 } 1342 1343 static void 1344 gen_intr2(void *arg) 1345 { 1346 struct gen_softc *sc = arg; 1347 1348 device_printf(sc->dev, "gen_intr2\n"); 1349 } 1350 1351 static int 1352 gen_newbuf_rx(struct gen_softc *sc, struct rx_queue *q, int index) 1353 { 1354 struct mbuf *m; 1355 1356 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1357 if (m == NULL) 1358 return (ENOBUFS); 1359 1360 m->m_pkthdr.len = m->m_len = m->m_ext.ext_size; 1361 m_adj(m, ETHER_ALIGN); 1362 1363 return (gen_mapbuf_rx(sc, q, index, m)); 1364 } 1365 1366 static int 1367 gen_mapbuf_rx(struct gen_softc *sc, struct rx_queue *q, int index, 1368 struct mbuf *m) 1369 { 1370 bus_dma_segment_t seg; 1371 bus_dmamap_t map; 1372 int nsegs; 1373 1374 map = q->entries[index].map; 1375 if (bus_dmamap_load_mbuf_sg(sc->rx_buf_tag, map, m, &seg, &nsegs, 1376 BUS_DMA_NOWAIT) != 0) { 1377 m_freem(m); 1378 return (ENOBUFS); 1379 } 1380 1381 bus_dmamap_sync(sc->rx_buf_tag, map, BUS_DMASYNC_PREREAD); 1382 1383 q->entries[index].mbuf = m; 1384 WR4(sc, GENET_RX_DESC_ADDRESS_LO(index), (uint32_t)seg.ds_addr); 1385 WR4(sc, GENET_RX_DESC_ADDRESS_HI(index), (uint32_t)(seg.ds_addr >> 32)); 1386 1387 return (0); 1388 } 1389 1390 static int 1391 gen_ioctl(if_t ifp, u_long cmd, caddr_t data) 1392 { 1393 struct gen_softc *sc; 1394 struct mii_data *mii; 1395 struct ifreq *ifr; 1396 int flags, enable, error; 1397 1398 sc = if_getsoftc(ifp); 1399 mii = device_get_softc(sc->miibus); 1400 ifr = (struct ifreq *)data; 1401 error = 0; 1402 1403 switch (cmd) { 1404 case SIOCSIFFLAGS: 1405 GEN_LOCK(sc); 1406 if (if_getflags(ifp) & IFF_UP) { 1407 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1408 flags = if_getflags(ifp) ^ sc->if_flags; 1409 if ((flags & (IFF_PROMISC|IFF_ALLMULTI)) != 0) 1410 gen_setup_rxfilter(sc); 1411 } else 1412 gen_init_locked(sc); 1413 } else { 1414 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 1415 gen_reset(sc); 1416 } 1417 sc->if_flags = if_getflags(ifp); 1418 GEN_UNLOCK(sc); 1419 break; 1420 1421 case SIOCADDMULTI: 1422 case SIOCDELMULTI: 1423 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1424 GEN_LOCK(sc); 1425 gen_setup_rxfilter(sc); 1426 GEN_UNLOCK(sc); 1427 } 1428 break; 1429 1430 case SIOCSIFMEDIA: 1431 case SIOCGIFMEDIA: 1432 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); 1433 break; 1434 1435 case SIOCSIFCAP: 1436 enable = if_getcapenable(ifp); 1437 flags = ifr->ifr_reqcap ^ enable; 1438 if (flags & IFCAP_RXCSUM) 1439 enable ^= IFCAP_RXCSUM; 1440 if (flags & IFCAP_RXCSUM_IPV6) 1441 enable ^= IFCAP_RXCSUM_IPV6; 1442 if (flags & IFCAP_TXCSUM) 1443 enable ^= IFCAP_TXCSUM; 1444 if (flags & IFCAP_TXCSUM_IPV6) 1445 enable ^= IFCAP_TXCSUM_IPV6; 1446 if (enable & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) 1447 if_sethwassist(ifp, GEN_CSUM_FEATURES); 1448 else 1449 if_sethwassist(ifp, 0); 1450 if_setcapenable(ifp, enable); 1451 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 1452 gen_enable_offload(sc); 1453 break; 1454 1455 default: 1456 error = ether_ioctl(ifp, cmd, data); 1457 break; 1458 } 1459 return (error); 1460 } 1461 1462 static void 1463 gen_tick(void *softc) 1464 { 1465 struct gen_softc *sc; 1466 struct mii_data *mii; 1467 if_t ifp; 1468 int link; 1469 1470 sc = softc; 1471 ifp = sc->ifp; 1472 mii = device_get_softc(sc->miibus); 1473 1474 GEN_ASSERT_LOCKED(sc); 1475 1476 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 1477 return; 1478 1479 link = sc->link; 1480 mii_tick(mii); 1481 if (sc->link && !link) 1482 gen_start_locked(sc); 1483 1484 callout_reset(&sc->stat_ch, hz, gen_tick, sc); 1485 } 1486 1487 #define MII_BUSY_RETRY 1000 1488 1489 static int 1490 gen_miibus_readreg(device_t dev, int phy, int reg) 1491 { 1492 struct gen_softc *sc; 1493 int retry, val; 1494 1495 sc = device_get_softc(dev); 1496 val = 0; 1497 1498 WR4(sc, GENET_MDIO_CMD, GENET_MDIO_READ | 1499 (phy << GENET_MDIO_ADDR_SHIFT) | (reg << GENET_MDIO_REG_SHIFT)); 1500 val = RD4(sc, GENET_MDIO_CMD); 1501 WR4(sc, GENET_MDIO_CMD, val | GENET_MDIO_START_BUSY); 1502 for (retry = MII_BUSY_RETRY; retry > 0; retry--) { 1503 if (((val = RD4(sc, GENET_MDIO_CMD)) & 1504 GENET_MDIO_START_BUSY) == 0) { 1505 if (val & GENET_MDIO_READ_FAILED) 1506 return (0); /* -1? */ 1507 val &= GENET_MDIO_VAL_MASK; 1508 break; 1509 } 1510 DELAY(10); 1511 } 1512 1513 if (retry == 0) 1514 device_printf(dev, "phy read timeout, phy=%d reg=%d\n", 1515 phy, reg); 1516 1517 return (val); 1518 } 1519 1520 static int 1521 gen_miibus_writereg(device_t dev, int phy, int reg, int val) 1522 { 1523 struct gen_softc *sc; 1524 int retry; 1525 1526 sc = device_get_softc(dev); 1527 1528 WR4(sc, GENET_MDIO_CMD, GENET_MDIO_WRITE | 1529 (phy << GENET_MDIO_ADDR_SHIFT) | (reg << GENET_MDIO_REG_SHIFT) | 1530 (val & GENET_MDIO_VAL_MASK)); 1531 val = RD4(sc, GENET_MDIO_CMD); 1532 WR4(sc, GENET_MDIO_CMD, val | GENET_MDIO_START_BUSY); 1533 for (retry = MII_BUSY_RETRY; retry > 0; retry--) { 1534 val = RD4(sc, GENET_MDIO_CMD); 1535 if ((val & GENET_MDIO_START_BUSY) == 0) 1536 break; 1537 DELAY(10); 1538 } 1539 if (retry == 0) 1540 device_printf(dev, "phy write timeout, phy=%d reg=%d\n", 1541 phy, reg); 1542 1543 return (0); 1544 } 1545 1546 static void 1547 gen_update_link_locked(struct gen_softc *sc) 1548 { 1549 struct mii_data *mii; 1550 uint32_t val; 1551 u_int speed; 1552 1553 GEN_ASSERT_LOCKED(sc); 1554 1555 if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) == 0) 1556 return; 1557 mii = device_get_softc(sc->miibus); 1558 1559 if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) == 1560 (IFM_ACTIVE | IFM_AVALID)) { 1561 switch (IFM_SUBTYPE(mii->mii_media_active)) { 1562 case IFM_1000_T: 1563 case IFM_1000_SX: 1564 speed = GENET_UMAC_CMD_SPEED_1000; 1565 sc->link = 1; 1566 break; 1567 case IFM_100_TX: 1568 speed = GENET_UMAC_CMD_SPEED_100; 1569 sc->link = 1; 1570 break; 1571 case IFM_10_T: 1572 speed = GENET_UMAC_CMD_SPEED_10; 1573 sc->link = 1; 1574 break; 1575 default: 1576 sc->link = 0; 1577 break; 1578 } 1579 } else 1580 sc->link = 0; 1581 1582 if (sc->link == 0) 1583 return; 1584 1585 val = RD4(sc, GENET_EXT_RGMII_OOB_CTRL); 1586 val &= ~GENET_EXT_RGMII_OOB_OOB_DISABLE; 1587 val |= GENET_EXT_RGMII_OOB_RGMII_LINK; 1588 val |= GENET_EXT_RGMII_OOB_RGMII_MODE_EN; 1589 if (sc->phy_mode == MII_CONTYPE_RGMII) 1590 val |= GENET_EXT_RGMII_OOB_ID_MODE_DISABLE; 1591 WR4(sc, GENET_EXT_RGMII_OOB_CTRL, val); 1592 1593 val = RD4(sc, GENET_UMAC_CMD); 1594 val &= ~GENET_UMAC_CMD_SPEED; 1595 val |= speed; 1596 WR4(sc, GENET_UMAC_CMD, val); 1597 } 1598 1599 static void 1600 gen_link_task(void *arg, int pending) 1601 { 1602 struct gen_softc *sc; 1603 1604 sc = arg; 1605 1606 GEN_LOCK(sc); 1607 gen_update_link_locked(sc); 1608 GEN_UNLOCK(sc); 1609 } 1610 1611 static void 1612 gen_miibus_statchg(device_t dev) 1613 { 1614 struct gen_softc *sc; 1615 1616 sc = device_get_softc(dev); 1617 1618 taskqueue_enqueue(taskqueue_swi, &sc->link_task); 1619 } 1620 1621 static void 1622 gen_media_status(if_t ifp, struct ifmediareq *ifmr) 1623 { 1624 struct gen_softc *sc; 1625 struct mii_data *mii; 1626 1627 sc = if_getsoftc(ifp); 1628 mii = device_get_softc(sc->miibus); 1629 1630 GEN_LOCK(sc); 1631 mii_pollstat(mii); 1632 ifmr->ifm_active = mii->mii_media_active; 1633 ifmr->ifm_status = mii->mii_media_status; 1634 GEN_UNLOCK(sc); 1635 } 1636 1637 static int 1638 gen_media_change(if_t ifp) 1639 { 1640 struct gen_softc *sc; 1641 struct mii_data *mii; 1642 int error; 1643 1644 sc = if_getsoftc(ifp); 1645 mii = device_get_softc(sc->miibus); 1646 1647 GEN_LOCK(sc); 1648 error = mii_mediachg(mii); 1649 GEN_UNLOCK(sc); 1650 1651 return (error); 1652 } 1653 1654 static device_method_t gen_methods[] = { 1655 /* Device interface */ 1656 DEVMETHOD(device_probe, gen_probe), 1657 DEVMETHOD(device_attach, gen_attach), 1658 1659 /* MII interface */ 1660 DEVMETHOD(miibus_readreg, gen_miibus_readreg), 1661 DEVMETHOD(miibus_writereg, gen_miibus_writereg), 1662 DEVMETHOD(miibus_statchg, gen_miibus_statchg), 1663 1664 DEVMETHOD_END 1665 }; 1666 1667 static driver_t gen_driver = { 1668 "genet", 1669 gen_methods, 1670 sizeof(struct gen_softc), 1671 }; 1672 1673 static devclass_t gen_devclass; 1674 1675 DRIVER_MODULE(genet, simplebus, gen_driver, gen_devclass, 0, 0); 1676 DRIVER_MODULE(miibus, genet, miibus_driver, miibus_devclass, 0, 0); 1677 MODULE_DEPEND(genet, ether, 1, 1, 1); 1678 MODULE_DEPEND(genet, miibus, 1, 1, 1); 1679