1 /*- 2 * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com> 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include <sys/param.h> 27 #include <sys/systm.h> 28 #include <sys/bus.h> 29 #include <sys/kernel.h> 30 #include <sys/lock.h> 31 #include <sys/malloc.h> 32 #include <sys/mbuf.h> 33 #include <sys/module.h> 34 #include <sys/mutex.h> 35 #include <sys/rman.h> 36 #include <sys/socket.h> 37 38 #include <net/bpf.h> 39 #include <net/if.h> 40 #include <net/ethernet.h> 41 #include <net/if_dl.h> 42 #include <net/if_media.h> 43 #include <net/if_types.h> 44 #include <net/if_var.h> 45 46 #include <machine/bus.h> 47 48 #include <dev/clk/clk.h> 49 #include <dev/hwreset/hwreset.h> 50 51 #include <dev/ofw/ofw_bus.h> 52 #include <dev/ofw/ofw_bus_subr.h> 53 54 #include <dev/dwc/if_dwcvar.h> 55 #include <dev/dwc/dwc4_reg.h> 56 #include <dev/dwc/dwc4_dma.h> 57 #include <dev/dwc/dwc4_mtl.h> 58 59 #define WATCHDOG_TIMEOUT_SECS 5 60 #define DMA_RESET_TIMEOUT 100 61 62 /* RDES1 Write-back format */ 63 #define RDES1_IPCE (1 << 7) /* IP Payload Error; 64 * 16-bit IP payload checksum mismatch. 65 */ 66 #define RDES1_IPCB (1 << 6) /* Checksum offload engine is bypassed. */ 67 #define RDES1_IPHE (1 << 3) /* 68 * IP Header Error; 69 * 16-bit IP header checksum mismatch. 70 */ 71 /* RX Read format */ 72 #define RDES3_OWN (1 << 31) /* DMA owns the descriptor */ 73 #define RDES3_IOC (1 << 30) /* Interrupt Enabled on Completion */ 74 #define RDES3_BUF2V (1 << 25) /* Buffer 2 Address Valid */ 75 #define RDES3_BUF1V (1 << 24) /* Buffer 1 Address Valid */ 76 77 /* RX Write-back format */ 78 #define RDES3_FD (1 << 29) /* First Descriptor */ 79 #define RDES3_LD (1 << 28) /* Last Descriptor */ 80 #define RDES3_RE (1 << 20) /* Receive Error */ 81 #define RDES3_PL_S 0 /* Packet Length */ 82 #define RDES3_PL_M (0x7fff << RDES3_PL_S) 83 #define RDES3_CE (1 << 24) /* CRC Error */ 84 #define RDES3_LT_S 16 /* Length/Type Field */ 85 #define RDES3_LT_M (0x3 << RDES3_LT_S) 86 87 /* TX Read format */ 88 #define TDES2_IOC (1 << 31) /* Interrupt on completion */ 89 #define TDES2_TTSE (1 << 30) /* Transmit Timestamp Enable */ 90 #define TDES2_B2L_S 16 /* Buffer 2 Length */ 91 #define TDES2_B2L_M (0x3fff << TDES2_B2L_S) 92 #define TDES2_VTIR_S 14 /* VLAN Tag Insertion or Replacement */ 93 #define TDES2_VTIR_M (0x3 << TDES2_VTIR_S) 94 95 #define TDES3_OWN (1 << 31) /* the DMA owns the descriptor */ 96 #define TDES3_CTXT (1 << 30) /* Context Type */ 97 #define TDES3_FD (1 << 29) /* First Descriptor */ 98 #define TDES3_LD (1 << 28) /* Last Descriptor */ 99 #define TDES3_CPC_S 26 100 #define TDES3_CPC_M (0x3 << TDES3_CPC_S) 101 #define TDES3_CPC_CRC_PAD (0x0 << TDES3_CPC_S) 102 #define TDES3_CPC_CRC (0x1 << TDES3_CPC_S) 103 #define TDES3_CPC_CRC_DISABLE (0x2 << TDES3_CPC_S) 104 #define TDES3_CPC_CRC_REPLACE (0x3 << TDES3_CPC_S) 105 #define TDES3_SAIC_S 23 106 #define TDES3_SAIC_M (0x7 << TDES3_SAIC_S) 107 #define TDES3_TSE 18 /* TCP Segmentation Enable */ 108 #define TDES3_CIC_S 16 109 #define TDES3_CIC_M (0x3 << TDES3_CIC_S) 110 #define TDES3_CIC_HDR (0x1 << TDES3_CIC_S) /* Only IP header csum inserted */ 111 #define TDES3_CIC_FULL (0x2 << TDES3_CIC_S) /* IP header & payload */ 112 #define TDES3_CIC_FULLP (0x3 << TDES3_CIC_S) /* IP header, payload, pseudo-hdr*/ 113 #define TDES3_TPL_S 0 /* TCP Payload Length */ 114 #define TDES3_TPL_M (0x3ffff << TDES3_TPL_S) 115 116 /* 117 * A hardware buffer descriptor. Rx and Tx buffers have the same descriptor 118 * layout, but the bits in the fields have different meanings. 119 */ 120 struct dwc_hwdesc 121 { 122 uint32_t tdes0; /* Header or Buffer 1 Address[31:0] */ 123 uint32_t tdes1; /* Buffer 2 Address [31:0] or Buffer 1 Address[63:32] */ 124 uint32_t tdes2; 125 uint32_t tdes3; 126 }; 127 128 #define RX_DESC_SIZE (sizeof(struct dwc_hwdesc) * RX_DESC_COUNT) 129 #define TX_DESC_SIZE (sizeof(struct dwc_hwdesc) * TX_DESC_COUNT) 130 131 /* 132 * The hardware imposes alignment restrictions on various objects involved in 133 * DMA transfers. These values are expressed in bytes (not bits). 134 */ 135 #define DWC_DESC_RING_ALIGN 2048 136 137 static inline uint32_t 138 next_txidx(struct dwc_softc *sc, uint32_t curidx) 139 { 140 141 return ((curidx + 1) % TX_DESC_COUNT); 142 } 143 144 static inline uint32_t 145 next_rxidx(struct dwc_softc *sc, uint32_t curidx) 146 { 147 148 return ((curidx + 1) % RX_DESC_COUNT); 149 } 150 151 static void 152 dwc_get1paddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 153 { 154 155 if (error != 0) 156 return; 157 *(bus_addr_t *)arg = segs[0].ds_addr; 158 } 159 160 inline static void 161 txdesc_clear(struct dwc_softc *sc, int idx) 162 { 163 164 sc->tx_desccount--; 165 sc->txdesc_ring[idx].tdes0 = 0; 166 sc->txdesc_ring[idx].tdes1 = 0; 167 sc->txdesc_ring[idx].tdes2 = 0; 168 sc->txdesc_ring[idx].tdes3 = 0; 169 } 170 171 inline static void 172 txdesc_setup(struct dwc_softc *sc, int idx, bus_addr_t paddr, 173 uint32_t len, uint32_t flags, bool first, bool last) 174 { 175 uint32_t tdes0, tdes1, tdes2, tdes3; 176 177 if (!sc->dma_ext_desc) { 178 tdes0 = (uint32_t)paddr; 179 tdes1 = (uint32_t)((uint64_t)paddr >> 32); 180 tdes2 = len | TDES2_TTSE; 181 tdes3 = flags; 182 183 if (first) 184 tdes3 |= TDES3_FD; 185 if (last) { 186 tdes2 |= TDES2_IOC; 187 tdes3 |= TDES3_LD; 188 } 189 } else { 190 tdes0 = 0; 191 tdes1 = 0; 192 tdes2 = 0; 193 tdes3 = 0; 194 } 195 ++sc->tx_desccount; 196 sc->txdesc_ring[idx].tdes0 = tdes0; 197 sc->txdesc_ring[idx].tdes1 = tdes1; 198 sc->txdesc_ring[idx].tdes2 = tdes2; 199 sc->txdesc_ring[idx].tdes3 = tdes3; 200 201 wmb(); 202 sc->txdesc_ring[idx].tdes3 |= TDES3_OWN; 203 wmb(); 204 } 205 206 inline static uint32_t 207 rxdesc_setup(struct dwc_softc *sc, int idx, bus_addr_t paddr) 208 { 209 210 sc->rxdesc_ring[idx].tdes0 = (uint32_t)paddr; 211 sc->rxdesc_ring[idx].tdes1 = (uint32_t)((uint64_t)paddr >> 32); 212 sc->rxdesc_ring[idx].tdes2 = 0; 213 sc->rxdesc_ring[idx].tdes3 = RDES3_IOC | RDES3_BUF1V; 214 215 wmb(); 216 sc->rxdesc_ring[idx].tdes3 |= RDES3_OWN; 217 wmb(); 218 219 return (0); 220 } 221 222 int 223 dma4_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp) 224 { 225 struct bus_dma_segment segs[TX_MAP_MAX_SEGS]; 226 int error, nsegs; 227 struct mbuf * m; 228 uint32_t flags; 229 uint32_t csum_flags; 230 int i; 231 int last; 232 233 flags = 0; 234 error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map, 235 *mp, segs, &nsegs, 0); 236 if (error == EFBIG) { 237 /* 238 * The map may be partially mapped from the first call. 239 * Make sure to reset it. 240 */ 241 bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map); 242 if ((m = m_defrag(*mp, M_NOWAIT)) == NULL) 243 return (ENOMEM); 244 *mp = m; 245 error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, 246 sc->txbuf_map[idx].map, *mp, segs, &nsegs, 0); 247 } 248 if (error != 0) 249 return (ENOMEM); 250 251 if (sc->tx_desccount + nsegs > TX_DESC_COUNT) { 252 bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map); 253 return (ENOMEM); 254 } 255 256 m = *mp; 257 258 csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP | CSUM_IP6_UDP; 259 if ((m->m_pkthdr.csum_flags & csum_flags) != 0) { 260 if (!sc->dma_ext_desc) 261 flags = TDES3_CIC_FULLP; 262 else 263 panic("implement me"); 264 } else if ((m->m_pkthdr.csum_flags & CSUM_IP) != 0) { 265 if (!sc->dma_ext_desc) 266 flags = TDES3_CIC_HDR; 267 else 268 panic("implement me"); 269 } 270 271 bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map, 272 BUS_DMASYNC_PREWRITE); 273 274 sc->txbuf_map[idx].mbuf = m; 275 276 for (i = 0; i < nsegs; i++) { 277 txdesc_setup(sc, sc->tx_desc_head, 278 segs[i].ds_addr, segs[i].ds_len, 279 (i == 0) ? flags : 0, /* only first desc needs flags */ 280 (i == 0), 281 (i == nsegs - 1)); 282 last = sc->tx_desc_head; 283 sc->tx_desc_head = next_txidx(sc, sc->tx_desc_head); 284 } 285 286 sc->txbuf_map[idx].last_desc_idx = last; 287 288 wmb(); 289 290 WRITE4(sc, ETH_DMACTXDTPR(0), sc->txdesc_ring_paddr + 291 sc->tx_desc_head * sizeof(struct dwc_hwdesc)); 292 293 return (0); 294 } 295 296 static int 297 dma4_setup_rxbuf(struct dwc_softc *sc, int idx, struct mbuf *m) 298 { 299 struct bus_dma_segment seg; 300 int error, nsegs; 301 302 m_adj(m, ETHER_ALIGN); 303 304 error = bus_dmamap_load_mbuf_sg(sc->rxbuf_tag, sc->rxbuf_map[idx].map, 305 m, &seg, &nsegs, 0); 306 if (error != 0) 307 return (error); 308 309 KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); 310 311 bus_dmamap_sync(sc->rxbuf_tag, sc->rxbuf_map[idx].map, 312 BUS_DMASYNC_PREREAD); 313 314 sc->rxbuf_map[idx].mbuf = m; 315 rxdesc_setup(sc, idx, seg.ds_addr); 316 317 return (0); 318 } 319 320 static struct mbuf * 321 dwc_alloc_mbufcl(struct dwc_softc *sc) 322 { 323 struct mbuf *m; 324 325 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 326 if (m != NULL) 327 m->m_pkthdr.len = m->m_len = m->m_ext.ext_size; 328 329 return (m); 330 } 331 332 static struct mbuf * 333 dwc_rxfinish_one(struct dwc_softc *sc, struct dwc_hwdesc *desc, 334 struct dwc_bufmap *map) 335 { 336 if_t ifp; 337 struct mbuf *m, *m0; 338 int len; 339 uint32_t rdes1; 340 uint32_t rdes3; 341 342 m = map->mbuf; 343 ifp = sc->ifp; 344 rdes1 = desc->tdes1; 345 rdes3 = desc->tdes3; 346 347 if ((rdes3 & (RDES3_FD | RDES3_LD)) != (RDES3_FD | RDES3_LD)) { 348 /* 349 * Something very wrong happens. The whole packet should be 350 * received in one descriptor. Report problem. 351 */ 352 device_printf(sc->dev, 353 "%s: RX descriptor without FIRST and LAST bit set: 0x%08X", 354 __func__, rdes3); 355 return (NULL); 356 } 357 358 len = (rdes3 & RDES3_PL_M) >> RDES3_PL_S; 359 360 /* Allocate new buffer */ 361 m0 = dwc_alloc_mbufcl(sc); 362 if (m0 == NULL) { 363 device_printf(sc->dev, "no mbufs\n"); 364 365 /* no new mbuf available, recycle old */ 366 if_inc_counter(sc->ifp, IFCOUNTER_IQDROPS, 1); 367 return (NULL); 368 } 369 /* Do dmasync for newly received packet */ 370 bus_dmamap_sync(sc->rxbuf_tag, map->map, BUS_DMASYNC_POSTREAD); 371 bus_dmamap_unload(sc->rxbuf_tag, map->map); 372 373 /* Received packet is valid, process it */ 374 m->m_pkthdr.rcvif = ifp; 375 m->m_pkthdr.len = len; 376 m->m_len = len; 377 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 378 379 if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) != 0 && 380 (rdes1 & RDES1_IPCB) == 0) { 381 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 382 if ((rdes1 & RDES1_IPCE) == 0) 383 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 384 if ((rdes1 & RDES1_IPHE) == 0) { 385 m->m_pkthdr.csum_flags |= 386 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 387 m->m_pkthdr.csum_data = 0xffff; 388 } 389 } 390 391 /* Remove trailing FCS */ 392 m_adj(m, -ETHER_CRC_LEN); 393 394 DWC_UNLOCK(sc); 395 if_input(ifp, m); 396 DWC_LOCK(sc); 397 return (m0); 398 } 399 400 void 401 dma4_txfinish_locked(struct dwc_softc *sc) 402 { 403 struct dwc_bufmap *bmap; 404 struct dwc_hwdesc *desc; 405 if_t ifp; 406 int idx, last_idx; 407 bool map_finished; 408 409 DWC_ASSERT_LOCKED(sc); 410 411 ifp = sc->ifp; 412 /* check if all descriptors of the map are done */ 413 while (sc->tx_map_tail != sc->tx_map_head) { 414 map_finished = true; 415 bmap = &sc->txbuf_map[sc->tx_map_tail]; 416 idx = sc->tx_desc_tail; 417 last_idx = next_txidx(sc, bmap->last_desc_idx); 418 while (idx != last_idx) { 419 desc = &sc->txdesc_ring[idx]; 420 if ((desc->tdes3 & TDES3_OWN) != 0) { 421 map_finished = false; 422 break; 423 } 424 idx = next_txidx(sc, idx); 425 } 426 427 if (!map_finished) 428 break; 429 bus_dmamap_sync(sc->txbuf_tag, bmap->map, 430 BUS_DMASYNC_POSTWRITE); 431 bus_dmamap_unload(sc->txbuf_tag, bmap->map); 432 m_freem(bmap->mbuf); 433 bmap->mbuf = NULL; 434 sc->tx_mapcount--; 435 while (sc->tx_desc_tail != last_idx) { 436 txdesc_clear(sc, sc->tx_desc_tail); 437 sc->tx_desc_tail = next_txidx(sc, sc->tx_desc_tail); 438 } 439 sc->tx_map_tail = next_txidx(sc, sc->tx_map_tail); 440 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 441 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 442 } 443 444 /* If there are no buffers outstanding, muzzle the watchdog. */ 445 if (sc->tx_desc_tail == sc->tx_desc_head) { 446 sc->tx_watchdog_count = 0; 447 } 448 } 449 450 void 451 dma4_txstart(struct dwc_softc *sc) 452 { 453 int enqueued; 454 struct mbuf *m; 455 456 enqueued = 0; 457 458 for (;;) { 459 if (sc->tx_desccount > (TX_DESC_COUNT - TX_MAP_MAX_SEGS + 1)) { 460 if_setdrvflagbits(sc->ifp, IFF_DRV_OACTIVE, 0); 461 break; 462 } 463 464 if (sc->tx_mapcount == (TX_MAP_COUNT - 1)) { 465 if_setdrvflagbits(sc->ifp, IFF_DRV_OACTIVE, 0); 466 break; 467 } 468 469 m = if_dequeue(sc->ifp); 470 if (m == NULL) 471 break; 472 if (dma4_setup_txbuf(sc, sc->tx_map_head, &m) != 0) { 473 if_sendq_prepend(sc->ifp, m); 474 if_setdrvflagbits(sc->ifp, IFF_DRV_OACTIVE, 0); 475 break; 476 } 477 bpf_mtap_if(sc->ifp, m); 478 sc->tx_map_head = next_txidx(sc, sc->tx_map_head); 479 sc->tx_mapcount++; 480 ++enqueued; 481 } 482 483 if (enqueued != 0) 484 sc->tx_watchdog_count = WATCHDOG_TIMEOUT_SECS; 485 } 486 487 void 488 dma4_rxfinish_locked(struct dwc_softc *sc) 489 { 490 struct mbuf *m; 491 int error, idx; 492 struct dwc_hwdesc *desc; 493 494 DWC_ASSERT_LOCKED(sc); 495 496 for (;;) { 497 idx = sc->rx_idx; 498 desc = sc->rxdesc_ring + idx; 499 if (desc->tdes3 & RDES3_OWN) 500 break; 501 m = dwc_rxfinish_one(sc, desc, sc->rxbuf_map + idx); 502 if (m == NULL) { 503 desc->tdes3 = RDES3_BUF1V | RDES3_IOC | RDES3_OWN; 504 wmb(); 505 } else { 506 /* We cannot create hole in RX ring */ 507 error = dma4_setup_rxbuf(sc, idx, m); 508 if (error != 0) 509 panic("dma4_setup_rxbuf failed: error %d\n", 510 error); 511 } 512 513 WRITE4(sc, ETH_DMACRXDTPR(0), sc->rxdesc_ring_paddr + 514 idx * sizeof(struct dwc_hwdesc)); 515 516 sc->rx_idx = next_rxidx(sc, sc->rx_idx); 517 } 518 } 519 520 /* 521 * Start the DMA controller 522 */ 523 void 524 dma4_start(struct dwc_softc *sc) 525 { 526 uint32_t reg; 527 528 DWC_ASSERT_LOCKED(sc); 529 530 /* Initialize DMA and enable transmitters */ 531 532 /* Configure TX channel 0 */ 533 reg = READ4(sc, ETH_DMACTXCR(0)); 534 reg |= DMACTXCR_OSF; 535 WRITE4(sc, ETH_DMACTXCR(0), reg); 536 537 reg = MTLTXQOMR_TSF; /* Transmit Store and Forward */ 538 reg |= MTLTXQOMR_EN; 539 reg |= MTLTXQOMR_TQS_8K; 540 WRITE4(sc, ETH_MTLTXQOMR(0), reg); 541 542 /* Configure RX channel 0 */ 543 reg = MTLRXQOMR_RSF; 544 reg |= MTLRXQOMR_EHFC; 545 reg |= MTLRXQOMR_RQS_4K; 546 WRITE4(sc, ETH_MTLRXQOMR(0), reg); 547 548 reg = READ4(sc, ETH_MACRXQC0R(0)); 549 reg |= MACRXQC0R_RXQ0EN_GEN; 550 WRITE4(sc, ETH_MACRXQC0R(0), reg); 551 552 /* Enable Interrupts. */ 553 reg = DMACIER_TIE | DMACIER_TXSE | DMACIER_TBUE | DMACIER_RIE | 554 DMACIER_RBUE | DMACIER_RSE | DMACIER_RWTE | DMACIER_ETIE | 555 DMACIER_ERIE | DMACIER_FBEE | DMACIER_CDEE | DMACIER_AIE | 556 DMACIER_NIE; 557 WRITE4(sc, ETH_DMACIER(0), reg); 558 559 /* Start DMA RX. */ 560 reg = READ4(sc, ETH_DMACRXCR(0)); 561 reg |= DMACRXCR_SR; 562 WRITE4(sc, ETH_DMACRXCR(0), reg); 563 564 /* Start DMA TX. */ 565 reg = READ4(sc, ETH_DMACTXCR(0)); 566 reg |= DMACTXCR_ST; 567 WRITE4(sc, ETH_DMACTXCR(0), reg); 568 } 569 570 /* 571 * Stop the DMA controller 572 */ 573 void 574 dma4_stop(struct dwc_softc *sc) 575 { 576 uint32_t reg; 577 578 DWC_ASSERT_LOCKED(sc); 579 580 /* Stop DMA TX */ 581 reg = READ4(sc, ETH_DMACTXCR(0)); 582 reg &= ~DMACTXCR_ST; 583 WRITE4(sc, ETH_DMACTXCR(0), reg); 584 585 /* Flush TX */ 586 reg = READ4(sc, ETH_MTLTXQOMR(0)); 587 reg |= MTLTXQOMR_FTQ; 588 WRITE4(sc, ETH_MTLTXQOMR(0), reg); 589 590 /* Stop DMA RX */ 591 reg = READ4(sc, ETH_DMACRXCR(0)); 592 reg &= ~DMACRXCR_SR; 593 WRITE4(sc, ETH_DMACRXCR(0), reg); 594 } 595 596 int 597 dma4_reset(struct dwc_softc *sc) 598 { 599 uint32_t reg; 600 int i; 601 602 reg = READ4(sc, ETH_DMAMR); 603 reg |= DMAMR_SWR; 604 WRITE4(sc, ETH_DMAMR, reg); 605 606 for (i = 0; i < DMA_RESET_TIMEOUT; i++) { 607 if ((READ4(sc, ETH_DMAMR) & DMAMR_SWR) == 0) 608 break; 609 DELAY(10); 610 } 611 if (i >= DMA_RESET_TIMEOUT) 612 return (ENXIO); 613 614 return (0); 615 } 616 617 /* 618 * Create the bus_dma resources 619 */ 620 int 621 dma4_init(struct dwc_softc *sc) 622 { 623 struct mbuf *m; 624 uint32_t reg; 625 int error; 626 int idx; 627 628 WRITE4(sc, ETH_MACQ0TXFCR, 0xffff << 16 | 1); 629 WRITE4(sc, ETH_MACRXFCR, 1); 630 631 reg = READ4(sc, ETH_DMACCR(0)); 632 if (sc->nopblx8) 633 reg &= ~DMACCR_PBLX8; 634 else 635 reg |= DMACCR_PBLX8; 636 WRITE4(sc, ETH_DMACCR(0), reg); 637 638 WRITE4(sc, ETH_DMACTXCR(0), 0); 639 640 reg = (MCLBYTES << DMACRXCR_RBSZ_S); 641 WRITE4(sc, ETH_DMACRXCR(0), reg); 642 643 reg = DMASBMR_BLEN4; 644 reg |= DMASBMR_BLEN8; 645 reg |= DMASBMR_BLEN16; 646 reg |= DMASBMR_EAME; 647 WRITE4(sc, ETH_DMASBMR, reg); 648 649 reg = READ4(sc, ETH_DMASBMR); 650 if (sc->fixed_burst) 651 reg |= DMASBMR_FB; 652 else 653 reg &= ~DMASBMR_FB; 654 if (sc->aal) 655 reg |= DMASBMR_AAL; 656 else 657 reg &= ~DMASBMR_AAL; 658 WRITE4(sc, ETH_DMASBMR, reg); 659 660 /* No support in the driver. */ 661 sc->dma_ext_desc = false; 662 663 /* 664 * DMA must be stop while changing descriptor list addresses. 665 */ 666 667 /* Stop DMA TX */ 668 reg = READ4(sc, ETH_DMACTXCR(0)); 669 reg &= ~DMACTXCR_ST; 670 WRITE4(sc, ETH_DMACTXCR(0), reg); 671 672 /* Stop DMA RX */ 673 reg = READ4(sc, ETH_DMACRXCR(0)); 674 reg &= ~DMACRXCR_SR; 675 WRITE4(sc, ETH_DMACRXCR(0), reg); 676 677 /* 678 * Set up TX descriptor ring, descriptors, and dma maps. 679 */ 680 error = bus_dma_tag_create( 681 bus_get_dma_tag(sc->dev), /* Parent tag. */ 682 DWC_DESC_RING_ALIGN, 0, /* alignment, boundary */ 683 BUS_SPACE_MAXADDR, /* lowaddr */ 684 BUS_SPACE_MAXADDR, /* highaddr */ 685 NULL, NULL, /* filter, filterarg */ 686 TX_DESC_SIZE, 1, /* maxsize, nsegments */ 687 TX_DESC_SIZE, /* maxsegsize */ 688 0, /* flags */ 689 NULL, NULL, /* lockfunc, lockarg */ 690 &sc->txdesc_tag); 691 if (error != 0) { 692 device_printf(sc->dev, 693 "could not create TX ring DMA tag.\n"); 694 goto out; 695 } 696 697 error = bus_dmamem_alloc(sc->txdesc_tag, (void **)&sc->txdesc_ring, 698 BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, 699 &sc->txdesc_map); 700 if (error != 0) { 701 device_printf(sc->dev, 702 "could not allocate TX descriptor ring.\n"); 703 goto out; 704 } 705 706 error = bus_dmamap_load(sc->txdesc_tag, sc->txdesc_map, 707 sc->txdesc_ring, TX_DESC_SIZE, dwc_get1paddr, 708 &sc->txdesc_ring_paddr, 0); 709 if (error != 0) { 710 device_printf(sc->dev, 711 "could not load TX descriptor ring map.\n"); 712 goto out; 713 } 714 715 error = bus_dma_tag_create( 716 bus_get_dma_tag(sc->dev), /* Parent tag. */ 717 1, 0, /* alignment, boundary */ 718 BUS_SPACE_MAXADDR, /* lowaddr */ 719 BUS_SPACE_MAXADDR, /* highaddr */ 720 NULL, NULL, /* filter, filterarg */ 721 MCLBYTES * TX_MAP_MAX_SEGS, /* maxsize */ 722 TX_MAP_MAX_SEGS, /* nsegments */ 723 MCLBYTES, /* maxsegsize */ 724 0, /* flags */ 725 NULL, NULL, /* lockfunc, lockarg */ 726 &sc->txbuf_tag); 727 if (error != 0) { 728 device_printf(sc->dev, 729 "could not create TX ring DMA tag.\n"); 730 goto out; 731 } 732 733 for (idx = 0; idx < TX_MAP_COUNT; idx++) { 734 error = bus_dmamap_create(sc->txbuf_tag, 0, 735 &sc->txbuf_map[idx].map); 736 if (error != 0) { 737 device_printf(sc->dev, 738 "could not create TX buffer DMA map.\n"); 739 goto out; 740 } 741 } 742 743 for (idx = 0; idx < TX_DESC_COUNT; idx++) 744 txdesc_clear(sc, idx); 745 746 WRITE4(sc, ETH_DMACTXDLAR_HI(0), 747 (uint32_t)((uint64_t)sc->txdesc_ring_paddr >> 32)); 748 WRITE4(sc, ETH_DMACTXDLAR(0), (uint32_t)sc->txdesc_ring_paddr); 749 WRITE4(sc, ETH_DMACTXRLR(0), TX_DESC_COUNT - 1); 750 751 /* 752 * Set up RX descriptor ring, descriptors, dma maps, and mbufs. 753 */ 754 error = bus_dma_tag_create( 755 bus_get_dma_tag(sc->dev), /* Parent tag. */ 756 DWC_DESC_RING_ALIGN, 0, /* alignment, boundary */ 757 BUS_SPACE_MAXADDR, /* lowaddr */ 758 BUS_SPACE_MAXADDR, /* highaddr */ 759 NULL, NULL, /* filter, filterarg */ 760 RX_DESC_SIZE, 1, /* maxsize, nsegments */ 761 RX_DESC_SIZE, /* maxsegsize */ 762 0, /* flags */ 763 NULL, NULL, /* lockfunc, lockarg */ 764 &sc->rxdesc_tag); 765 if (error != 0) { 766 device_printf(sc->dev, 767 "could not create RX ring DMA tag.\n"); 768 goto out; 769 } 770 771 error = bus_dmamem_alloc(sc->rxdesc_tag, (void **)&sc->rxdesc_ring, 772 BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, 773 &sc->rxdesc_map); 774 if (error != 0) { 775 device_printf(sc->dev, 776 "could not allocate RX descriptor ring.\n"); 777 goto out; 778 } 779 780 error = bus_dmamap_load(sc->rxdesc_tag, sc->rxdesc_map, 781 sc->rxdesc_ring, RX_DESC_SIZE, dwc_get1paddr, 782 &sc->rxdesc_ring_paddr, 0); 783 if (error != 0) { 784 device_printf(sc->dev, 785 "could not load RX descriptor ring map.\n"); 786 goto out; 787 } 788 789 error = bus_dma_tag_create( 790 bus_get_dma_tag(sc->dev), /* Parent tag. */ 791 1, 0, /* alignment, boundary */ 792 BUS_SPACE_MAXADDR, /* lowaddr */ 793 BUS_SPACE_MAXADDR, /* highaddr */ 794 NULL, NULL, /* filter, filterarg */ 795 MCLBYTES, 1, /* maxsize, nsegments */ 796 MCLBYTES, /* maxsegsize */ 797 0, /* flags */ 798 NULL, NULL, /* lockfunc, lockarg */ 799 &sc->rxbuf_tag); 800 if (error != 0) { 801 device_printf(sc->dev, 802 "could not create RX buf DMA tag.\n"); 803 goto out; 804 } 805 806 for (idx = 0; idx < RX_MAP_COUNT; idx++) { 807 error = bus_dmamap_create(sc->rxbuf_tag, 0, 808 &sc->rxbuf_map[idx].map); 809 if (error != 0) { 810 device_printf(sc->dev, 811 "could not create RX buffer DMA map.\n"); 812 goto out; 813 } 814 if ((m = dwc_alloc_mbufcl(sc)) == NULL) { 815 device_printf(sc->dev, "Could not alloc mbuf\n"); 816 error = ENOMEM; 817 goto out; 818 } 819 if ((error = dma4_setup_rxbuf(sc, idx, m)) != 0) { 820 device_printf(sc->dev, 821 "could not create new RX buffer.\n"); 822 goto out; 823 } 824 } 825 826 WRITE4(sc, ETH_DMACRXDLAR_HI(0), 827 (uint32_t)((uint64_t)sc->rxdesc_ring_paddr >> 32)); 828 WRITE4(sc, ETH_DMACRXDLAR(0), (uint32_t)sc->rxdesc_ring_paddr); 829 830 WRITE4(sc, ETH_DMACRXDTPR(0), sc->rxdesc_ring_paddr + 831 (RX_DESC_COUNT - 1) * sizeof(struct dwc_hwdesc)); 832 WRITE4(sc, ETH_DMACRXRLR(0), RX_DESC_COUNT - 1); 833 834 out: 835 if (error != 0) 836 return (ENXIO); 837 838 return (0); 839 } 840 841 /* 842 * Free the bus_dma resources 843 */ 844 void 845 dma4_free(struct dwc_softc *sc) 846 { 847 bus_dmamap_t map; 848 int idx; 849 850 /* Clean up RX DMA resources and free mbufs. */ 851 for (idx = 0; idx < RX_DESC_COUNT; ++idx) { 852 if ((map = sc->rxbuf_map[idx].map) != NULL) { 853 bus_dmamap_unload(sc->rxbuf_tag, map); 854 bus_dmamap_destroy(sc->rxbuf_tag, map); 855 m_freem(sc->rxbuf_map[idx].mbuf); 856 } 857 } 858 if (sc->rxbuf_tag != NULL) 859 bus_dma_tag_destroy(sc->rxbuf_tag); 860 if (sc->rxdesc_map != NULL) { 861 bus_dmamap_unload(sc->rxdesc_tag, sc->rxdesc_map); 862 bus_dmamem_free(sc->rxdesc_tag, sc->rxdesc_ring, 863 sc->rxdesc_map); 864 } 865 if (sc->rxdesc_tag != NULL) 866 bus_dma_tag_destroy(sc->rxdesc_tag); 867 868 /* Clean up TX DMA resources. */ 869 for (idx = 0; idx < TX_DESC_COUNT; ++idx) { 870 if ((map = sc->txbuf_map[idx].map) != NULL) { 871 /* TX maps are already unloaded. */ 872 bus_dmamap_destroy(sc->txbuf_tag, map); 873 } 874 } 875 if (sc->txbuf_tag != NULL) 876 bus_dma_tag_destroy(sc->txbuf_tag); 877 if (sc->txdesc_map != NULL) { 878 bus_dmamap_unload(sc->txdesc_tag, sc->txdesc_map); 879 bus_dmamem_free(sc->txdesc_tag, sc->txdesc_ring, 880 sc->txdesc_map); 881 } 882 if (sc->txdesc_tag != NULL) 883 bus_dma_tag_destroy(sc->txdesc_tag); 884 } 885 886 /* 887 * Interrupt function 888 */ 889 890 int 891 dma4_intr(struct dwc_softc *sc) 892 { 893 uint32_t reg; 894 int rv; 895 896 rv = 0; 897 898 DWC_ASSERT_LOCKED(sc); 899 900 reg = READ4(sc, ETH_DMACSR(0)); 901 902 if (reg & DMACSR_NIS) { 903 if (reg & DMACSR_RI) 904 dma4_rxfinish_locked(sc); 905 906 if (reg & DMACSR_TI) { 907 dma4_txfinish_locked(sc); 908 dma4_txstart(sc); 909 } 910 } 911 912 if (reg & DMACSR_AIS) { 913 if (reg & DMACSR_FBE) { 914 /* Fatal Bus Error. */ 915 rv = EIO; 916 } 917 } 918 919 WRITE4(sc, ETH_DMACSR(0), reg); 920 921 return (rv); 922 } 923