1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * 4 * This software was developed by SRI International and the University of 5 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 6 * ("CTSRD"), as part of the DARPA CRASH research programme. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/bus.h> 33 #include <sys/kernel.h> 34 #include <sys/lock.h> 35 #include <sys/malloc.h> 36 #include <sys/mbuf.h> 37 #include <sys/module.h> 38 #include <sys/mutex.h> 39 #include <sys/rman.h> 40 #include <sys/socket.h> 41 42 #include <net/bpf.h> 43 #include <net/if.h> 44 #include <net/ethernet.h> 45 #include <net/if_dl.h> 46 #include <net/if_media.h> 47 #include <net/if_types.h> 48 #include <net/if_var.h> 49 50 #include <machine/bus.h> 51 52 #include <dev/clk/clk.h> 53 #include <dev/hwreset/hwreset.h> 54 55 #include <dev/ofw/ofw_bus.h> 56 #include <dev/ofw/ofw_bus_subr.h> 57 58 #include <dev/dwc/if_dwcvar.h> 59 #include <dev/dwc/dwc1000_reg.h> 60 #include <dev/dwc/dwc1000_dma.h> 61 62 #define WATCHDOG_TIMEOUT_SECS 5 63 #define DMA_RESET_TIMEOUT 100 64 65 /* TX descriptors - TDESC0 is almost unified */ 66 #define TDESC0_OWN (1U << 31) 67 #define TDESC0_IHE (1U << 16) /* IP Header Error */ 68 #define TDESC0_ES (1U << 15) /* Error Summary */ 69 #define TDESC0_JT (1U << 14) /* Jabber Timeout */ 70 #define TDESC0_FF (1U << 13) /* Frame Flushed */ 71 #define TDESC0_PCE (1U << 12) /* Payload Checksum Error */ 72 #define TDESC0_LOC (1U << 11) /* Loss of Carrier */ 73 #define TDESC0_NC (1U << 10) /* No Carrier */ 74 #define TDESC0_LC (1U << 9) /* Late Collision */ 75 #define TDESC0_EC (1U << 8) /* Excessive Collision */ 76 #define TDESC0_VF (1U << 7) /* VLAN Frame */ 77 #define TDESC0_CC_MASK 0xf 78 #define TDESC0_CC_SHIFT 3 /* Collision Count */ 79 #define TDESC0_ED (1U << 2) /* Excessive Deferral */ 80 #define TDESC0_UF (1U << 1) /* Underflow Error */ 81 #define TDESC0_DB (1U << 0) /* Deferred Bit */ 82 /* TX descriptors - TDESC0 extended format only */ 83 #define ETDESC0_IC (1U << 30) /* Interrupt on Completion */ 84 #define ETDESC0_LS (1U << 29) /* Last Segment */ 85 #define ETDESC0_FS (1U << 28) /* First Segment */ 86 #define ETDESC0_DC (1U << 27) /* Disable CRC */ 87 #define ETDESC0_DP (1U << 26) /* Disable Padding */ 88 #define ETDESC0_CIC_NONE (0U << 22) /* Checksum Insertion Control */ 89 #define ETDESC0_CIC_HDR (1U << 22) 90 #define ETDESC0_CIC_SEG (2U << 22) 91 #define ETDESC0_CIC_FULL (3U << 22) 92 #define ETDESC0_TER (1U << 21) /* Transmit End of Ring */ 93 #define ETDESC0_TCH (1U << 20) /* Second Address Chained */ 94 95 /* TX descriptors - TDESC1 normal format */ 96 #define NTDESC1_IC (1U << 31) /* Interrupt on Completion */ 97 #define NTDESC1_LS (1U << 30) /* Last Segment */ 98 #define NTDESC1_FS (1U << 29) /* First Segment */ 99 #define NTDESC1_CIC_NONE (0U << 27) /* Checksum Insertion Control */ 100 #define NTDESC1_CIC_HDR (1U << 27) 101 #define NTDESC1_CIC_SEG (2U << 27) 102 #define NTDESC1_CIC_FULL (3U << 27) 103 #define NTDESC1_DC (1U << 26) /* Disable CRC */ 104 #define NTDESC1_TER (1U << 25) /* Transmit End of Ring */ 105 #define NTDESC1_TCH (1U << 24) /* Second Address Chained */ 106 /* TX descriptors - TDESC1 extended format */ 107 #define ETDESC1_DP (1U << 23) /* Disable Padding */ 108 #define ETDESC1_TBS2_MASK 0x7ff 109 #define ETDESC1_TBS2_SHIFT 11 /* Receive Buffer 2 Size */ 110 #define ETDESC1_TBS1_MASK 0x7ff 111 #define ETDESC1_TBS1_SHIFT 0 /* Receive Buffer 1 Size */ 112 113 /* RX descriptor - RDESC0 is unified */ 114 #define RDESC0_OWN (1U << 31) 115 #define RDESC0_AFM (1U << 30) /* Dest. Address Filter Fail */ 116 #define RDESC0_FL_MASK 0x3fff 117 #define RDESC0_FL_SHIFT 16 /* Frame Length */ 118 #define RDESC0_ES (1U << 15) /* Error Summary */ 119 #define RDESC0_DE (1U << 14) /* Descriptor Error */ 120 #define RDESC0_SAF (1U << 13) /* Source Address Filter Fail */ 121 #define RDESC0_LE (1U << 12) /* Length Error */ 122 #define RDESC0_OE (1U << 11) /* Overflow Error */ 123 #define RDESC0_VLAN (1U << 10) /* VLAN Tag */ 124 #define RDESC0_FS (1U << 9) /* First Descriptor */ 125 #define RDESC0_LS (1U << 8) /* Last Descriptor */ 126 #define RDESC0_ICE (1U << 7) /* IPC Checksum Error */ 127 #define RDESC0_LC (1U << 6) /* Late Collision */ 128 #define RDESC0_FT (1U << 5) /* Frame Type */ 129 #define RDESC0_RWT (1U << 4) /* Receive Watchdog Timeout */ 130 #define RDESC0_RE (1U << 3) /* Receive Error */ 131 #define RDESC0_DBE (1U << 2) /* Dribble Bit Error */ 132 #define RDESC0_CE (1U << 1) /* CRC Error */ 133 #define RDESC0_PCE (1U << 0) /* Payload Checksum Error */ 134 #define RDESC0_RXMA (1U << 0) /* Rx MAC Address */ 135 136 /* RX descriptors - RDESC1 normal format */ 137 #define NRDESC1_DIC (1U << 31) /* Disable Intr on Completion */ 138 #define NRDESC1_RER (1U << 25) /* Receive End of Ring */ 139 #define NRDESC1_RCH (1U << 24) /* Second Address Chained */ 140 #define NRDESC1_RBS2_MASK 0x7ff 141 #define NRDESC1_RBS2_SHIFT 11 /* Receive Buffer 2 Size */ 142 #define NRDESC1_RBS1_MASK 0x7ff 143 #define NRDESC1_RBS1_SHIFT 0 /* Receive Buffer 1 Size */ 144 145 /* RX descriptors - RDESC1 enhanced format */ 146 #define ERDESC1_DIC (1U << 31) /* Disable Intr on Completion */ 147 #define ERDESC1_RBS2_MASK 0x7ffff 148 #define ERDESC1_RBS2_SHIFT 16 /* Receive Buffer 2 Size */ 149 #define ERDESC1_RER (1U << 15) /* Receive End of Ring */ 150 #define ERDESC1_RCH (1U << 14) /* Second Address Chained */ 151 #define ERDESC1_RBS1_MASK 0x7ffff 152 #define ERDESC1_RBS1_SHIFT 0 /* Receive Buffer 1 Size */ 153 154 /* 155 * A hardware buffer descriptor. Rx and Tx buffers have the same descriptor 156 * layout, but the bits in the fields have different meanings. 157 */ 158 struct dwc_hwdesc 159 { 160 uint32_t desc0; 161 uint32_t desc1; 162 uint32_t addr1; /* ptr to first buffer data */ 163 uint32_t addr2; /* ptr to next descriptor / second buffer data*/ 164 }; 165 166 #define RX_DESC_SIZE (sizeof(struct dwc_hwdesc) * RX_DESC_COUNT) 167 #define TX_DESC_SIZE (sizeof(struct dwc_hwdesc) * TX_DESC_COUNT) 168 169 /* 170 * The hardware imposes alignment restrictions on various objects involved in 171 * DMA transfers. These values are expressed in bytes (not bits). 172 */ 173 #define DWC_DESC_RING_ALIGN 2048 174 175 static inline uint32_t 176 next_txidx(struct dwc_softc *sc, uint32_t curidx) 177 { 178 179 return ((curidx + 1) % TX_DESC_COUNT); 180 } 181 182 static inline uint32_t 183 next_rxidx(struct dwc_softc *sc, uint32_t curidx) 184 { 185 186 return ((curidx + 1) % RX_DESC_COUNT); 187 } 188 189 static void 190 dwc_get1paddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 191 { 192 193 if (error != 0) 194 return; 195 *(bus_addr_t *)arg = segs[0].ds_addr; 196 } 197 198 inline static void 199 txdesc_clear(struct dwc_softc *sc, int idx) 200 { 201 202 sc->tx_desccount--; 203 sc->txdesc_ring[idx].addr1 = (uint32_t)(0); 204 sc->txdesc_ring[idx].desc0 = 0; 205 sc->txdesc_ring[idx].desc1 = 0; 206 } 207 208 inline static void 209 txdesc_setup(struct dwc_softc *sc, int idx, bus_addr_t paddr, 210 uint32_t len, uint32_t flags, bool first, bool last) 211 { 212 uint32_t desc0, desc1; 213 214 if (!sc->dma_ext_desc) { 215 desc0 = 0; 216 desc1 = NTDESC1_TCH | len | flags; 217 if (first) 218 desc1 |= NTDESC1_FS; 219 if (last) 220 desc1 |= NTDESC1_LS | NTDESC1_IC; 221 } else { 222 desc0 = ETDESC0_TCH | flags; 223 if (first) 224 desc0 |= ETDESC0_FS; 225 if (last) 226 desc0 |= ETDESC0_LS | ETDESC0_IC; 227 desc1 = len; 228 } 229 ++sc->tx_desccount; 230 sc->txdesc_ring[idx].addr1 = (uint32_t)(paddr); 231 sc->txdesc_ring[idx].desc0 = desc0; 232 sc->txdesc_ring[idx].desc1 = desc1; 233 wmb(); 234 sc->txdesc_ring[idx].desc0 |= TDESC0_OWN; 235 wmb(); 236 } 237 238 inline static uint32_t 239 rxdesc_setup(struct dwc_softc *sc, int idx, bus_addr_t paddr) 240 { 241 uint32_t nidx; 242 243 sc->rxdesc_ring[idx].addr1 = (uint32_t)paddr; 244 nidx = next_rxidx(sc, idx); 245 sc->rxdesc_ring[idx].addr2 = sc->rxdesc_ring_paddr + 246 (nidx * sizeof(struct dwc_hwdesc)); 247 if (!sc->dma_ext_desc) 248 sc->rxdesc_ring[idx].desc1 = NRDESC1_RCH | 249 MIN(MCLBYTES, NRDESC1_RBS1_MASK); 250 else 251 sc->rxdesc_ring[idx].desc1 = ERDESC1_RCH | 252 MIN(MCLBYTES, ERDESC1_RBS1_MASK); 253 254 wmb(); 255 sc->rxdesc_ring[idx].desc0 = RDESC0_OWN; 256 wmb(); 257 return (nidx); 258 } 259 260 int 261 dma1000_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp) 262 { 263 struct bus_dma_segment segs[TX_MAP_MAX_SEGS]; 264 int error, nsegs; 265 struct mbuf * m; 266 uint32_t flags; 267 int i; 268 int last; 269 270 error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map, 271 *mp, segs, &nsegs, 0); 272 if (error == EFBIG) { 273 /* 274 * The map may be partially mapped from the first call. 275 * Make sure to reset it. 276 */ 277 bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map); 278 if ((m = m_defrag(*mp, M_NOWAIT)) == NULL) 279 return (ENOMEM); 280 *mp = m; 281 error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map, 282 *mp, segs, &nsegs, 0); 283 } 284 if (error != 0) 285 return (ENOMEM); 286 287 if (sc->tx_desccount + nsegs > TX_DESC_COUNT) { 288 bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map); 289 return (ENOMEM); 290 } 291 292 m = *mp; 293 294 if ((m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) != 0) 295 flags = sc->dma_ext_desc ? ETDESC0_CIC_SEG : NTDESC1_CIC_SEG; 296 else if ((m->m_pkthdr.csum_flags & CSUM_IP) != 0) 297 flags = sc->dma_ext_desc ? ETDESC0_CIC_HDR : NTDESC1_CIC_HDR; 298 else 299 flags = sc->dma_ext_desc ? ETDESC0_CIC_NONE : NTDESC1_CIC_NONE; 300 301 bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map, 302 BUS_DMASYNC_PREWRITE); 303 304 sc->txbuf_map[idx].mbuf = m; 305 306 for (i = 0; i < nsegs; i++) { 307 txdesc_setup(sc, sc->tx_desc_head, 308 segs[i].ds_addr, segs[i].ds_len, 309 (i == 0) ? flags : 0, /* only first desc needs flags */ 310 (i == 0), 311 (i == nsegs - 1)); 312 last = sc->tx_desc_head; 313 sc->tx_desc_head = next_txidx(sc, sc->tx_desc_head); 314 } 315 316 sc->txbuf_map[idx].last_desc_idx = last; 317 318 return (0); 319 } 320 321 static int 322 dma1000_setup_rxbuf(struct dwc_softc *sc, int idx, struct mbuf *m) 323 { 324 struct bus_dma_segment seg; 325 int error, nsegs; 326 327 m_adj(m, ETHER_ALIGN); 328 329 error = bus_dmamap_load_mbuf_sg(sc->rxbuf_tag, sc->rxbuf_map[idx].map, 330 m, &seg, &nsegs, 0); 331 if (error != 0) 332 return (error); 333 334 KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); 335 336 bus_dmamap_sync(sc->rxbuf_tag, sc->rxbuf_map[idx].map, 337 BUS_DMASYNC_PREREAD); 338 339 sc->rxbuf_map[idx].mbuf = m; 340 rxdesc_setup(sc, idx, seg.ds_addr); 341 342 return (0); 343 } 344 345 static struct mbuf * 346 dwc_alloc_mbufcl(struct dwc_softc *sc) 347 { 348 struct mbuf *m; 349 350 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 351 if (m != NULL) 352 m->m_pkthdr.len = m->m_len = m->m_ext.ext_size; 353 354 return (m); 355 } 356 357 static struct mbuf * 358 dwc_rxfinish_one(struct dwc_softc *sc, struct dwc_hwdesc *desc, 359 struct dwc_bufmap *map) 360 { 361 if_t ifp; 362 struct mbuf *m, *m0; 363 int len; 364 uint32_t rdesc0; 365 366 m = map->mbuf; 367 ifp = sc->ifp; 368 rdesc0 = desc ->desc0; 369 370 if ((rdesc0 & (RDESC0_FS | RDESC0_LS)) != 371 (RDESC0_FS | RDESC0_LS)) { 372 /* 373 * Something very wrong happens. The whole packet should be 374 * received in one descriptor. Report problem. 375 */ 376 device_printf(sc->dev, 377 "%s: RX descriptor without FIRST and LAST bit set: 0x%08X", 378 __func__, rdesc0); 379 return (NULL); 380 } 381 382 len = (rdesc0 >> RDESC0_FL_SHIFT) & RDESC0_FL_MASK; 383 if (len < 64) { 384 /* 385 * Lenght is invalid, recycle old mbuf 386 * Probably impossible case 387 */ 388 return (NULL); 389 } 390 391 /* Allocate new buffer */ 392 m0 = dwc_alloc_mbufcl(sc); 393 if (m0 == NULL) { 394 /* no new mbuf available, recycle old */ 395 if_inc_counter(sc->ifp, IFCOUNTER_IQDROPS, 1); 396 return (NULL); 397 } 398 /* Do dmasync for newly received packet */ 399 bus_dmamap_sync(sc->rxbuf_tag, map->map, BUS_DMASYNC_POSTREAD); 400 bus_dmamap_unload(sc->rxbuf_tag, map->map); 401 402 /* Received packet is valid, process it */ 403 m->m_pkthdr.rcvif = ifp; 404 m->m_pkthdr.len = len; 405 m->m_len = len; 406 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 407 408 if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) != 0 && 409 (rdesc0 & RDESC0_FT) != 0) { 410 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 411 if ((rdesc0 & RDESC0_ICE) == 0) 412 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 413 if ((rdesc0 & RDESC0_PCE) == 0) { 414 m->m_pkthdr.csum_flags |= 415 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 416 m->m_pkthdr.csum_data = 0xffff; 417 } 418 } 419 420 /* Remove trailing FCS */ 421 m_adj(m, -ETHER_CRC_LEN); 422 423 DWC_UNLOCK(sc); 424 if_input(ifp, m); 425 DWC_LOCK(sc); 426 return (m0); 427 } 428 429 void 430 dma1000_txfinish_locked(struct dwc_softc *sc) 431 { 432 struct dwc_bufmap *bmap; 433 struct dwc_hwdesc *desc; 434 if_t ifp; 435 int idx, last_idx; 436 bool map_finished; 437 438 DWC_ASSERT_LOCKED(sc); 439 440 ifp = sc->ifp; 441 /* check if all descriptors of the map are done */ 442 while (sc->tx_map_tail != sc->tx_map_head) { 443 map_finished = true; 444 bmap = &sc->txbuf_map[sc->tx_map_tail]; 445 idx = sc->tx_desc_tail; 446 last_idx = next_txidx(sc, bmap->last_desc_idx); 447 while (idx != last_idx) { 448 desc = &sc->txdesc_ring[idx]; 449 if ((desc->desc0 & TDESC0_OWN) != 0) { 450 map_finished = false; 451 break; 452 } 453 idx = next_txidx(sc, idx); 454 } 455 456 if (!map_finished) 457 break; 458 bus_dmamap_sync(sc->txbuf_tag, bmap->map, 459 BUS_DMASYNC_POSTWRITE); 460 bus_dmamap_unload(sc->txbuf_tag, bmap->map); 461 m_freem(bmap->mbuf); 462 bmap->mbuf = NULL; 463 sc->tx_mapcount--; 464 while (sc->tx_desc_tail != last_idx) { 465 txdesc_clear(sc, sc->tx_desc_tail); 466 sc->tx_desc_tail = next_txidx(sc, sc->tx_desc_tail); 467 } 468 sc->tx_map_tail = next_txidx(sc, sc->tx_map_tail); 469 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 470 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 471 } 472 473 /* If there are no buffers outstanding, muzzle the watchdog. */ 474 if (sc->tx_desc_tail == sc->tx_desc_head) { 475 sc->tx_watchdog_count = 0; 476 } 477 } 478 479 void 480 dma1000_txstart(struct dwc_softc *sc) 481 { 482 int enqueued; 483 struct mbuf *m; 484 485 enqueued = 0; 486 487 for (;;) { 488 if (sc->tx_desccount > (TX_DESC_COUNT - TX_MAP_MAX_SEGS + 1)) { 489 if_setdrvflagbits(sc->ifp, IFF_DRV_OACTIVE, 0); 490 break; 491 } 492 493 if (sc->tx_mapcount == (TX_MAP_COUNT - 1)) { 494 if_setdrvflagbits(sc->ifp, IFF_DRV_OACTIVE, 0); 495 break; 496 } 497 498 m = if_dequeue(sc->ifp); 499 if (m == NULL) 500 break; 501 if (dma1000_setup_txbuf(sc, sc->tx_map_head, &m) != 0) { 502 if_sendq_prepend(sc->ifp, m); 503 if_setdrvflagbits(sc->ifp, IFF_DRV_OACTIVE, 0); 504 break; 505 } 506 bpf_mtap_if(sc->ifp, m); 507 sc->tx_map_head = next_txidx(sc, sc->tx_map_head); 508 sc->tx_mapcount++; 509 ++enqueued; 510 } 511 512 if (enqueued != 0) { 513 WRITE4(sc, TRANSMIT_POLL_DEMAND, 0x1); 514 sc->tx_watchdog_count = WATCHDOG_TIMEOUT_SECS; 515 } 516 } 517 518 void 519 dma1000_rxfinish_locked(struct dwc_softc *sc) 520 { 521 struct mbuf *m; 522 int error, idx; 523 struct dwc_hwdesc *desc; 524 525 DWC_ASSERT_LOCKED(sc); 526 for (;;) { 527 idx = sc->rx_idx; 528 desc = sc->rxdesc_ring + idx; 529 if ((desc->desc0 & RDESC0_OWN) != 0) 530 break; 531 532 m = dwc_rxfinish_one(sc, desc, sc->rxbuf_map + idx); 533 if (m == NULL) { 534 wmb(); 535 desc->desc0 = RDESC0_OWN; 536 wmb(); 537 } else { 538 /* We cannot create hole in RX ring */ 539 error = dma1000_setup_rxbuf(sc, idx, m); 540 if (error != 0) 541 panic("dma1000_setup_rxbuf failed: error %d\n", 542 error); 543 544 } 545 sc->rx_idx = next_rxidx(sc, sc->rx_idx); 546 } 547 } 548 549 /* 550 * Start the DMA controller 551 */ 552 void 553 dma1000_start(struct dwc_softc *sc) 554 { 555 uint32_t reg; 556 557 DWC_ASSERT_LOCKED(sc); 558 559 /* Initializa DMA and enable transmitters */ 560 reg = READ4(sc, OPERATION_MODE); 561 reg |= (MODE_TSF | MODE_OSF | MODE_FUF); 562 reg &= ~(MODE_RSF); 563 reg |= (MODE_RTC_LEV32 << MODE_RTC_SHIFT); 564 WRITE4(sc, OPERATION_MODE, reg); 565 566 WRITE4(sc, INTERRUPT_ENABLE, INT_EN_DEFAULT); 567 568 /* Start DMA */ 569 reg = READ4(sc, OPERATION_MODE); 570 reg |= (MODE_ST | MODE_SR); 571 WRITE4(sc, OPERATION_MODE, reg); 572 } 573 574 /* 575 * Stop the DMA controller 576 */ 577 void 578 dma1000_stop(struct dwc_softc *sc) 579 { 580 uint32_t reg; 581 582 DWC_ASSERT_LOCKED(sc); 583 584 /* Stop DMA TX */ 585 reg = READ4(sc, OPERATION_MODE); 586 reg &= ~(MODE_ST); 587 WRITE4(sc, OPERATION_MODE, reg); 588 589 /* Flush TX */ 590 reg = READ4(sc, OPERATION_MODE); 591 reg |= (MODE_FTF); 592 WRITE4(sc, OPERATION_MODE, reg); 593 594 /* Stop DMA RX */ 595 reg = READ4(sc, OPERATION_MODE); 596 reg &= ~(MODE_SR); 597 WRITE4(sc, OPERATION_MODE, reg); 598 } 599 600 int 601 dma1000_reset(struct dwc_softc *sc) 602 { 603 uint32_t reg; 604 int i; 605 606 reg = READ4(sc, BUS_MODE); 607 reg |= (BUS_MODE_SWR); 608 WRITE4(sc, BUS_MODE, reg); 609 610 for (i = 0; i < DMA_RESET_TIMEOUT; i++) { 611 if ((READ4(sc, BUS_MODE) & BUS_MODE_SWR) == 0) 612 break; 613 DELAY(10); 614 } 615 if (i >= DMA_RESET_TIMEOUT) { 616 return (ENXIO); 617 } 618 619 return (0); 620 } 621 622 /* 623 * Create the bus_dma resources 624 */ 625 int 626 dma1000_init(struct dwc_softc *sc) 627 { 628 struct mbuf *m; 629 uint32_t reg; 630 int error; 631 int nidx; 632 int idx; 633 634 reg = BUS_MODE_USP; 635 if (!sc->nopblx8) 636 reg |= BUS_MODE_EIGHTXPBL; 637 reg |= (sc->txpbl << BUS_MODE_PBL_SHIFT); 638 reg |= (sc->rxpbl << BUS_MODE_RPBL_SHIFT); 639 if (sc->fixed_burst) 640 reg |= BUS_MODE_FIXEDBURST; 641 if (sc->mixed_burst) 642 reg |= BUS_MODE_MIXEDBURST; 643 if (sc->aal) 644 reg |= BUS_MODE_AAL; 645 646 WRITE4(sc, BUS_MODE, reg); 647 648 reg = READ4(sc, HW_FEATURE); 649 if (reg & HW_FEATURE_EXT_DESCRIPTOR) 650 sc->dma_ext_desc = true; 651 652 /* 653 * DMA must be stop while changing descriptor list addresses. 654 */ 655 reg = READ4(sc, OPERATION_MODE); 656 reg &= ~(MODE_ST | MODE_SR); 657 WRITE4(sc, OPERATION_MODE, reg); 658 659 /* 660 * Set up TX descriptor ring, descriptors, and dma maps. 661 */ 662 error = bus_dma_tag_create( 663 bus_get_dma_tag(sc->dev), /* Parent tag. */ 664 DWC_DESC_RING_ALIGN, 0, /* alignment, boundary */ 665 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 666 BUS_SPACE_MAXADDR, /* highaddr */ 667 NULL, NULL, /* filter, filterarg */ 668 TX_DESC_SIZE, 1, /* maxsize, nsegments */ 669 TX_DESC_SIZE, /* maxsegsize */ 670 0, /* flags */ 671 NULL, NULL, /* lockfunc, lockarg */ 672 &sc->txdesc_tag); 673 if (error != 0) { 674 device_printf(sc->dev, 675 "could not create TX ring DMA tag.\n"); 676 goto out; 677 } 678 679 error = bus_dmamem_alloc(sc->txdesc_tag, (void**)&sc->txdesc_ring, 680 BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, 681 &sc->txdesc_map); 682 if (error != 0) { 683 device_printf(sc->dev, 684 "could not allocate TX descriptor ring.\n"); 685 goto out; 686 } 687 688 error = bus_dmamap_load(sc->txdesc_tag, sc->txdesc_map, 689 sc->txdesc_ring, TX_DESC_SIZE, dwc_get1paddr, 690 &sc->txdesc_ring_paddr, 0); 691 if (error != 0) { 692 device_printf(sc->dev, 693 "could not load TX descriptor ring map.\n"); 694 goto out; 695 } 696 697 for (idx = 0; idx < TX_DESC_COUNT; idx++) { 698 nidx = next_txidx(sc, idx); 699 sc->txdesc_ring[idx].addr2 = sc->txdesc_ring_paddr + 700 (nidx * sizeof(struct dwc_hwdesc)); 701 } 702 703 error = bus_dma_tag_create( 704 bus_get_dma_tag(sc->dev), /* Parent tag. */ 705 1, 0, /* alignment, boundary */ 706 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 707 BUS_SPACE_MAXADDR, /* highaddr */ 708 NULL, NULL, /* filter, filterarg */ 709 MCLBYTES*TX_MAP_MAX_SEGS, /* maxsize */ 710 TX_MAP_MAX_SEGS, /* nsegments */ 711 MCLBYTES, /* maxsegsize */ 712 0, /* flags */ 713 NULL, NULL, /* lockfunc, lockarg */ 714 &sc->txbuf_tag); 715 if (error != 0) { 716 device_printf(sc->dev, 717 "could not create TX ring DMA tag.\n"); 718 goto out; 719 } 720 721 for (idx = 0; idx < TX_MAP_COUNT; idx++) { 722 error = bus_dmamap_create(sc->txbuf_tag, BUS_DMA_COHERENT, 723 &sc->txbuf_map[idx].map); 724 if (error != 0) { 725 device_printf(sc->dev, 726 "could not create TX buffer DMA map.\n"); 727 goto out; 728 } 729 } 730 731 for (idx = 0; idx < TX_DESC_COUNT; idx++) 732 txdesc_clear(sc, idx); 733 734 WRITE4(sc, TX_DESCR_LIST_ADDR, sc->txdesc_ring_paddr); 735 736 /* 737 * Set up RX descriptor ring, descriptors, dma maps, and mbufs. 738 */ 739 error = bus_dma_tag_create( 740 bus_get_dma_tag(sc->dev), /* Parent tag. */ 741 DWC_DESC_RING_ALIGN, 0, /* alignment, boundary */ 742 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 743 BUS_SPACE_MAXADDR, /* highaddr */ 744 NULL, NULL, /* filter, filterarg */ 745 RX_DESC_SIZE, 1, /* maxsize, nsegments */ 746 RX_DESC_SIZE, /* maxsegsize */ 747 0, /* flags */ 748 NULL, NULL, /* lockfunc, lockarg */ 749 &sc->rxdesc_tag); 750 if (error != 0) { 751 device_printf(sc->dev, 752 "could not create RX ring DMA tag.\n"); 753 goto out; 754 } 755 756 error = bus_dmamem_alloc(sc->rxdesc_tag, (void **)&sc->rxdesc_ring, 757 BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, 758 &sc->rxdesc_map); 759 if (error != 0) { 760 device_printf(sc->dev, 761 "could not allocate RX descriptor ring.\n"); 762 goto out; 763 } 764 765 error = bus_dmamap_load(sc->rxdesc_tag, sc->rxdesc_map, 766 sc->rxdesc_ring, RX_DESC_SIZE, dwc_get1paddr, 767 &sc->rxdesc_ring_paddr, 0); 768 if (error != 0) { 769 device_printf(sc->dev, 770 "could not load RX descriptor ring map.\n"); 771 goto out; 772 } 773 774 error = bus_dma_tag_create( 775 bus_get_dma_tag(sc->dev), /* Parent tag. */ 776 1, 0, /* alignment, boundary */ 777 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 778 BUS_SPACE_MAXADDR, /* highaddr */ 779 NULL, NULL, /* filter, filterarg */ 780 MCLBYTES, 1, /* maxsize, nsegments */ 781 MCLBYTES, /* maxsegsize */ 782 0, /* flags */ 783 NULL, NULL, /* lockfunc, lockarg */ 784 &sc->rxbuf_tag); 785 if (error != 0) { 786 device_printf(sc->dev, 787 "could not create RX buf DMA tag.\n"); 788 goto out; 789 } 790 791 for (idx = 0; idx < RX_DESC_COUNT; idx++) { 792 error = bus_dmamap_create(sc->rxbuf_tag, BUS_DMA_COHERENT, 793 &sc->rxbuf_map[idx].map); 794 if (error != 0) { 795 device_printf(sc->dev, 796 "could not create RX buffer DMA map.\n"); 797 goto out; 798 } 799 if ((m = dwc_alloc_mbufcl(sc)) == NULL) { 800 device_printf(sc->dev, "Could not alloc mbuf\n"); 801 error = ENOMEM; 802 goto out; 803 } 804 if ((error = dma1000_setup_rxbuf(sc, idx, m)) != 0) { 805 device_printf(sc->dev, 806 "could not create new RX buffer.\n"); 807 goto out; 808 } 809 } 810 WRITE4(sc, RX_DESCR_LIST_ADDR, sc->rxdesc_ring_paddr); 811 812 out: 813 if (error != 0) 814 return (ENXIO); 815 816 return (0); 817 } 818 819 /* 820 * Free the bus_dma resources 821 */ 822 void 823 dma1000_free(struct dwc_softc *sc) 824 { 825 bus_dmamap_t map; 826 int idx; 827 828 /* Clean up RX DMA resources and free mbufs. */ 829 for (idx = 0; idx < RX_DESC_COUNT; ++idx) { 830 if ((map = sc->rxbuf_map[idx].map) != NULL) { 831 bus_dmamap_unload(sc->rxbuf_tag, map); 832 bus_dmamap_destroy(sc->rxbuf_tag, map); 833 m_freem(sc->rxbuf_map[idx].mbuf); 834 } 835 } 836 if (sc->rxbuf_tag != NULL) 837 bus_dma_tag_destroy(sc->rxbuf_tag); 838 if (sc->rxdesc_map != NULL) { 839 bus_dmamap_unload(sc->rxdesc_tag, sc->rxdesc_map); 840 bus_dmamem_free(sc->rxdesc_tag, sc->rxdesc_ring, 841 sc->rxdesc_map); 842 } 843 if (sc->rxdesc_tag != NULL) 844 bus_dma_tag_destroy(sc->rxdesc_tag); 845 846 /* Clean up TX DMA resources. */ 847 for (idx = 0; idx < TX_DESC_COUNT; ++idx) { 848 if ((map = sc->txbuf_map[idx].map) != NULL) { 849 /* TX maps are already unloaded. */ 850 bus_dmamap_destroy(sc->txbuf_tag, map); 851 } 852 } 853 if (sc->txbuf_tag != NULL) 854 bus_dma_tag_destroy(sc->txbuf_tag); 855 if (sc->txdesc_map != NULL) { 856 bus_dmamap_unload(sc->txdesc_tag, sc->txdesc_map); 857 bus_dmamem_free(sc->txdesc_tag, sc->txdesc_ring, 858 sc->txdesc_map); 859 } 860 if (sc->txdesc_tag != NULL) 861 bus_dma_tag_destroy(sc->txdesc_tag); 862 } 863 864 /* 865 * Interrupt function 866 */ 867 868 int 869 dma1000_intr(struct dwc_softc *sc) 870 { 871 uint32_t reg; 872 int rv; 873 874 DWC_ASSERT_LOCKED(sc); 875 876 rv = 0; 877 reg = READ4(sc, DMA_STATUS); 878 if (reg & DMA_STATUS_NIS) { 879 if (reg & DMA_STATUS_RI) 880 dma1000_rxfinish_locked(sc); 881 882 if (reg & DMA_STATUS_TI) { 883 dma1000_txfinish_locked(sc); 884 dma1000_txstart(sc); 885 } 886 } 887 888 if (reg & DMA_STATUS_AIS) { 889 if (reg & DMA_STATUS_FBI) { 890 /* Fatal bus error */ 891 rv = EIO; 892 } 893 } 894 895 WRITE4(sc, DMA_STATUS, reg & DMA_STATUS_INTR_MASK); 896 return (rv); 897 } 898