1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/clk.h> 10 #include <linux/clk-provider.h> 11 #include <linux/crc32.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/kernel.h> 15 #include <linux/types.h> 16 #include <linux/circ_buf.h> 17 #include <linux/slab.h> 18 #include <linux/init.h> 19 #include <linux/io.h> 20 #include <linux/gpio.h> 21 #include <linux/gpio/consumer.h> 22 #include <linux/interrupt.h> 23 #include <linux/netdevice.h> 24 #include <linux/etherdevice.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/platform_data/macb.h> 27 #include <linux/platform_device.h> 28 #include <linux/phy.h> 29 #include <linux/of.h> 30 #include <linux/of_device.h> 31 #include <linux/of_gpio.h> 32 #include <linux/of_mdio.h> 33 #include <linux/of_net.h> 34 #include <linux/ip.h> 35 #include <linux/udp.h> 36 #include <linux/tcp.h> 37 #include <linux/iopoll.h> 38 #include <linux/pm_runtime.h> 39 #include "macb.h" 40 41 /* This structure is only used for MACB on SiFive FU540 devices */ 42 struct sifive_fu540_macb_mgmt { 43 void __iomem *reg; 44 unsigned long rate; 45 struct clk_hw hw; 46 }; 47 48 #define MACB_RX_BUFFER_SIZE 128 49 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 50 51 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 52 #define MIN_RX_RING_SIZE 64 53 #define MAX_RX_RING_SIZE 8192 54 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 55 * (bp)->rx_ring_size) 56 57 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 58 #define MIN_TX_RING_SIZE 64 59 #define MAX_TX_RING_SIZE 4096 60 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 61 * (bp)->tx_ring_size) 62 63 /* level of occupied TX descriptors under which we wake up TX process */ 64 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 65 66 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 67 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 68 | MACB_BIT(ISR_RLE) \ 69 | MACB_BIT(TXERR)) 70 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 71 | MACB_BIT(TXUBR)) 72 73 /* Max length of transmit frame must be a multiple of 8 bytes */ 74 #define MACB_TX_LEN_ALIGN 8 75 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 76 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 77 78 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 79 #define MACB_NETIF_LSO NETIF_F_TSO 80 81 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 82 #define MACB_WOL_ENABLED (0x1 << 1) 83 84 /* Graceful stop timeouts in us. We should allow up to 85 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 86 */ 87 #define MACB_HALT_TIMEOUT 1230 88 89 #define MACB_PM_TIMEOUT 100 /* ms */ 90 91 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 92 93 /* DMA buffer descriptor might be different size 94 * depends on hardware configuration: 95 * 96 * 1. dma address width 32 bits: 97 * word 1: 32 bit address of Data Buffer 98 * word 2: control 99 * 100 * 2. dma address width 64 bits: 101 * word 1: 32 bit address of Data Buffer 102 * word 2: control 103 * word 3: upper 32 bit address of Data Buffer 104 * word 4: unused 105 * 106 * 3. dma address width 32 bits with hardware timestamping: 107 * word 1: 32 bit address of Data Buffer 108 * word 2: control 109 * word 3: timestamp word 1 110 * word 4: timestamp word 2 111 * 112 * 4. dma address width 64 bits with hardware timestamping: 113 * word 1: 32 bit address of Data Buffer 114 * word 2: control 115 * word 3: upper 32 bit address of Data Buffer 116 * word 4: unused 117 * word 5: timestamp word 1 118 * word 6: timestamp word 2 119 */ 120 static unsigned int macb_dma_desc_get_size(struct macb *bp) 121 { 122 #ifdef MACB_EXT_DESC 123 unsigned int desc_size; 124 125 switch (bp->hw_dma_cap) { 126 case HW_DMA_CAP_64B: 127 desc_size = sizeof(struct macb_dma_desc) 128 + sizeof(struct macb_dma_desc_64); 129 break; 130 case HW_DMA_CAP_PTP: 131 desc_size = sizeof(struct macb_dma_desc) 132 + sizeof(struct macb_dma_desc_ptp); 133 break; 134 case HW_DMA_CAP_64B_PTP: 135 desc_size = sizeof(struct macb_dma_desc) 136 + sizeof(struct macb_dma_desc_64) 137 + sizeof(struct macb_dma_desc_ptp); 138 break; 139 default: 140 desc_size = sizeof(struct macb_dma_desc); 141 } 142 return desc_size; 143 #endif 144 return sizeof(struct macb_dma_desc); 145 } 146 147 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 148 { 149 #ifdef MACB_EXT_DESC 150 switch (bp->hw_dma_cap) { 151 case HW_DMA_CAP_64B: 152 case HW_DMA_CAP_PTP: 153 desc_idx <<= 1; 154 break; 155 case HW_DMA_CAP_64B_PTP: 156 desc_idx *= 3; 157 break; 158 default: 159 break; 160 } 161 #endif 162 return desc_idx; 163 } 164 165 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 166 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 167 { 168 return (struct macb_dma_desc_64 *)((void *)desc 169 + sizeof(struct macb_dma_desc)); 170 } 171 #endif 172 173 /* Ring buffer accessors */ 174 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 175 { 176 return index & (bp->tx_ring_size - 1); 177 } 178 179 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 180 unsigned int index) 181 { 182 index = macb_tx_ring_wrap(queue->bp, index); 183 index = macb_adj_dma_desc_idx(queue->bp, index); 184 return &queue->tx_ring[index]; 185 } 186 187 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 188 unsigned int index) 189 { 190 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 191 } 192 193 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 194 { 195 dma_addr_t offset; 196 197 offset = macb_tx_ring_wrap(queue->bp, index) * 198 macb_dma_desc_get_size(queue->bp); 199 200 return queue->tx_ring_dma + offset; 201 } 202 203 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 204 { 205 return index & (bp->rx_ring_size - 1); 206 } 207 208 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 209 { 210 index = macb_rx_ring_wrap(queue->bp, index); 211 index = macb_adj_dma_desc_idx(queue->bp, index); 212 return &queue->rx_ring[index]; 213 } 214 215 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 216 { 217 return queue->rx_buffers + queue->bp->rx_buffer_size * 218 macb_rx_ring_wrap(queue->bp, index); 219 } 220 221 /* I/O accessors */ 222 static u32 hw_readl_native(struct macb *bp, int offset) 223 { 224 return __raw_readl(bp->regs + offset); 225 } 226 227 static void hw_writel_native(struct macb *bp, int offset, u32 value) 228 { 229 __raw_writel(value, bp->regs + offset); 230 } 231 232 static u32 hw_readl(struct macb *bp, int offset) 233 { 234 return readl_relaxed(bp->regs + offset); 235 } 236 237 static void hw_writel(struct macb *bp, int offset, u32 value) 238 { 239 writel_relaxed(value, bp->regs + offset); 240 } 241 242 /* Find the CPU endianness by using the loopback bit of NCR register. When the 243 * CPU is in big endian we need to program swapped mode for management 244 * descriptor access. 245 */ 246 static bool hw_is_native_io(void __iomem *addr) 247 { 248 u32 value = MACB_BIT(LLB); 249 250 __raw_writel(value, addr + MACB_NCR); 251 value = __raw_readl(addr + MACB_NCR); 252 253 /* Write 0 back to disable everything */ 254 __raw_writel(0, addr + MACB_NCR); 255 256 return value == MACB_BIT(LLB); 257 } 258 259 static bool hw_is_gem(void __iomem *addr, bool native_io) 260 { 261 u32 id; 262 263 if (native_io) 264 id = __raw_readl(addr + MACB_MID); 265 else 266 id = readl_relaxed(addr + MACB_MID); 267 268 return MACB_BFEXT(IDNUM, id) >= 0x2; 269 } 270 271 static void macb_set_hwaddr(struct macb *bp) 272 { 273 u32 bottom; 274 u16 top; 275 276 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 277 macb_or_gem_writel(bp, SA1B, bottom); 278 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 279 macb_or_gem_writel(bp, SA1T, top); 280 281 /* Clear unused address register sets */ 282 macb_or_gem_writel(bp, SA2B, 0); 283 macb_or_gem_writel(bp, SA2T, 0); 284 macb_or_gem_writel(bp, SA3B, 0); 285 macb_or_gem_writel(bp, SA3T, 0); 286 macb_or_gem_writel(bp, SA4B, 0); 287 macb_or_gem_writel(bp, SA4T, 0); 288 } 289 290 static void macb_get_hwaddr(struct macb *bp) 291 { 292 u32 bottom; 293 u16 top; 294 u8 addr[6]; 295 int i; 296 297 /* Check all 4 address register for valid address */ 298 for (i = 0; i < 4; i++) { 299 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 300 top = macb_or_gem_readl(bp, SA1T + i * 8); 301 302 addr[0] = bottom & 0xff; 303 addr[1] = (bottom >> 8) & 0xff; 304 addr[2] = (bottom >> 16) & 0xff; 305 addr[3] = (bottom >> 24) & 0xff; 306 addr[4] = top & 0xff; 307 addr[5] = (top >> 8) & 0xff; 308 309 if (is_valid_ether_addr(addr)) { 310 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 311 return; 312 } 313 } 314 315 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 316 eth_hw_addr_random(bp->dev); 317 } 318 319 static int macb_mdio_wait_for_idle(struct macb *bp) 320 { 321 u32 val; 322 323 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 324 1, MACB_MDIO_TIMEOUT); 325 } 326 327 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 328 { 329 struct macb *bp = bus->priv; 330 int status; 331 332 status = pm_runtime_get_sync(&bp->pdev->dev); 333 if (status < 0) 334 goto mdio_pm_exit; 335 336 status = macb_mdio_wait_for_idle(bp); 337 if (status < 0) 338 goto mdio_read_exit; 339 340 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 341 | MACB_BF(RW, MACB_MAN_READ) 342 | MACB_BF(PHYA, mii_id) 343 | MACB_BF(REGA, regnum) 344 | MACB_BF(CODE, MACB_MAN_CODE))); 345 346 status = macb_mdio_wait_for_idle(bp); 347 if (status < 0) 348 goto mdio_read_exit; 349 350 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 351 352 mdio_read_exit: 353 pm_runtime_mark_last_busy(&bp->pdev->dev); 354 pm_runtime_put_autosuspend(&bp->pdev->dev); 355 mdio_pm_exit: 356 return status; 357 } 358 359 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 360 u16 value) 361 { 362 struct macb *bp = bus->priv; 363 int status; 364 365 status = pm_runtime_get_sync(&bp->pdev->dev); 366 if (status < 0) 367 goto mdio_pm_exit; 368 369 status = macb_mdio_wait_for_idle(bp); 370 if (status < 0) 371 goto mdio_write_exit; 372 373 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 374 | MACB_BF(RW, MACB_MAN_WRITE) 375 | MACB_BF(PHYA, mii_id) 376 | MACB_BF(REGA, regnum) 377 | MACB_BF(CODE, MACB_MAN_CODE) 378 | MACB_BF(DATA, value))); 379 380 status = macb_mdio_wait_for_idle(bp); 381 if (status < 0) 382 goto mdio_write_exit; 383 384 mdio_write_exit: 385 pm_runtime_mark_last_busy(&bp->pdev->dev); 386 pm_runtime_put_autosuspend(&bp->pdev->dev); 387 mdio_pm_exit: 388 return status; 389 } 390 391 /** 392 * macb_set_tx_clk() - Set a clock to a new frequency 393 * @clk Pointer to the clock to change 394 * @rate New frequency in Hz 395 * @dev Pointer to the struct net_device 396 */ 397 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 398 { 399 long ferr, rate, rate_rounded; 400 401 if (!clk) 402 return; 403 404 switch (speed) { 405 case SPEED_10: 406 rate = 2500000; 407 break; 408 case SPEED_100: 409 rate = 25000000; 410 break; 411 case SPEED_1000: 412 rate = 125000000; 413 break; 414 default: 415 return; 416 } 417 418 rate_rounded = clk_round_rate(clk, rate); 419 if (rate_rounded < 0) 420 return; 421 422 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 423 * is not satisfied. 424 */ 425 ferr = abs(rate_rounded - rate); 426 ferr = DIV_ROUND_UP(ferr, rate / 100000); 427 if (ferr > 5) 428 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 429 rate); 430 431 if (clk_set_rate(clk, rate_rounded)) 432 netdev_err(dev, "adjusting tx_clk failed.\n"); 433 } 434 435 static void macb_handle_link_change(struct net_device *dev) 436 { 437 struct macb *bp = netdev_priv(dev); 438 struct phy_device *phydev = dev->phydev; 439 unsigned long flags; 440 int status_change = 0; 441 442 spin_lock_irqsave(&bp->lock, flags); 443 444 if (phydev->link) { 445 if ((bp->speed != phydev->speed) || 446 (bp->duplex != phydev->duplex)) { 447 u32 reg; 448 449 reg = macb_readl(bp, NCFGR); 450 reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 451 if (macb_is_gem(bp)) 452 reg &= ~GEM_BIT(GBE); 453 454 if (phydev->duplex) 455 reg |= MACB_BIT(FD); 456 if (phydev->speed == SPEED_100) 457 reg |= MACB_BIT(SPD); 458 if (phydev->speed == SPEED_1000 && 459 bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 460 reg |= GEM_BIT(GBE); 461 462 macb_or_gem_writel(bp, NCFGR, reg); 463 464 bp->speed = phydev->speed; 465 bp->duplex = phydev->duplex; 466 status_change = 1; 467 } 468 } 469 470 if (phydev->link != bp->link) { 471 if (!phydev->link) { 472 bp->speed = 0; 473 bp->duplex = -1; 474 } 475 bp->link = phydev->link; 476 477 status_change = 1; 478 } 479 480 spin_unlock_irqrestore(&bp->lock, flags); 481 482 if (status_change) { 483 if (phydev->link) { 484 /* Update the TX clock rate if and only if the link is 485 * up and there has been a link change. 486 */ 487 macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); 488 489 netif_carrier_on(dev); 490 netdev_info(dev, "link up (%d/%s)\n", 491 phydev->speed, 492 phydev->duplex == DUPLEX_FULL ? 493 "Full" : "Half"); 494 } else { 495 netif_carrier_off(dev); 496 netdev_info(dev, "link down\n"); 497 } 498 } 499 } 500 501 /* based on au1000_eth. c*/ 502 static int macb_mii_probe(struct net_device *dev) 503 { 504 struct macb *bp = netdev_priv(dev); 505 struct phy_device *phydev; 506 struct device_node *np; 507 int ret, i; 508 509 np = bp->pdev->dev.of_node; 510 ret = 0; 511 512 if (np) { 513 if (of_phy_is_fixed_link(np)) { 514 bp->phy_node = of_node_get(np); 515 } else { 516 bp->phy_node = of_parse_phandle(np, "phy-handle", 0); 517 /* fallback to standard phy registration if no 518 * phy-handle was found nor any phy found during 519 * dt phy registration 520 */ 521 if (!bp->phy_node && !phy_find_first(bp->mii_bus)) { 522 for (i = 0; i < PHY_MAX_ADDR; i++) { 523 phydev = mdiobus_scan(bp->mii_bus, i); 524 if (IS_ERR(phydev) && 525 PTR_ERR(phydev) != -ENODEV) { 526 ret = PTR_ERR(phydev); 527 break; 528 } 529 } 530 531 if (ret) 532 return -ENODEV; 533 } 534 } 535 } 536 537 if (bp->phy_node) { 538 phydev = of_phy_connect(dev, bp->phy_node, 539 &macb_handle_link_change, 0, 540 bp->phy_interface); 541 if (!phydev) 542 return -ENODEV; 543 } else { 544 phydev = phy_find_first(bp->mii_bus); 545 if (!phydev) { 546 netdev_err(dev, "no PHY found\n"); 547 return -ENXIO; 548 } 549 550 /* attach the mac to the phy */ 551 ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, 552 bp->phy_interface); 553 if (ret) { 554 netdev_err(dev, "Could not attach to PHY\n"); 555 return ret; 556 } 557 } 558 559 /* mask with MAC supported features */ 560 if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 561 phy_set_max_speed(phydev, SPEED_1000); 562 else 563 phy_set_max_speed(phydev, SPEED_100); 564 565 if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) 566 phy_remove_link_mode(phydev, 567 ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 568 569 bp->link = 0; 570 bp->speed = 0; 571 bp->duplex = -1; 572 573 return 0; 574 } 575 576 static int macb_mii_init(struct macb *bp) 577 { 578 struct device_node *np; 579 int err = -ENXIO; 580 581 /* Enable management port */ 582 macb_writel(bp, NCR, MACB_BIT(MPE)); 583 584 bp->mii_bus = mdiobus_alloc(); 585 if (!bp->mii_bus) { 586 err = -ENOMEM; 587 goto err_out; 588 } 589 590 bp->mii_bus->name = "MACB_mii_bus"; 591 bp->mii_bus->read = &macb_mdio_read; 592 bp->mii_bus->write = &macb_mdio_write; 593 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 594 bp->pdev->name, bp->pdev->id); 595 bp->mii_bus->priv = bp; 596 bp->mii_bus->parent = &bp->pdev->dev; 597 598 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 599 600 np = bp->pdev->dev.of_node; 601 if (np && of_phy_is_fixed_link(np)) { 602 if (of_phy_register_fixed_link(np) < 0) { 603 dev_err(&bp->pdev->dev, 604 "broken fixed-link specification %pOF\n", np); 605 goto err_out_free_mdiobus; 606 } 607 608 err = mdiobus_register(bp->mii_bus); 609 } else { 610 err = of_mdiobus_register(bp->mii_bus, np); 611 } 612 613 if (err) 614 goto err_out_free_fixed_link; 615 616 err = macb_mii_probe(bp->dev); 617 if (err) 618 goto err_out_unregister_bus; 619 620 return 0; 621 622 err_out_unregister_bus: 623 mdiobus_unregister(bp->mii_bus); 624 err_out_free_fixed_link: 625 if (np && of_phy_is_fixed_link(np)) 626 of_phy_deregister_fixed_link(np); 627 err_out_free_mdiobus: 628 of_node_put(bp->phy_node); 629 mdiobus_free(bp->mii_bus); 630 err_out: 631 return err; 632 } 633 634 static void macb_update_stats(struct macb *bp) 635 { 636 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 637 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 638 int offset = MACB_PFR; 639 640 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 641 642 for (; p < end; p++, offset += 4) 643 *p += bp->macb_reg_readl(bp, offset); 644 } 645 646 static int macb_halt_tx(struct macb *bp) 647 { 648 unsigned long halt_time, timeout; 649 u32 status; 650 651 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 652 653 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 654 do { 655 halt_time = jiffies; 656 status = macb_readl(bp, TSR); 657 if (!(status & MACB_BIT(TGO))) 658 return 0; 659 660 udelay(250); 661 } while (time_before(halt_time, timeout)); 662 663 return -ETIMEDOUT; 664 } 665 666 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 667 { 668 if (tx_skb->mapping) { 669 if (tx_skb->mapped_as_page) 670 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 671 tx_skb->size, DMA_TO_DEVICE); 672 else 673 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 674 tx_skb->size, DMA_TO_DEVICE); 675 tx_skb->mapping = 0; 676 } 677 678 if (tx_skb->skb) { 679 dev_kfree_skb_any(tx_skb->skb); 680 tx_skb->skb = NULL; 681 } 682 } 683 684 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 685 { 686 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 687 struct macb_dma_desc_64 *desc_64; 688 689 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 690 desc_64 = macb_64b_desc(bp, desc); 691 desc_64->addrh = upper_32_bits(addr); 692 /* The low bits of RX address contain the RX_USED bit, clearing 693 * of which allows packet RX. Make sure the high bits are also 694 * visible to HW at that point. 695 */ 696 dma_wmb(); 697 } 698 #endif 699 desc->addr = lower_32_bits(addr); 700 } 701 702 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 703 { 704 dma_addr_t addr = 0; 705 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 706 struct macb_dma_desc_64 *desc_64; 707 708 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 709 desc_64 = macb_64b_desc(bp, desc); 710 addr = ((u64)(desc_64->addrh) << 32); 711 } 712 #endif 713 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 714 return addr; 715 } 716 717 static void macb_tx_error_task(struct work_struct *work) 718 { 719 struct macb_queue *queue = container_of(work, struct macb_queue, 720 tx_error_task); 721 struct macb *bp = queue->bp; 722 struct macb_tx_skb *tx_skb; 723 struct macb_dma_desc *desc; 724 struct sk_buff *skb; 725 unsigned int tail; 726 unsigned long flags; 727 728 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 729 (unsigned int)(queue - bp->queues), 730 queue->tx_tail, queue->tx_head); 731 732 /* Prevent the queue IRQ handlers from running: each of them may call 733 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 734 * As explained below, we have to halt the transmission before updating 735 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 736 * network engine about the macb/gem being halted. 737 */ 738 spin_lock_irqsave(&bp->lock, flags); 739 740 /* Make sure nobody is trying to queue up new packets */ 741 netif_tx_stop_all_queues(bp->dev); 742 743 /* Stop transmission now 744 * (in case we have just queued new packets) 745 * macb/gem must be halted to write TBQP register 746 */ 747 if (macb_halt_tx(bp)) 748 /* Just complain for now, reinitializing TX path can be good */ 749 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 750 751 /* Treat frames in TX queue including the ones that caused the error. 752 * Free transmit buffers in upper layer. 753 */ 754 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 755 u32 ctrl; 756 757 desc = macb_tx_desc(queue, tail); 758 ctrl = desc->ctrl; 759 tx_skb = macb_tx_skb(queue, tail); 760 skb = tx_skb->skb; 761 762 if (ctrl & MACB_BIT(TX_USED)) { 763 /* skb is set for the last buffer of the frame */ 764 while (!skb) { 765 macb_tx_unmap(bp, tx_skb); 766 tail++; 767 tx_skb = macb_tx_skb(queue, tail); 768 skb = tx_skb->skb; 769 } 770 771 /* ctrl still refers to the first buffer descriptor 772 * since it's the only one written back by the hardware 773 */ 774 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 775 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 776 macb_tx_ring_wrap(bp, tail), 777 skb->data); 778 bp->dev->stats.tx_packets++; 779 queue->stats.tx_packets++; 780 bp->dev->stats.tx_bytes += skb->len; 781 queue->stats.tx_bytes += skb->len; 782 } 783 } else { 784 /* "Buffers exhausted mid-frame" errors may only happen 785 * if the driver is buggy, so complain loudly about 786 * those. Statistics are updated by hardware. 787 */ 788 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 789 netdev_err(bp->dev, 790 "BUG: TX buffers exhausted mid-frame\n"); 791 792 desc->ctrl = ctrl | MACB_BIT(TX_USED); 793 } 794 795 macb_tx_unmap(bp, tx_skb); 796 } 797 798 /* Set end of TX queue */ 799 desc = macb_tx_desc(queue, 0); 800 macb_set_addr(bp, desc, 0); 801 desc->ctrl = MACB_BIT(TX_USED); 802 803 /* Make descriptor updates visible to hardware */ 804 wmb(); 805 806 /* Reinitialize the TX desc queue */ 807 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 808 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 809 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 810 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 811 #endif 812 /* Make TX ring reflect state of hardware */ 813 queue->tx_head = 0; 814 queue->tx_tail = 0; 815 816 /* Housework before enabling TX IRQ */ 817 macb_writel(bp, TSR, macb_readl(bp, TSR)); 818 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 819 820 /* Now we are ready to start transmission again */ 821 netif_tx_start_all_queues(bp->dev); 822 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 823 824 spin_unlock_irqrestore(&bp->lock, flags); 825 } 826 827 static void macb_tx_interrupt(struct macb_queue *queue) 828 { 829 unsigned int tail; 830 unsigned int head; 831 u32 status; 832 struct macb *bp = queue->bp; 833 u16 queue_index = queue - bp->queues; 834 835 status = macb_readl(bp, TSR); 836 macb_writel(bp, TSR, status); 837 838 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 839 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 840 841 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 842 (unsigned long)status); 843 844 head = queue->tx_head; 845 for (tail = queue->tx_tail; tail != head; tail++) { 846 struct macb_tx_skb *tx_skb; 847 struct sk_buff *skb; 848 struct macb_dma_desc *desc; 849 u32 ctrl; 850 851 desc = macb_tx_desc(queue, tail); 852 853 /* Make hw descriptor updates visible to CPU */ 854 rmb(); 855 856 ctrl = desc->ctrl; 857 858 /* TX_USED bit is only set by hardware on the very first buffer 859 * descriptor of the transmitted frame. 860 */ 861 if (!(ctrl & MACB_BIT(TX_USED))) 862 break; 863 864 /* Process all buffers of the current transmitted frame */ 865 for (;; tail++) { 866 tx_skb = macb_tx_skb(queue, tail); 867 skb = tx_skb->skb; 868 869 /* First, update TX stats if needed */ 870 if (skb) { 871 if (unlikely(skb_shinfo(skb)->tx_flags & 872 SKBTX_HW_TSTAMP) && 873 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 874 /* skb now belongs to timestamp buffer 875 * and will be removed later 876 */ 877 tx_skb->skb = NULL; 878 } 879 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 880 macb_tx_ring_wrap(bp, tail), 881 skb->data); 882 bp->dev->stats.tx_packets++; 883 queue->stats.tx_packets++; 884 bp->dev->stats.tx_bytes += skb->len; 885 queue->stats.tx_bytes += skb->len; 886 } 887 888 /* Now we can safely release resources */ 889 macb_tx_unmap(bp, tx_skb); 890 891 /* skb is set only for the last buffer of the frame. 892 * WARNING: at this point skb has been freed by 893 * macb_tx_unmap(). 894 */ 895 if (skb) 896 break; 897 } 898 } 899 900 queue->tx_tail = tail; 901 if (__netif_subqueue_stopped(bp->dev, queue_index) && 902 CIRC_CNT(queue->tx_head, queue->tx_tail, 903 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 904 netif_wake_subqueue(bp->dev, queue_index); 905 } 906 907 static void gem_rx_refill(struct macb_queue *queue) 908 { 909 unsigned int entry; 910 struct sk_buff *skb; 911 dma_addr_t paddr; 912 struct macb *bp = queue->bp; 913 struct macb_dma_desc *desc; 914 915 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 916 bp->rx_ring_size) > 0) { 917 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 918 919 /* Make hw descriptor updates visible to CPU */ 920 rmb(); 921 922 queue->rx_prepared_head++; 923 desc = macb_rx_desc(queue, entry); 924 925 if (!queue->rx_skbuff[entry]) { 926 /* allocate sk_buff for this free entry in ring */ 927 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 928 if (unlikely(!skb)) { 929 netdev_err(bp->dev, 930 "Unable to allocate sk_buff\n"); 931 break; 932 } 933 934 /* now fill corresponding descriptor entry */ 935 paddr = dma_map_single(&bp->pdev->dev, skb->data, 936 bp->rx_buffer_size, 937 DMA_FROM_DEVICE); 938 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 939 dev_kfree_skb(skb); 940 break; 941 } 942 943 queue->rx_skbuff[entry] = skb; 944 945 if (entry == bp->rx_ring_size - 1) 946 paddr |= MACB_BIT(RX_WRAP); 947 desc->ctrl = 0; 948 /* Setting addr clears RX_USED and allows reception, 949 * make sure ctrl is cleared first to avoid a race. 950 */ 951 dma_wmb(); 952 macb_set_addr(bp, desc, paddr); 953 954 /* properly align Ethernet header */ 955 skb_reserve(skb, NET_IP_ALIGN); 956 } else { 957 desc->ctrl = 0; 958 dma_wmb(); 959 desc->addr &= ~MACB_BIT(RX_USED); 960 } 961 } 962 963 /* Make descriptor updates visible to hardware */ 964 wmb(); 965 966 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 967 queue, queue->rx_prepared_head, queue->rx_tail); 968 } 969 970 /* Mark DMA descriptors from begin up to and not including end as unused */ 971 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 972 unsigned int end) 973 { 974 unsigned int frag; 975 976 for (frag = begin; frag != end; frag++) { 977 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 978 979 desc->addr &= ~MACB_BIT(RX_USED); 980 } 981 982 /* Make descriptor updates visible to hardware */ 983 wmb(); 984 985 /* When this happens, the hardware stats registers for 986 * whatever caused this is updated, so we don't have to record 987 * anything. 988 */ 989 } 990 991 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 992 int budget) 993 { 994 struct macb *bp = queue->bp; 995 unsigned int len; 996 unsigned int entry; 997 struct sk_buff *skb; 998 struct macb_dma_desc *desc; 999 int count = 0; 1000 1001 while (count < budget) { 1002 u32 ctrl; 1003 dma_addr_t addr; 1004 bool rxused; 1005 1006 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1007 desc = macb_rx_desc(queue, entry); 1008 1009 /* Make hw descriptor updates visible to CPU */ 1010 rmb(); 1011 1012 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1013 addr = macb_get_addr(bp, desc); 1014 1015 if (!rxused) 1016 break; 1017 1018 /* Ensure ctrl is at least as up-to-date as rxused */ 1019 dma_rmb(); 1020 1021 ctrl = desc->ctrl; 1022 1023 queue->rx_tail++; 1024 count++; 1025 1026 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1027 netdev_err(bp->dev, 1028 "not whole frame pointed by descriptor\n"); 1029 bp->dev->stats.rx_dropped++; 1030 queue->stats.rx_dropped++; 1031 break; 1032 } 1033 skb = queue->rx_skbuff[entry]; 1034 if (unlikely(!skb)) { 1035 netdev_err(bp->dev, 1036 "inconsistent Rx descriptor chain\n"); 1037 bp->dev->stats.rx_dropped++; 1038 queue->stats.rx_dropped++; 1039 break; 1040 } 1041 /* now everything is ready for receiving packet */ 1042 queue->rx_skbuff[entry] = NULL; 1043 len = ctrl & bp->rx_frm_len_mask; 1044 1045 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1046 1047 skb_put(skb, len); 1048 dma_unmap_single(&bp->pdev->dev, addr, 1049 bp->rx_buffer_size, DMA_FROM_DEVICE); 1050 1051 skb->protocol = eth_type_trans(skb, bp->dev); 1052 skb_checksum_none_assert(skb); 1053 if (bp->dev->features & NETIF_F_RXCSUM && 1054 !(bp->dev->flags & IFF_PROMISC) && 1055 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1056 skb->ip_summed = CHECKSUM_UNNECESSARY; 1057 1058 bp->dev->stats.rx_packets++; 1059 queue->stats.rx_packets++; 1060 bp->dev->stats.rx_bytes += skb->len; 1061 queue->stats.rx_bytes += skb->len; 1062 1063 gem_ptp_do_rxstamp(bp, skb, desc); 1064 1065 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1066 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1067 skb->len, skb->csum); 1068 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1069 skb_mac_header(skb), 16, true); 1070 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1071 skb->data, 32, true); 1072 #endif 1073 1074 napi_gro_receive(napi, skb); 1075 } 1076 1077 gem_rx_refill(queue); 1078 1079 return count; 1080 } 1081 1082 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1083 unsigned int first_frag, unsigned int last_frag) 1084 { 1085 unsigned int len; 1086 unsigned int frag; 1087 unsigned int offset; 1088 struct sk_buff *skb; 1089 struct macb_dma_desc *desc; 1090 struct macb *bp = queue->bp; 1091 1092 desc = macb_rx_desc(queue, last_frag); 1093 len = desc->ctrl & bp->rx_frm_len_mask; 1094 1095 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1096 macb_rx_ring_wrap(bp, first_frag), 1097 macb_rx_ring_wrap(bp, last_frag), len); 1098 1099 /* The ethernet header starts NET_IP_ALIGN bytes into the 1100 * first buffer. Since the header is 14 bytes, this makes the 1101 * payload word-aligned. 1102 * 1103 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1104 * the two padding bytes into the skb so that we avoid hitting 1105 * the slowpath in memcpy(), and pull them off afterwards. 1106 */ 1107 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1108 if (!skb) { 1109 bp->dev->stats.rx_dropped++; 1110 for (frag = first_frag; ; frag++) { 1111 desc = macb_rx_desc(queue, frag); 1112 desc->addr &= ~MACB_BIT(RX_USED); 1113 if (frag == last_frag) 1114 break; 1115 } 1116 1117 /* Make descriptor updates visible to hardware */ 1118 wmb(); 1119 1120 return 1; 1121 } 1122 1123 offset = 0; 1124 len += NET_IP_ALIGN; 1125 skb_checksum_none_assert(skb); 1126 skb_put(skb, len); 1127 1128 for (frag = first_frag; ; frag++) { 1129 unsigned int frag_len = bp->rx_buffer_size; 1130 1131 if (offset + frag_len > len) { 1132 if (unlikely(frag != last_frag)) { 1133 dev_kfree_skb_any(skb); 1134 return -1; 1135 } 1136 frag_len = len - offset; 1137 } 1138 skb_copy_to_linear_data_offset(skb, offset, 1139 macb_rx_buffer(queue, frag), 1140 frag_len); 1141 offset += bp->rx_buffer_size; 1142 desc = macb_rx_desc(queue, frag); 1143 desc->addr &= ~MACB_BIT(RX_USED); 1144 1145 if (frag == last_frag) 1146 break; 1147 } 1148 1149 /* Make descriptor updates visible to hardware */ 1150 wmb(); 1151 1152 __skb_pull(skb, NET_IP_ALIGN); 1153 skb->protocol = eth_type_trans(skb, bp->dev); 1154 1155 bp->dev->stats.rx_packets++; 1156 bp->dev->stats.rx_bytes += skb->len; 1157 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1158 skb->len, skb->csum); 1159 napi_gro_receive(napi, skb); 1160 1161 return 0; 1162 } 1163 1164 static inline void macb_init_rx_ring(struct macb_queue *queue) 1165 { 1166 struct macb *bp = queue->bp; 1167 dma_addr_t addr; 1168 struct macb_dma_desc *desc = NULL; 1169 int i; 1170 1171 addr = queue->rx_buffers_dma; 1172 for (i = 0; i < bp->rx_ring_size; i++) { 1173 desc = macb_rx_desc(queue, i); 1174 macb_set_addr(bp, desc, addr); 1175 desc->ctrl = 0; 1176 addr += bp->rx_buffer_size; 1177 } 1178 desc->addr |= MACB_BIT(RX_WRAP); 1179 queue->rx_tail = 0; 1180 } 1181 1182 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1183 int budget) 1184 { 1185 struct macb *bp = queue->bp; 1186 bool reset_rx_queue = false; 1187 int received = 0; 1188 unsigned int tail; 1189 int first_frag = -1; 1190 1191 for (tail = queue->rx_tail; budget > 0; tail++) { 1192 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1193 u32 ctrl; 1194 1195 /* Make hw descriptor updates visible to CPU */ 1196 rmb(); 1197 1198 if (!(desc->addr & MACB_BIT(RX_USED))) 1199 break; 1200 1201 /* Ensure ctrl is at least as up-to-date as addr */ 1202 dma_rmb(); 1203 1204 ctrl = desc->ctrl; 1205 1206 if (ctrl & MACB_BIT(RX_SOF)) { 1207 if (first_frag != -1) 1208 discard_partial_frame(queue, first_frag, tail); 1209 first_frag = tail; 1210 } 1211 1212 if (ctrl & MACB_BIT(RX_EOF)) { 1213 int dropped; 1214 1215 if (unlikely(first_frag == -1)) { 1216 reset_rx_queue = true; 1217 continue; 1218 } 1219 1220 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1221 first_frag = -1; 1222 if (unlikely(dropped < 0)) { 1223 reset_rx_queue = true; 1224 continue; 1225 } 1226 if (!dropped) { 1227 received++; 1228 budget--; 1229 } 1230 } 1231 } 1232 1233 if (unlikely(reset_rx_queue)) { 1234 unsigned long flags; 1235 u32 ctrl; 1236 1237 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1238 1239 spin_lock_irqsave(&bp->lock, flags); 1240 1241 ctrl = macb_readl(bp, NCR); 1242 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1243 1244 macb_init_rx_ring(queue); 1245 queue_writel(queue, RBQP, queue->rx_ring_dma); 1246 1247 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1248 1249 spin_unlock_irqrestore(&bp->lock, flags); 1250 return received; 1251 } 1252 1253 if (first_frag != -1) 1254 queue->rx_tail = first_frag; 1255 else 1256 queue->rx_tail = tail; 1257 1258 return received; 1259 } 1260 1261 static int macb_poll(struct napi_struct *napi, int budget) 1262 { 1263 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1264 struct macb *bp = queue->bp; 1265 int work_done; 1266 u32 status; 1267 1268 status = macb_readl(bp, RSR); 1269 macb_writel(bp, RSR, status); 1270 1271 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1272 (unsigned long)status, budget); 1273 1274 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 1275 if (work_done < budget) { 1276 napi_complete_done(napi, work_done); 1277 1278 /* Packets received while interrupts were disabled */ 1279 status = macb_readl(bp, RSR); 1280 if (status) { 1281 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1282 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1283 napi_reschedule(napi); 1284 } else { 1285 queue_writel(queue, IER, bp->rx_intr_mask); 1286 } 1287 } 1288 1289 /* TODO: Handle errors */ 1290 1291 return work_done; 1292 } 1293 1294 static void macb_hresp_error_task(unsigned long data) 1295 { 1296 struct macb *bp = (struct macb *)data; 1297 struct net_device *dev = bp->dev; 1298 struct macb_queue *queue = bp->queues; 1299 unsigned int q; 1300 u32 ctrl; 1301 1302 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1303 queue_writel(queue, IDR, bp->rx_intr_mask | 1304 MACB_TX_INT_FLAGS | 1305 MACB_BIT(HRESP)); 1306 } 1307 ctrl = macb_readl(bp, NCR); 1308 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1309 macb_writel(bp, NCR, ctrl); 1310 1311 netif_tx_stop_all_queues(dev); 1312 netif_carrier_off(dev); 1313 1314 bp->macbgem_ops.mog_init_rings(bp); 1315 1316 /* Initialize TX and RX buffers */ 1317 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1318 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 1319 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1320 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1321 queue_writel(queue, RBQPH, 1322 upper_32_bits(queue->rx_ring_dma)); 1323 #endif 1324 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1325 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1326 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1327 queue_writel(queue, TBQPH, 1328 upper_32_bits(queue->tx_ring_dma)); 1329 #endif 1330 1331 /* Enable interrupts */ 1332 queue_writel(queue, IER, 1333 bp->rx_intr_mask | 1334 MACB_TX_INT_FLAGS | 1335 MACB_BIT(HRESP)); 1336 } 1337 1338 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1339 macb_writel(bp, NCR, ctrl); 1340 1341 netif_carrier_on(dev); 1342 netif_tx_start_all_queues(dev); 1343 } 1344 1345 static void macb_tx_restart(struct macb_queue *queue) 1346 { 1347 unsigned int head = queue->tx_head; 1348 unsigned int tail = queue->tx_tail; 1349 struct macb *bp = queue->bp; 1350 1351 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1352 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1353 1354 if (head == tail) 1355 return; 1356 1357 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1358 } 1359 1360 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1361 { 1362 struct macb_queue *queue = dev_id; 1363 struct macb *bp = queue->bp; 1364 struct net_device *dev = bp->dev; 1365 u32 status, ctrl; 1366 1367 status = queue_readl(queue, ISR); 1368 1369 if (unlikely(!status)) 1370 return IRQ_NONE; 1371 1372 spin_lock(&bp->lock); 1373 1374 while (status) { 1375 /* close possible race with dev_close */ 1376 if (unlikely(!netif_running(dev))) { 1377 queue_writel(queue, IDR, -1); 1378 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1379 queue_writel(queue, ISR, -1); 1380 break; 1381 } 1382 1383 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1384 (unsigned int)(queue - bp->queues), 1385 (unsigned long)status); 1386 1387 if (status & bp->rx_intr_mask) { 1388 /* There's no point taking any more interrupts 1389 * until we have processed the buffers. The 1390 * scheduling call may fail if the poll routine 1391 * is already scheduled, so disable interrupts 1392 * now. 1393 */ 1394 queue_writel(queue, IDR, bp->rx_intr_mask); 1395 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1396 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1397 1398 if (napi_schedule_prep(&queue->napi)) { 1399 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1400 __napi_schedule(&queue->napi); 1401 } 1402 } 1403 1404 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1405 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1406 schedule_work(&queue->tx_error_task); 1407 1408 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1409 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1410 1411 break; 1412 } 1413 1414 if (status & MACB_BIT(TCOMP)) 1415 macb_tx_interrupt(queue); 1416 1417 if (status & MACB_BIT(TXUBR)) 1418 macb_tx_restart(queue); 1419 1420 /* Link change detection isn't possible with RMII, so we'll 1421 * add that if/when we get our hands on a full-blown MII PHY. 1422 */ 1423 1424 /* There is a hardware issue under heavy load where DMA can 1425 * stop, this causes endless "used buffer descriptor read" 1426 * interrupts but it can be cleared by re-enabling RX. See 1427 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1428 * section 16.7.4 for details. RXUBR is only enabled for 1429 * these two versions. 1430 */ 1431 if (status & MACB_BIT(RXUBR)) { 1432 ctrl = macb_readl(bp, NCR); 1433 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1434 wmb(); 1435 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1436 1437 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1438 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1439 } 1440 1441 if (status & MACB_BIT(ISR_ROVR)) { 1442 /* We missed at least one packet */ 1443 if (macb_is_gem(bp)) 1444 bp->hw_stats.gem.rx_overruns++; 1445 else 1446 bp->hw_stats.macb.rx_overruns++; 1447 1448 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1449 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1450 } 1451 1452 if (status & MACB_BIT(HRESP)) { 1453 tasklet_schedule(&bp->hresp_err_tasklet); 1454 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1455 1456 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1457 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1458 } 1459 status = queue_readl(queue, ISR); 1460 } 1461 1462 spin_unlock(&bp->lock); 1463 1464 return IRQ_HANDLED; 1465 } 1466 1467 #ifdef CONFIG_NET_POLL_CONTROLLER 1468 /* Polling receive - used by netconsole and other diagnostic tools 1469 * to allow network i/o with interrupts disabled. 1470 */ 1471 static void macb_poll_controller(struct net_device *dev) 1472 { 1473 struct macb *bp = netdev_priv(dev); 1474 struct macb_queue *queue; 1475 unsigned long flags; 1476 unsigned int q; 1477 1478 local_irq_save(flags); 1479 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1480 macb_interrupt(dev->irq, queue); 1481 local_irq_restore(flags); 1482 } 1483 #endif 1484 1485 static unsigned int macb_tx_map(struct macb *bp, 1486 struct macb_queue *queue, 1487 struct sk_buff *skb, 1488 unsigned int hdrlen) 1489 { 1490 dma_addr_t mapping; 1491 unsigned int len, entry, i, tx_head = queue->tx_head; 1492 struct macb_tx_skb *tx_skb = NULL; 1493 struct macb_dma_desc *desc; 1494 unsigned int offset, size, count = 0; 1495 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1496 unsigned int eof = 1, mss_mfs = 0; 1497 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1498 1499 /* LSO */ 1500 if (skb_shinfo(skb)->gso_size != 0) { 1501 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1502 /* UDP - UFO */ 1503 lso_ctrl = MACB_LSO_UFO_ENABLE; 1504 else 1505 /* TCP - TSO */ 1506 lso_ctrl = MACB_LSO_TSO_ENABLE; 1507 } 1508 1509 /* First, map non-paged data */ 1510 len = skb_headlen(skb); 1511 1512 /* first buffer length */ 1513 size = hdrlen; 1514 1515 offset = 0; 1516 while (len) { 1517 entry = macb_tx_ring_wrap(bp, tx_head); 1518 tx_skb = &queue->tx_skb[entry]; 1519 1520 mapping = dma_map_single(&bp->pdev->dev, 1521 skb->data + offset, 1522 size, DMA_TO_DEVICE); 1523 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1524 goto dma_error; 1525 1526 /* Save info to properly release resources */ 1527 tx_skb->skb = NULL; 1528 tx_skb->mapping = mapping; 1529 tx_skb->size = size; 1530 tx_skb->mapped_as_page = false; 1531 1532 len -= size; 1533 offset += size; 1534 count++; 1535 tx_head++; 1536 1537 size = min(len, bp->max_tx_length); 1538 } 1539 1540 /* Then, map paged data from fragments */ 1541 for (f = 0; f < nr_frags; f++) { 1542 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1543 1544 len = skb_frag_size(frag); 1545 offset = 0; 1546 while (len) { 1547 size = min(len, bp->max_tx_length); 1548 entry = macb_tx_ring_wrap(bp, tx_head); 1549 tx_skb = &queue->tx_skb[entry]; 1550 1551 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1552 offset, size, DMA_TO_DEVICE); 1553 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1554 goto dma_error; 1555 1556 /* Save info to properly release resources */ 1557 tx_skb->skb = NULL; 1558 tx_skb->mapping = mapping; 1559 tx_skb->size = size; 1560 tx_skb->mapped_as_page = true; 1561 1562 len -= size; 1563 offset += size; 1564 count++; 1565 tx_head++; 1566 } 1567 } 1568 1569 /* Should never happen */ 1570 if (unlikely(!tx_skb)) { 1571 netdev_err(bp->dev, "BUG! empty skb!\n"); 1572 return 0; 1573 } 1574 1575 /* This is the last buffer of the frame: save socket buffer */ 1576 tx_skb->skb = skb; 1577 1578 /* Update TX ring: update buffer descriptors in reverse order 1579 * to avoid race condition 1580 */ 1581 1582 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1583 * to set the end of TX queue 1584 */ 1585 i = tx_head; 1586 entry = macb_tx_ring_wrap(bp, i); 1587 ctrl = MACB_BIT(TX_USED); 1588 desc = macb_tx_desc(queue, entry); 1589 desc->ctrl = ctrl; 1590 1591 if (lso_ctrl) { 1592 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1593 /* include header and FCS in value given to h/w */ 1594 mss_mfs = skb_shinfo(skb)->gso_size + 1595 skb_transport_offset(skb) + 1596 ETH_FCS_LEN; 1597 else /* TSO */ { 1598 mss_mfs = skb_shinfo(skb)->gso_size; 1599 /* TCP Sequence Number Source Select 1600 * can be set only for TSO 1601 */ 1602 seq_ctrl = 0; 1603 } 1604 } 1605 1606 do { 1607 i--; 1608 entry = macb_tx_ring_wrap(bp, i); 1609 tx_skb = &queue->tx_skb[entry]; 1610 desc = macb_tx_desc(queue, entry); 1611 1612 ctrl = (u32)tx_skb->size; 1613 if (eof) { 1614 ctrl |= MACB_BIT(TX_LAST); 1615 eof = 0; 1616 } 1617 if (unlikely(entry == (bp->tx_ring_size - 1))) 1618 ctrl |= MACB_BIT(TX_WRAP); 1619 1620 /* First descriptor is header descriptor */ 1621 if (i == queue->tx_head) { 1622 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1623 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1624 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1625 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1626 ctrl |= MACB_BIT(TX_NOCRC); 1627 } else 1628 /* Only set MSS/MFS on payload descriptors 1629 * (second or later descriptor) 1630 */ 1631 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1632 1633 /* Set TX buffer descriptor */ 1634 macb_set_addr(bp, desc, tx_skb->mapping); 1635 /* desc->addr must be visible to hardware before clearing 1636 * 'TX_USED' bit in desc->ctrl. 1637 */ 1638 wmb(); 1639 desc->ctrl = ctrl; 1640 } while (i != queue->tx_head); 1641 1642 queue->tx_head = tx_head; 1643 1644 return count; 1645 1646 dma_error: 1647 netdev_err(bp->dev, "TX DMA map failed\n"); 1648 1649 for (i = queue->tx_head; i != tx_head; i++) { 1650 tx_skb = macb_tx_skb(queue, i); 1651 1652 macb_tx_unmap(bp, tx_skb); 1653 } 1654 1655 return 0; 1656 } 1657 1658 static netdev_features_t macb_features_check(struct sk_buff *skb, 1659 struct net_device *dev, 1660 netdev_features_t features) 1661 { 1662 unsigned int nr_frags, f; 1663 unsigned int hdrlen; 1664 1665 /* Validate LSO compatibility */ 1666 1667 /* there is only one buffer */ 1668 if (!skb_is_nonlinear(skb)) 1669 return features; 1670 1671 /* length of header */ 1672 hdrlen = skb_transport_offset(skb); 1673 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1674 hdrlen += tcp_hdrlen(skb); 1675 1676 /* For LSO: 1677 * When software supplies two or more payload buffers all payload buffers 1678 * apart from the last must be a multiple of 8 bytes in size. 1679 */ 1680 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1681 return features & ~MACB_NETIF_LSO; 1682 1683 nr_frags = skb_shinfo(skb)->nr_frags; 1684 /* No need to check last fragment */ 1685 nr_frags--; 1686 for (f = 0; f < nr_frags; f++) { 1687 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1688 1689 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1690 return features & ~MACB_NETIF_LSO; 1691 } 1692 return features; 1693 } 1694 1695 static inline int macb_clear_csum(struct sk_buff *skb) 1696 { 1697 /* no change for packets without checksum offloading */ 1698 if (skb->ip_summed != CHECKSUM_PARTIAL) 1699 return 0; 1700 1701 /* make sure we can modify the header */ 1702 if (unlikely(skb_cow_head(skb, 0))) 1703 return -1; 1704 1705 /* initialize checksum field 1706 * This is required - at least for Zynq, which otherwise calculates 1707 * wrong UDP header checksums for UDP packets with UDP data len <=2 1708 */ 1709 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1710 return 0; 1711 } 1712 1713 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1714 { 1715 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1716 int padlen = ETH_ZLEN - (*skb)->len; 1717 int headroom = skb_headroom(*skb); 1718 int tailroom = skb_tailroom(*skb); 1719 struct sk_buff *nskb; 1720 u32 fcs; 1721 1722 if (!(ndev->features & NETIF_F_HW_CSUM) || 1723 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1724 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1725 return 0; 1726 1727 if (padlen <= 0) { 1728 /* FCS could be appeded to tailroom. */ 1729 if (tailroom >= ETH_FCS_LEN) 1730 goto add_fcs; 1731 /* FCS could be appeded by moving data to headroom. */ 1732 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1733 padlen = 0; 1734 /* No room for FCS, need to reallocate skb. */ 1735 else 1736 padlen = ETH_FCS_LEN; 1737 } else { 1738 /* Add room for FCS. */ 1739 padlen += ETH_FCS_LEN; 1740 } 1741 1742 if (!cloned && headroom + tailroom >= padlen) { 1743 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1744 skb_set_tail_pointer(*skb, (*skb)->len); 1745 } else { 1746 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1747 if (!nskb) 1748 return -ENOMEM; 1749 1750 dev_consume_skb_any(*skb); 1751 *skb = nskb; 1752 } 1753 1754 if (padlen > ETH_FCS_LEN) 1755 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1756 1757 add_fcs: 1758 /* set FCS to packet */ 1759 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1760 fcs = ~fcs; 1761 1762 skb_put_u8(*skb, fcs & 0xff); 1763 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1764 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1765 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1766 1767 return 0; 1768 } 1769 1770 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1771 { 1772 u16 queue_index = skb_get_queue_mapping(skb); 1773 struct macb *bp = netdev_priv(dev); 1774 struct macb_queue *queue = &bp->queues[queue_index]; 1775 unsigned long flags; 1776 unsigned int desc_cnt, nr_frags, frag_size, f; 1777 unsigned int hdrlen; 1778 bool is_lso, is_udp = 0; 1779 netdev_tx_t ret = NETDEV_TX_OK; 1780 1781 if (macb_clear_csum(skb)) { 1782 dev_kfree_skb_any(skb); 1783 return ret; 1784 } 1785 1786 if (macb_pad_and_fcs(&skb, dev)) { 1787 dev_kfree_skb_any(skb); 1788 return ret; 1789 } 1790 1791 is_lso = (skb_shinfo(skb)->gso_size != 0); 1792 1793 if (is_lso) { 1794 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1795 1796 /* length of headers */ 1797 if (is_udp) 1798 /* only queue eth + ip headers separately for UDP */ 1799 hdrlen = skb_transport_offset(skb); 1800 else 1801 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1802 if (skb_headlen(skb) < hdrlen) { 1803 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1804 /* if this is required, would need to copy to single buffer */ 1805 return NETDEV_TX_BUSY; 1806 } 1807 } else 1808 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1809 1810 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1811 netdev_vdbg(bp->dev, 1812 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1813 queue_index, skb->len, skb->head, skb->data, 1814 skb_tail_pointer(skb), skb_end_pointer(skb)); 1815 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1816 skb->data, 16, true); 1817 #endif 1818 1819 /* Count how many TX buffer descriptors are needed to send this 1820 * socket buffer: skb fragments of jumbo frames may need to be 1821 * split into many buffer descriptors. 1822 */ 1823 if (is_lso && (skb_headlen(skb) > hdrlen)) 1824 /* extra header descriptor if also payload in first buffer */ 1825 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1826 else 1827 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1828 nr_frags = skb_shinfo(skb)->nr_frags; 1829 for (f = 0; f < nr_frags; f++) { 1830 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1831 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1832 } 1833 1834 spin_lock_irqsave(&bp->lock, flags); 1835 1836 /* This is a hard error, log it. */ 1837 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1838 bp->tx_ring_size) < desc_cnt) { 1839 netif_stop_subqueue(dev, queue_index); 1840 spin_unlock_irqrestore(&bp->lock, flags); 1841 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1842 queue->tx_head, queue->tx_tail); 1843 return NETDEV_TX_BUSY; 1844 } 1845 1846 /* Map socket buffer for DMA transfer */ 1847 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1848 dev_kfree_skb_any(skb); 1849 goto unlock; 1850 } 1851 1852 /* Make newly initialized descriptor visible to hardware */ 1853 wmb(); 1854 skb_tx_timestamp(skb); 1855 1856 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1857 1858 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1859 netif_stop_subqueue(dev, queue_index); 1860 1861 unlock: 1862 spin_unlock_irqrestore(&bp->lock, flags); 1863 1864 return ret; 1865 } 1866 1867 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1868 { 1869 if (!macb_is_gem(bp)) { 1870 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1871 } else { 1872 bp->rx_buffer_size = size; 1873 1874 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1875 netdev_dbg(bp->dev, 1876 "RX buffer must be multiple of %d bytes, expanding\n", 1877 RX_BUFFER_MULTIPLE); 1878 bp->rx_buffer_size = 1879 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1880 } 1881 } 1882 1883 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1884 bp->dev->mtu, bp->rx_buffer_size); 1885 } 1886 1887 static void gem_free_rx_buffers(struct macb *bp) 1888 { 1889 struct sk_buff *skb; 1890 struct macb_dma_desc *desc; 1891 struct macb_queue *queue; 1892 dma_addr_t addr; 1893 unsigned int q; 1894 int i; 1895 1896 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1897 if (!queue->rx_skbuff) 1898 continue; 1899 1900 for (i = 0; i < bp->rx_ring_size; i++) { 1901 skb = queue->rx_skbuff[i]; 1902 1903 if (!skb) 1904 continue; 1905 1906 desc = macb_rx_desc(queue, i); 1907 addr = macb_get_addr(bp, desc); 1908 1909 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1910 DMA_FROM_DEVICE); 1911 dev_kfree_skb_any(skb); 1912 skb = NULL; 1913 } 1914 1915 kfree(queue->rx_skbuff); 1916 queue->rx_skbuff = NULL; 1917 } 1918 } 1919 1920 static void macb_free_rx_buffers(struct macb *bp) 1921 { 1922 struct macb_queue *queue = &bp->queues[0]; 1923 1924 if (queue->rx_buffers) { 1925 dma_free_coherent(&bp->pdev->dev, 1926 bp->rx_ring_size * bp->rx_buffer_size, 1927 queue->rx_buffers, queue->rx_buffers_dma); 1928 queue->rx_buffers = NULL; 1929 } 1930 } 1931 1932 static void macb_free_consistent(struct macb *bp) 1933 { 1934 struct macb_queue *queue; 1935 unsigned int q; 1936 int size; 1937 1938 bp->macbgem_ops.mog_free_rx_buffers(bp); 1939 1940 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1941 kfree(queue->tx_skb); 1942 queue->tx_skb = NULL; 1943 if (queue->tx_ring) { 1944 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1945 dma_free_coherent(&bp->pdev->dev, size, 1946 queue->tx_ring, queue->tx_ring_dma); 1947 queue->tx_ring = NULL; 1948 } 1949 if (queue->rx_ring) { 1950 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 1951 dma_free_coherent(&bp->pdev->dev, size, 1952 queue->rx_ring, queue->rx_ring_dma); 1953 queue->rx_ring = NULL; 1954 } 1955 } 1956 } 1957 1958 static int gem_alloc_rx_buffers(struct macb *bp) 1959 { 1960 struct macb_queue *queue; 1961 unsigned int q; 1962 int size; 1963 1964 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1965 size = bp->rx_ring_size * sizeof(struct sk_buff *); 1966 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 1967 if (!queue->rx_skbuff) 1968 return -ENOMEM; 1969 else 1970 netdev_dbg(bp->dev, 1971 "Allocated %d RX struct sk_buff entries at %p\n", 1972 bp->rx_ring_size, queue->rx_skbuff); 1973 } 1974 return 0; 1975 } 1976 1977 static int macb_alloc_rx_buffers(struct macb *bp) 1978 { 1979 struct macb_queue *queue = &bp->queues[0]; 1980 int size; 1981 1982 size = bp->rx_ring_size * bp->rx_buffer_size; 1983 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 1984 &queue->rx_buffers_dma, GFP_KERNEL); 1985 if (!queue->rx_buffers) 1986 return -ENOMEM; 1987 1988 netdev_dbg(bp->dev, 1989 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 1990 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 1991 return 0; 1992 } 1993 1994 static int macb_alloc_consistent(struct macb *bp) 1995 { 1996 struct macb_queue *queue; 1997 unsigned int q; 1998 int size; 1999 2000 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2001 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2002 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2003 &queue->tx_ring_dma, 2004 GFP_KERNEL); 2005 if (!queue->tx_ring) 2006 goto out_err; 2007 netdev_dbg(bp->dev, 2008 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2009 q, size, (unsigned long)queue->tx_ring_dma, 2010 queue->tx_ring); 2011 2012 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2013 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2014 if (!queue->tx_skb) 2015 goto out_err; 2016 2017 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2018 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2019 &queue->rx_ring_dma, GFP_KERNEL); 2020 if (!queue->rx_ring) 2021 goto out_err; 2022 netdev_dbg(bp->dev, 2023 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2024 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2025 } 2026 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2027 goto out_err; 2028 2029 return 0; 2030 2031 out_err: 2032 macb_free_consistent(bp); 2033 return -ENOMEM; 2034 } 2035 2036 static void gem_init_rings(struct macb *bp) 2037 { 2038 struct macb_queue *queue; 2039 struct macb_dma_desc *desc = NULL; 2040 unsigned int q; 2041 int i; 2042 2043 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2044 for (i = 0; i < bp->tx_ring_size; i++) { 2045 desc = macb_tx_desc(queue, i); 2046 macb_set_addr(bp, desc, 0); 2047 desc->ctrl = MACB_BIT(TX_USED); 2048 } 2049 desc->ctrl |= MACB_BIT(TX_WRAP); 2050 queue->tx_head = 0; 2051 queue->tx_tail = 0; 2052 2053 queue->rx_tail = 0; 2054 queue->rx_prepared_head = 0; 2055 2056 gem_rx_refill(queue); 2057 } 2058 2059 } 2060 2061 static void macb_init_rings(struct macb *bp) 2062 { 2063 int i; 2064 struct macb_dma_desc *desc = NULL; 2065 2066 macb_init_rx_ring(&bp->queues[0]); 2067 2068 for (i = 0; i < bp->tx_ring_size; i++) { 2069 desc = macb_tx_desc(&bp->queues[0], i); 2070 macb_set_addr(bp, desc, 0); 2071 desc->ctrl = MACB_BIT(TX_USED); 2072 } 2073 bp->queues[0].tx_head = 0; 2074 bp->queues[0].tx_tail = 0; 2075 desc->ctrl |= MACB_BIT(TX_WRAP); 2076 } 2077 2078 static void macb_reset_hw(struct macb *bp) 2079 { 2080 struct macb_queue *queue; 2081 unsigned int q; 2082 u32 ctrl = macb_readl(bp, NCR); 2083 2084 /* Disable RX and TX (XXX: Should we halt the transmission 2085 * more gracefully?) 2086 */ 2087 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2088 2089 /* Clear the stats registers (XXX: Update stats first?) */ 2090 ctrl |= MACB_BIT(CLRSTAT); 2091 2092 macb_writel(bp, NCR, ctrl); 2093 2094 /* Clear all status flags */ 2095 macb_writel(bp, TSR, -1); 2096 macb_writel(bp, RSR, -1); 2097 2098 /* Disable all interrupts */ 2099 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2100 queue_writel(queue, IDR, -1); 2101 queue_readl(queue, ISR); 2102 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2103 queue_writel(queue, ISR, -1); 2104 } 2105 } 2106 2107 static u32 gem_mdc_clk_div(struct macb *bp) 2108 { 2109 u32 config; 2110 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2111 2112 if (pclk_hz <= 20000000) 2113 config = GEM_BF(CLK, GEM_CLK_DIV8); 2114 else if (pclk_hz <= 40000000) 2115 config = GEM_BF(CLK, GEM_CLK_DIV16); 2116 else if (pclk_hz <= 80000000) 2117 config = GEM_BF(CLK, GEM_CLK_DIV32); 2118 else if (pclk_hz <= 120000000) 2119 config = GEM_BF(CLK, GEM_CLK_DIV48); 2120 else if (pclk_hz <= 160000000) 2121 config = GEM_BF(CLK, GEM_CLK_DIV64); 2122 else 2123 config = GEM_BF(CLK, GEM_CLK_DIV96); 2124 2125 return config; 2126 } 2127 2128 static u32 macb_mdc_clk_div(struct macb *bp) 2129 { 2130 u32 config; 2131 unsigned long pclk_hz; 2132 2133 if (macb_is_gem(bp)) 2134 return gem_mdc_clk_div(bp); 2135 2136 pclk_hz = clk_get_rate(bp->pclk); 2137 if (pclk_hz <= 20000000) 2138 config = MACB_BF(CLK, MACB_CLK_DIV8); 2139 else if (pclk_hz <= 40000000) 2140 config = MACB_BF(CLK, MACB_CLK_DIV16); 2141 else if (pclk_hz <= 80000000) 2142 config = MACB_BF(CLK, MACB_CLK_DIV32); 2143 else 2144 config = MACB_BF(CLK, MACB_CLK_DIV64); 2145 2146 return config; 2147 } 2148 2149 /* Get the DMA bus width field of the network configuration register that we 2150 * should program. We find the width from decoding the design configuration 2151 * register to find the maximum supported data bus width. 2152 */ 2153 static u32 macb_dbw(struct macb *bp) 2154 { 2155 if (!macb_is_gem(bp)) 2156 return 0; 2157 2158 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2159 case 4: 2160 return GEM_BF(DBW, GEM_DBW128); 2161 case 2: 2162 return GEM_BF(DBW, GEM_DBW64); 2163 case 1: 2164 default: 2165 return GEM_BF(DBW, GEM_DBW32); 2166 } 2167 } 2168 2169 /* Configure the receive DMA engine 2170 * - use the correct receive buffer size 2171 * - set best burst length for DMA operations 2172 * (if not supported by FIFO, it will fallback to default) 2173 * - set both rx/tx packet buffers to full memory size 2174 * These are configurable parameters for GEM. 2175 */ 2176 static void macb_configure_dma(struct macb *bp) 2177 { 2178 struct macb_queue *queue; 2179 u32 buffer_size; 2180 unsigned int q; 2181 u32 dmacfg; 2182 2183 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2184 if (macb_is_gem(bp)) { 2185 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2186 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2187 if (q) 2188 queue_writel(queue, RBQS, buffer_size); 2189 else 2190 dmacfg |= GEM_BF(RXBS, buffer_size); 2191 } 2192 if (bp->dma_burst_length) 2193 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2194 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2195 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2196 2197 if (bp->native_io) 2198 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2199 else 2200 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2201 2202 if (bp->dev->features & NETIF_F_HW_CSUM) 2203 dmacfg |= GEM_BIT(TXCOEN); 2204 else 2205 dmacfg &= ~GEM_BIT(TXCOEN); 2206 2207 dmacfg &= ~GEM_BIT(ADDR64); 2208 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2209 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2210 dmacfg |= GEM_BIT(ADDR64); 2211 #endif 2212 #ifdef CONFIG_MACB_USE_HWSTAMP 2213 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2214 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2215 #endif 2216 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2217 dmacfg); 2218 gem_writel(bp, DMACFG, dmacfg); 2219 } 2220 } 2221 2222 static void macb_init_hw(struct macb *bp) 2223 { 2224 struct macb_queue *queue; 2225 unsigned int q; 2226 2227 u32 config; 2228 2229 macb_reset_hw(bp); 2230 macb_set_hwaddr(bp); 2231 2232 config = macb_mdc_clk_div(bp); 2233 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2234 config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2235 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2236 config |= MACB_BIT(PAE); /* PAuse Enable */ 2237 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2238 if (bp->caps & MACB_CAPS_JUMBO) 2239 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2240 else 2241 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2242 if (bp->dev->flags & IFF_PROMISC) 2243 config |= MACB_BIT(CAF); /* Copy All Frames */ 2244 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2245 config |= GEM_BIT(RXCOEN); 2246 if (!(bp->dev->flags & IFF_BROADCAST)) 2247 config |= MACB_BIT(NBC); /* No BroadCast */ 2248 config |= macb_dbw(bp); 2249 macb_writel(bp, NCFGR, config); 2250 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2251 gem_writel(bp, JML, bp->jumbo_max_len); 2252 bp->speed = SPEED_10; 2253 bp->duplex = DUPLEX_HALF; 2254 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2255 if (bp->caps & MACB_CAPS_JUMBO) 2256 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2257 2258 macb_configure_dma(bp); 2259 2260 /* Initialize TX and RX buffers */ 2261 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2262 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 2263 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2264 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2265 queue_writel(queue, RBQPH, upper_32_bits(queue->rx_ring_dma)); 2266 #endif 2267 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 2268 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2269 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2270 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 2271 #endif 2272 2273 /* Enable interrupts */ 2274 queue_writel(queue, IER, 2275 bp->rx_intr_mask | 2276 MACB_TX_INT_FLAGS | 2277 MACB_BIT(HRESP)); 2278 } 2279 2280 /* Enable TX and RX */ 2281 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 2282 } 2283 2284 /* The hash address register is 64 bits long and takes up two 2285 * locations in the memory map. The least significant bits are stored 2286 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2287 * 2288 * The unicast hash enable and the multicast hash enable bits in the 2289 * network configuration register enable the reception of hash matched 2290 * frames. The destination address is reduced to a 6 bit index into 2291 * the 64 bit hash register using the following hash function. The 2292 * hash function is an exclusive or of every sixth bit of the 2293 * destination address. 2294 * 2295 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2296 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2297 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2298 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2299 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2300 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2301 * 2302 * da[0] represents the least significant bit of the first byte 2303 * received, that is, the multicast/unicast indicator, and da[47] 2304 * represents the most significant bit of the last byte received. If 2305 * the hash index, hi[n], points to a bit that is set in the hash 2306 * register then the frame will be matched according to whether the 2307 * frame is multicast or unicast. A multicast match will be signalled 2308 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2309 * index points to a bit set in the hash register. A unicast match 2310 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2311 * and the hash index points to a bit set in the hash register. To 2312 * receive all multicast frames, the hash register should be set with 2313 * all ones and the multicast hash enable bit should be set in the 2314 * network configuration register. 2315 */ 2316 2317 static inline int hash_bit_value(int bitnr, __u8 *addr) 2318 { 2319 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2320 return 1; 2321 return 0; 2322 } 2323 2324 /* Return the hash index value for the specified address. */ 2325 static int hash_get_index(__u8 *addr) 2326 { 2327 int i, j, bitval; 2328 int hash_index = 0; 2329 2330 for (j = 0; j < 6; j++) { 2331 for (i = 0, bitval = 0; i < 8; i++) 2332 bitval ^= hash_bit_value(i * 6 + j, addr); 2333 2334 hash_index |= (bitval << j); 2335 } 2336 2337 return hash_index; 2338 } 2339 2340 /* Add multicast addresses to the internal multicast-hash table. */ 2341 static void macb_sethashtable(struct net_device *dev) 2342 { 2343 struct netdev_hw_addr *ha; 2344 unsigned long mc_filter[2]; 2345 unsigned int bitnr; 2346 struct macb *bp = netdev_priv(dev); 2347 2348 mc_filter[0] = 0; 2349 mc_filter[1] = 0; 2350 2351 netdev_for_each_mc_addr(ha, dev) { 2352 bitnr = hash_get_index(ha->addr); 2353 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2354 } 2355 2356 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2357 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2358 } 2359 2360 /* Enable/Disable promiscuous and multicast modes. */ 2361 static void macb_set_rx_mode(struct net_device *dev) 2362 { 2363 unsigned long cfg; 2364 struct macb *bp = netdev_priv(dev); 2365 2366 cfg = macb_readl(bp, NCFGR); 2367 2368 if (dev->flags & IFF_PROMISC) { 2369 /* Enable promiscuous mode */ 2370 cfg |= MACB_BIT(CAF); 2371 2372 /* Disable RX checksum offload */ 2373 if (macb_is_gem(bp)) 2374 cfg &= ~GEM_BIT(RXCOEN); 2375 } else { 2376 /* Disable promiscuous mode */ 2377 cfg &= ~MACB_BIT(CAF); 2378 2379 /* Enable RX checksum offload only if requested */ 2380 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2381 cfg |= GEM_BIT(RXCOEN); 2382 } 2383 2384 if (dev->flags & IFF_ALLMULTI) { 2385 /* Enable all multicast mode */ 2386 macb_or_gem_writel(bp, HRB, -1); 2387 macb_or_gem_writel(bp, HRT, -1); 2388 cfg |= MACB_BIT(NCFGR_MTI); 2389 } else if (!netdev_mc_empty(dev)) { 2390 /* Enable specific multicasts */ 2391 macb_sethashtable(dev); 2392 cfg |= MACB_BIT(NCFGR_MTI); 2393 } else if (dev->flags & (~IFF_ALLMULTI)) { 2394 /* Disable all multicast mode */ 2395 macb_or_gem_writel(bp, HRB, 0); 2396 macb_or_gem_writel(bp, HRT, 0); 2397 cfg &= ~MACB_BIT(NCFGR_MTI); 2398 } 2399 2400 macb_writel(bp, NCFGR, cfg); 2401 } 2402 2403 static int macb_open(struct net_device *dev) 2404 { 2405 struct macb *bp = netdev_priv(dev); 2406 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2407 struct macb_queue *queue; 2408 unsigned int q; 2409 int err; 2410 2411 netdev_dbg(bp->dev, "open\n"); 2412 2413 err = pm_runtime_get_sync(&bp->pdev->dev); 2414 if (err < 0) 2415 goto pm_exit; 2416 2417 /* carrier starts down */ 2418 netif_carrier_off(dev); 2419 2420 /* if the phy is not yet register, retry later*/ 2421 if (!dev->phydev) { 2422 err = -EAGAIN; 2423 goto pm_exit; 2424 } 2425 2426 /* RX buffers initialization */ 2427 macb_init_rx_buffer_size(bp, bufsz); 2428 2429 err = macb_alloc_consistent(bp); 2430 if (err) { 2431 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2432 err); 2433 goto pm_exit; 2434 } 2435 2436 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2437 napi_enable(&queue->napi); 2438 2439 bp->macbgem_ops.mog_init_rings(bp); 2440 macb_init_hw(bp); 2441 2442 /* schedule a link state check */ 2443 phy_start(dev->phydev); 2444 2445 netif_tx_start_all_queues(dev); 2446 2447 if (bp->ptp_info) 2448 bp->ptp_info->ptp_init(dev); 2449 2450 pm_exit: 2451 if (err) { 2452 pm_runtime_put_sync(&bp->pdev->dev); 2453 return err; 2454 } 2455 return 0; 2456 } 2457 2458 static int macb_close(struct net_device *dev) 2459 { 2460 struct macb *bp = netdev_priv(dev); 2461 struct macb_queue *queue; 2462 unsigned long flags; 2463 unsigned int q; 2464 2465 netif_tx_stop_all_queues(dev); 2466 2467 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2468 napi_disable(&queue->napi); 2469 2470 if (dev->phydev) 2471 phy_stop(dev->phydev); 2472 2473 spin_lock_irqsave(&bp->lock, flags); 2474 macb_reset_hw(bp); 2475 netif_carrier_off(dev); 2476 spin_unlock_irqrestore(&bp->lock, flags); 2477 2478 macb_free_consistent(bp); 2479 2480 if (bp->ptp_info) 2481 bp->ptp_info->ptp_remove(dev); 2482 2483 pm_runtime_put(&bp->pdev->dev); 2484 2485 return 0; 2486 } 2487 2488 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2489 { 2490 if (netif_running(dev)) 2491 return -EBUSY; 2492 2493 dev->mtu = new_mtu; 2494 2495 return 0; 2496 } 2497 2498 static void gem_update_stats(struct macb *bp) 2499 { 2500 struct macb_queue *queue; 2501 unsigned int i, q, idx; 2502 unsigned long *stat; 2503 2504 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2505 2506 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2507 u32 offset = gem_statistics[i].offset; 2508 u64 val = bp->macb_reg_readl(bp, offset); 2509 2510 bp->ethtool_stats[i] += val; 2511 *p += val; 2512 2513 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2514 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2515 val = bp->macb_reg_readl(bp, offset + 4); 2516 bp->ethtool_stats[i] += ((u64)val) << 32; 2517 *(++p) += val; 2518 } 2519 } 2520 2521 idx = GEM_STATS_LEN; 2522 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2523 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2524 bp->ethtool_stats[idx++] = *stat; 2525 } 2526 2527 static struct net_device_stats *gem_get_stats(struct macb *bp) 2528 { 2529 struct gem_stats *hwstat = &bp->hw_stats.gem; 2530 struct net_device_stats *nstat = &bp->dev->stats; 2531 2532 gem_update_stats(bp); 2533 2534 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2535 hwstat->rx_alignment_errors + 2536 hwstat->rx_resource_errors + 2537 hwstat->rx_overruns + 2538 hwstat->rx_oversize_frames + 2539 hwstat->rx_jabbers + 2540 hwstat->rx_undersized_frames + 2541 hwstat->rx_length_field_frame_errors); 2542 nstat->tx_errors = (hwstat->tx_late_collisions + 2543 hwstat->tx_excessive_collisions + 2544 hwstat->tx_underrun + 2545 hwstat->tx_carrier_sense_errors); 2546 nstat->multicast = hwstat->rx_multicast_frames; 2547 nstat->collisions = (hwstat->tx_single_collision_frames + 2548 hwstat->tx_multiple_collision_frames + 2549 hwstat->tx_excessive_collisions); 2550 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2551 hwstat->rx_jabbers + 2552 hwstat->rx_undersized_frames + 2553 hwstat->rx_length_field_frame_errors); 2554 nstat->rx_over_errors = hwstat->rx_resource_errors; 2555 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2556 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2557 nstat->rx_fifo_errors = hwstat->rx_overruns; 2558 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2559 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2560 nstat->tx_fifo_errors = hwstat->tx_underrun; 2561 2562 return nstat; 2563 } 2564 2565 static void gem_get_ethtool_stats(struct net_device *dev, 2566 struct ethtool_stats *stats, u64 *data) 2567 { 2568 struct macb *bp; 2569 2570 bp = netdev_priv(dev); 2571 gem_update_stats(bp); 2572 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2573 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2574 } 2575 2576 static int gem_get_sset_count(struct net_device *dev, int sset) 2577 { 2578 struct macb *bp = netdev_priv(dev); 2579 2580 switch (sset) { 2581 case ETH_SS_STATS: 2582 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2583 default: 2584 return -EOPNOTSUPP; 2585 } 2586 } 2587 2588 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2589 { 2590 char stat_string[ETH_GSTRING_LEN]; 2591 struct macb *bp = netdev_priv(dev); 2592 struct macb_queue *queue; 2593 unsigned int i; 2594 unsigned int q; 2595 2596 switch (sset) { 2597 case ETH_SS_STATS: 2598 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2599 memcpy(p, gem_statistics[i].stat_string, 2600 ETH_GSTRING_LEN); 2601 2602 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2603 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2604 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2605 q, queue_statistics[i].stat_string); 2606 memcpy(p, stat_string, ETH_GSTRING_LEN); 2607 } 2608 } 2609 break; 2610 } 2611 } 2612 2613 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2614 { 2615 struct macb *bp = netdev_priv(dev); 2616 struct net_device_stats *nstat = &bp->dev->stats; 2617 struct macb_stats *hwstat = &bp->hw_stats.macb; 2618 2619 if (macb_is_gem(bp)) 2620 return gem_get_stats(bp); 2621 2622 /* read stats from hardware */ 2623 macb_update_stats(bp); 2624 2625 /* Convert HW stats into netdevice stats */ 2626 nstat->rx_errors = (hwstat->rx_fcs_errors + 2627 hwstat->rx_align_errors + 2628 hwstat->rx_resource_errors + 2629 hwstat->rx_overruns + 2630 hwstat->rx_oversize_pkts + 2631 hwstat->rx_jabbers + 2632 hwstat->rx_undersize_pkts + 2633 hwstat->rx_length_mismatch); 2634 nstat->tx_errors = (hwstat->tx_late_cols + 2635 hwstat->tx_excessive_cols + 2636 hwstat->tx_underruns + 2637 hwstat->tx_carrier_errors + 2638 hwstat->sqe_test_errors); 2639 nstat->collisions = (hwstat->tx_single_cols + 2640 hwstat->tx_multiple_cols + 2641 hwstat->tx_excessive_cols); 2642 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2643 hwstat->rx_jabbers + 2644 hwstat->rx_undersize_pkts + 2645 hwstat->rx_length_mismatch); 2646 nstat->rx_over_errors = hwstat->rx_resource_errors + 2647 hwstat->rx_overruns; 2648 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2649 nstat->rx_frame_errors = hwstat->rx_align_errors; 2650 nstat->rx_fifo_errors = hwstat->rx_overruns; 2651 /* XXX: What does "missed" mean? */ 2652 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2653 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2654 nstat->tx_fifo_errors = hwstat->tx_underruns; 2655 /* Don't know about heartbeat or window errors... */ 2656 2657 return nstat; 2658 } 2659 2660 static int macb_get_regs_len(struct net_device *netdev) 2661 { 2662 return MACB_GREGS_NBR * sizeof(u32); 2663 } 2664 2665 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2666 void *p) 2667 { 2668 struct macb *bp = netdev_priv(dev); 2669 unsigned int tail, head; 2670 u32 *regs_buff = p; 2671 2672 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2673 | MACB_GREGS_VERSION; 2674 2675 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2676 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2677 2678 regs_buff[0] = macb_readl(bp, NCR); 2679 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2680 regs_buff[2] = macb_readl(bp, NSR); 2681 regs_buff[3] = macb_readl(bp, TSR); 2682 regs_buff[4] = macb_readl(bp, RBQP); 2683 regs_buff[5] = macb_readl(bp, TBQP); 2684 regs_buff[6] = macb_readl(bp, RSR); 2685 regs_buff[7] = macb_readl(bp, IMR); 2686 2687 regs_buff[8] = tail; 2688 regs_buff[9] = head; 2689 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2690 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2691 2692 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2693 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2694 if (macb_is_gem(bp)) 2695 regs_buff[13] = gem_readl(bp, DMACFG); 2696 } 2697 2698 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2699 { 2700 struct macb *bp = netdev_priv(netdev); 2701 2702 wol->supported = 0; 2703 wol->wolopts = 0; 2704 2705 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2706 wol->supported = WAKE_MAGIC; 2707 2708 if (bp->wol & MACB_WOL_ENABLED) 2709 wol->wolopts |= WAKE_MAGIC; 2710 } 2711 } 2712 2713 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2714 { 2715 struct macb *bp = netdev_priv(netdev); 2716 2717 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2718 (wol->wolopts & ~WAKE_MAGIC)) 2719 return -EOPNOTSUPP; 2720 2721 if (wol->wolopts & WAKE_MAGIC) 2722 bp->wol |= MACB_WOL_ENABLED; 2723 else 2724 bp->wol &= ~MACB_WOL_ENABLED; 2725 2726 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2727 2728 return 0; 2729 } 2730 2731 static void macb_get_ringparam(struct net_device *netdev, 2732 struct ethtool_ringparam *ring) 2733 { 2734 struct macb *bp = netdev_priv(netdev); 2735 2736 ring->rx_max_pending = MAX_RX_RING_SIZE; 2737 ring->tx_max_pending = MAX_TX_RING_SIZE; 2738 2739 ring->rx_pending = bp->rx_ring_size; 2740 ring->tx_pending = bp->tx_ring_size; 2741 } 2742 2743 static int macb_set_ringparam(struct net_device *netdev, 2744 struct ethtool_ringparam *ring) 2745 { 2746 struct macb *bp = netdev_priv(netdev); 2747 u32 new_rx_size, new_tx_size; 2748 unsigned int reset = 0; 2749 2750 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2751 return -EINVAL; 2752 2753 new_rx_size = clamp_t(u32, ring->rx_pending, 2754 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2755 new_rx_size = roundup_pow_of_two(new_rx_size); 2756 2757 new_tx_size = clamp_t(u32, ring->tx_pending, 2758 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2759 new_tx_size = roundup_pow_of_two(new_tx_size); 2760 2761 if ((new_tx_size == bp->tx_ring_size) && 2762 (new_rx_size == bp->rx_ring_size)) { 2763 /* nothing to do */ 2764 return 0; 2765 } 2766 2767 if (netif_running(bp->dev)) { 2768 reset = 1; 2769 macb_close(bp->dev); 2770 } 2771 2772 bp->rx_ring_size = new_rx_size; 2773 bp->tx_ring_size = new_tx_size; 2774 2775 if (reset) 2776 macb_open(bp->dev); 2777 2778 return 0; 2779 } 2780 2781 #ifdef CONFIG_MACB_USE_HWSTAMP 2782 static unsigned int gem_get_tsu_rate(struct macb *bp) 2783 { 2784 struct clk *tsu_clk; 2785 unsigned int tsu_rate; 2786 2787 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2788 if (!IS_ERR(tsu_clk)) 2789 tsu_rate = clk_get_rate(tsu_clk); 2790 /* try pclk instead */ 2791 else if (!IS_ERR(bp->pclk)) { 2792 tsu_clk = bp->pclk; 2793 tsu_rate = clk_get_rate(tsu_clk); 2794 } else 2795 return -ENOTSUPP; 2796 return tsu_rate; 2797 } 2798 2799 static s32 gem_get_ptp_max_adj(void) 2800 { 2801 return 64000000; 2802 } 2803 2804 static int gem_get_ts_info(struct net_device *dev, 2805 struct ethtool_ts_info *info) 2806 { 2807 struct macb *bp = netdev_priv(dev); 2808 2809 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2810 ethtool_op_get_ts_info(dev, info); 2811 return 0; 2812 } 2813 2814 info->so_timestamping = 2815 SOF_TIMESTAMPING_TX_SOFTWARE | 2816 SOF_TIMESTAMPING_RX_SOFTWARE | 2817 SOF_TIMESTAMPING_SOFTWARE | 2818 SOF_TIMESTAMPING_TX_HARDWARE | 2819 SOF_TIMESTAMPING_RX_HARDWARE | 2820 SOF_TIMESTAMPING_RAW_HARDWARE; 2821 info->tx_types = 2822 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2823 (1 << HWTSTAMP_TX_OFF) | 2824 (1 << HWTSTAMP_TX_ON); 2825 info->rx_filters = 2826 (1 << HWTSTAMP_FILTER_NONE) | 2827 (1 << HWTSTAMP_FILTER_ALL); 2828 2829 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2830 2831 return 0; 2832 } 2833 2834 static struct macb_ptp_info gem_ptp_info = { 2835 .ptp_init = gem_ptp_init, 2836 .ptp_remove = gem_ptp_remove, 2837 .get_ptp_max_adj = gem_get_ptp_max_adj, 2838 .get_tsu_rate = gem_get_tsu_rate, 2839 .get_ts_info = gem_get_ts_info, 2840 .get_hwtst = gem_get_hwtst, 2841 .set_hwtst = gem_set_hwtst, 2842 }; 2843 #endif 2844 2845 static int macb_get_ts_info(struct net_device *netdev, 2846 struct ethtool_ts_info *info) 2847 { 2848 struct macb *bp = netdev_priv(netdev); 2849 2850 if (bp->ptp_info) 2851 return bp->ptp_info->get_ts_info(netdev, info); 2852 2853 return ethtool_op_get_ts_info(netdev, info); 2854 } 2855 2856 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2857 { 2858 struct net_device *netdev = bp->dev; 2859 struct ethtool_rx_fs_item *item; 2860 u32 t2_scr; 2861 int num_t2_scr; 2862 2863 if (!(netdev->features & NETIF_F_NTUPLE)) 2864 return; 2865 2866 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2867 2868 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2869 struct ethtool_rx_flow_spec *fs = &item->fs; 2870 struct ethtool_tcpip4_spec *tp4sp_m; 2871 2872 if (fs->location >= num_t2_scr) 2873 continue; 2874 2875 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2876 2877 /* enable/disable screener regs for the flow entry */ 2878 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2879 2880 /* only enable fields with no masking */ 2881 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2882 2883 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2884 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2885 else 2886 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2887 2888 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2889 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2890 else 2891 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2892 2893 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2894 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2895 else 2896 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 2897 2898 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 2899 } 2900 } 2901 2902 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 2903 { 2904 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 2905 uint16_t index = fs->location; 2906 u32 w0, w1, t2_scr; 2907 bool cmp_a = false; 2908 bool cmp_b = false; 2909 bool cmp_c = false; 2910 2911 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 2912 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2913 2914 /* ignore field if any masking set */ 2915 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 2916 /* 1st compare reg - IP source address */ 2917 w0 = 0; 2918 w1 = 0; 2919 w0 = tp4sp_v->ip4src; 2920 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2921 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2922 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 2923 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 2924 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 2925 cmp_a = true; 2926 } 2927 2928 /* ignore field if any masking set */ 2929 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 2930 /* 2nd compare reg - IP destination address */ 2931 w0 = 0; 2932 w1 = 0; 2933 w0 = tp4sp_v->ip4dst; 2934 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2935 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2936 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 2937 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 2938 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 2939 cmp_b = true; 2940 } 2941 2942 /* ignore both port fields if masking set in both */ 2943 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 2944 /* 3rd compare reg - source port, destination port */ 2945 w0 = 0; 2946 w1 = 0; 2947 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 2948 if (tp4sp_m->psrc == tp4sp_m->pdst) { 2949 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 2950 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2951 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2952 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2953 } else { 2954 /* only one port definition */ 2955 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 2956 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 2957 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 2958 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 2959 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2960 } else { /* dst port */ 2961 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2962 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 2963 } 2964 } 2965 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 2966 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 2967 cmp_c = true; 2968 } 2969 2970 t2_scr = 0; 2971 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 2972 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 2973 if (cmp_a) 2974 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 2975 if (cmp_b) 2976 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 2977 if (cmp_c) 2978 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 2979 gem_writel_n(bp, SCRT2, index, t2_scr); 2980 } 2981 2982 static int gem_add_flow_filter(struct net_device *netdev, 2983 struct ethtool_rxnfc *cmd) 2984 { 2985 struct macb *bp = netdev_priv(netdev); 2986 struct ethtool_rx_flow_spec *fs = &cmd->fs; 2987 struct ethtool_rx_fs_item *item, *newfs; 2988 unsigned long flags; 2989 int ret = -EINVAL; 2990 bool added = false; 2991 2992 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 2993 if (newfs == NULL) 2994 return -ENOMEM; 2995 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 2996 2997 netdev_dbg(netdev, 2998 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2999 fs->flow_type, (int)fs->ring_cookie, fs->location, 3000 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3001 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3002 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 3003 3004 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3005 3006 /* find correct place to add in list */ 3007 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3008 if (item->fs.location > newfs->fs.location) { 3009 list_add_tail(&newfs->list, &item->list); 3010 added = true; 3011 break; 3012 } else if (item->fs.location == fs->location) { 3013 netdev_err(netdev, "Rule not added: location %d not free!\n", 3014 fs->location); 3015 ret = -EBUSY; 3016 goto err; 3017 } 3018 } 3019 if (!added) 3020 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3021 3022 gem_prog_cmp_regs(bp, fs); 3023 bp->rx_fs_list.count++; 3024 /* enable filtering if NTUPLE on */ 3025 gem_enable_flow_filters(bp, 1); 3026 3027 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3028 return 0; 3029 3030 err: 3031 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3032 kfree(newfs); 3033 return ret; 3034 } 3035 3036 static int gem_del_flow_filter(struct net_device *netdev, 3037 struct ethtool_rxnfc *cmd) 3038 { 3039 struct macb *bp = netdev_priv(netdev); 3040 struct ethtool_rx_fs_item *item; 3041 struct ethtool_rx_flow_spec *fs; 3042 unsigned long flags; 3043 3044 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3045 3046 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3047 if (item->fs.location == cmd->fs.location) { 3048 /* disable screener regs for the flow entry */ 3049 fs = &(item->fs); 3050 netdev_dbg(netdev, 3051 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3052 fs->flow_type, (int)fs->ring_cookie, fs->location, 3053 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3054 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3055 htons(fs->h_u.tcp_ip4_spec.psrc), 3056 htons(fs->h_u.tcp_ip4_spec.pdst)); 3057 3058 gem_writel_n(bp, SCRT2, fs->location, 0); 3059 3060 list_del(&item->list); 3061 bp->rx_fs_list.count--; 3062 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3063 kfree(item); 3064 return 0; 3065 } 3066 } 3067 3068 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3069 return -EINVAL; 3070 } 3071 3072 static int gem_get_flow_entry(struct net_device *netdev, 3073 struct ethtool_rxnfc *cmd) 3074 { 3075 struct macb *bp = netdev_priv(netdev); 3076 struct ethtool_rx_fs_item *item; 3077 3078 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3079 if (item->fs.location == cmd->fs.location) { 3080 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3081 return 0; 3082 } 3083 } 3084 return -EINVAL; 3085 } 3086 3087 static int gem_get_all_flow_entries(struct net_device *netdev, 3088 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3089 { 3090 struct macb *bp = netdev_priv(netdev); 3091 struct ethtool_rx_fs_item *item; 3092 uint32_t cnt = 0; 3093 3094 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3095 if (cnt == cmd->rule_cnt) 3096 return -EMSGSIZE; 3097 rule_locs[cnt] = item->fs.location; 3098 cnt++; 3099 } 3100 cmd->data = bp->max_tuples; 3101 cmd->rule_cnt = cnt; 3102 3103 return 0; 3104 } 3105 3106 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3107 u32 *rule_locs) 3108 { 3109 struct macb *bp = netdev_priv(netdev); 3110 int ret = 0; 3111 3112 switch (cmd->cmd) { 3113 case ETHTOOL_GRXRINGS: 3114 cmd->data = bp->num_queues; 3115 break; 3116 case ETHTOOL_GRXCLSRLCNT: 3117 cmd->rule_cnt = bp->rx_fs_list.count; 3118 break; 3119 case ETHTOOL_GRXCLSRULE: 3120 ret = gem_get_flow_entry(netdev, cmd); 3121 break; 3122 case ETHTOOL_GRXCLSRLALL: 3123 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3124 break; 3125 default: 3126 netdev_err(netdev, 3127 "Command parameter %d is not supported\n", cmd->cmd); 3128 ret = -EOPNOTSUPP; 3129 } 3130 3131 return ret; 3132 } 3133 3134 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3135 { 3136 struct macb *bp = netdev_priv(netdev); 3137 int ret; 3138 3139 switch (cmd->cmd) { 3140 case ETHTOOL_SRXCLSRLINS: 3141 if ((cmd->fs.location >= bp->max_tuples) 3142 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3143 ret = -EINVAL; 3144 break; 3145 } 3146 ret = gem_add_flow_filter(netdev, cmd); 3147 break; 3148 case ETHTOOL_SRXCLSRLDEL: 3149 ret = gem_del_flow_filter(netdev, cmd); 3150 break; 3151 default: 3152 netdev_err(netdev, 3153 "Command parameter %d is not supported\n", cmd->cmd); 3154 ret = -EOPNOTSUPP; 3155 } 3156 3157 return ret; 3158 } 3159 3160 static const struct ethtool_ops macb_ethtool_ops = { 3161 .get_regs_len = macb_get_regs_len, 3162 .get_regs = macb_get_regs, 3163 .get_link = ethtool_op_get_link, 3164 .get_ts_info = ethtool_op_get_ts_info, 3165 .get_wol = macb_get_wol, 3166 .set_wol = macb_set_wol, 3167 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3168 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3169 .get_ringparam = macb_get_ringparam, 3170 .set_ringparam = macb_set_ringparam, 3171 }; 3172 3173 static const struct ethtool_ops gem_ethtool_ops = { 3174 .get_regs_len = macb_get_regs_len, 3175 .get_regs = macb_get_regs, 3176 .get_link = ethtool_op_get_link, 3177 .get_ts_info = macb_get_ts_info, 3178 .get_ethtool_stats = gem_get_ethtool_stats, 3179 .get_strings = gem_get_ethtool_strings, 3180 .get_sset_count = gem_get_sset_count, 3181 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3182 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3183 .get_ringparam = macb_get_ringparam, 3184 .set_ringparam = macb_set_ringparam, 3185 .get_rxnfc = gem_get_rxnfc, 3186 .set_rxnfc = gem_set_rxnfc, 3187 }; 3188 3189 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3190 { 3191 struct phy_device *phydev = dev->phydev; 3192 struct macb *bp = netdev_priv(dev); 3193 3194 if (!netif_running(dev)) 3195 return -EINVAL; 3196 3197 if (!phydev) 3198 return -ENODEV; 3199 3200 if (!bp->ptp_info) 3201 return phy_mii_ioctl(phydev, rq, cmd); 3202 3203 switch (cmd) { 3204 case SIOCSHWTSTAMP: 3205 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3206 case SIOCGHWTSTAMP: 3207 return bp->ptp_info->get_hwtst(dev, rq); 3208 default: 3209 return phy_mii_ioctl(phydev, rq, cmd); 3210 } 3211 } 3212 3213 static inline void macb_set_txcsum_feature(struct macb *bp, 3214 netdev_features_t features) 3215 { 3216 u32 val; 3217 3218 if (!macb_is_gem(bp)) 3219 return; 3220 3221 val = gem_readl(bp, DMACFG); 3222 if (features & NETIF_F_HW_CSUM) 3223 val |= GEM_BIT(TXCOEN); 3224 else 3225 val &= ~GEM_BIT(TXCOEN); 3226 3227 gem_writel(bp, DMACFG, val); 3228 } 3229 3230 static inline void macb_set_rxcsum_feature(struct macb *bp, 3231 netdev_features_t features) 3232 { 3233 struct net_device *netdev = bp->dev; 3234 u32 val; 3235 3236 if (!macb_is_gem(bp)) 3237 return; 3238 3239 val = gem_readl(bp, NCFGR); 3240 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 3241 val |= GEM_BIT(RXCOEN); 3242 else 3243 val &= ~GEM_BIT(RXCOEN); 3244 3245 gem_writel(bp, NCFGR, val); 3246 } 3247 3248 static inline void macb_set_rxflow_feature(struct macb *bp, 3249 netdev_features_t features) 3250 { 3251 if (!macb_is_gem(bp)) 3252 return; 3253 3254 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 3255 } 3256 3257 static int macb_set_features(struct net_device *netdev, 3258 netdev_features_t features) 3259 { 3260 struct macb *bp = netdev_priv(netdev); 3261 netdev_features_t changed = features ^ netdev->features; 3262 3263 /* TX checksum offload */ 3264 if (changed & NETIF_F_HW_CSUM) 3265 macb_set_txcsum_feature(bp, features); 3266 3267 /* RX checksum offload */ 3268 if (changed & NETIF_F_RXCSUM) 3269 macb_set_rxcsum_feature(bp, features); 3270 3271 /* RX Flow Filters */ 3272 if (changed & NETIF_F_NTUPLE) 3273 macb_set_rxflow_feature(bp, features); 3274 3275 return 0; 3276 } 3277 3278 static void macb_restore_features(struct macb *bp) 3279 { 3280 struct net_device *netdev = bp->dev; 3281 netdev_features_t features = netdev->features; 3282 3283 /* TX checksum offload */ 3284 macb_set_txcsum_feature(bp, features); 3285 3286 /* RX checksum offload */ 3287 macb_set_rxcsum_feature(bp, features); 3288 3289 /* RX Flow Filters */ 3290 macb_set_rxflow_feature(bp, features); 3291 } 3292 3293 static const struct net_device_ops macb_netdev_ops = { 3294 .ndo_open = macb_open, 3295 .ndo_stop = macb_close, 3296 .ndo_start_xmit = macb_start_xmit, 3297 .ndo_set_rx_mode = macb_set_rx_mode, 3298 .ndo_get_stats = macb_get_stats, 3299 .ndo_do_ioctl = macb_ioctl, 3300 .ndo_validate_addr = eth_validate_addr, 3301 .ndo_change_mtu = macb_change_mtu, 3302 .ndo_set_mac_address = eth_mac_addr, 3303 #ifdef CONFIG_NET_POLL_CONTROLLER 3304 .ndo_poll_controller = macb_poll_controller, 3305 #endif 3306 .ndo_set_features = macb_set_features, 3307 .ndo_features_check = macb_features_check, 3308 }; 3309 3310 /* Configure peripheral capabilities according to device tree 3311 * and integration options used 3312 */ 3313 static void macb_configure_caps(struct macb *bp, 3314 const struct macb_config *dt_conf) 3315 { 3316 u32 dcfg; 3317 3318 if (dt_conf) 3319 bp->caps = dt_conf->caps; 3320 3321 if (hw_is_gem(bp->regs, bp->native_io)) { 3322 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3323 3324 dcfg = gem_readl(bp, DCFG1); 3325 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3326 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3327 dcfg = gem_readl(bp, DCFG2); 3328 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3329 bp->caps |= MACB_CAPS_FIFO_MODE; 3330 #ifdef CONFIG_MACB_USE_HWSTAMP 3331 if (gem_has_ptp(bp)) { 3332 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3333 pr_err("GEM doesn't support hardware ptp.\n"); 3334 else { 3335 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3336 bp->ptp_info = &gem_ptp_info; 3337 } 3338 } 3339 #endif 3340 } 3341 3342 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3343 } 3344 3345 static void macb_probe_queues(void __iomem *mem, 3346 bool native_io, 3347 unsigned int *queue_mask, 3348 unsigned int *num_queues) 3349 { 3350 unsigned int hw_q; 3351 3352 *queue_mask = 0x1; 3353 *num_queues = 1; 3354 3355 /* is it macb or gem ? 3356 * 3357 * We need to read directly from the hardware here because 3358 * we are early in the probe process and don't have the 3359 * MACB_CAPS_MACB_IS_GEM flag positioned 3360 */ 3361 if (!hw_is_gem(mem, native_io)) 3362 return; 3363 3364 /* bit 0 is never set but queue 0 always exists */ 3365 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3366 3367 *queue_mask |= 0x1; 3368 3369 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3370 if (*queue_mask & (1 << hw_q)) 3371 (*num_queues)++; 3372 } 3373 3374 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3375 struct clk **hclk, struct clk **tx_clk, 3376 struct clk **rx_clk, struct clk **tsu_clk) 3377 { 3378 struct macb_platform_data *pdata; 3379 int err; 3380 3381 pdata = dev_get_platdata(&pdev->dev); 3382 if (pdata) { 3383 *pclk = pdata->pclk; 3384 *hclk = pdata->hclk; 3385 } else { 3386 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3387 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3388 } 3389 3390 if (IS_ERR_OR_NULL(*pclk)) { 3391 err = PTR_ERR(*pclk); 3392 if (!err) 3393 err = -ENODEV; 3394 3395 dev_err(&pdev->dev, "failed to get macb_clk (%d)\n", err); 3396 return err; 3397 } 3398 3399 if (IS_ERR_OR_NULL(*hclk)) { 3400 err = PTR_ERR(*hclk); 3401 if (!err) 3402 err = -ENODEV; 3403 3404 dev_err(&pdev->dev, "failed to get hclk (%d)\n", err); 3405 return err; 3406 } 3407 3408 *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); 3409 if (IS_ERR(*tx_clk)) 3410 *tx_clk = NULL; 3411 3412 *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); 3413 if (IS_ERR(*rx_clk)) 3414 *rx_clk = NULL; 3415 3416 *tsu_clk = devm_clk_get(&pdev->dev, "tsu_clk"); 3417 if (IS_ERR(*tsu_clk)) 3418 *tsu_clk = NULL; 3419 3420 err = clk_prepare_enable(*pclk); 3421 if (err) { 3422 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3423 return err; 3424 } 3425 3426 err = clk_prepare_enable(*hclk); 3427 if (err) { 3428 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 3429 goto err_disable_pclk; 3430 } 3431 3432 err = clk_prepare_enable(*tx_clk); 3433 if (err) { 3434 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 3435 goto err_disable_hclk; 3436 } 3437 3438 err = clk_prepare_enable(*rx_clk); 3439 if (err) { 3440 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 3441 goto err_disable_txclk; 3442 } 3443 3444 err = clk_prepare_enable(*tsu_clk); 3445 if (err) { 3446 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 3447 goto err_disable_rxclk; 3448 } 3449 3450 return 0; 3451 3452 err_disable_rxclk: 3453 clk_disable_unprepare(*rx_clk); 3454 3455 err_disable_txclk: 3456 clk_disable_unprepare(*tx_clk); 3457 3458 err_disable_hclk: 3459 clk_disable_unprepare(*hclk); 3460 3461 err_disable_pclk: 3462 clk_disable_unprepare(*pclk); 3463 3464 return err; 3465 } 3466 3467 static int macb_init(struct platform_device *pdev) 3468 { 3469 struct net_device *dev = platform_get_drvdata(pdev); 3470 unsigned int hw_q, q; 3471 struct macb *bp = netdev_priv(dev); 3472 struct macb_queue *queue; 3473 int err; 3474 u32 val, reg; 3475 3476 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3477 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3478 3479 /* set the queue register mapping once for all: queue0 has a special 3480 * register mapping but we don't want to test the queue index then 3481 * compute the corresponding register offset at run time. 3482 */ 3483 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3484 if (!(bp->queue_mask & (1 << hw_q))) 3485 continue; 3486 3487 queue = &bp->queues[q]; 3488 queue->bp = bp; 3489 netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT); 3490 if (hw_q) { 3491 queue->ISR = GEM_ISR(hw_q - 1); 3492 queue->IER = GEM_IER(hw_q - 1); 3493 queue->IDR = GEM_IDR(hw_q - 1); 3494 queue->IMR = GEM_IMR(hw_q - 1); 3495 queue->TBQP = GEM_TBQP(hw_q - 1); 3496 queue->RBQP = GEM_RBQP(hw_q - 1); 3497 queue->RBQS = GEM_RBQS(hw_q - 1); 3498 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3499 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3500 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3501 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3502 } 3503 #endif 3504 } else { 3505 /* queue0 uses legacy registers */ 3506 queue->ISR = MACB_ISR; 3507 queue->IER = MACB_IER; 3508 queue->IDR = MACB_IDR; 3509 queue->IMR = MACB_IMR; 3510 queue->TBQP = MACB_TBQP; 3511 queue->RBQP = MACB_RBQP; 3512 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3513 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3514 queue->TBQPH = MACB_TBQPH; 3515 queue->RBQPH = MACB_RBQPH; 3516 } 3517 #endif 3518 } 3519 3520 /* get irq: here we use the linux queue index, not the hardware 3521 * queue index. the queue irq definitions in the device tree 3522 * must remove the optional gaps that could exist in the 3523 * hardware queue mask. 3524 */ 3525 queue->irq = platform_get_irq(pdev, q); 3526 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3527 IRQF_SHARED, dev->name, queue); 3528 if (err) { 3529 dev_err(&pdev->dev, 3530 "Unable to request IRQ %d (error %d)\n", 3531 queue->irq, err); 3532 return err; 3533 } 3534 3535 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3536 q++; 3537 } 3538 3539 dev->netdev_ops = &macb_netdev_ops; 3540 3541 /* setup appropriated routines according to adapter type */ 3542 if (macb_is_gem(bp)) { 3543 bp->max_tx_length = GEM_MAX_TX_LEN; 3544 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3545 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3546 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3547 bp->macbgem_ops.mog_rx = gem_rx; 3548 dev->ethtool_ops = &gem_ethtool_ops; 3549 } else { 3550 bp->max_tx_length = MACB_MAX_TX_LEN; 3551 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3552 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3553 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3554 bp->macbgem_ops.mog_rx = macb_rx; 3555 dev->ethtool_ops = &macb_ethtool_ops; 3556 } 3557 3558 /* Set features */ 3559 dev->hw_features = NETIF_F_SG; 3560 3561 /* Check LSO capability */ 3562 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3563 dev->hw_features |= MACB_NETIF_LSO; 3564 3565 /* Checksum offload is only available on gem with packet buffer */ 3566 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3567 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3568 if (bp->caps & MACB_CAPS_SG_DISABLED) 3569 dev->hw_features &= ~NETIF_F_SG; 3570 dev->features = dev->hw_features; 3571 3572 /* Check RX Flow Filters support. 3573 * Max Rx flows set by availability of screeners & compare regs: 3574 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3575 */ 3576 reg = gem_readl(bp, DCFG8); 3577 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3578 GEM_BFEXT(T2SCR, reg)); 3579 if (bp->max_tuples > 0) { 3580 /* also needs one ethtype match to check IPv4 */ 3581 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3582 /* program this reg now */ 3583 reg = 0; 3584 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3585 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3586 /* Filtering is supported in hw but don't enable it in kernel now */ 3587 dev->hw_features |= NETIF_F_NTUPLE; 3588 /* init Rx flow definitions */ 3589 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3590 bp->rx_fs_list.count = 0; 3591 spin_lock_init(&bp->rx_fs_lock); 3592 } else 3593 bp->max_tuples = 0; 3594 } 3595 3596 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3597 val = 0; 3598 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3599 val = GEM_BIT(RGMII); 3600 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3601 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3602 val = MACB_BIT(RMII); 3603 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3604 val = MACB_BIT(MII); 3605 3606 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3607 val |= MACB_BIT(CLKEN); 3608 3609 macb_or_gem_writel(bp, USRIO, val); 3610 } 3611 3612 /* Set MII management clock divider */ 3613 val = macb_mdc_clk_div(bp); 3614 val |= macb_dbw(bp); 3615 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3616 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3617 macb_writel(bp, NCFGR, val); 3618 3619 return 0; 3620 } 3621 3622 #if defined(CONFIG_OF) 3623 /* 1518 rounded up */ 3624 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3625 /* max number of receive buffers */ 3626 #define AT91ETHER_MAX_RX_DESCR 9 3627 3628 static struct sifive_fu540_macb_mgmt *mgmt; 3629 3630 /* Initialize and start the Receiver and Transmit subsystems */ 3631 static int at91ether_start(struct net_device *dev) 3632 { 3633 struct macb *lp = netdev_priv(dev); 3634 struct macb_queue *q = &lp->queues[0]; 3635 struct macb_dma_desc *desc; 3636 dma_addr_t addr; 3637 u32 ctl; 3638 int i; 3639 3640 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3641 (AT91ETHER_MAX_RX_DESCR * 3642 macb_dma_desc_get_size(lp)), 3643 &q->rx_ring_dma, GFP_KERNEL); 3644 if (!q->rx_ring) 3645 return -ENOMEM; 3646 3647 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3648 AT91ETHER_MAX_RX_DESCR * 3649 AT91ETHER_MAX_RBUFF_SZ, 3650 &q->rx_buffers_dma, GFP_KERNEL); 3651 if (!q->rx_buffers) { 3652 dma_free_coherent(&lp->pdev->dev, 3653 AT91ETHER_MAX_RX_DESCR * 3654 macb_dma_desc_get_size(lp), 3655 q->rx_ring, q->rx_ring_dma); 3656 q->rx_ring = NULL; 3657 return -ENOMEM; 3658 } 3659 3660 addr = q->rx_buffers_dma; 3661 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3662 desc = macb_rx_desc(q, i); 3663 macb_set_addr(lp, desc, addr); 3664 desc->ctrl = 0; 3665 addr += AT91ETHER_MAX_RBUFF_SZ; 3666 } 3667 3668 /* Set the Wrap bit on the last descriptor */ 3669 desc->addr |= MACB_BIT(RX_WRAP); 3670 3671 /* Reset buffer index */ 3672 q->rx_tail = 0; 3673 3674 /* Program address of descriptor list in Rx Buffer Queue register */ 3675 macb_writel(lp, RBQP, q->rx_ring_dma); 3676 3677 /* Enable Receive and Transmit */ 3678 ctl = macb_readl(lp, NCR); 3679 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3680 3681 return 0; 3682 } 3683 3684 /* Open the ethernet interface */ 3685 static int at91ether_open(struct net_device *dev) 3686 { 3687 struct macb *lp = netdev_priv(dev); 3688 u32 ctl; 3689 int ret; 3690 3691 /* Clear internal statistics */ 3692 ctl = macb_readl(lp, NCR); 3693 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3694 3695 macb_set_hwaddr(lp); 3696 3697 ret = at91ether_start(dev); 3698 if (ret) 3699 return ret; 3700 3701 /* Enable MAC interrupts */ 3702 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3703 MACB_BIT(RXUBR) | 3704 MACB_BIT(ISR_TUND) | 3705 MACB_BIT(ISR_RLE) | 3706 MACB_BIT(TCOMP) | 3707 MACB_BIT(ISR_ROVR) | 3708 MACB_BIT(HRESP)); 3709 3710 /* schedule a link state check */ 3711 phy_start(dev->phydev); 3712 3713 netif_start_queue(dev); 3714 3715 return 0; 3716 } 3717 3718 /* Close the interface */ 3719 static int at91ether_close(struct net_device *dev) 3720 { 3721 struct macb *lp = netdev_priv(dev); 3722 struct macb_queue *q = &lp->queues[0]; 3723 u32 ctl; 3724 3725 /* Disable Receiver and Transmitter */ 3726 ctl = macb_readl(lp, NCR); 3727 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3728 3729 /* Disable MAC interrupts */ 3730 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3731 MACB_BIT(RXUBR) | 3732 MACB_BIT(ISR_TUND) | 3733 MACB_BIT(ISR_RLE) | 3734 MACB_BIT(TCOMP) | 3735 MACB_BIT(ISR_ROVR) | 3736 MACB_BIT(HRESP)); 3737 3738 netif_stop_queue(dev); 3739 3740 dma_free_coherent(&lp->pdev->dev, 3741 AT91ETHER_MAX_RX_DESCR * 3742 macb_dma_desc_get_size(lp), 3743 q->rx_ring, q->rx_ring_dma); 3744 q->rx_ring = NULL; 3745 3746 dma_free_coherent(&lp->pdev->dev, 3747 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3748 q->rx_buffers, q->rx_buffers_dma); 3749 q->rx_buffers = NULL; 3750 3751 return 0; 3752 } 3753 3754 /* Transmit packet */ 3755 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3756 struct net_device *dev) 3757 { 3758 struct macb *lp = netdev_priv(dev); 3759 3760 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3761 netif_stop_queue(dev); 3762 3763 /* Store packet information (to free when Tx completed) */ 3764 lp->skb = skb; 3765 lp->skb_length = skb->len; 3766 lp->skb_physaddr = dma_map_single(&lp->pdev->dev, skb->data, 3767 skb->len, DMA_TO_DEVICE); 3768 if (dma_mapping_error(&lp->pdev->dev, lp->skb_physaddr)) { 3769 dev_kfree_skb_any(skb); 3770 dev->stats.tx_dropped++; 3771 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3772 return NETDEV_TX_OK; 3773 } 3774 3775 /* Set address of the data in the Transmit Address register */ 3776 macb_writel(lp, TAR, lp->skb_physaddr); 3777 /* Set length of the packet in the Transmit Control register */ 3778 macb_writel(lp, TCR, skb->len); 3779 3780 } else { 3781 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3782 return NETDEV_TX_BUSY; 3783 } 3784 3785 return NETDEV_TX_OK; 3786 } 3787 3788 /* Extract received frame from buffer descriptors and sent to upper layers. 3789 * (Called from interrupt context) 3790 */ 3791 static void at91ether_rx(struct net_device *dev) 3792 { 3793 struct macb *lp = netdev_priv(dev); 3794 struct macb_queue *q = &lp->queues[0]; 3795 struct macb_dma_desc *desc; 3796 unsigned char *p_recv; 3797 struct sk_buff *skb; 3798 unsigned int pktlen; 3799 3800 desc = macb_rx_desc(q, q->rx_tail); 3801 while (desc->addr & MACB_BIT(RX_USED)) { 3802 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3803 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3804 skb = netdev_alloc_skb(dev, pktlen + 2); 3805 if (skb) { 3806 skb_reserve(skb, 2); 3807 skb_put_data(skb, p_recv, pktlen); 3808 3809 skb->protocol = eth_type_trans(skb, dev); 3810 dev->stats.rx_packets++; 3811 dev->stats.rx_bytes += pktlen; 3812 netif_rx(skb); 3813 } else { 3814 dev->stats.rx_dropped++; 3815 } 3816 3817 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3818 dev->stats.multicast++; 3819 3820 /* reset ownership bit */ 3821 desc->addr &= ~MACB_BIT(RX_USED); 3822 3823 /* wrap after last buffer */ 3824 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3825 q->rx_tail = 0; 3826 else 3827 q->rx_tail++; 3828 3829 desc = macb_rx_desc(q, q->rx_tail); 3830 } 3831 } 3832 3833 /* MAC interrupt handler */ 3834 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3835 { 3836 struct net_device *dev = dev_id; 3837 struct macb *lp = netdev_priv(dev); 3838 u32 intstatus, ctl; 3839 3840 /* MAC Interrupt Status register indicates what interrupts are pending. 3841 * It is automatically cleared once read. 3842 */ 3843 intstatus = macb_readl(lp, ISR); 3844 3845 /* Receive complete */ 3846 if (intstatus & MACB_BIT(RCOMP)) 3847 at91ether_rx(dev); 3848 3849 /* Transmit complete */ 3850 if (intstatus & MACB_BIT(TCOMP)) { 3851 /* The TCOM bit is set even if the transmission failed */ 3852 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3853 dev->stats.tx_errors++; 3854 3855 if (lp->skb) { 3856 dev_consume_skb_irq(lp->skb); 3857 lp->skb = NULL; 3858 dma_unmap_single(&lp->pdev->dev, lp->skb_physaddr, 3859 lp->skb_length, DMA_TO_DEVICE); 3860 dev->stats.tx_packets++; 3861 dev->stats.tx_bytes += lp->skb_length; 3862 } 3863 netif_wake_queue(dev); 3864 } 3865 3866 /* Work-around for EMAC Errata section 41.3.1 */ 3867 if (intstatus & MACB_BIT(RXUBR)) { 3868 ctl = macb_readl(lp, NCR); 3869 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3870 wmb(); 3871 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3872 } 3873 3874 if (intstatus & MACB_BIT(ISR_ROVR)) 3875 netdev_err(dev, "ROVR error\n"); 3876 3877 return IRQ_HANDLED; 3878 } 3879 3880 #ifdef CONFIG_NET_POLL_CONTROLLER 3881 static void at91ether_poll_controller(struct net_device *dev) 3882 { 3883 unsigned long flags; 3884 3885 local_irq_save(flags); 3886 at91ether_interrupt(dev->irq, dev); 3887 local_irq_restore(flags); 3888 } 3889 #endif 3890 3891 static const struct net_device_ops at91ether_netdev_ops = { 3892 .ndo_open = at91ether_open, 3893 .ndo_stop = at91ether_close, 3894 .ndo_start_xmit = at91ether_start_xmit, 3895 .ndo_get_stats = macb_get_stats, 3896 .ndo_set_rx_mode = macb_set_rx_mode, 3897 .ndo_set_mac_address = eth_mac_addr, 3898 .ndo_do_ioctl = macb_ioctl, 3899 .ndo_validate_addr = eth_validate_addr, 3900 #ifdef CONFIG_NET_POLL_CONTROLLER 3901 .ndo_poll_controller = at91ether_poll_controller, 3902 #endif 3903 }; 3904 3905 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3906 struct clk **hclk, struct clk **tx_clk, 3907 struct clk **rx_clk, struct clk **tsu_clk) 3908 { 3909 int err; 3910 3911 *hclk = NULL; 3912 *tx_clk = NULL; 3913 *rx_clk = NULL; 3914 *tsu_clk = NULL; 3915 3916 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3917 if (IS_ERR(*pclk)) 3918 return PTR_ERR(*pclk); 3919 3920 err = clk_prepare_enable(*pclk); 3921 if (err) { 3922 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3923 return err; 3924 } 3925 3926 return 0; 3927 } 3928 3929 static int at91ether_init(struct platform_device *pdev) 3930 { 3931 struct net_device *dev = platform_get_drvdata(pdev); 3932 struct macb *bp = netdev_priv(dev); 3933 int err; 3934 u32 reg; 3935 3936 bp->queues[0].bp = bp; 3937 3938 dev->netdev_ops = &at91ether_netdev_ops; 3939 dev->ethtool_ops = &macb_ethtool_ops; 3940 3941 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3942 0, dev->name, dev); 3943 if (err) 3944 return err; 3945 3946 macb_writel(bp, NCR, 0); 3947 3948 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3949 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3950 reg |= MACB_BIT(RM9200_RMII); 3951 3952 macb_writel(bp, NCFGR, reg); 3953 3954 return 0; 3955 } 3956 3957 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 3958 unsigned long parent_rate) 3959 { 3960 return mgmt->rate; 3961 } 3962 3963 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, 3964 unsigned long *parent_rate) 3965 { 3966 if (WARN_ON(rate < 2500000)) 3967 return 2500000; 3968 else if (rate == 2500000) 3969 return 2500000; 3970 else if (WARN_ON(rate < 13750000)) 3971 return 2500000; 3972 else if (WARN_ON(rate < 25000000)) 3973 return 25000000; 3974 else if (rate == 25000000) 3975 return 25000000; 3976 else if (WARN_ON(rate < 75000000)) 3977 return 25000000; 3978 else if (WARN_ON(rate < 125000000)) 3979 return 125000000; 3980 else if (rate == 125000000) 3981 return 125000000; 3982 3983 WARN_ON(rate > 125000000); 3984 3985 return 125000000; 3986 } 3987 3988 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 3989 unsigned long parent_rate) 3990 { 3991 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); 3992 if (rate != 125000000) 3993 iowrite32(1, mgmt->reg); 3994 else 3995 iowrite32(0, mgmt->reg); 3996 mgmt->rate = rate; 3997 3998 return 0; 3999 } 4000 4001 static const struct clk_ops fu540_c000_ops = { 4002 .recalc_rate = fu540_macb_tx_recalc_rate, 4003 .round_rate = fu540_macb_tx_round_rate, 4004 .set_rate = fu540_macb_tx_set_rate, 4005 }; 4006 4007 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 4008 struct clk **hclk, struct clk **tx_clk, 4009 struct clk **rx_clk, struct clk **tsu_clk) 4010 { 4011 struct clk_init_data init; 4012 int err = 0; 4013 4014 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 4015 if (err) 4016 return err; 4017 4018 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 4019 if (!mgmt) 4020 return -ENOMEM; 4021 4022 init.name = "sifive-gemgxl-mgmt"; 4023 init.ops = &fu540_c000_ops; 4024 init.flags = 0; 4025 init.num_parents = 0; 4026 4027 mgmt->rate = 0; 4028 mgmt->hw.init = &init; 4029 4030 *tx_clk = clk_register(NULL, &mgmt->hw); 4031 if (IS_ERR(*tx_clk)) 4032 return PTR_ERR(*tx_clk); 4033 4034 err = clk_prepare_enable(*tx_clk); 4035 if (err) 4036 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 4037 else 4038 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 4039 4040 return 0; 4041 } 4042 4043 static int fu540_c000_init(struct platform_device *pdev) 4044 { 4045 struct resource *res; 4046 4047 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 4048 if (!res) 4049 return -ENODEV; 4050 4051 mgmt->reg = ioremap(res->start, resource_size(res)); 4052 if (!mgmt->reg) 4053 return -ENOMEM; 4054 4055 return macb_init(pdev); 4056 } 4057 4058 static const struct macb_config fu540_c000_config = { 4059 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 4060 MACB_CAPS_GEM_HAS_PTP, 4061 .dma_burst_length = 16, 4062 .clk_init = fu540_c000_clk_init, 4063 .init = fu540_c000_init, 4064 .jumbo_max_len = 10240, 4065 }; 4066 4067 static const struct macb_config at91sam9260_config = { 4068 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4069 .clk_init = macb_clk_init, 4070 .init = macb_init, 4071 }; 4072 4073 static const struct macb_config sama5d3macb_config = { 4074 .caps = MACB_CAPS_SG_DISABLED 4075 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4076 .clk_init = macb_clk_init, 4077 .init = macb_init, 4078 }; 4079 4080 static const struct macb_config pc302gem_config = { 4081 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 4082 .dma_burst_length = 16, 4083 .clk_init = macb_clk_init, 4084 .init = macb_init, 4085 }; 4086 4087 static const struct macb_config sama5d2_config = { 4088 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4089 .dma_burst_length = 16, 4090 .clk_init = macb_clk_init, 4091 .init = macb_init, 4092 }; 4093 4094 static const struct macb_config sama5d3_config = { 4095 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 4096 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 4097 .dma_burst_length = 16, 4098 .clk_init = macb_clk_init, 4099 .init = macb_init, 4100 .jumbo_max_len = 10240, 4101 }; 4102 4103 static const struct macb_config sama5d4_config = { 4104 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4105 .dma_burst_length = 4, 4106 .clk_init = macb_clk_init, 4107 .init = macb_init, 4108 }; 4109 4110 static const struct macb_config emac_config = { 4111 .caps = MACB_CAPS_NEEDS_RSTONUBR, 4112 .clk_init = at91ether_clk_init, 4113 .init = at91ether_init, 4114 }; 4115 4116 static const struct macb_config np4_config = { 4117 .caps = MACB_CAPS_USRIO_DISABLED, 4118 .clk_init = macb_clk_init, 4119 .init = macb_init, 4120 }; 4121 4122 static const struct macb_config zynqmp_config = { 4123 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4124 MACB_CAPS_JUMBO | 4125 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 4126 .dma_burst_length = 16, 4127 .clk_init = macb_clk_init, 4128 .init = macb_init, 4129 .jumbo_max_len = 10240, 4130 }; 4131 4132 static const struct macb_config zynq_config = { 4133 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 4134 MACB_CAPS_NEEDS_RSTONUBR, 4135 .dma_burst_length = 16, 4136 .clk_init = macb_clk_init, 4137 .init = macb_init, 4138 }; 4139 4140 static const struct of_device_id macb_dt_ids[] = { 4141 { .compatible = "cdns,at32ap7000-macb" }, 4142 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 4143 { .compatible = "cdns,macb" }, 4144 { .compatible = "cdns,np4-macb", .data = &np4_config }, 4145 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 4146 { .compatible = "cdns,gem", .data = &pc302gem_config }, 4147 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 4148 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 4149 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 4150 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 4151 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 4152 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 4153 { .compatible = "cdns,emac", .data = &emac_config }, 4154 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 4155 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 4156 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 4157 { /* sentinel */ } 4158 }; 4159 MODULE_DEVICE_TABLE(of, macb_dt_ids); 4160 #endif /* CONFIG_OF */ 4161 4162 static const struct macb_config default_gem_config = { 4163 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4164 MACB_CAPS_JUMBO | 4165 MACB_CAPS_GEM_HAS_PTP, 4166 .dma_burst_length = 16, 4167 .clk_init = macb_clk_init, 4168 .init = macb_init, 4169 .jumbo_max_len = 10240, 4170 }; 4171 4172 static int macb_probe(struct platform_device *pdev) 4173 { 4174 const struct macb_config *macb_config = &default_gem_config; 4175 int (*clk_init)(struct platform_device *, struct clk **, 4176 struct clk **, struct clk **, struct clk **, 4177 struct clk **) = macb_config->clk_init; 4178 int (*init)(struct platform_device *) = macb_config->init; 4179 struct device_node *np = pdev->dev.of_node; 4180 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4181 struct clk *tsu_clk = NULL; 4182 unsigned int queue_mask, num_queues; 4183 bool native_io; 4184 struct phy_device *phydev; 4185 struct net_device *dev; 4186 struct resource *regs; 4187 void __iomem *mem; 4188 const char *mac; 4189 struct macb *bp; 4190 int err, val; 4191 4192 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4193 mem = devm_ioremap_resource(&pdev->dev, regs); 4194 if (IS_ERR(mem)) 4195 return PTR_ERR(mem); 4196 4197 if (np) { 4198 const struct of_device_id *match; 4199 4200 match = of_match_node(macb_dt_ids, np); 4201 if (match && match->data) { 4202 macb_config = match->data; 4203 clk_init = macb_config->clk_init; 4204 init = macb_config->init; 4205 } 4206 } 4207 4208 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4209 if (err) 4210 return err; 4211 4212 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4213 pm_runtime_use_autosuspend(&pdev->dev); 4214 pm_runtime_get_noresume(&pdev->dev); 4215 pm_runtime_set_active(&pdev->dev); 4216 pm_runtime_enable(&pdev->dev); 4217 native_io = hw_is_native_io(mem); 4218 4219 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4220 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4221 if (!dev) { 4222 err = -ENOMEM; 4223 goto err_disable_clocks; 4224 } 4225 4226 dev->base_addr = regs->start; 4227 4228 SET_NETDEV_DEV(dev, &pdev->dev); 4229 4230 bp = netdev_priv(dev); 4231 bp->pdev = pdev; 4232 bp->dev = dev; 4233 bp->regs = mem; 4234 bp->native_io = native_io; 4235 if (native_io) { 4236 bp->macb_reg_readl = hw_readl_native; 4237 bp->macb_reg_writel = hw_writel_native; 4238 } else { 4239 bp->macb_reg_readl = hw_readl; 4240 bp->macb_reg_writel = hw_writel; 4241 } 4242 bp->num_queues = num_queues; 4243 bp->queue_mask = queue_mask; 4244 if (macb_config) 4245 bp->dma_burst_length = macb_config->dma_burst_length; 4246 bp->pclk = pclk; 4247 bp->hclk = hclk; 4248 bp->tx_clk = tx_clk; 4249 bp->rx_clk = rx_clk; 4250 bp->tsu_clk = tsu_clk; 4251 if (macb_config) 4252 bp->jumbo_max_len = macb_config->jumbo_max_len; 4253 4254 bp->wol = 0; 4255 if (of_get_property(np, "magic-packet", NULL)) 4256 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4257 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4258 4259 spin_lock_init(&bp->lock); 4260 4261 /* setup capabilities */ 4262 macb_configure_caps(bp, macb_config); 4263 4264 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4265 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4266 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4267 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4268 } 4269 #endif 4270 platform_set_drvdata(pdev, dev); 4271 4272 dev->irq = platform_get_irq(pdev, 0); 4273 if (dev->irq < 0) { 4274 err = dev->irq; 4275 goto err_out_free_netdev; 4276 } 4277 4278 /* MTU range: 68 - 1500 or 10240 */ 4279 dev->min_mtu = GEM_MTU_MIN_SIZE; 4280 if (bp->caps & MACB_CAPS_JUMBO) 4281 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4282 else 4283 dev->max_mtu = ETH_DATA_LEN; 4284 4285 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4286 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4287 if (val) 4288 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4289 macb_dma_desc_get_size(bp); 4290 4291 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4292 if (val) 4293 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4294 macb_dma_desc_get_size(bp); 4295 } 4296 4297 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4298 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4299 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4300 4301 mac = of_get_mac_address(np); 4302 if (PTR_ERR(mac) == -EPROBE_DEFER) { 4303 err = -EPROBE_DEFER; 4304 goto err_out_free_netdev; 4305 } else if (!IS_ERR_OR_NULL(mac)) { 4306 ether_addr_copy(bp->dev->dev_addr, mac); 4307 } else { 4308 macb_get_hwaddr(bp); 4309 } 4310 4311 err = of_get_phy_mode(np); 4312 if (err < 0) 4313 /* not found in DT, MII by default */ 4314 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4315 else 4316 bp->phy_interface = err; 4317 4318 /* IP specific init */ 4319 err = init(pdev); 4320 if (err) 4321 goto err_out_free_netdev; 4322 4323 err = macb_mii_init(bp); 4324 if (err) 4325 goto err_out_free_netdev; 4326 4327 phydev = dev->phydev; 4328 4329 netif_carrier_off(dev); 4330 4331 err = register_netdev(dev); 4332 if (err) { 4333 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4334 goto err_out_unregister_mdio; 4335 } 4336 4337 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4338 (unsigned long)bp); 4339 4340 phy_attached_info(phydev); 4341 4342 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4343 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4344 dev->base_addr, dev->irq, dev->dev_addr); 4345 4346 pm_runtime_mark_last_busy(&bp->pdev->dev); 4347 pm_runtime_put_autosuspend(&bp->pdev->dev); 4348 4349 return 0; 4350 4351 err_out_unregister_mdio: 4352 phy_disconnect(dev->phydev); 4353 mdiobus_unregister(bp->mii_bus); 4354 of_node_put(bp->phy_node); 4355 if (np && of_phy_is_fixed_link(np)) 4356 of_phy_deregister_fixed_link(np); 4357 mdiobus_free(bp->mii_bus); 4358 4359 err_out_free_netdev: 4360 free_netdev(dev); 4361 4362 err_disable_clocks: 4363 clk_disable_unprepare(tx_clk); 4364 clk_unregister(tx_clk); 4365 clk_disable_unprepare(hclk); 4366 clk_disable_unprepare(pclk); 4367 clk_disable_unprepare(rx_clk); 4368 clk_disable_unprepare(tsu_clk); 4369 pm_runtime_disable(&pdev->dev); 4370 pm_runtime_set_suspended(&pdev->dev); 4371 pm_runtime_dont_use_autosuspend(&pdev->dev); 4372 4373 return err; 4374 } 4375 4376 static int macb_remove(struct platform_device *pdev) 4377 { 4378 struct net_device *dev; 4379 struct macb *bp; 4380 struct device_node *np = pdev->dev.of_node; 4381 4382 dev = platform_get_drvdata(pdev); 4383 4384 if (dev) { 4385 bp = netdev_priv(dev); 4386 if (dev->phydev) 4387 phy_disconnect(dev->phydev); 4388 mdiobus_unregister(bp->mii_bus); 4389 if (np && of_phy_is_fixed_link(np)) 4390 of_phy_deregister_fixed_link(np); 4391 dev->phydev = NULL; 4392 mdiobus_free(bp->mii_bus); 4393 4394 unregister_netdev(dev); 4395 pm_runtime_disable(&pdev->dev); 4396 pm_runtime_dont_use_autosuspend(&pdev->dev); 4397 if (!pm_runtime_suspended(&pdev->dev)) { 4398 clk_disable_unprepare(bp->tx_clk); 4399 clk_unregister(bp->tx_clk); 4400 clk_disable_unprepare(bp->hclk); 4401 clk_disable_unprepare(bp->pclk); 4402 clk_disable_unprepare(bp->rx_clk); 4403 clk_disable_unprepare(bp->tsu_clk); 4404 pm_runtime_set_suspended(&pdev->dev); 4405 } 4406 of_node_put(bp->phy_node); 4407 free_netdev(dev); 4408 } 4409 4410 return 0; 4411 } 4412 4413 static int __maybe_unused macb_suspend(struct device *dev) 4414 { 4415 struct net_device *netdev = dev_get_drvdata(dev); 4416 struct macb *bp = netdev_priv(netdev); 4417 struct macb_queue *queue = bp->queues; 4418 unsigned long flags; 4419 unsigned int q; 4420 4421 if (!netif_running(netdev)) 4422 return 0; 4423 4424 4425 if (bp->wol & MACB_WOL_ENABLED) { 4426 macb_writel(bp, IER, MACB_BIT(WOL)); 4427 macb_writel(bp, WOL, MACB_BIT(MAG)); 4428 enable_irq_wake(bp->queues[0].irq); 4429 netif_device_detach(netdev); 4430 } else { 4431 netif_device_detach(netdev); 4432 for (q = 0, queue = bp->queues; q < bp->num_queues; 4433 ++q, ++queue) 4434 napi_disable(&queue->napi); 4435 phy_stop(netdev->phydev); 4436 phy_suspend(netdev->phydev); 4437 spin_lock_irqsave(&bp->lock, flags); 4438 macb_reset_hw(bp); 4439 spin_unlock_irqrestore(&bp->lock, flags); 4440 4441 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4442 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 4443 4444 if (netdev->hw_features & NETIF_F_NTUPLE) 4445 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 4446 } 4447 4448 netif_carrier_off(netdev); 4449 if (bp->ptp_info) 4450 bp->ptp_info->ptp_remove(netdev); 4451 pm_runtime_force_suspend(dev); 4452 4453 return 0; 4454 } 4455 4456 static int __maybe_unused macb_resume(struct device *dev) 4457 { 4458 struct net_device *netdev = dev_get_drvdata(dev); 4459 struct macb *bp = netdev_priv(netdev); 4460 struct macb_queue *queue = bp->queues; 4461 unsigned int q; 4462 4463 if (!netif_running(netdev)) 4464 return 0; 4465 4466 pm_runtime_force_resume(dev); 4467 4468 if (bp->wol & MACB_WOL_ENABLED) { 4469 macb_writel(bp, IDR, MACB_BIT(WOL)); 4470 macb_writel(bp, WOL, 0); 4471 disable_irq_wake(bp->queues[0].irq); 4472 } else { 4473 macb_writel(bp, NCR, MACB_BIT(MPE)); 4474 4475 if (netdev->hw_features & NETIF_F_NTUPLE) 4476 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 4477 4478 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4479 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 4480 4481 for (q = 0, queue = bp->queues; q < bp->num_queues; 4482 ++q, ++queue) 4483 napi_enable(&queue->napi); 4484 phy_resume(netdev->phydev); 4485 phy_init_hw(netdev->phydev); 4486 phy_start(netdev->phydev); 4487 } 4488 4489 bp->macbgem_ops.mog_init_rings(bp); 4490 macb_init_hw(bp); 4491 macb_set_rx_mode(netdev); 4492 macb_restore_features(bp); 4493 netif_device_attach(netdev); 4494 if (bp->ptp_info) 4495 bp->ptp_info->ptp_init(netdev); 4496 4497 return 0; 4498 } 4499 4500 static int __maybe_unused macb_runtime_suspend(struct device *dev) 4501 { 4502 struct net_device *netdev = dev_get_drvdata(dev); 4503 struct macb *bp = netdev_priv(netdev); 4504 4505 if (!(device_may_wakeup(&bp->dev->dev))) { 4506 clk_disable_unprepare(bp->tx_clk); 4507 clk_disable_unprepare(bp->hclk); 4508 clk_disable_unprepare(bp->pclk); 4509 clk_disable_unprepare(bp->rx_clk); 4510 } 4511 clk_disable_unprepare(bp->tsu_clk); 4512 4513 return 0; 4514 } 4515 4516 static int __maybe_unused macb_runtime_resume(struct device *dev) 4517 { 4518 struct net_device *netdev = dev_get_drvdata(dev); 4519 struct macb *bp = netdev_priv(netdev); 4520 4521 if (!(device_may_wakeup(&bp->dev->dev))) { 4522 clk_prepare_enable(bp->pclk); 4523 clk_prepare_enable(bp->hclk); 4524 clk_prepare_enable(bp->tx_clk); 4525 clk_prepare_enable(bp->rx_clk); 4526 } 4527 clk_prepare_enable(bp->tsu_clk); 4528 4529 return 0; 4530 } 4531 4532 static const struct dev_pm_ops macb_pm_ops = { 4533 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 4534 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 4535 }; 4536 4537 static struct platform_driver macb_driver = { 4538 .probe = macb_probe, 4539 .remove = macb_remove, 4540 .driver = { 4541 .name = "macb", 4542 .of_match_table = of_match_ptr(macb_dt_ids), 4543 .pm = &macb_pm_ops, 4544 }, 4545 }; 4546 4547 module_platform_driver(macb_driver); 4548 4549 MODULE_LICENSE("GPL"); 4550 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4551 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4552 MODULE_ALIAS("platform:macb"); 4553