1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/clk.h> 10 #include <linux/clk-provider.h> 11 #include <linux/crc32.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/kernel.h> 15 #include <linux/types.h> 16 #include <linux/circ_buf.h> 17 #include <linux/slab.h> 18 #include <linux/init.h> 19 #include <linux/io.h> 20 #include <linux/gpio.h> 21 #include <linux/gpio/consumer.h> 22 #include <linux/interrupt.h> 23 #include <linux/netdevice.h> 24 #include <linux/etherdevice.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/platform_data/macb.h> 27 #include <linux/platform_device.h> 28 #include <linux/phylink.h> 29 #include <linux/of.h> 30 #include <linux/of_device.h> 31 #include <linux/of_gpio.h> 32 #include <linux/of_mdio.h> 33 #include <linux/of_net.h> 34 #include <linux/ip.h> 35 #include <linux/udp.h> 36 #include <linux/tcp.h> 37 #include <linux/iopoll.h> 38 #include <linux/pm_runtime.h> 39 #include "macb.h" 40 41 /* This structure is only used for MACB on SiFive FU540 devices */ 42 struct sifive_fu540_macb_mgmt { 43 void __iomem *reg; 44 unsigned long rate; 45 struct clk_hw hw; 46 }; 47 48 #define MACB_RX_BUFFER_SIZE 128 49 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 50 51 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 52 #define MIN_RX_RING_SIZE 64 53 #define MAX_RX_RING_SIZE 8192 54 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 55 * (bp)->rx_ring_size) 56 57 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 58 #define MIN_TX_RING_SIZE 64 59 #define MAX_TX_RING_SIZE 4096 60 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 61 * (bp)->tx_ring_size) 62 63 /* level of occupied TX descriptors under which we wake up TX process */ 64 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 65 66 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 67 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 68 | MACB_BIT(ISR_RLE) \ 69 | MACB_BIT(TXERR)) 70 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 71 | MACB_BIT(TXUBR)) 72 73 /* Max length of transmit frame must be a multiple of 8 bytes */ 74 #define MACB_TX_LEN_ALIGN 8 75 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 76 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 77 78 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 79 #define MACB_NETIF_LSO NETIF_F_TSO 80 81 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 82 #define MACB_WOL_ENABLED (0x1 << 1) 83 84 /* Graceful stop timeouts in us. We should allow up to 85 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 86 */ 87 #define MACB_HALT_TIMEOUT 1230 88 89 #define MACB_PM_TIMEOUT 100 /* ms */ 90 91 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 92 93 /* DMA buffer descriptor might be different size 94 * depends on hardware configuration: 95 * 96 * 1. dma address width 32 bits: 97 * word 1: 32 bit address of Data Buffer 98 * word 2: control 99 * 100 * 2. dma address width 64 bits: 101 * word 1: 32 bit address of Data Buffer 102 * word 2: control 103 * word 3: upper 32 bit address of Data Buffer 104 * word 4: unused 105 * 106 * 3. dma address width 32 bits with hardware timestamping: 107 * word 1: 32 bit address of Data Buffer 108 * word 2: control 109 * word 3: timestamp word 1 110 * word 4: timestamp word 2 111 * 112 * 4. dma address width 64 bits with hardware timestamping: 113 * word 1: 32 bit address of Data Buffer 114 * word 2: control 115 * word 3: upper 32 bit address of Data Buffer 116 * word 4: unused 117 * word 5: timestamp word 1 118 * word 6: timestamp word 2 119 */ 120 static unsigned int macb_dma_desc_get_size(struct macb *bp) 121 { 122 #ifdef MACB_EXT_DESC 123 unsigned int desc_size; 124 125 switch (bp->hw_dma_cap) { 126 case HW_DMA_CAP_64B: 127 desc_size = sizeof(struct macb_dma_desc) 128 + sizeof(struct macb_dma_desc_64); 129 break; 130 case HW_DMA_CAP_PTP: 131 desc_size = sizeof(struct macb_dma_desc) 132 + sizeof(struct macb_dma_desc_ptp); 133 break; 134 case HW_DMA_CAP_64B_PTP: 135 desc_size = sizeof(struct macb_dma_desc) 136 + sizeof(struct macb_dma_desc_64) 137 + sizeof(struct macb_dma_desc_ptp); 138 break; 139 default: 140 desc_size = sizeof(struct macb_dma_desc); 141 } 142 return desc_size; 143 #endif 144 return sizeof(struct macb_dma_desc); 145 } 146 147 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 148 { 149 #ifdef MACB_EXT_DESC 150 switch (bp->hw_dma_cap) { 151 case HW_DMA_CAP_64B: 152 case HW_DMA_CAP_PTP: 153 desc_idx <<= 1; 154 break; 155 case HW_DMA_CAP_64B_PTP: 156 desc_idx *= 3; 157 break; 158 default: 159 break; 160 } 161 #endif 162 return desc_idx; 163 } 164 165 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 166 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 167 { 168 return (struct macb_dma_desc_64 *)((void *)desc 169 + sizeof(struct macb_dma_desc)); 170 } 171 #endif 172 173 /* Ring buffer accessors */ 174 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 175 { 176 return index & (bp->tx_ring_size - 1); 177 } 178 179 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 180 unsigned int index) 181 { 182 index = macb_tx_ring_wrap(queue->bp, index); 183 index = macb_adj_dma_desc_idx(queue->bp, index); 184 return &queue->tx_ring[index]; 185 } 186 187 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 188 unsigned int index) 189 { 190 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 191 } 192 193 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 194 { 195 dma_addr_t offset; 196 197 offset = macb_tx_ring_wrap(queue->bp, index) * 198 macb_dma_desc_get_size(queue->bp); 199 200 return queue->tx_ring_dma + offset; 201 } 202 203 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 204 { 205 return index & (bp->rx_ring_size - 1); 206 } 207 208 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 209 { 210 index = macb_rx_ring_wrap(queue->bp, index); 211 index = macb_adj_dma_desc_idx(queue->bp, index); 212 return &queue->rx_ring[index]; 213 } 214 215 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 216 { 217 return queue->rx_buffers + queue->bp->rx_buffer_size * 218 macb_rx_ring_wrap(queue->bp, index); 219 } 220 221 /* I/O accessors */ 222 static u32 hw_readl_native(struct macb *bp, int offset) 223 { 224 return __raw_readl(bp->regs + offset); 225 } 226 227 static void hw_writel_native(struct macb *bp, int offset, u32 value) 228 { 229 __raw_writel(value, bp->regs + offset); 230 } 231 232 static u32 hw_readl(struct macb *bp, int offset) 233 { 234 return readl_relaxed(bp->regs + offset); 235 } 236 237 static void hw_writel(struct macb *bp, int offset, u32 value) 238 { 239 writel_relaxed(value, bp->regs + offset); 240 } 241 242 /* Find the CPU endianness by using the loopback bit of NCR register. When the 243 * CPU is in big endian we need to program swapped mode for management 244 * descriptor access. 245 */ 246 static bool hw_is_native_io(void __iomem *addr) 247 { 248 u32 value = MACB_BIT(LLB); 249 250 __raw_writel(value, addr + MACB_NCR); 251 value = __raw_readl(addr + MACB_NCR); 252 253 /* Write 0 back to disable everything */ 254 __raw_writel(0, addr + MACB_NCR); 255 256 return value == MACB_BIT(LLB); 257 } 258 259 static bool hw_is_gem(void __iomem *addr, bool native_io) 260 { 261 u32 id; 262 263 if (native_io) 264 id = __raw_readl(addr + MACB_MID); 265 else 266 id = readl_relaxed(addr + MACB_MID); 267 268 return MACB_BFEXT(IDNUM, id) >= 0x2; 269 } 270 271 static void macb_set_hwaddr(struct macb *bp) 272 { 273 u32 bottom; 274 u16 top; 275 276 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 277 macb_or_gem_writel(bp, SA1B, bottom); 278 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 279 macb_or_gem_writel(bp, SA1T, top); 280 281 /* Clear unused address register sets */ 282 macb_or_gem_writel(bp, SA2B, 0); 283 macb_or_gem_writel(bp, SA2T, 0); 284 macb_or_gem_writel(bp, SA3B, 0); 285 macb_or_gem_writel(bp, SA3T, 0); 286 macb_or_gem_writel(bp, SA4B, 0); 287 macb_or_gem_writel(bp, SA4T, 0); 288 } 289 290 static void macb_get_hwaddr(struct macb *bp) 291 { 292 u32 bottom; 293 u16 top; 294 u8 addr[6]; 295 int i; 296 297 /* Check all 4 address register for valid address */ 298 for (i = 0; i < 4; i++) { 299 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 300 top = macb_or_gem_readl(bp, SA1T + i * 8); 301 302 addr[0] = bottom & 0xff; 303 addr[1] = (bottom >> 8) & 0xff; 304 addr[2] = (bottom >> 16) & 0xff; 305 addr[3] = (bottom >> 24) & 0xff; 306 addr[4] = top & 0xff; 307 addr[5] = (top >> 8) & 0xff; 308 309 if (is_valid_ether_addr(addr)) { 310 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 311 return; 312 } 313 } 314 315 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 316 eth_hw_addr_random(bp->dev); 317 } 318 319 static int macb_mdio_wait_for_idle(struct macb *bp) 320 { 321 u32 val; 322 323 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 324 1, MACB_MDIO_TIMEOUT); 325 } 326 327 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 328 { 329 struct macb *bp = bus->priv; 330 int status; 331 332 status = pm_runtime_get_sync(&bp->pdev->dev); 333 if (status < 0) 334 goto mdio_pm_exit; 335 336 status = macb_mdio_wait_for_idle(bp); 337 if (status < 0) 338 goto mdio_read_exit; 339 340 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 341 | MACB_BF(RW, MACB_MAN_READ) 342 | MACB_BF(PHYA, mii_id) 343 | MACB_BF(REGA, regnum) 344 | MACB_BF(CODE, MACB_MAN_CODE))); 345 346 status = macb_mdio_wait_for_idle(bp); 347 if (status < 0) 348 goto mdio_read_exit; 349 350 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 351 352 mdio_read_exit: 353 pm_runtime_mark_last_busy(&bp->pdev->dev); 354 pm_runtime_put_autosuspend(&bp->pdev->dev); 355 mdio_pm_exit: 356 return status; 357 } 358 359 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 360 u16 value) 361 { 362 struct macb *bp = bus->priv; 363 int status; 364 365 status = pm_runtime_get_sync(&bp->pdev->dev); 366 if (status < 0) 367 goto mdio_pm_exit; 368 369 status = macb_mdio_wait_for_idle(bp); 370 if (status < 0) 371 goto mdio_write_exit; 372 373 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 374 | MACB_BF(RW, MACB_MAN_WRITE) 375 | MACB_BF(PHYA, mii_id) 376 | MACB_BF(REGA, regnum) 377 | MACB_BF(CODE, MACB_MAN_CODE) 378 | MACB_BF(DATA, value))); 379 380 status = macb_mdio_wait_for_idle(bp); 381 if (status < 0) 382 goto mdio_write_exit; 383 384 mdio_write_exit: 385 pm_runtime_mark_last_busy(&bp->pdev->dev); 386 pm_runtime_put_autosuspend(&bp->pdev->dev); 387 mdio_pm_exit: 388 return status; 389 } 390 391 static void macb_init_buffers(struct macb *bp) 392 { 393 struct macb_queue *queue; 394 unsigned int q; 395 396 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 397 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 398 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 399 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 400 queue_writel(queue, RBQPH, 401 upper_32_bits(queue->rx_ring_dma)); 402 #endif 403 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 404 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 405 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 406 queue_writel(queue, TBQPH, 407 upper_32_bits(queue->tx_ring_dma)); 408 #endif 409 } 410 } 411 412 /** 413 * macb_set_tx_clk() - Set a clock to a new frequency 414 * @clk Pointer to the clock to change 415 * @rate New frequency in Hz 416 * @dev Pointer to the struct net_device 417 */ 418 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 419 { 420 long ferr, rate, rate_rounded; 421 422 if (!clk) 423 return; 424 425 switch (speed) { 426 case SPEED_10: 427 rate = 2500000; 428 break; 429 case SPEED_100: 430 rate = 25000000; 431 break; 432 case SPEED_1000: 433 rate = 125000000; 434 break; 435 default: 436 return; 437 } 438 439 rate_rounded = clk_round_rate(clk, rate); 440 if (rate_rounded < 0) 441 return; 442 443 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 444 * is not satisfied. 445 */ 446 ferr = abs(rate_rounded - rate); 447 ferr = DIV_ROUND_UP(ferr, rate / 100000); 448 if (ferr > 5) 449 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 450 rate); 451 452 if (clk_set_rate(clk, rate_rounded)) 453 netdev_err(dev, "adjusting tx_clk failed.\n"); 454 } 455 456 static void macb_validate(struct phylink_config *config, 457 unsigned long *supported, 458 struct phylink_link_state *state) 459 { 460 struct net_device *ndev = to_net_dev(config->dev); 461 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; 462 struct macb *bp = netdev_priv(ndev); 463 464 /* We only support MII, RMII, GMII, RGMII & SGMII. */ 465 if (state->interface != PHY_INTERFACE_MODE_NA && 466 state->interface != PHY_INTERFACE_MODE_MII && 467 state->interface != PHY_INTERFACE_MODE_RMII && 468 state->interface != PHY_INTERFACE_MODE_GMII && 469 state->interface != PHY_INTERFACE_MODE_SGMII && 470 !phy_interface_mode_is_rgmii(state->interface)) { 471 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); 472 return; 473 } 474 475 if (!macb_is_gem(bp) && 476 (state->interface == PHY_INTERFACE_MODE_GMII || 477 phy_interface_mode_is_rgmii(state->interface))) { 478 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); 479 return; 480 } 481 482 phylink_set_port_modes(mask); 483 phylink_set(mask, Autoneg); 484 phylink_set(mask, Asym_Pause); 485 486 phylink_set(mask, 10baseT_Half); 487 phylink_set(mask, 10baseT_Full); 488 phylink_set(mask, 100baseT_Half); 489 phylink_set(mask, 100baseT_Full); 490 491 if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && 492 (state->interface == PHY_INTERFACE_MODE_NA || 493 state->interface == PHY_INTERFACE_MODE_GMII || 494 state->interface == PHY_INTERFACE_MODE_SGMII || 495 phy_interface_mode_is_rgmii(state->interface))) { 496 phylink_set(mask, 1000baseT_Full); 497 phylink_set(mask, 1000baseX_Full); 498 499 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) 500 phylink_set(mask, 1000baseT_Half); 501 } 502 503 bitmap_and(supported, supported, mask, __ETHTOOL_LINK_MODE_MASK_NBITS); 504 bitmap_and(state->advertising, state->advertising, mask, 505 __ETHTOOL_LINK_MODE_MASK_NBITS); 506 } 507 508 static void macb_mac_pcs_get_state(struct phylink_config *config, 509 struct phylink_link_state *state) 510 { 511 state->link = 0; 512 } 513 514 static void macb_mac_an_restart(struct phylink_config *config) 515 { 516 /* Not supported */ 517 } 518 519 static void macb_mac_config(struct phylink_config *config, unsigned int mode, 520 const struct phylink_link_state *state) 521 { 522 struct net_device *ndev = to_net_dev(config->dev); 523 struct macb *bp = netdev_priv(ndev); 524 unsigned long flags; 525 u32 old_ctrl, ctrl; 526 527 spin_lock_irqsave(&bp->lock, flags); 528 529 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); 530 531 /* Clear all the bits we might set later */ 532 ctrl &= ~(GEM_BIT(GBE) | MACB_BIT(SPD) | MACB_BIT(FD) | MACB_BIT(PAE) | 533 GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); 534 535 if (state->speed == SPEED_1000) 536 ctrl |= GEM_BIT(GBE); 537 else if (state->speed == SPEED_100) 538 ctrl |= MACB_BIT(SPD); 539 540 if (state->duplex) 541 ctrl |= MACB_BIT(FD); 542 543 /* We do not support MLO_PAUSE_RX yet */ 544 if (state->pause & MLO_PAUSE_TX) 545 ctrl |= MACB_BIT(PAE); 546 547 if (state->interface == PHY_INTERFACE_MODE_SGMII) 548 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 549 550 /* Apply the new configuration, if any */ 551 if (old_ctrl ^ ctrl) 552 macb_or_gem_writel(bp, NCFGR, ctrl); 553 554 bp->speed = state->speed; 555 556 spin_unlock_irqrestore(&bp->lock, flags); 557 } 558 559 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, 560 phy_interface_t interface) 561 { 562 struct net_device *ndev = to_net_dev(config->dev); 563 struct macb *bp = netdev_priv(ndev); 564 struct macb_queue *queue; 565 unsigned int q; 566 u32 ctrl; 567 568 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 569 queue_writel(queue, IDR, 570 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 571 572 /* Disable Rx and Tx */ 573 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); 574 macb_writel(bp, NCR, ctrl); 575 576 netif_tx_stop_all_queues(ndev); 577 } 578 579 static void macb_mac_link_up(struct phylink_config *config, unsigned int mode, 580 phy_interface_t interface, struct phy_device *phy) 581 { 582 struct net_device *ndev = to_net_dev(config->dev); 583 struct macb *bp = netdev_priv(ndev); 584 struct macb_queue *queue; 585 unsigned int q; 586 587 macb_set_tx_clk(bp->tx_clk, bp->speed, ndev); 588 589 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down 590 * cleared the pipeline and control registers. 591 */ 592 bp->macbgem_ops.mog_init_rings(bp); 593 macb_init_buffers(bp); 594 595 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 596 queue_writel(queue, IER, 597 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 598 599 /* Enable Rx and Tx */ 600 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 601 602 netif_tx_wake_all_queues(ndev); 603 } 604 605 static const struct phylink_mac_ops macb_phylink_ops = { 606 .validate = macb_validate, 607 .mac_pcs_get_state = macb_mac_pcs_get_state, 608 .mac_an_restart = macb_mac_an_restart, 609 .mac_config = macb_mac_config, 610 .mac_link_down = macb_mac_link_down, 611 .mac_link_up = macb_mac_link_up, 612 }; 613 614 static int macb_phylink_connect(struct macb *bp) 615 { 616 struct net_device *dev = bp->dev; 617 struct phy_device *phydev; 618 int ret; 619 620 if (bp->pdev->dev.of_node && 621 of_parse_phandle(bp->pdev->dev.of_node, "phy-handle", 0)) { 622 ret = phylink_of_phy_connect(bp->phylink, bp->pdev->dev.of_node, 623 0); 624 if (ret) { 625 netdev_err(dev, "Could not attach PHY (%d)\n", ret); 626 return ret; 627 } 628 } else { 629 phydev = phy_find_first(bp->mii_bus); 630 if (!phydev) { 631 netdev_err(dev, "no PHY found\n"); 632 return -ENXIO; 633 } 634 635 /* attach the mac to the phy */ 636 ret = phylink_connect_phy(bp->phylink, phydev); 637 if (ret) { 638 netdev_err(dev, "Could not attach to PHY (%d)\n", ret); 639 return ret; 640 } 641 } 642 643 phylink_start(bp->phylink); 644 645 return 0; 646 } 647 648 /* based on au1000_eth. c*/ 649 static int macb_mii_probe(struct net_device *dev) 650 { 651 struct macb *bp = netdev_priv(dev); 652 653 bp->phylink_config.dev = &dev->dev; 654 bp->phylink_config.type = PHYLINK_NETDEV; 655 656 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, 657 bp->phy_interface, &macb_phylink_ops); 658 if (IS_ERR(bp->phylink)) { 659 netdev_err(dev, "Could not create a phylink instance (%ld)\n", 660 PTR_ERR(bp->phylink)); 661 return PTR_ERR(bp->phylink); 662 } 663 664 return 0; 665 } 666 667 static int macb_mii_init(struct macb *bp) 668 { 669 struct device_node *np; 670 int err = -ENXIO; 671 672 /* Enable management port */ 673 macb_writel(bp, NCR, MACB_BIT(MPE)); 674 675 bp->mii_bus = mdiobus_alloc(); 676 if (!bp->mii_bus) { 677 err = -ENOMEM; 678 goto err_out; 679 } 680 681 bp->mii_bus->name = "MACB_mii_bus"; 682 bp->mii_bus->read = &macb_mdio_read; 683 bp->mii_bus->write = &macb_mdio_write; 684 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 685 bp->pdev->name, bp->pdev->id); 686 bp->mii_bus->priv = bp; 687 bp->mii_bus->parent = &bp->pdev->dev; 688 689 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 690 691 np = bp->pdev->dev.of_node; 692 693 err = of_mdiobus_register(bp->mii_bus, np); 694 if (err) 695 goto err_out_free_mdiobus; 696 697 err = macb_mii_probe(bp->dev); 698 if (err) 699 goto err_out_unregister_bus; 700 701 return 0; 702 703 err_out_unregister_bus: 704 mdiobus_unregister(bp->mii_bus); 705 err_out_free_mdiobus: 706 mdiobus_free(bp->mii_bus); 707 err_out: 708 return err; 709 } 710 711 static void macb_update_stats(struct macb *bp) 712 { 713 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 714 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 715 int offset = MACB_PFR; 716 717 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 718 719 for (; p < end; p++, offset += 4) 720 *p += bp->macb_reg_readl(bp, offset); 721 } 722 723 static int macb_halt_tx(struct macb *bp) 724 { 725 unsigned long halt_time, timeout; 726 u32 status; 727 728 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 729 730 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 731 do { 732 halt_time = jiffies; 733 status = macb_readl(bp, TSR); 734 if (!(status & MACB_BIT(TGO))) 735 return 0; 736 737 udelay(250); 738 } while (time_before(halt_time, timeout)); 739 740 return -ETIMEDOUT; 741 } 742 743 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 744 { 745 if (tx_skb->mapping) { 746 if (tx_skb->mapped_as_page) 747 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 748 tx_skb->size, DMA_TO_DEVICE); 749 else 750 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 751 tx_skb->size, DMA_TO_DEVICE); 752 tx_skb->mapping = 0; 753 } 754 755 if (tx_skb->skb) { 756 dev_kfree_skb_any(tx_skb->skb); 757 tx_skb->skb = NULL; 758 } 759 } 760 761 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 762 { 763 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 764 struct macb_dma_desc_64 *desc_64; 765 766 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 767 desc_64 = macb_64b_desc(bp, desc); 768 desc_64->addrh = upper_32_bits(addr); 769 /* The low bits of RX address contain the RX_USED bit, clearing 770 * of which allows packet RX. Make sure the high bits are also 771 * visible to HW at that point. 772 */ 773 dma_wmb(); 774 } 775 #endif 776 desc->addr = lower_32_bits(addr); 777 } 778 779 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 780 { 781 dma_addr_t addr = 0; 782 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 783 struct macb_dma_desc_64 *desc_64; 784 785 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 786 desc_64 = macb_64b_desc(bp, desc); 787 addr = ((u64)(desc_64->addrh) << 32); 788 } 789 #endif 790 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 791 return addr; 792 } 793 794 static void macb_tx_error_task(struct work_struct *work) 795 { 796 struct macb_queue *queue = container_of(work, struct macb_queue, 797 tx_error_task); 798 struct macb *bp = queue->bp; 799 struct macb_tx_skb *tx_skb; 800 struct macb_dma_desc *desc; 801 struct sk_buff *skb; 802 unsigned int tail; 803 unsigned long flags; 804 805 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 806 (unsigned int)(queue - bp->queues), 807 queue->tx_tail, queue->tx_head); 808 809 /* Prevent the queue IRQ handlers from running: each of them may call 810 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 811 * As explained below, we have to halt the transmission before updating 812 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 813 * network engine about the macb/gem being halted. 814 */ 815 spin_lock_irqsave(&bp->lock, flags); 816 817 /* Make sure nobody is trying to queue up new packets */ 818 netif_tx_stop_all_queues(bp->dev); 819 820 /* Stop transmission now 821 * (in case we have just queued new packets) 822 * macb/gem must be halted to write TBQP register 823 */ 824 if (macb_halt_tx(bp)) 825 /* Just complain for now, reinitializing TX path can be good */ 826 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 827 828 /* Treat frames in TX queue including the ones that caused the error. 829 * Free transmit buffers in upper layer. 830 */ 831 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 832 u32 ctrl; 833 834 desc = macb_tx_desc(queue, tail); 835 ctrl = desc->ctrl; 836 tx_skb = macb_tx_skb(queue, tail); 837 skb = tx_skb->skb; 838 839 if (ctrl & MACB_BIT(TX_USED)) { 840 /* skb is set for the last buffer of the frame */ 841 while (!skb) { 842 macb_tx_unmap(bp, tx_skb); 843 tail++; 844 tx_skb = macb_tx_skb(queue, tail); 845 skb = tx_skb->skb; 846 } 847 848 /* ctrl still refers to the first buffer descriptor 849 * since it's the only one written back by the hardware 850 */ 851 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 852 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 853 macb_tx_ring_wrap(bp, tail), 854 skb->data); 855 bp->dev->stats.tx_packets++; 856 queue->stats.tx_packets++; 857 bp->dev->stats.tx_bytes += skb->len; 858 queue->stats.tx_bytes += skb->len; 859 } 860 } else { 861 /* "Buffers exhausted mid-frame" errors may only happen 862 * if the driver is buggy, so complain loudly about 863 * those. Statistics are updated by hardware. 864 */ 865 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 866 netdev_err(bp->dev, 867 "BUG: TX buffers exhausted mid-frame\n"); 868 869 desc->ctrl = ctrl | MACB_BIT(TX_USED); 870 } 871 872 macb_tx_unmap(bp, tx_skb); 873 } 874 875 /* Set end of TX queue */ 876 desc = macb_tx_desc(queue, 0); 877 macb_set_addr(bp, desc, 0); 878 desc->ctrl = MACB_BIT(TX_USED); 879 880 /* Make descriptor updates visible to hardware */ 881 wmb(); 882 883 /* Reinitialize the TX desc queue */ 884 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 885 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 886 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 887 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 888 #endif 889 /* Make TX ring reflect state of hardware */ 890 queue->tx_head = 0; 891 queue->tx_tail = 0; 892 893 /* Housework before enabling TX IRQ */ 894 macb_writel(bp, TSR, macb_readl(bp, TSR)); 895 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 896 897 /* Now we are ready to start transmission again */ 898 netif_tx_start_all_queues(bp->dev); 899 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 900 901 spin_unlock_irqrestore(&bp->lock, flags); 902 } 903 904 static void macb_tx_interrupt(struct macb_queue *queue) 905 { 906 unsigned int tail; 907 unsigned int head; 908 u32 status; 909 struct macb *bp = queue->bp; 910 u16 queue_index = queue - bp->queues; 911 912 status = macb_readl(bp, TSR); 913 macb_writel(bp, TSR, status); 914 915 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 916 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 917 918 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 919 (unsigned long)status); 920 921 head = queue->tx_head; 922 for (tail = queue->tx_tail; tail != head; tail++) { 923 struct macb_tx_skb *tx_skb; 924 struct sk_buff *skb; 925 struct macb_dma_desc *desc; 926 u32 ctrl; 927 928 desc = macb_tx_desc(queue, tail); 929 930 /* Make hw descriptor updates visible to CPU */ 931 rmb(); 932 933 ctrl = desc->ctrl; 934 935 /* TX_USED bit is only set by hardware on the very first buffer 936 * descriptor of the transmitted frame. 937 */ 938 if (!(ctrl & MACB_BIT(TX_USED))) 939 break; 940 941 /* Process all buffers of the current transmitted frame */ 942 for (;; tail++) { 943 tx_skb = macb_tx_skb(queue, tail); 944 skb = tx_skb->skb; 945 946 /* First, update TX stats if needed */ 947 if (skb) { 948 if (unlikely(skb_shinfo(skb)->tx_flags & 949 SKBTX_HW_TSTAMP) && 950 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 951 /* skb now belongs to timestamp buffer 952 * and will be removed later 953 */ 954 tx_skb->skb = NULL; 955 } 956 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 957 macb_tx_ring_wrap(bp, tail), 958 skb->data); 959 bp->dev->stats.tx_packets++; 960 queue->stats.tx_packets++; 961 bp->dev->stats.tx_bytes += skb->len; 962 queue->stats.tx_bytes += skb->len; 963 } 964 965 /* Now we can safely release resources */ 966 macb_tx_unmap(bp, tx_skb); 967 968 /* skb is set only for the last buffer of the frame. 969 * WARNING: at this point skb has been freed by 970 * macb_tx_unmap(). 971 */ 972 if (skb) 973 break; 974 } 975 } 976 977 queue->tx_tail = tail; 978 if (__netif_subqueue_stopped(bp->dev, queue_index) && 979 CIRC_CNT(queue->tx_head, queue->tx_tail, 980 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 981 netif_wake_subqueue(bp->dev, queue_index); 982 } 983 984 static void gem_rx_refill(struct macb_queue *queue) 985 { 986 unsigned int entry; 987 struct sk_buff *skb; 988 dma_addr_t paddr; 989 struct macb *bp = queue->bp; 990 struct macb_dma_desc *desc; 991 992 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 993 bp->rx_ring_size) > 0) { 994 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 995 996 /* Make hw descriptor updates visible to CPU */ 997 rmb(); 998 999 queue->rx_prepared_head++; 1000 desc = macb_rx_desc(queue, entry); 1001 1002 if (!queue->rx_skbuff[entry]) { 1003 /* allocate sk_buff for this free entry in ring */ 1004 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 1005 if (unlikely(!skb)) { 1006 netdev_err(bp->dev, 1007 "Unable to allocate sk_buff\n"); 1008 break; 1009 } 1010 1011 /* now fill corresponding descriptor entry */ 1012 paddr = dma_map_single(&bp->pdev->dev, skb->data, 1013 bp->rx_buffer_size, 1014 DMA_FROM_DEVICE); 1015 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 1016 dev_kfree_skb(skb); 1017 break; 1018 } 1019 1020 queue->rx_skbuff[entry] = skb; 1021 1022 if (entry == bp->rx_ring_size - 1) 1023 paddr |= MACB_BIT(RX_WRAP); 1024 desc->ctrl = 0; 1025 /* Setting addr clears RX_USED and allows reception, 1026 * make sure ctrl is cleared first to avoid a race. 1027 */ 1028 dma_wmb(); 1029 macb_set_addr(bp, desc, paddr); 1030 1031 /* properly align Ethernet header */ 1032 skb_reserve(skb, NET_IP_ALIGN); 1033 } else { 1034 desc->ctrl = 0; 1035 dma_wmb(); 1036 desc->addr &= ~MACB_BIT(RX_USED); 1037 } 1038 } 1039 1040 /* Make descriptor updates visible to hardware */ 1041 wmb(); 1042 1043 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 1044 queue, queue->rx_prepared_head, queue->rx_tail); 1045 } 1046 1047 /* Mark DMA descriptors from begin up to and not including end as unused */ 1048 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 1049 unsigned int end) 1050 { 1051 unsigned int frag; 1052 1053 for (frag = begin; frag != end; frag++) { 1054 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 1055 1056 desc->addr &= ~MACB_BIT(RX_USED); 1057 } 1058 1059 /* Make descriptor updates visible to hardware */ 1060 wmb(); 1061 1062 /* When this happens, the hardware stats registers for 1063 * whatever caused this is updated, so we don't have to record 1064 * anything. 1065 */ 1066 } 1067 1068 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 1069 int budget) 1070 { 1071 struct macb *bp = queue->bp; 1072 unsigned int len; 1073 unsigned int entry; 1074 struct sk_buff *skb; 1075 struct macb_dma_desc *desc; 1076 int count = 0; 1077 1078 while (count < budget) { 1079 u32 ctrl; 1080 dma_addr_t addr; 1081 bool rxused; 1082 1083 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1084 desc = macb_rx_desc(queue, entry); 1085 1086 /* Make hw descriptor updates visible to CPU */ 1087 rmb(); 1088 1089 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1090 addr = macb_get_addr(bp, desc); 1091 1092 if (!rxused) 1093 break; 1094 1095 /* Ensure ctrl is at least as up-to-date as rxused */ 1096 dma_rmb(); 1097 1098 ctrl = desc->ctrl; 1099 1100 queue->rx_tail++; 1101 count++; 1102 1103 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1104 netdev_err(bp->dev, 1105 "not whole frame pointed by descriptor\n"); 1106 bp->dev->stats.rx_dropped++; 1107 queue->stats.rx_dropped++; 1108 break; 1109 } 1110 skb = queue->rx_skbuff[entry]; 1111 if (unlikely(!skb)) { 1112 netdev_err(bp->dev, 1113 "inconsistent Rx descriptor chain\n"); 1114 bp->dev->stats.rx_dropped++; 1115 queue->stats.rx_dropped++; 1116 break; 1117 } 1118 /* now everything is ready for receiving packet */ 1119 queue->rx_skbuff[entry] = NULL; 1120 len = ctrl & bp->rx_frm_len_mask; 1121 1122 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1123 1124 skb_put(skb, len); 1125 dma_unmap_single(&bp->pdev->dev, addr, 1126 bp->rx_buffer_size, DMA_FROM_DEVICE); 1127 1128 skb->protocol = eth_type_trans(skb, bp->dev); 1129 skb_checksum_none_assert(skb); 1130 if (bp->dev->features & NETIF_F_RXCSUM && 1131 !(bp->dev->flags & IFF_PROMISC) && 1132 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1133 skb->ip_summed = CHECKSUM_UNNECESSARY; 1134 1135 bp->dev->stats.rx_packets++; 1136 queue->stats.rx_packets++; 1137 bp->dev->stats.rx_bytes += skb->len; 1138 queue->stats.rx_bytes += skb->len; 1139 1140 gem_ptp_do_rxstamp(bp, skb, desc); 1141 1142 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1143 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1144 skb->len, skb->csum); 1145 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1146 skb_mac_header(skb), 16, true); 1147 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1148 skb->data, 32, true); 1149 #endif 1150 1151 napi_gro_receive(napi, skb); 1152 } 1153 1154 gem_rx_refill(queue); 1155 1156 return count; 1157 } 1158 1159 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1160 unsigned int first_frag, unsigned int last_frag) 1161 { 1162 unsigned int len; 1163 unsigned int frag; 1164 unsigned int offset; 1165 struct sk_buff *skb; 1166 struct macb_dma_desc *desc; 1167 struct macb *bp = queue->bp; 1168 1169 desc = macb_rx_desc(queue, last_frag); 1170 len = desc->ctrl & bp->rx_frm_len_mask; 1171 1172 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1173 macb_rx_ring_wrap(bp, first_frag), 1174 macb_rx_ring_wrap(bp, last_frag), len); 1175 1176 /* The ethernet header starts NET_IP_ALIGN bytes into the 1177 * first buffer. Since the header is 14 bytes, this makes the 1178 * payload word-aligned. 1179 * 1180 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1181 * the two padding bytes into the skb so that we avoid hitting 1182 * the slowpath in memcpy(), and pull them off afterwards. 1183 */ 1184 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1185 if (!skb) { 1186 bp->dev->stats.rx_dropped++; 1187 for (frag = first_frag; ; frag++) { 1188 desc = macb_rx_desc(queue, frag); 1189 desc->addr &= ~MACB_BIT(RX_USED); 1190 if (frag == last_frag) 1191 break; 1192 } 1193 1194 /* Make descriptor updates visible to hardware */ 1195 wmb(); 1196 1197 return 1; 1198 } 1199 1200 offset = 0; 1201 len += NET_IP_ALIGN; 1202 skb_checksum_none_assert(skb); 1203 skb_put(skb, len); 1204 1205 for (frag = first_frag; ; frag++) { 1206 unsigned int frag_len = bp->rx_buffer_size; 1207 1208 if (offset + frag_len > len) { 1209 if (unlikely(frag != last_frag)) { 1210 dev_kfree_skb_any(skb); 1211 return -1; 1212 } 1213 frag_len = len - offset; 1214 } 1215 skb_copy_to_linear_data_offset(skb, offset, 1216 macb_rx_buffer(queue, frag), 1217 frag_len); 1218 offset += bp->rx_buffer_size; 1219 desc = macb_rx_desc(queue, frag); 1220 desc->addr &= ~MACB_BIT(RX_USED); 1221 1222 if (frag == last_frag) 1223 break; 1224 } 1225 1226 /* Make descriptor updates visible to hardware */ 1227 wmb(); 1228 1229 __skb_pull(skb, NET_IP_ALIGN); 1230 skb->protocol = eth_type_trans(skb, bp->dev); 1231 1232 bp->dev->stats.rx_packets++; 1233 bp->dev->stats.rx_bytes += skb->len; 1234 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1235 skb->len, skb->csum); 1236 napi_gro_receive(napi, skb); 1237 1238 return 0; 1239 } 1240 1241 static inline void macb_init_rx_ring(struct macb_queue *queue) 1242 { 1243 struct macb *bp = queue->bp; 1244 dma_addr_t addr; 1245 struct macb_dma_desc *desc = NULL; 1246 int i; 1247 1248 addr = queue->rx_buffers_dma; 1249 for (i = 0; i < bp->rx_ring_size; i++) { 1250 desc = macb_rx_desc(queue, i); 1251 macb_set_addr(bp, desc, addr); 1252 desc->ctrl = 0; 1253 addr += bp->rx_buffer_size; 1254 } 1255 desc->addr |= MACB_BIT(RX_WRAP); 1256 queue->rx_tail = 0; 1257 } 1258 1259 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1260 int budget) 1261 { 1262 struct macb *bp = queue->bp; 1263 bool reset_rx_queue = false; 1264 int received = 0; 1265 unsigned int tail; 1266 int first_frag = -1; 1267 1268 for (tail = queue->rx_tail; budget > 0; tail++) { 1269 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1270 u32 ctrl; 1271 1272 /* Make hw descriptor updates visible to CPU */ 1273 rmb(); 1274 1275 if (!(desc->addr & MACB_BIT(RX_USED))) 1276 break; 1277 1278 /* Ensure ctrl is at least as up-to-date as addr */ 1279 dma_rmb(); 1280 1281 ctrl = desc->ctrl; 1282 1283 if (ctrl & MACB_BIT(RX_SOF)) { 1284 if (first_frag != -1) 1285 discard_partial_frame(queue, first_frag, tail); 1286 first_frag = tail; 1287 } 1288 1289 if (ctrl & MACB_BIT(RX_EOF)) { 1290 int dropped; 1291 1292 if (unlikely(first_frag == -1)) { 1293 reset_rx_queue = true; 1294 continue; 1295 } 1296 1297 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1298 first_frag = -1; 1299 if (unlikely(dropped < 0)) { 1300 reset_rx_queue = true; 1301 continue; 1302 } 1303 if (!dropped) { 1304 received++; 1305 budget--; 1306 } 1307 } 1308 } 1309 1310 if (unlikely(reset_rx_queue)) { 1311 unsigned long flags; 1312 u32 ctrl; 1313 1314 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1315 1316 spin_lock_irqsave(&bp->lock, flags); 1317 1318 ctrl = macb_readl(bp, NCR); 1319 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1320 1321 macb_init_rx_ring(queue); 1322 queue_writel(queue, RBQP, queue->rx_ring_dma); 1323 1324 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1325 1326 spin_unlock_irqrestore(&bp->lock, flags); 1327 return received; 1328 } 1329 1330 if (first_frag != -1) 1331 queue->rx_tail = first_frag; 1332 else 1333 queue->rx_tail = tail; 1334 1335 return received; 1336 } 1337 1338 static int macb_poll(struct napi_struct *napi, int budget) 1339 { 1340 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1341 struct macb *bp = queue->bp; 1342 int work_done; 1343 u32 status; 1344 1345 status = macb_readl(bp, RSR); 1346 macb_writel(bp, RSR, status); 1347 1348 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1349 (unsigned long)status, budget); 1350 1351 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 1352 if (work_done < budget) { 1353 napi_complete_done(napi, work_done); 1354 1355 /* Packets received while interrupts were disabled */ 1356 status = macb_readl(bp, RSR); 1357 if (status) { 1358 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1359 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1360 napi_reschedule(napi); 1361 } else { 1362 queue_writel(queue, IER, bp->rx_intr_mask); 1363 } 1364 } 1365 1366 /* TODO: Handle errors */ 1367 1368 return work_done; 1369 } 1370 1371 static void macb_hresp_error_task(unsigned long data) 1372 { 1373 struct macb *bp = (struct macb *)data; 1374 struct net_device *dev = bp->dev; 1375 struct macb_queue *queue = bp->queues; 1376 unsigned int q; 1377 u32 ctrl; 1378 1379 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1380 queue_writel(queue, IDR, bp->rx_intr_mask | 1381 MACB_TX_INT_FLAGS | 1382 MACB_BIT(HRESP)); 1383 } 1384 ctrl = macb_readl(bp, NCR); 1385 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1386 macb_writel(bp, NCR, ctrl); 1387 1388 netif_tx_stop_all_queues(dev); 1389 netif_carrier_off(dev); 1390 1391 bp->macbgem_ops.mog_init_rings(bp); 1392 1393 /* Initialize TX and RX buffers */ 1394 macb_init_buffers(bp); 1395 1396 /* Enable interrupts */ 1397 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1398 queue_writel(queue, IER, 1399 bp->rx_intr_mask | 1400 MACB_TX_INT_FLAGS | 1401 MACB_BIT(HRESP)); 1402 1403 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1404 macb_writel(bp, NCR, ctrl); 1405 1406 netif_carrier_on(dev); 1407 netif_tx_start_all_queues(dev); 1408 } 1409 1410 static void macb_tx_restart(struct macb_queue *queue) 1411 { 1412 unsigned int head = queue->tx_head; 1413 unsigned int tail = queue->tx_tail; 1414 struct macb *bp = queue->bp; 1415 1416 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1417 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1418 1419 if (head == tail) 1420 return; 1421 1422 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1423 } 1424 1425 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1426 { 1427 struct macb_queue *queue = dev_id; 1428 struct macb *bp = queue->bp; 1429 struct net_device *dev = bp->dev; 1430 u32 status, ctrl; 1431 1432 status = queue_readl(queue, ISR); 1433 1434 if (unlikely(!status)) 1435 return IRQ_NONE; 1436 1437 spin_lock(&bp->lock); 1438 1439 while (status) { 1440 /* close possible race with dev_close */ 1441 if (unlikely(!netif_running(dev))) { 1442 queue_writel(queue, IDR, -1); 1443 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1444 queue_writel(queue, ISR, -1); 1445 break; 1446 } 1447 1448 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1449 (unsigned int)(queue - bp->queues), 1450 (unsigned long)status); 1451 1452 if (status & bp->rx_intr_mask) { 1453 /* There's no point taking any more interrupts 1454 * until we have processed the buffers. The 1455 * scheduling call may fail if the poll routine 1456 * is already scheduled, so disable interrupts 1457 * now. 1458 */ 1459 queue_writel(queue, IDR, bp->rx_intr_mask); 1460 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1461 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1462 1463 if (napi_schedule_prep(&queue->napi)) { 1464 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1465 __napi_schedule(&queue->napi); 1466 } 1467 } 1468 1469 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1470 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1471 schedule_work(&queue->tx_error_task); 1472 1473 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1474 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1475 1476 break; 1477 } 1478 1479 if (status & MACB_BIT(TCOMP)) 1480 macb_tx_interrupt(queue); 1481 1482 if (status & MACB_BIT(TXUBR)) 1483 macb_tx_restart(queue); 1484 1485 /* Link change detection isn't possible with RMII, so we'll 1486 * add that if/when we get our hands on a full-blown MII PHY. 1487 */ 1488 1489 /* There is a hardware issue under heavy load where DMA can 1490 * stop, this causes endless "used buffer descriptor read" 1491 * interrupts but it can be cleared by re-enabling RX. See 1492 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1493 * section 16.7.4 for details. RXUBR is only enabled for 1494 * these two versions. 1495 */ 1496 if (status & MACB_BIT(RXUBR)) { 1497 ctrl = macb_readl(bp, NCR); 1498 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1499 wmb(); 1500 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1501 1502 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1503 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1504 } 1505 1506 if (status & MACB_BIT(ISR_ROVR)) { 1507 /* We missed at least one packet */ 1508 if (macb_is_gem(bp)) 1509 bp->hw_stats.gem.rx_overruns++; 1510 else 1511 bp->hw_stats.macb.rx_overruns++; 1512 1513 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1514 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1515 } 1516 1517 if (status & MACB_BIT(HRESP)) { 1518 tasklet_schedule(&bp->hresp_err_tasklet); 1519 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1520 1521 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1522 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1523 } 1524 status = queue_readl(queue, ISR); 1525 } 1526 1527 spin_unlock(&bp->lock); 1528 1529 return IRQ_HANDLED; 1530 } 1531 1532 #ifdef CONFIG_NET_POLL_CONTROLLER 1533 /* Polling receive - used by netconsole and other diagnostic tools 1534 * to allow network i/o with interrupts disabled. 1535 */ 1536 static void macb_poll_controller(struct net_device *dev) 1537 { 1538 struct macb *bp = netdev_priv(dev); 1539 struct macb_queue *queue; 1540 unsigned long flags; 1541 unsigned int q; 1542 1543 local_irq_save(flags); 1544 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1545 macb_interrupt(dev->irq, queue); 1546 local_irq_restore(flags); 1547 } 1548 #endif 1549 1550 static unsigned int macb_tx_map(struct macb *bp, 1551 struct macb_queue *queue, 1552 struct sk_buff *skb, 1553 unsigned int hdrlen) 1554 { 1555 dma_addr_t mapping; 1556 unsigned int len, entry, i, tx_head = queue->tx_head; 1557 struct macb_tx_skb *tx_skb = NULL; 1558 struct macb_dma_desc *desc; 1559 unsigned int offset, size, count = 0; 1560 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1561 unsigned int eof = 1, mss_mfs = 0; 1562 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1563 1564 /* LSO */ 1565 if (skb_shinfo(skb)->gso_size != 0) { 1566 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1567 /* UDP - UFO */ 1568 lso_ctrl = MACB_LSO_UFO_ENABLE; 1569 else 1570 /* TCP - TSO */ 1571 lso_ctrl = MACB_LSO_TSO_ENABLE; 1572 } 1573 1574 /* First, map non-paged data */ 1575 len = skb_headlen(skb); 1576 1577 /* first buffer length */ 1578 size = hdrlen; 1579 1580 offset = 0; 1581 while (len) { 1582 entry = macb_tx_ring_wrap(bp, tx_head); 1583 tx_skb = &queue->tx_skb[entry]; 1584 1585 mapping = dma_map_single(&bp->pdev->dev, 1586 skb->data + offset, 1587 size, DMA_TO_DEVICE); 1588 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1589 goto dma_error; 1590 1591 /* Save info to properly release resources */ 1592 tx_skb->skb = NULL; 1593 tx_skb->mapping = mapping; 1594 tx_skb->size = size; 1595 tx_skb->mapped_as_page = false; 1596 1597 len -= size; 1598 offset += size; 1599 count++; 1600 tx_head++; 1601 1602 size = min(len, bp->max_tx_length); 1603 } 1604 1605 /* Then, map paged data from fragments */ 1606 for (f = 0; f < nr_frags; f++) { 1607 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1608 1609 len = skb_frag_size(frag); 1610 offset = 0; 1611 while (len) { 1612 size = min(len, bp->max_tx_length); 1613 entry = macb_tx_ring_wrap(bp, tx_head); 1614 tx_skb = &queue->tx_skb[entry]; 1615 1616 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1617 offset, size, DMA_TO_DEVICE); 1618 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1619 goto dma_error; 1620 1621 /* Save info to properly release resources */ 1622 tx_skb->skb = NULL; 1623 tx_skb->mapping = mapping; 1624 tx_skb->size = size; 1625 tx_skb->mapped_as_page = true; 1626 1627 len -= size; 1628 offset += size; 1629 count++; 1630 tx_head++; 1631 } 1632 } 1633 1634 /* Should never happen */ 1635 if (unlikely(!tx_skb)) { 1636 netdev_err(bp->dev, "BUG! empty skb!\n"); 1637 return 0; 1638 } 1639 1640 /* This is the last buffer of the frame: save socket buffer */ 1641 tx_skb->skb = skb; 1642 1643 /* Update TX ring: update buffer descriptors in reverse order 1644 * to avoid race condition 1645 */ 1646 1647 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1648 * to set the end of TX queue 1649 */ 1650 i = tx_head; 1651 entry = macb_tx_ring_wrap(bp, i); 1652 ctrl = MACB_BIT(TX_USED); 1653 desc = macb_tx_desc(queue, entry); 1654 desc->ctrl = ctrl; 1655 1656 if (lso_ctrl) { 1657 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1658 /* include header and FCS in value given to h/w */ 1659 mss_mfs = skb_shinfo(skb)->gso_size + 1660 skb_transport_offset(skb) + 1661 ETH_FCS_LEN; 1662 else /* TSO */ { 1663 mss_mfs = skb_shinfo(skb)->gso_size; 1664 /* TCP Sequence Number Source Select 1665 * can be set only for TSO 1666 */ 1667 seq_ctrl = 0; 1668 } 1669 } 1670 1671 do { 1672 i--; 1673 entry = macb_tx_ring_wrap(bp, i); 1674 tx_skb = &queue->tx_skb[entry]; 1675 desc = macb_tx_desc(queue, entry); 1676 1677 ctrl = (u32)tx_skb->size; 1678 if (eof) { 1679 ctrl |= MACB_BIT(TX_LAST); 1680 eof = 0; 1681 } 1682 if (unlikely(entry == (bp->tx_ring_size - 1))) 1683 ctrl |= MACB_BIT(TX_WRAP); 1684 1685 /* First descriptor is header descriptor */ 1686 if (i == queue->tx_head) { 1687 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1688 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1689 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1690 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1691 ctrl |= MACB_BIT(TX_NOCRC); 1692 } else 1693 /* Only set MSS/MFS on payload descriptors 1694 * (second or later descriptor) 1695 */ 1696 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1697 1698 /* Set TX buffer descriptor */ 1699 macb_set_addr(bp, desc, tx_skb->mapping); 1700 /* desc->addr must be visible to hardware before clearing 1701 * 'TX_USED' bit in desc->ctrl. 1702 */ 1703 wmb(); 1704 desc->ctrl = ctrl; 1705 } while (i != queue->tx_head); 1706 1707 queue->tx_head = tx_head; 1708 1709 return count; 1710 1711 dma_error: 1712 netdev_err(bp->dev, "TX DMA map failed\n"); 1713 1714 for (i = queue->tx_head; i != tx_head; i++) { 1715 tx_skb = macb_tx_skb(queue, i); 1716 1717 macb_tx_unmap(bp, tx_skb); 1718 } 1719 1720 return 0; 1721 } 1722 1723 static netdev_features_t macb_features_check(struct sk_buff *skb, 1724 struct net_device *dev, 1725 netdev_features_t features) 1726 { 1727 unsigned int nr_frags, f; 1728 unsigned int hdrlen; 1729 1730 /* Validate LSO compatibility */ 1731 1732 /* there is only one buffer */ 1733 if (!skb_is_nonlinear(skb)) 1734 return features; 1735 1736 /* length of header */ 1737 hdrlen = skb_transport_offset(skb); 1738 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1739 hdrlen += tcp_hdrlen(skb); 1740 1741 /* For LSO: 1742 * When software supplies two or more payload buffers all payload buffers 1743 * apart from the last must be a multiple of 8 bytes in size. 1744 */ 1745 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1746 return features & ~MACB_NETIF_LSO; 1747 1748 nr_frags = skb_shinfo(skb)->nr_frags; 1749 /* No need to check last fragment */ 1750 nr_frags--; 1751 for (f = 0; f < nr_frags; f++) { 1752 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1753 1754 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1755 return features & ~MACB_NETIF_LSO; 1756 } 1757 return features; 1758 } 1759 1760 static inline int macb_clear_csum(struct sk_buff *skb) 1761 { 1762 /* no change for packets without checksum offloading */ 1763 if (skb->ip_summed != CHECKSUM_PARTIAL) 1764 return 0; 1765 1766 /* make sure we can modify the header */ 1767 if (unlikely(skb_cow_head(skb, 0))) 1768 return -1; 1769 1770 /* initialize checksum field 1771 * This is required - at least for Zynq, which otherwise calculates 1772 * wrong UDP header checksums for UDP packets with UDP data len <=2 1773 */ 1774 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1775 return 0; 1776 } 1777 1778 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1779 { 1780 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1781 int padlen = ETH_ZLEN - (*skb)->len; 1782 int headroom = skb_headroom(*skb); 1783 int tailroom = skb_tailroom(*skb); 1784 struct sk_buff *nskb; 1785 u32 fcs; 1786 1787 if (!(ndev->features & NETIF_F_HW_CSUM) || 1788 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1789 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1790 return 0; 1791 1792 if (padlen <= 0) { 1793 /* FCS could be appeded to tailroom. */ 1794 if (tailroom >= ETH_FCS_LEN) 1795 goto add_fcs; 1796 /* FCS could be appeded by moving data to headroom. */ 1797 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1798 padlen = 0; 1799 /* No room for FCS, need to reallocate skb. */ 1800 else 1801 padlen = ETH_FCS_LEN; 1802 } else { 1803 /* Add room for FCS. */ 1804 padlen += ETH_FCS_LEN; 1805 } 1806 1807 if (!cloned && headroom + tailroom >= padlen) { 1808 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1809 skb_set_tail_pointer(*skb, (*skb)->len); 1810 } else { 1811 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1812 if (!nskb) 1813 return -ENOMEM; 1814 1815 dev_consume_skb_any(*skb); 1816 *skb = nskb; 1817 } 1818 1819 if (padlen > ETH_FCS_LEN) 1820 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1821 1822 add_fcs: 1823 /* set FCS to packet */ 1824 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1825 fcs = ~fcs; 1826 1827 skb_put_u8(*skb, fcs & 0xff); 1828 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1829 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1830 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1831 1832 return 0; 1833 } 1834 1835 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1836 { 1837 u16 queue_index = skb_get_queue_mapping(skb); 1838 struct macb *bp = netdev_priv(dev); 1839 struct macb_queue *queue = &bp->queues[queue_index]; 1840 unsigned long flags; 1841 unsigned int desc_cnt, nr_frags, frag_size, f; 1842 unsigned int hdrlen; 1843 bool is_lso, is_udp = 0; 1844 netdev_tx_t ret = NETDEV_TX_OK; 1845 1846 if (macb_clear_csum(skb)) { 1847 dev_kfree_skb_any(skb); 1848 return ret; 1849 } 1850 1851 if (macb_pad_and_fcs(&skb, dev)) { 1852 dev_kfree_skb_any(skb); 1853 return ret; 1854 } 1855 1856 is_lso = (skb_shinfo(skb)->gso_size != 0); 1857 1858 if (is_lso) { 1859 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1860 1861 /* length of headers */ 1862 if (is_udp) 1863 /* only queue eth + ip headers separately for UDP */ 1864 hdrlen = skb_transport_offset(skb); 1865 else 1866 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1867 if (skb_headlen(skb) < hdrlen) { 1868 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1869 /* if this is required, would need to copy to single buffer */ 1870 return NETDEV_TX_BUSY; 1871 } 1872 } else 1873 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1874 1875 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1876 netdev_vdbg(bp->dev, 1877 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1878 queue_index, skb->len, skb->head, skb->data, 1879 skb_tail_pointer(skb), skb_end_pointer(skb)); 1880 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1881 skb->data, 16, true); 1882 #endif 1883 1884 /* Count how many TX buffer descriptors are needed to send this 1885 * socket buffer: skb fragments of jumbo frames may need to be 1886 * split into many buffer descriptors. 1887 */ 1888 if (is_lso && (skb_headlen(skb) > hdrlen)) 1889 /* extra header descriptor if also payload in first buffer */ 1890 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1891 else 1892 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1893 nr_frags = skb_shinfo(skb)->nr_frags; 1894 for (f = 0; f < nr_frags; f++) { 1895 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1896 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1897 } 1898 1899 spin_lock_irqsave(&bp->lock, flags); 1900 1901 /* This is a hard error, log it. */ 1902 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1903 bp->tx_ring_size) < desc_cnt) { 1904 netif_stop_subqueue(dev, queue_index); 1905 spin_unlock_irqrestore(&bp->lock, flags); 1906 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1907 queue->tx_head, queue->tx_tail); 1908 return NETDEV_TX_BUSY; 1909 } 1910 1911 /* Map socket buffer for DMA transfer */ 1912 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1913 dev_kfree_skb_any(skb); 1914 goto unlock; 1915 } 1916 1917 /* Make newly initialized descriptor visible to hardware */ 1918 wmb(); 1919 skb_tx_timestamp(skb); 1920 1921 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1922 1923 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1924 netif_stop_subqueue(dev, queue_index); 1925 1926 unlock: 1927 spin_unlock_irqrestore(&bp->lock, flags); 1928 1929 return ret; 1930 } 1931 1932 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1933 { 1934 if (!macb_is_gem(bp)) { 1935 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1936 } else { 1937 bp->rx_buffer_size = size; 1938 1939 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1940 netdev_dbg(bp->dev, 1941 "RX buffer must be multiple of %d bytes, expanding\n", 1942 RX_BUFFER_MULTIPLE); 1943 bp->rx_buffer_size = 1944 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1945 } 1946 } 1947 1948 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1949 bp->dev->mtu, bp->rx_buffer_size); 1950 } 1951 1952 static void gem_free_rx_buffers(struct macb *bp) 1953 { 1954 struct sk_buff *skb; 1955 struct macb_dma_desc *desc; 1956 struct macb_queue *queue; 1957 dma_addr_t addr; 1958 unsigned int q; 1959 int i; 1960 1961 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1962 if (!queue->rx_skbuff) 1963 continue; 1964 1965 for (i = 0; i < bp->rx_ring_size; i++) { 1966 skb = queue->rx_skbuff[i]; 1967 1968 if (!skb) 1969 continue; 1970 1971 desc = macb_rx_desc(queue, i); 1972 addr = macb_get_addr(bp, desc); 1973 1974 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1975 DMA_FROM_DEVICE); 1976 dev_kfree_skb_any(skb); 1977 skb = NULL; 1978 } 1979 1980 kfree(queue->rx_skbuff); 1981 queue->rx_skbuff = NULL; 1982 } 1983 } 1984 1985 static void macb_free_rx_buffers(struct macb *bp) 1986 { 1987 struct macb_queue *queue = &bp->queues[0]; 1988 1989 if (queue->rx_buffers) { 1990 dma_free_coherent(&bp->pdev->dev, 1991 bp->rx_ring_size * bp->rx_buffer_size, 1992 queue->rx_buffers, queue->rx_buffers_dma); 1993 queue->rx_buffers = NULL; 1994 } 1995 } 1996 1997 static void macb_free_consistent(struct macb *bp) 1998 { 1999 struct macb_queue *queue; 2000 unsigned int q; 2001 int size; 2002 2003 bp->macbgem_ops.mog_free_rx_buffers(bp); 2004 2005 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2006 kfree(queue->tx_skb); 2007 queue->tx_skb = NULL; 2008 if (queue->tx_ring) { 2009 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2010 dma_free_coherent(&bp->pdev->dev, size, 2011 queue->tx_ring, queue->tx_ring_dma); 2012 queue->tx_ring = NULL; 2013 } 2014 if (queue->rx_ring) { 2015 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2016 dma_free_coherent(&bp->pdev->dev, size, 2017 queue->rx_ring, queue->rx_ring_dma); 2018 queue->rx_ring = NULL; 2019 } 2020 } 2021 } 2022 2023 static int gem_alloc_rx_buffers(struct macb *bp) 2024 { 2025 struct macb_queue *queue; 2026 unsigned int q; 2027 int size; 2028 2029 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2030 size = bp->rx_ring_size * sizeof(struct sk_buff *); 2031 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 2032 if (!queue->rx_skbuff) 2033 return -ENOMEM; 2034 else 2035 netdev_dbg(bp->dev, 2036 "Allocated %d RX struct sk_buff entries at %p\n", 2037 bp->rx_ring_size, queue->rx_skbuff); 2038 } 2039 return 0; 2040 } 2041 2042 static int macb_alloc_rx_buffers(struct macb *bp) 2043 { 2044 struct macb_queue *queue = &bp->queues[0]; 2045 int size; 2046 2047 size = bp->rx_ring_size * bp->rx_buffer_size; 2048 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 2049 &queue->rx_buffers_dma, GFP_KERNEL); 2050 if (!queue->rx_buffers) 2051 return -ENOMEM; 2052 2053 netdev_dbg(bp->dev, 2054 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 2055 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 2056 return 0; 2057 } 2058 2059 static int macb_alloc_consistent(struct macb *bp) 2060 { 2061 struct macb_queue *queue; 2062 unsigned int q; 2063 int size; 2064 2065 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2066 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2067 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2068 &queue->tx_ring_dma, 2069 GFP_KERNEL); 2070 if (!queue->tx_ring) 2071 goto out_err; 2072 netdev_dbg(bp->dev, 2073 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2074 q, size, (unsigned long)queue->tx_ring_dma, 2075 queue->tx_ring); 2076 2077 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2078 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2079 if (!queue->tx_skb) 2080 goto out_err; 2081 2082 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2083 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2084 &queue->rx_ring_dma, GFP_KERNEL); 2085 if (!queue->rx_ring) 2086 goto out_err; 2087 netdev_dbg(bp->dev, 2088 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2089 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2090 } 2091 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2092 goto out_err; 2093 2094 return 0; 2095 2096 out_err: 2097 macb_free_consistent(bp); 2098 return -ENOMEM; 2099 } 2100 2101 static void gem_init_rings(struct macb *bp) 2102 { 2103 struct macb_queue *queue; 2104 struct macb_dma_desc *desc = NULL; 2105 unsigned int q; 2106 int i; 2107 2108 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2109 for (i = 0; i < bp->tx_ring_size; i++) { 2110 desc = macb_tx_desc(queue, i); 2111 macb_set_addr(bp, desc, 0); 2112 desc->ctrl = MACB_BIT(TX_USED); 2113 } 2114 desc->ctrl |= MACB_BIT(TX_WRAP); 2115 queue->tx_head = 0; 2116 queue->tx_tail = 0; 2117 2118 queue->rx_tail = 0; 2119 queue->rx_prepared_head = 0; 2120 2121 gem_rx_refill(queue); 2122 } 2123 2124 } 2125 2126 static void macb_init_rings(struct macb *bp) 2127 { 2128 int i; 2129 struct macb_dma_desc *desc = NULL; 2130 2131 macb_init_rx_ring(&bp->queues[0]); 2132 2133 for (i = 0; i < bp->tx_ring_size; i++) { 2134 desc = macb_tx_desc(&bp->queues[0], i); 2135 macb_set_addr(bp, desc, 0); 2136 desc->ctrl = MACB_BIT(TX_USED); 2137 } 2138 bp->queues[0].tx_head = 0; 2139 bp->queues[0].tx_tail = 0; 2140 desc->ctrl |= MACB_BIT(TX_WRAP); 2141 } 2142 2143 static void macb_reset_hw(struct macb *bp) 2144 { 2145 struct macb_queue *queue; 2146 unsigned int q; 2147 u32 ctrl = macb_readl(bp, NCR); 2148 2149 /* Disable RX and TX (XXX: Should we halt the transmission 2150 * more gracefully?) 2151 */ 2152 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2153 2154 /* Clear the stats registers (XXX: Update stats first?) */ 2155 ctrl |= MACB_BIT(CLRSTAT); 2156 2157 macb_writel(bp, NCR, ctrl); 2158 2159 /* Clear all status flags */ 2160 macb_writel(bp, TSR, -1); 2161 macb_writel(bp, RSR, -1); 2162 2163 /* Disable all interrupts */ 2164 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2165 queue_writel(queue, IDR, -1); 2166 queue_readl(queue, ISR); 2167 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2168 queue_writel(queue, ISR, -1); 2169 } 2170 } 2171 2172 static u32 gem_mdc_clk_div(struct macb *bp) 2173 { 2174 u32 config; 2175 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2176 2177 if (pclk_hz <= 20000000) 2178 config = GEM_BF(CLK, GEM_CLK_DIV8); 2179 else if (pclk_hz <= 40000000) 2180 config = GEM_BF(CLK, GEM_CLK_DIV16); 2181 else if (pclk_hz <= 80000000) 2182 config = GEM_BF(CLK, GEM_CLK_DIV32); 2183 else if (pclk_hz <= 120000000) 2184 config = GEM_BF(CLK, GEM_CLK_DIV48); 2185 else if (pclk_hz <= 160000000) 2186 config = GEM_BF(CLK, GEM_CLK_DIV64); 2187 else 2188 config = GEM_BF(CLK, GEM_CLK_DIV96); 2189 2190 return config; 2191 } 2192 2193 static u32 macb_mdc_clk_div(struct macb *bp) 2194 { 2195 u32 config; 2196 unsigned long pclk_hz; 2197 2198 if (macb_is_gem(bp)) 2199 return gem_mdc_clk_div(bp); 2200 2201 pclk_hz = clk_get_rate(bp->pclk); 2202 if (pclk_hz <= 20000000) 2203 config = MACB_BF(CLK, MACB_CLK_DIV8); 2204 else if (pclk_hz <= 40000000) 2205 config = MACB_BF(CLK, MACB_CLK_DIV16); 2206 else if (pclk_hz <= 80000000) 2207 config = MACB_BF(CLK, MACB_CLK_DIV32); 2208 else 2209 config = MACB_BF(CLK, MACB_CLK_DIV64); 2210 2211 return config; 2212 } 2213 2214 /* Get the DMA bus width field of the network configuration register that we 2215 * should program. We find the width from decoding the design configuration 2216 * register to find the maximum supported data bus width. 2217 */ 2218 static u32 macb_dbw(struct macb *bp) 2219 { 2220 if (!macb_is_gem(bp)) 2221 return 0; 2222 2223 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2224 case 4: 2225 return GEM_BF(DBW, GEM_DBW128); 2226 case 2: 2227 return GEM_BF(DBW, GEM_DBW64); 2228 case 1: 2229 default: 2230 return GEM_BF(DBW, GEM_DBW32); 2231 } 2232 } 2233 2234 /* Configure the receive DMA engine 2235 * - use the correct receive buffer size 2236 * - set best burst length for DMA operations 2237 * (if not supported by FIFO, it will fallback to default) 2238 * - set both rx/tx packet buffers to full memory size 2239 * These are configurable parameters for GEM. 2240 */ 2241 static void macb_configure_dma(struct macb *bp) 2242 { 2243 struct macb_queue *queue; 2244 u32 buffer_size; 2245 unsigned int q; 2246 u32 dmacfg; 2247 2248 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2249 if (macb_is_gem(bp)) { 2250 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2251 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2252 if (q) 2253 queue_writel(queue, RBQS, buffer_size); 2254 else 2255 dmacfg |= GEM_BF(RXBS, buffer_size); 2256 } 2257 if (bp->dma_burst_length) 2258 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2259 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2260 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2261 2262 if (bp->native_io) 2263 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2264 else 2265 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2266 2267 if (bp->dev->features & NETIF_F_HW_CSUM) 2268 dmacfg |= GEM_BIT(TXCOEN); 2269 else 2270 dmacfg &= ~GEM_BIT(TXCOEN); 2271 2272 dmacfg &= ~GEM_BIT(ADDR64); 2273 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2274 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2275 dmacfg |= GEM_BIT(ADDR64); 2276 #endif 2277 #ifdef CONFIG_MACB_USE_HWSTAMP 2278 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2279 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2280 #endif 2281 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2282 dmacfg); 2283 gem_writel(bp, DMACFG, dmacfg); 2284 } 2285 } 2286 2287 static void macb_init_hw(struct macb *bp) 2288 { 2289 u32 config; 2290 2291 macb_reset_hw(bp); 2292 macb_set_hwaddr(bp); 2293 2294 config = macb_mdc_clk_div(bp); 2295 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2296 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2297 if (bp->caps & MACB_CAPS_JUMBO) 2298 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2299 else 2300 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2301 if (bp->dev->flags & IFF_PROMISC) 2302 config |= MACB_BIT(CAF); /* Copy All Frames */ 2303 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2304 config |= GEM_BIT(RXCOEN); 2305 if (!(bp->dev->flags & IFF_BROADCAST)) 2306 config |= MACB_BIT(NBC); /* No BroadCast */ 2307 config |= macb_dbw(bp); 2308 macb_writel(bp, NCFGR, config); 2309 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2310 gem_writel(bp, JML, bp->jumbo_max_len); 2311 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2312 if (bp->caps & MACB_CAPS_JUMBO) 2313 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2314 2315 macb_configure_dma(bp); 2316 } 2317 2318 /* The hash address register is 64 bits long and takes up two 2319 * locations in the memory map. The least significant bits are stored 2320 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2321 * 2322 * The unicast hash enable and the multicast hash enable bits in the 2323 * network configuration register enable the reception of hash matched 2324 * frames. The destination address is reduced to a 6 bit index into 2325 * the 64 bit hash register using the following hash function. The 2326 * hash function is an exclusive or of every sixth bit of the 2327 * destination address. 2328 * 2329 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2330 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2331 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2332 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2333 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2334 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2335 * 2336 * da[0] represents the least significant bit of the first byte 2337 * received, that is, the multicast/unicast indicator, and da[47] 2338 * represents the most significant bit of the last byte received. If 2339 * the hash index, hi[n], points to a bit that is set in the hash 2340 * register then the frame will be matched according to whether the 2341 * frame is multicast or unicast. A multicast match will be signalled 2342 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2343 * index points to a bit set in the hash register. A unicast match 2344 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2345 * and the hash index points to a bit set in the hash register. To 2346 * receive all multicast frames, the hash register should be set with 2347 * all ones and the multicast hash enable bit should be set in the 2348 * network configuration register. 2349 */ 2350 2351 static inline int hash_bit_value(int bitnr, __u8 *addr) 2352 { 2353 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2354 return 1; 2355 return 0; 2356 } 2357 2358 /* Return the hash index value for the specified address. */ 2359 static int hash_get_index(__u8 *addr) 2360 { 2361 int i, j, bitval; 2362 int hash_index = 0; 2363 2364 for (j = 0; j < 6; j++) { 2365 for (i = 0, bitval = 0; i < 8; i++) 2366 bitval ^= hash_bit_value(i * 6 + j, addr); 2367 2368 hash_index |= (bitval << j); 2369 } 2370 2371 return hash_index; 2372 } 2373 2374 /* Add multicast addresses to the internal multicast-hash table. */ 2375 static void macb_sethashtable(struct net_device *dev) 2376 { 2377 struct netdev_hw_addr *ha; 2378 unsigned long mc_filter[2]; 2379 unsigned int bitnr; 2380 struct macb *bp = netdev_priv(dev); 2381 2382 mc_filter[0] = 0; 2383 mc_filter[1] = 0; 2384 2385 netdev_for_each_mc_addr(ha, dev) { 2386 bitnr = hash_get_index(ha->addr); 2387 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2388 } 2389 2390 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2391 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2392 } 2393 2394 /* Enable/Disable promiscuous and multicast modes. */ 2395 static void macb_set_rx_mode(struct net_device *dev) 2396 { 2397 unsigned long cfg; 2398 struct macb *bp = netdev_priv(dev); 2399 2400 cfg = macb_readl(bp, NCFGR); 2401 2402 if (dev->flags & IFF_PROMISC) { 2403 /* Enable promiscuous mode */ 2404 cfg |= MACB_BIT(CAF); 2405 2406 /* Disable RX checksum offload */ 2407 if (macb_is_gem(bp)) 2408 cfg &= ~GEM_BIT(RXCOEN); 2409 } else { 2410 /* Disable promiscuous mode */ 2411 cfg &= ~MACB_BIT(CAF); 2412 2413 /* Enable RX checksum offload only if requested */ 2414 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2415 cfg |= GEM_BIT(RXCOEN); 2416 } 2417 2418 if (dev->flags & IFF_ALLMULTI) { 2419 /* Enable all multicast mode */ 2420 macb_or_gem_writel(bp, HRB, -1); 2421 macb_or_gem_writel(bp, HRT, -1); 2422 cfg |= MACB_BIT(NCFGR_MTI); 2423 } else if (!netdev_mc_empty(dev)) { 2424 /* Enable specific multicasts */ 2425 macb_sethashtable(dev); 2426 cfg |= MACB_BIT(NCFGR_MTI); 2427 } else if (dev->flags & (~IFF_ALLMULTI)) { 2428 /* Disable all multicast mode */ 2429 macb_or_gem_writel(bp, HRB, 0); 2430 macb_or_gem_writel(bp, HRT, 0); 2431 cfg &= ~MACB_BIT(NCFGR_MTI); 2432 } 2433 2434 macb_writel(bp, NCFGR, cfg); 2435 } 2436 2437 static int macb_open(struct net_device *dev) 2438 { 2439 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2440 struct macb *bp = netdev_priv(dev); 2441 struct macb_queue *queue; 2442 unsigned int q; 2443 int err; 2444 2445 netdev_dbg(bp->dev, "open\n"); 2446 2447 err = pm_runtime_get_sync(&bp->pdev->dev); 2448 if (err < 0) 2449 goto pm_exit; 2450 2451 /* RX buffers initialization */ 2452 macb_init_rx_buffer_size(bp, bufsz); 2453 2454 err = macb_alloc_consistent(bp); 2455 if (err) { 2456 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2457 err); 2458 goto pm_exit; 2459 } 2460 2461 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2462 napi_enable(&queue->napi); 2463 2464 macb_init_hw(bp); 2465 2466 err = macb_phylink_connect(bp); 2467 if (err) 2468 goto pm_exit; 2469 2470 netif_tx_start_all_queues(dev); 2471 2472 if (bp->ptp_info) 2473 bp->ptp_info->ptp_init(dev); 2474 2475 pm_exit: 2476 if (err) { 2477 pm_runtime_put_sync(&bp->pdev->dev); 2478 return err; 2479 } 2480 return 0; 2481 } 2482 2483 static int macb_close(struct net_device *dev) 2484 { 2485 struct macb *bp = netdev_priv(dev); 2486 struct macb_queue *queue; 2487 unsigned long flags; 2488 unsigned int q; 2489 2490 netif_tx_stop_all_queues(dev); 2491 2492 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2493 napi_disable(&queue->napi); 2494 2495 phylink_stop(bp->phylink); 2496 phylink_disconnect_phy(bp->phylink); 2497 2498 spin_lock_irqsave(&bp->lock, flags); 2499 macb_reset_hw(bp); 2500 netif_carrier_off(dev); 2501 spin_unlock_irqrestore(&bp->lock, flags); 2502 2503 macb_free_consistent(bp); 2504 2505 if (bp->ptp_info) 2506 bp->ptp_info->ptp_remove(dev); 2507 2508 pm_runtime_put(&bp->pdev->dev); 2509 2510 return 0; 2511 } 2512 2513 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2514 { 2515 if (netif_running(dev)) 2516 return -EBUSY; 2517 2518 dev->mtu = new_mtu; 2519 2520 return 0; 2521 } 2522 2523 static void gem_update_stats(struct macb *bp) 2524 { 2525 struct macb_queue *queue; 2526 unsigned int i, q, idx; 2527 unsigned long *stat; 2528 2529 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2530 2531 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2532 u32 offset = gem_statistics[i].offset; 2533 u64 val = bp->macb_reg_readl(bp, offset); 2534 2535 bp->ethtool_stats[i] += val; 2536 *p += val; 2537 2538 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2539 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2540 val = bp->macb_reg_readl(bp, offset + 4); 2541 bp->ethtool_stats[i] += ((u64)val) << 32; 2542 *(++p) += val; 2543 } 2544 } 2545 2546 idx = GEM_STATS_LEN; 2547 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2548 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2549 bp->ethtool_stats[idx++] = *stat; 2550 } 2551 2552 static struct net_device_stats *gem_get_stats(struct macb *bp) 2553 { 2554 struct gem_stats *hwstat = &bp->hw_stats.gem; 2555 struct net_device_stats *nstat = &bp->dev->stats; 2556 2557 gem_update_stats(bp); 2558 2559 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2560 hwstat->rx_alignment_errors + 2561 hwstat->rx_resource_errors + 2562 hwstat->rx_overruns + 2563 hwstat->rx_oversize_frames + 2564 hwstat->rx_jabbers + 2565 hwstat->rx_undersized_frames + 2566 hwstat->rx_length_field_frame_errors); 2567 nstat->tx_errors = (hwstat->tx_late_collisions + 2568 hwstat->tx_excessive_collisions + 2569 hwstat->tx_underrun + 2570 hwstat->tx_carrier_sense_errors); 2571 nstat->multicast = hwstat->rx_multicast_frames; 2572 nstat->collisions = (hwstat->tx_single_collision_frames + 2573 hwstat->tx_multiple_collision_frames + 2574 hwstat->tx_excessive_collisions); 2575 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2576 hwstat->rx_jabbers + 2577 hwstat->rx_undersized_frames + 2578 hwstat->rx_length_field_frame_errors); 2579 nstat->rx_over_errors = hwstat->rx_resource_errors; 2580 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2581 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2582 nstat->rx_fifo_errors = hwstat->rx_overruns; 2583 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2584 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2585 nstat->tx_fifo_errors = hwstat->tx_underrun; 2586 2587 return nstat; 2588 } 2589 2590 static void gem_get_ethtool_stats(struct net_device *dev, 2591 struct ethtool_stats *stats, u64 *data) 2592 { 2593 struct macb *bp; 2594 2595 bp = netdev_priv(dev); 2596 gem_update_stats(bp); 2597 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2598 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2599 } 2600 2601 static int gem_get_sset_count(struct net_device *dev, int sset) 2602 { 2603 struct macb *bp = netdev_priv(dev); 2604 2605 switch (sset) { 2606 case ETH_SS_STATS: 2607 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2608 default: 2609 return -EOPNOTSUPP; 2610 } 2611 } 2612 2613 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2614 { 2615 char stat_string[ETH_GSTRING_LEN]; 2616 struct macb *bp = netdev_priv(dev); 2617 struct macb_queue *queue; 2618 unsigned int i; 2619 unsigned int q; 2620 2621 switch (sset) { 2622 case ETH_SS_STATS: 2623 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2624 memcpy(p, gem_statistics[i].stat_string, 2625 ETH_GSTRING_LEN); 2626 2627 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2628 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2629 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2630 q, queue_statistics[i].stat_string); 2631 memcpy(p, stat_string, ETH_GSTRING_LEN); 2632 } 2633 } 2634 break; 2635 } 2636 } 2637 2638 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2639 { 2640 struct macb *bp = netdev_priv(dev); 2641 struct net_device_stats *nstat = &bp->dev->stats; 2642 struct macb_stats *hwstat = &bp->hw_stats.macb; 2643 2644 if (macb_is_gem(bp)) 2645 return gem_get_stats(bp); 2646 2647 /* read stats from hardware */ 2648 macb_update_stats(bp); 2649 2650 /* Convert HW stats into netdevice stats */ 2651 nstat->rx_errors = (hwstat->rx_fcs_errors + 2652 hwstat->rx_align_errors + 2653 hwstat->rx_resource_errors + 2654 hwstat->rx_overruns + 2655 hwstat->rx_oversize_pkts + 2656 hwstat->rx_jabbers + 2657 hwstat->rx_undersize_pkts + 2658 hwstat->rx_length_mismatch); 2659 nstat->tx_errors = (hwstat->tx_late_cols + 2660 hwstat->tx_excessive_cols + 2661 hwstat->tx_underruns + 2662 hwstat->tx_carrier_errors + 2663 hwstat->sqe_test_errors); 2664 nstat->collisions = (hwstat->tx_single_cols + 2665 hwstat->tx_multiple_cols + 2666 hwstat->tx_excessive_cols); 2667 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2668 hwstat->rx_jabbers + 2669 hwstat->rx_undersize_pkts + 2670 hwstat->rx_length_mismatch); 2671 nstat->rx_over_errors = hwstat->rx_resource_errors + 2672 hwstat->rx_overruns; 2673 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2674 nstat->rx_frame_errors = hwstat->rx_align_errors; 2675 nstat->rx_fifo_errors = hwstat->rx_overruns; 2676 /* XXX: What does "missed" mean? */ 2677 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2678 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2679 nstat->tx_fifo_errors = hwstat->tx_underruns; 2680 /* Don't know about heartbeat or window errors... */ 2681 2682 return nstat; 2683 } 2684 2685 static int macb_get_regs_len(struct net_device *netdev) 2686 { 2687 return MACB_GREGS_NBR * sizeof(u32); 2688 } 2689 2690 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2691 void *p) 2692 { 2693 struct macb *bp = netdev_priv(dev); 2694 unsigned int tail, head; 2695 u32 *regs_buff = p; 2696 2697 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2698 | MACB_GREGS_VERSION; 2699 2700 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2701 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2702 2703 regs_buff[0] = macb_readl(bp, NCR); 2704 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2705 regs_buff[2] = macb_readl(bp, NSR); 2706 regs_buff[3] = macb_readl(bp, TSR); 2707 regs_buff[4] = macb_readl(bp, RBQP); 2708 regs_buff[5] = macb_readl(bp, TBQP); 2709 regs_buff[6] = macb_readl(bp, RSR); 2710 regs_buff[7] = macb_readl(bp, IMR); 2711 2712 regs_buff[8] = tail; 2713 regs_buff[9] = head; 2714 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2715 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2716 2717 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2718 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2719 if (macb_is_gem(bp)) 2720 regs_buff[13] = gem_readl(bp, DMACFG); 2721 } 2722 2723 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2724 { 2725 struct macb *bp = netdev_priv(netdev); 2726 2727 wol->supported = 0; 2728 wol->wolopts = 0; 2729 2730 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) 2731 phylink_ethtool_get_wol(bp->phylink, wol); 2732 } 2733 2734 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2735 { 2736 struct macb *bp = netdev_priv(netdev); 2737 int ret; 2738 2739 ret = phylink_ethtool_set_wol(bp->phylink, wol); 2740 if (!ret) 2741 return 0; 2742 2743 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2744 (wol->wolopts & ~WAKE_MAGIC)) 2745 return -EOPNOTSUPP; 2746 2747 if (wol->wolopts & WAKE_MAGIC) 2748 bp->wol |= MACB_WOL_ENABLED; 2749 else 2750 bp->wol &= ~MACB_WOL_ENABLED; 2751 2752 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2753 2754 return 0; 2755 } 2756 2757 static int macb_get_link_ksettings(struct net_device *netdev, 2758 struct ethtool_link_ksettings *kset) 2759 { 2760 struct macb *bp = netdev_priv(netdev); 2761 2762 return phylink_ethtool_ksettings_get(bp->phylink, kset); 2763 } 2764 2765 static int macb_set_link_ksettings(struct net_device *netdev, 2766 const struct ethtool_link_ksettings *kset) 2767 { 2768 struct macb *bp = netdev_priv(netdev); 2769 2770 return phylink_ethtool_ksettings_set(bp->phylink, kset); 2771 } 2772 2773 static void macb_get_ringparam(struct net_device *netdev, 2774 struct ethtool_ringparam *ring) 2775 { 2776 struct macb *bp = netdev_priv(netdev); 2777 2778 ring->rx_max_pending = MAX_RX_RING_SIZE; 2779 ring->tx_max_pending = MAX_TX_RING_SIZE; 2780 2781 ring->rx_pending = bp->rx_ring_size; 2782 ring->tx_pending = bp->tx_ring_size; 2783 } 2784 2785 static int macb_set_ringparam(struct net_device *netdev, 2786 struct ethtool_ringparam *ring) 2787 { 2788 struct macb *bp = netdev_priv(netdev); 2789 u32 new_rx_size, new_tx_size; 2790 unsigned int reset = 0; 2791 2792 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2793 return -EINVAL; 2794 2795 new_rx_size = clamp_t(u32, ring->rx_pending, 2796 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2797 new_rx_size = roundup_pow_of_two(new_rx_size); 2798 2799 new_tx_size = clamp_t(u32, ring->tx_pending, 2800 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2801 new_tx_size = roundup_pow_of_two(new_tx_size); 2802 2803 if ((new_tx_size == bp->tx_ring_size) && 2804 (new_rx_size == bp->rx_ring_size)) { 2805 /* nothing to do */ 2806 return 0; 2807 } 2808 2809 if (netif_running(bp->dev)) { 2810 reset = 1; 2811 macb_close(bp->dev); 2812 } 2813 2814 bp->rx_ring_size = new_rx_size; 2815 bp->tx_ring_size = new_tx_size; 2816 2817 if (reset) 2818 macb_open(bp->dev); 2819 2820 return 0; 2821 } 2822 2823 #ifdef CONFIG_MACB_USE_HWSTAMP 2824 static unsigned int gem_get_tsu_rate(struct macb *bp) 2825 { 2826 struct clk *tsu_clk; 2827 unsigned int tsu_rate; 2828 2829 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2830 if (!IS_ERR(tsu_clk)) 2831 tsu_rate = clk_get_rate(tsu_clk); 2832 /* try pclk instead */ 2833 else if (!IS_ERR(bp->pclk)) { 2834 tsu_clk = bp->pclk; 2835 tsu_rate = clk_get_rate(tsu_clk); 2836 } else 2837 return -ENOTSUPP; 2838 return tsu_rate; 2839 } 2840 2841 static s32 gem_get_ptp_max_adj(void) 2842 { 2843 return 64000000; 2844 } 2845 2846 static int gem_get_ts_info(struct net_device *dev, 2847 struct ethtool_ts_info *info) 2848 { 2849 struct macb *bp = netdev_priv(dev); 2850 2851 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2852 ethtool_op_get_ts_info(dev, info); 2853 return 0; 2854 } 2855 2856 info->so_timestamping = 2857 SOF_TIMESTAMPING_TX_SOFTWARE | 2858 SOF_TIMESTAMPING_RX_SOFTWARE | 2859 SOF_TIMESTAMPING_SOFTWARE | 2860 SOF_TIMESTAMPING_TX_HARDWARE | 2861 SOF_TIMESTAMPING_RX_HARDWARE | 2862 SOF_TIMESTAMPING_RAW_HARDWARE; 2863 info->tx_types = 2864 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2865 (1 << HWTSTAMP_TX_OFF) | 2866 (1 << HWTSTAMP_TX_ON); 2867 info->rx_filters = 2868 (1 << HWTSTAMP_FILTER_NONE) | 2869 (1 << HWTSTAMP_FILTER_ALL); 2870 2871 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2872 2873 return 0; 2874 } 2875 2876 static struct macb_ptp_info gem_ptp_info = { 2877 .ptp_init = gem_ptp_init, 2878 .ptp_remove = gem_ptp_remove, 2879 .get_ptp_max_adj = gem_get_ptp_max_adj, 2880 .get_tsu_rate = gem_get_tsu_rate, 2881 .get_ts_info = gem_get_ts_info, 2882 .get_hwtst = gem_get_hwtst, 2883 .set_hwtst = gem_set_hwtst, 2884 }; 2885 #endif 2886 2887 static int macb_get_ts_info(struct net_device *netdev, 2888 struct ethtool_ts_info *info) 2889 { 2890 struct macb *bp = netdev_priv(netdev); 2891 2892 if (bp->ptp_info) 2893 return bp->ptp_info->get_ts_info(netdev, info); 2894 2895 return ethtool_op_get_ts_info(netdev, info); 2896 } 2897 2898 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2899 { 2900 struct net_device *netdev = bp->dev; 2901 struct ethtool_rx_fs_item *item; 2902 u32 t2_scr; 2903 int num_t2_scr; 2904 2905 if (!(netdev->features & NETIF_F_NTUPLE)) 2906 return; 2907 2908 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2909 2910 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2911 struct ethtool_rx_flow_spec *fs = &item->fs; 2912 struct ethtool_tcpip4_spec *tp4sp_m; 2913 2914 if (fs->location >= num_t2_scr) 2915 continue; 2916 2917 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2918 2919 /* enable/disable screener regs for the flow entry */ 2920 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2921 2922 /* only enable fields with no masking */ 2923 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2924 2925 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2926 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2927 else 2928 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2929 2930 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2931 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2932 else 2933 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2934 2935 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2936 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2937 else 2938 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 2939 2940 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 2941 } 2942 } 2943 2944 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 2945 { 2946 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 2947 uint16_t index = fs->location; 2948 u32 w0, w1, t2_scr; 2949 bool cmp_a = false; 2950 bool cmp_b = false; 2951 bool cmp_c = false; 2952 2953 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 2954 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2955 2956 /* ignore field if any masking set */ 2957 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 2958 /* 1st compare reg - IP source address */ 2959 w0 = 0; 2960 w1 = 0; 2961 w0 = tp4sp_v->ip4src; 2962 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2963 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2964 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 2965 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 2966 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 2967 cmp_a = true; 2968 } 2969 2970 /* ignore field if any masking set */ 2971 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 2972 /* 2nd compare reg - IP destination address */ 2973 w0 = 0; 2974 w1 = 0; 2975 w0 = tp4sp_v->ip4dst; 2976 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2977 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2978 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 2979 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 2980 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 2981 cmp_b = true; 2982 } 2983 2984 /* ignore both port fields if masking set in both */ 2985 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 2986 /* 3rd compare reg - source port, destination port */ 2987 w0 = 0; 2988 w1 = 0; 2989 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 2990 if (tp4sp_m->psrc == tp4sp_m->pdst) { 2991 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 2992 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2993 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2994 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2995 } else { 2996 /* only one port definition */ 2997 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 2998 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 2999 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 3000 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 3001 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3002 } else { /* dst port */ 3003 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3004 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 3005 } 3006 } 3007 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 3008 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 3009 cmp_c = true; 3010 } 3011 3012 t2_scr = 0; 3013 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 3014 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 3015 if (cmp_a) 3016 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 3017 if (cmp_b) 3018 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 3019 if (cmp_c) 3020 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 3021 gem_writel_n(bp, SCRT2, index, t2_scr); 3022 } 3023 3024 static int gem_add_flow_filter(struct net_device *netdev, 3025 struct ethtool_rxnfc *cmd) 3026 { 3027 struct macb *bp = netdev_priv(netdev); 3028 struct ethtool_rx_flow_spec *fs = &cmd->fs; 3029 struct ethtool_rx_fs_item *item, *newfs; 3030 unsigned long flags; 3031 int ret = -EINVAL; 3032 bool added = false; 3033 3034 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 3035 if (newfs == NULL) 3036 return -ENOMEM; 3037 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 3038 3039 netdev_dbg(netdev, 3040 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3041 fs->flow_type, (int)fs->ring_cookie, fs->location, 3042 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3043 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3044 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 3045 3046 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3047 3048 /* find correct place to add in list */ 3049 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3050 if (item->fs.location > newfs->fs.location) { 3051 list_add_tail(&newfs->list, &item->list); 3052 added = true; 3053 break; 3054 } else if (item->fs.location == fs->location) { 3055 netdev_err(netdev, "Rule not added: location %d not free!\n", 3056 fs->location); 3057 ret = -EBUSY; 3058 goto err; 3059 } 3060 } 3061 if (!added) 3062 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3063 3064 gem_prog_cmp_regs(bp, fs); 3065 bp->rx_fs_list.count++; 3066 /* enable filtering if NTUPLE on */ 3067 gem_enable_flow_filters(bp, 1); 3068 3069 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3070 return 0; 3071 3072 err: 3073 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3074 kfree(newfs); 3075 return ret; 3076 } 3077 3078 static int gem_del_flow_filter(struct net_device *netdev, 3079 struct ethtool_rxnfc *cmd) 3080 { 3081 struct macb *bp = netdev_priv(netdev); 3082 struct ethtool_rx_fs_item *item; 3083 struct ethtool_rx_flow_spec *fs; 3084 unsigned long flags; 3085 3086 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3087 3088 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3089 if (item->fs.location == cmd->fs.location) { 3090 /* disable screener regs for the flow entry */ 3091 fs = &(item->fs); 3092 netdev_dbg(netdev, 3093 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3094 fs->flow_type, (int)fs->ring_cookie, fs->location, 3095 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3096 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3097 htons(fs->h_u.tcp_ip4_spec.psrc), 3098 htons(fs->h_u.tcp_ip4_spec.pdst)); 3099 3100 gem_writel_n(bp, SCRT2, fs->location, 0); 3101 3102 list_del(&item->list); 3103 bp->rx_fs_list.count--; 3104 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3105 kfree(item); 3106 return 0; 3107 } 3108 } 3109 3110 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3111 return -EINVAL; 3112 } 3113 3114 static int gem_get_flow_entry(struct net_device *netdev, 3115 struct ethtool_rxnfc *cmd) 3116 { 3117 struct macb *bp = netdev_priv(netdev); 3118 struct ethtool_rx_fs_item *item; 3119 3120 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3121 if (item->fs.location == cmd->fs.location) { 3122 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3123 return 0; 3124 } 3125 } 3126 return -EINVAL; 3127 } 3128 3129 static int gem_get_all_flow_entries(struct net_device *netdev, 3130 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3131 { 3132 struct macb *bp = netdev_priv(netdev); 3133 struct ethtool_rx_fs_item *item; 3134 uint32_t cnt = 0; 3135 3136 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3137 if (cnt == cmd->rule_cnt) 3138 return -EMSGSIZE; 3139 rule_locs[cnt] = item->fs.location; 3140 cnt++; 3141 } 3142 cmd->data = bp->max_tuples; 3143 cmd->rule_cnt = cnt; 3144 3145 return 0; 3146 } 3147 3148 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3149 u32 *rule_locs) 3150 { 3151 struct macb *bp = netdev_priv(netdev); 3152 int ret = 0; 3153 3154 switch (cmd->cmd) { 3155 case ETHTOOL_GRXRINGS: 3156 cmd->data = bp->num_queues; 3157 break; 3158 case ETHTOOL_GRXCLSRLCNT: 3159 cmd->rule_cnt = bp->rx_fs_list.count; 3160 break; 3161 case ETHTOOL_GRXCLSRULE: 3162 ret = gem_get_flow_entry(netdev, cmd); 3163 break; 3164 case ETHTOOL_GRXCLSRLALL: 3165 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3166 break; 3167 default: 3168 netdev_err(netdev, 3169 "Command parameter %d is not supported\n", cmd->cmd); 3170 ret = -EOPNOTSUPP; 3171 } 3172 3173 return ret; 3174 } 3175 3176 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3177 { 3178 struct macb *bp = netdev_priv(netdev); 3179 int ret; 3180 3181 switch (cmd->cmd) { 3182 case ETHTOOL_SRXCLSRLINS: 3183 if ((cmd->fs.location >= bp->max_tuples) 3184 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3185 ret = -EINVAL; 3186 break; 3187 } 3188 ret = gem_add_flow_filter(netdev, cmd); 3189 break; 3190 case ETHTOOL_SRXCLSRLDEL: 3191 ret = gem_del_flow_filter(netdev, cmd); 3192 break; 3193 default: 3194 netdev_err(netdev, 3195 "Command parameter %d is not supported\n", cmd->cmd); 3196 ret = -EOPNOTSUPP; 3197 } 3198 3199 return ret; 3200 } 3201 3202 static const struct ethtool_ops macb_ethtool_ops = { 3203 .get_regs_len = macb_get_regs_len, 3204 .get_regs = macb_get_regs, 3205 .get_link = ethtool_op_get_link, 3206 .get_ts_info = ethtool_op_get_ts_info, 3207 .get_wol = macb_get_wol, 3208 .set_wol = macb_set_wol, 3209 .get_link_ksettings = macb_get_link_ksettings, 3210 .set_link_ksettings = macb_set_link_ksettings, 3211 .get_ringparam = macb_get_ringparam, 3212 .set_ringparam = macb_set_ringparam, 3213 }; 3214 3215 static const struct ethtool_ops gem_ethtool_ops = { 3216 .get_regs_len = macb_get_regs_len, 3217 .get_regs = macb_get_regs, 3218 .get_link = ethtool_op_get_link, 3219 .get_ts_info = macb_get_ts_info, 3220 .get_ethtool_stats = gem_get_ethtool_stats, 3221 .get_strings = gem_get_ethtool_strings, 3222 .get_sset_count = gem_get_sset_count, 3223 .get_link_ksettings = macb_get_link_ksettings, 3224 .set_link_ksettings = macb_set_link_ksettings, 3225 .get_ringparam = macb_get_ringparam, 3226 .set_ringparam = macb_set_ringparam, 3227 .get_rxnfc = gem_get_rxnfc, 3228 .set_rxnfc = gem_set_rxnfc, 3229 }; 3230 3231 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3232 { 3233 struct macb *bp = netdev_priv(dev); 3234 3235 if (!netif_running(dev)) 3236 return -EINVAL; 3237 3238 if (bp->ptp_info) { 3239 switch (cmd) { 3240 case SIOCSHWTSTAMP: 3241 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3242 case SIOCGHWTSTAMP: 3243 return bp->ptp_info->get_hwtst(dev, rq); 3244 } 3245 } 3246 3247 return phylink_mii_ioctl(bp->phylink, rq, cmd); 3248 } 3249 3250 static inline void macb_set_txcsum_feature(struct macb *bp, 3251 netdev_features_t features) 3252 { 3253 u32 val; 3254 3255 if (!macb_is_gem(bp)) 3256 return; 3257 3258 val = gem_readl(bp, DMACFG); 3259 if (features & NETIF_F_HW_CSUM) 3260 val |= GEM_BIT(TXCOEN); 3261 else 3262 val &= ~GEM_BIT(TXCOEN); 3263 3264 gem_writel(bp, DMACFG, val); 3265 } 3266 3267 static inline void macb_set_rxcsum_feature(struct macb *bp, 3268 netdev_features_t features) 3269 { 3270 struct net_device *netdev = bp->dev; 3271 u32 val; 3272 3273 if (!macb_is_gem(bp)) 3274 return; 3275 3276 val = gem_readl(bp, NCFGR); 3277 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 3278 val |= GEM_BIT(RXCOEN); 3279 else 3280 val &= ~GEM_BIT(RXCOEN); 3281 3282 gem_writel(bp, NCFGR, val); 3283 } 3284 3285 static inline void macb_set_rxflow_feature(struct macb *bp, 3286 netdev_features_t features) 3287 { 3288 if (!macb_is_gem(bp)) 3289 return; 3290 3291 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 3292 } 3293 3294 static int macb_set_features(struct net_device *netdev, 3295 netdev_features_t features) 3296 { 3297 struct macb *bp = netdev_priv(netdev); 3298 netdev_features_t changed = features ^ netdev->features; 3299 3300 /* TX checksum offload */ 3301 if (changed & NETIF_F_HW_CSUM) 3302 macb_set_txcsum_feature(bp, features); 3303 3304 /* RX checksum offload */ 3305 if (changed & NETIF_F_RXCSUM) 3306 macb_set_rxcsum_feature(bp, features); 3307 3308 /* RX Flow Filters */ 3309 if (changed & NETIF_F_NTUPLE) 3310 macb_set_rxflow_feature(bp, features); 3311 3312 return 0; 3313 } 3314 3315 static void macb_restore_features(struct macb *bp) 3316 { 3317 struct net_device *netdev = bp->dev; 3318 netdev_features_t features = netdev->features; 3319 3320 /* TX checksum offload */ 3321 macb_set_txcsum_feature(bp, features); 3322 3323 /* RX checksum offload */ 3324 macb_set_rxcsum_feature(bp, features); 3325 3326 /* RX Flow Filters */ 3327 macb_set_rxflow_feature(bp, features); 3328 } 3329 3330 static const struct net_device_ops macb_netdev_ops = { 3331 .ndo_open = macb_open, 3332 .ndo_stop = macb_close, 3333 .ndo_start_xmit = macb_start_xmit, 3334 .ndo_set_rx_mode = macb_set_rx_mode, 3335 .ndo_get_stats = macb_get_stats, 3336 .ndo_do_ioctl = macb_ioctl, 3337 .ndo_validate_addr = eth_validate_addr, 3338 .ndo_change_mtu = macb_change_mtu, 3339 .ndo_set_mac_address = eth_mac_addr, 3340 #ifdef CONFIG_NET_POLL_CONTROLLER 3341 .ndo_poll_controller = macb_poll_controller, 3342 #endif 3343 .ndo_set_features = macb_set_features, 3344 .ndo_features_check = macb_features_check, 3345 }; 3346 3347 /* Configure peripheral capabilities according to device tree 3348 * and integration options used 3349 */ 3350 static void macb_configure_caps(struct macb *bp, 3351 const struct macb_config *dt_conf) 3352 { 3353 u32 dcfg; 3354 3355 if (dt_conf) 3356 bp->caps = dt_conf->caps; 3357 3358 if (hw_is_gem(bp->regs, bp->native_io)) { 3359 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3360 3361 dcfg = gem_readl(bp, DCFG1); 3362 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3363 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3364 dcfg = gem_readl(bp, DCFG2); 3365 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3366 bp->caps |= MACB_CAPS_FIFO_MODE; 3367 #ifdef CONFIG_MACB_USE_HWSTAMP 3368 if (gem_has_ptp(bp)) { 3369 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3370 dev_err(&bp->pdev->dev, 3371 "GEM doesn't support hardware ptp.\n"); 3372 else { 3373 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3374 bp->ptp_info = &gem_ptp_info; 3375 } 3376 } 3377 #endif 3378 } 3379 3380 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3381 } 3382 3383 static void macb_probe_queues(void __iomem *mem, 3384 bool native_io, 3385 unsigned int *queue_mask, 3386 unsigned int *num_queues) 3387 { 3388 unsigned int hw_q; 3389 3390 *queue_mask = 0x1; 3391 *num_queues = 1; 3392 3393 /* is it macb or gem ? 3394 * 3395 * We need to read directly from the hardware here because 3396 * we are early in the probe process and don't have the 3397 * MACB_CAPS_MACB_IS_GEM flag positioned 3398 */ 3399 if (!hw_is_gem(mem, native_io)) 3400 return; 3401 3402 /* bit 0 is never set but queue 0 always exists */ 3403 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3404 3405 *queue_mask |= 0x1; 3406 3407 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3408 if (*queue_mask & (1 << hw_q)) 3409 (*num_queues)++; 3410 } 3411 3412 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3413 struct clk **hclk, struct clk **tx_clk, 3414 struct clk **rx_clk, struct clk **tsu_clk) 3415 { 3416 struct macb_platform_data *pdata; 3417 int err; 3418 3419 pdata = dev_get_platdata(&pdev->dev); 3420 if (pdata) { 3421 *pclk = pdata->pclk; 3422 *hclk = pdata->hclk; 3423 } else { 3424 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3425 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3426 } 3427 3428 if (IS_ERR_OR_NULL(*pclk)) { 3429 err = PTR_ERR(*pclk); 3430 if (!err) 3431 err = -ENODEV; 3432 3433 dev_err(&pdev->dev, "failed to get macb_clk (%d)\n", err); 3434 return err; 3435 } 3436 3437 if (IS_ERR_OR_NULL(*hclk)) { 3438 err = PTR_ERR(*hclk); 3439 if (!err) 3440 err = -ENODEV; 3441 3442 dev_err(&pdev->dev, "failed to get hclk (%d)\n", err); 3443 return err; 3444 } 3445 3446 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); 3447 if (IS_ERR(*tx_clk)) 3448 return PTR_ERR(*tx_clk); 3449 3450 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); 3451 if (IS_ERR(*rx_clk)) 3452 return PTR_ERR(*rx_clk); 3453 3454 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); 3455 if (IS_ERR(*tsu_clk)) 3456 return PTR_ERR(*tsu_clk); 3457 3458 err = clk_prepare_enable(*pclk); 3459 if (err) { 3460 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3461 return err; 3462 } 3463 3464 err = clk_prepare_enable(*hclk); 3465 if (err) { 3466 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 3467 goto err_disable_pclk; 3468 } 3469 3470 err = clk_prepare_enable(*tx_clk); 3471 if (err) { 3472 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 3473 goto err_disable_hclk; 3474 } 3475 3476 err = clk_prepare_enable(*rx_clk); 3477 if (err) { 3478 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 3479 goto err_disable_txclk; 3480 } 3481 3482 err = clk_prepare_enable(*tsu_clk); 3483 if (err) { 3484 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 3485 goto err_disable_rxclk; 3486 } 3487 3488 return 0; 3489 3490 err_disable_rxclk: 3491 clk_disable_unprepare(*rx_clk); 3492 3493 err_disable_txclk: 3494 clk_disable_unprepare(*tx_clk); 3495 3496 err_disable_hclk: 3497 clk_disable_unprepare(*hclk); 3498 3499 err_disable_pclk: 3500 clk_disable_unprepare(*pclk); 3501 3502 return err; 3503 } 3504 3505 static int macb_init(struct platform_device *pdev) 3506 { 3507 struct net_device *dev = platform_get_drvdata(pdev); 3508 unsigned int hw_q, q; 3509 struct macb *bp = netdev_priv(dev); 3510 struct macb_queue *queue; 3511 int err; 3512 u32 val, reg; 3513 3514 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3515 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3516 3517 /* set the queue register mapping once for all: queue0 has a special 3518 * register mapping but we don't want to test the queue index then 3519 * compute the corresponding register offset at run time. 3520 */ 3521 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3522 if (!(bp->queue_mask & (1 << hw_q))) 3523 continue; 3524 3525 queue = &bp->queues[q]; 3526 queue->bp = bp; 3527 netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT); 3528 if (hw_q) { 3529 queue->ISR = GEM_ISR(hw_q - 1); 3530 queue->IER = GEM_IER(hw_q - 1); 3531 queue->IDR = GEM_IDR(hw_q - 1); 3532 queue->IMR = GEM_IMR(hw_q - 1); 3533 queue->TBQP = GEM_TBQP(hw_q - 1); 3534 queue->RBQP = GEM_RBQP(hw_q - 1); 3535 queue->RBQS = GEM_RBQS(hw_q - 1); 3536 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3537 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3538 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3539 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3540 } 3541 #endif 3542 } else { 3543 /* queue0 uses legacy registers */ 3544 queue->ISR = MACB_ISR; 3545 queue->IER = MACB_IER; 3546 queue->IDR = MACB_IDR; 3547 queue->IMR = MACB_IMR; 3548 queue->TBQP = MACB_TBQP; 3549 queue->RBQP = MACB_RBQP; 3550 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3551 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3552 queue->TBQPH = MACB_TBQPH; 3553 queue->RBQPH = MACB_RBQPH; 3554 } 3555 #endif 3556 } 3557 3558 /* get irq: here we use the linux queue index, not the hardware 3559 * queue index. the queue irq definitions in the device tree 3560 * must remove the optional gaps that could exist in the 3561 * hardware queue mask. 3562 */ 3563 queue->irq = platform_get_irq(pdev, q); 3564 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3565 IRQF_SHARED, dev->name, queue); 3566 if (err) { 3567 dev_err(&pdev->dev, 3568 "Unable to request IRQ %d (error %d)\n", 3569 queue->irq, err); 3570 return err; 3571 } 3572 3573 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3574 q++; 3575 } 3576 3577 dev->netdev_ops = &macb_netdev_ops; 3578 3579 /* setup appropriated routines according to adapter type */ 3580 if (macb_is_gem(bp)) { 3581 bp->max_tx_length = GEM_MAX_TX_LEN; 3582 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3583 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3584 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3585 bp->macbgem_ops.mog_rx = gem_rx; 3586 dev->ethtool_ops = &gem_ethtool_ops; 3587 } else { 3588 bp->max_tx_length = MACB_MAX_TX_LEN; 3589 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3590 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3591 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3592 bp->macbgem_ops.mog_rx = macb_rx; 3593 dev->ethtool_ops = &macb_ethtool_ops; 3594 } 3595 3596 /* Set features */ 3597 dev->hw_features = NETIF_F_SG; 3598 3599 /* Check LSO capability */ 3600 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3601 dev->hw_features |= MACB_NETIF_LSO; 3602 3603 /* Checksum offload is only available on gem with packet buffer */ 3604 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3605 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3606 if (bp->caps & MACB_CAPS_SG_DISABLED) 3607 dev->hw_features &= ~NETIF_F_SG; 3608 dev->features = dev->hw_features; 3609 3610 /* Check RX Flow Filters support. 3611 * Max Rx flows set by availability of screeners & compare regs: 3612 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3613 */ 3614 reg = gem_readl(bp, DCFG8); 3615 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3616 GEM_BFEXT(T2SCR, reg)); 3617 if (bp->max_tuples > 0) { 3618 /* also needs one ethtype match to check IPv4 */ 3619 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3620 /* program this reg now */ 3621 reg = 0; 3622 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3623 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3624 /* Filtering is supported in hw but don't enable it in kernel now */ 3625 dev->hw_features |= NETIF_F_NTUPLE; 3626 /* init Rx flow definitions */ 3627 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3628 bp->rx_fs_list.count = 0; 3629 spin_lock_init(&bp->rx_fs_lock); 3630 } else 3631 bp->max_tuples = 0; 3632 } 3633 3634 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3635 val = 0; 3636 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3637 val = GEM_BIT(RGMII); 3638 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3639 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3640 val = MACB_BIT(RMII); 3641 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3642 val = MACB_BIT(MII); 3643 3644 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3645 val |= MACB_BIT(CLKEN); 3646 3647 macb_or_gem_writel(bp, USRIO, val); 3648 } 3649 3650 /* Set MII management clock divider */ 3651 val = macb_mdc_clk_div(bp); 3652 val |= macb_dbw(bp); 3653 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3654 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3655 macb_writel(bp, NCFGR, val); 3656 3657 return 0; 3658 } 3659 3660 #if defined(CONFIG_OF) 3661 /* 1518 rounded up */ 3662 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3663 /* max number of receive buffers */ 3664 #define AT91ETHER_MAX_RX_DESCR 9 3665 3666 static struct sifive_fu540_macb_mgmt *mgmt; 3667 3668 /* Initialize and start the Receiver and Transmit subsystems */ 3669 static int at91ether_start(struct net_device *dev) 3670 { 3671 struct macb *lp = netdev_priv(dev); 3672 struct macb_queue *q = &lp->queues[0]; 3673 struct macb_dma_desc *desc; 3674 dma_addr_t addr; 3675 u32 ctl; 3676 int i; 3677 3678 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3679 (AT91ETHER_MAX_RX_DESCR * 3680 macb_dma_desc_get_size(lp)), 3681 &q->rx_ring_dma, GFP_KERNEL); 3682 if (!q->rx_ring) 3683 return -ENOMEM; 3684 3685 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3686 AT91ETHER_MAX_RX_DESCR * 3687 AT91ETHER_MAX_RBUFF_SZ, 3688 &q->rx_buffers_dma, GFP_KERNEL); 3689 if (!q->rx_buffers) { 3690 dma_free_coherent(&lp->pdev->dev, 3691 AT91ETHER_MAX_RX_DESCR * 3692 macb_dma_desc_get_size(lp), 3693 q->rx_ring, q->rx_ring_dma); 3694 q->rx_ring = NULL; 3695 return -ENOMEM; 3696 } 3697 3698 addr = q->rx_buffers_dma; 3699 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3700 desc = macb_rx_desc(q, i); 3701 macb_set_addr(lp, desc, addr); 3702 desc->ctrl = 0; 3703 addr += AT91ETHER_MAX_RBUFF_SZ; 3704 } 3705 3706 /* Set the Wrap bit on the last descriptor */ 3707 desc->addr |= MACB_BIT(RX_WRAP); 3708 3709 /* Reset buffer index */ 3710 q->rx_tail = 0; 3711 3712 /* Program address of descriptor list in Rx Buffer Queue register */ 3713 macb_writel(lp, RBQP, q->rx_ring_dma); 3714 3715 /* Enable Receive and Transmit */ 3716 ctl = macb_readl(lp, NCR); 3717 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3718 3719 return 0; 3720 } 3721 3722 /* Open the ethernet interface */ 3723 static int at91ether_open(struct net_device *dev) 3724 { 3725 struct macb *lp = netdev_priv(dev); 3726 u32 ctl; 3727 int ret; 3728 3729 /* Clear internal statistics */ 3730 ctl = macb_readl(lp, NCR); 3731 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3732 3733 macb_set_hwaddr(lp); 3734 3735 ret = at91ether_start(dev); 3736 if (ret) 3737 return ret; 3738 3739 /* Enable MAC interrupts */ 3740 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3741 MACB_BIT(RXUBR) | 3742 MACB_BIT(ISR_TUND) | 3743 MACB_BIT(ISR_RLE) | 3744 MACB_BIT(TCOMP) | 3745 MACB_BIT(ISR_ROVR) | 3746 MACB_BIT(HRESP)); 3747 3748 ret = macb_phylink_connect(lp); 3749 if (ret) 3750 return ret; 3751 3752 netif_start_queue(dev); 3753 3754 return 0; 3755 } 3756 3757 /* Close the interface */ 3758 static int at91ether_close(struct net_device *dev) 3759 { 3760 struct macb *lp = netdev_priv(dev); 3761 struct macb_queue *q = &lp->queues[0]; 3762 u32 ctl; 3763 3764 /* Disable Receiver and Transmitter */ 3765 ctl = macb_readl(lp, NCR); 3766 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3767 3768 /* Disable MAC interrupts */ 3769 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3770 MACB_BIT(RXUBR) | 3771 MACB_BIT(ISR_TUND) | 3772 MACB_BIT(ISR_RLE) | 3773 MACB_BIT(TCOMP) | 3774 MACB_BIT(ISR_ROVR) | 3775 MACB_BIT(HRESP)); 3776 3777 netif_stop_queue(dev); 3778 3779 phylink_stop(lp->phylink); 3780 phylink_disconnect_phy(lp->phylink); 3781 3782 dma_free_coherent(&lp->pdev->dev, 3783 AT91ETHER_MAX_RX_DESCR * 3784 macb_dma_desc_get_size(lp), 3785 q->rx_ring, q->rx_ring_dma); 3786 q->rx_ring = NULL; 3787 3788 dma_free_coherent(&lp->pdev->dev, 3789 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3790 q->rx_buffers, q->rx_buffers_dma); 3791 q->rx_buffers = NULL; 3792 3793 return 0; 3794 } 3795 3796 /* Transmit packet */ 3797 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3798 struct net_device *dev) 3799 { 3800 struct macb *lp = netdev_priv(dev); 3801 3802 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3803 netif_stop_queue(dev); 3804 3805 /* Store packet information (to free when Tx completed) */ 3806 lp->skb = skb; 3807 lp->skb_length = skb->len; 3808 lp->skb_physaddr = dma_map_single(&lp->pdev->dev, skb->data, 3809 skb->len, DMA_TO_DEVICE); 3810 if (dma_mapping_error(&lp->pdev->dev, lp->skb_physaddr)) { 3811 dev_kfree_skb_any(skb); 3812 dev->stats.tx_dropped++; 3813 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3814 return NETDEV_TX_OK; 3815 } 3816 3817 /* Set address of the data in the Transmit Address register */ 3818 macb_writel(lp, TAR, lp->skb_physaddr); 3819 /* Set length of the packet in the Transmit Control register */ 3820 macb_writel(lp, TCR, skb->len); 3821 3822 } else { 3823 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3824 return NETDEV_TX_BUSY; 3825 } 3826 3827 return NETDEV_TX_OK; 3828 } 3829 3830 /* Extract received frame from buffer descriptors and sent to upper layers. 3831 * (Called from interrupt context) 3832 */ 3833 static void at91ether_rx(struct net_device *dev) 3834 { 3835 struct macb *lp = netdev_priv(dev); 3836 struct macb_queue *q = &lp->queues[0]; 3837 struct macb_dma_desc *desc; 3838 unsigned char *p_recv; 3839 struct sk_buff *skb; 3840 unsigned int pktlen; 3841 3842 desc = macb_rx_desc(q, q->rx_tail); 3843 while (desc->addr & MACB_BIT(RX_USED)) { 3844 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3845 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3846 skb = netdev_alloc_skb(dev, pktlen + 2); 3847 if (skb) { 3848 skb_reserve(skb, 2); 3849 skb_put_data(skb, p_recv, pktlen); 3850 3851 skb->protocol = eth_type_trans(skb, dev); 3852 dev->stats.rx_packets++; 3853 dev->stats.rx_bytes += pktlen; 3854 netif_rx(skb); 3855 } else { 3856 dev->stats.rx_dropped++; 3857 } 3858 3859 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3860 dev->stats.multicast++; 3861 3862 /* reset ownership bit */ 3863 desc->addr &= ~MACB_BIT(RX_USED); 3864 3865 /* wrap after last buffer */ 3866 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3867 q->rx_tail = 0; 3868 else 3869 q->rx_tail++; 3870 3871 desc = macb_rx_desc(q, q->rx_tail); 3872 } 3873 } 3874 3875 /* MAC interrupt handler */ 3876 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3877 { 3878 struct net_device *dev = dev_id; 3879 struct macb *lp = netdev_priv(dev); 3880 u32 intstatus, ctl; 3881 3882 /* MAC Interrupt Status register indicates what interrupts are pending. 3883 * It is automatically cleared once read. 3884 */ 3885 intstatus = macb_readl(lp, ISR); 3886 3887 /* Receive complete */ 3888 if (intstatus & MACB_BIT(RCOMP)) 3889 at91ether_rx(dev); 3890 3891 /* Transmit complete */ 3892 if (intstatus & MACB_BIT(TCOMP)) { 3893 /* The TCOM bit is set even if the transmission failed */ 3894 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3895 dev->stats.tx_errors++; 3896 3897 if (lp->skb) { 3898 dev_consume_skb_irq(lp->skb); 3899 lp->skb = NULL; 3900 dma_unmap_single(&lp->pdev->dev, lp->skb_physaddr, 3901 lp->skb_length, DMA_TO_DEVICE); 3902 dev->stats.tx_packets++; 3903 dev->stats.tx_bytes += lp->skb_length; 3904 } 3905 netif_wake_queue(dev); 3906 } 3907 3908 /* Work-around for EMAC Errata section 41.3.1 */ 3909 if (intstatus & MACB_BIT(RXUBR)) { 3910 ctl = macb_readl(lp, NCR); 3911 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3912 wmb(); 3913 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3914 } 3915 3916 if (intstatus & MACB_BIT(ISR_ROVR)) 3917 netdev_err(dev, "ROVR error\n"); 3918 3919 return IRQ_HANDLED; 3920 } 3921 3922 #ifdef CONFIG_NET_POLL_CONTROLLER 3923 static void at91ether_poll_controller(struct net_device *dev) 3924 { 3925 unsigned long flags; 3926 3927 local_irq_save(flags); 3928 at91ether_interrupt(dev->irq, dev); 3929 local_irq_restore(flags); 3930 } 3931 #endif 3932 3933 static const struct net_device_ops at91ether_netdev_ops = { 3934 .ndo_open = at91ether_open, 3935 .ndo_stop = at91ether_close, 3936 .ndo_start_xmit = at91ether_start_xmit, 3937 .ndo_get_stats = macb_get_stats, 3938 .ndo_set_rx_mode = macb_set_rx_mode, 3939 .ndo_set_mac_address = eth_mac_addr, 3940 .ndo_do_ioctl = macb_ioctl, 3941 .ndo_validate_addr = eth_validate_addr, 3942 #ifdef CONFIG_NET_POLL_CONTROLLER 3943 .ndo_poll_controller = at91ether_poll_controller, 3944 #endif 3945 }; 3946 3947 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3948 struct clk **hclk, struct clk **tx_clk, 3949 struct clk **rx_clk, struct clk **tsu_clk) 3950 { 3951 int err; 3952 3953 *hclk = NULL; 3954 *tx_clk = NULL; 3955 *rx_clk = NULL; 3956 *tsu_clk = NULL; 3957 3958 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3959 if (IS_ERR(*pclk)) 3960 return PTR_ERR(*pclk); 3961 3962 err = clk_prepare_enable(*pclk); 3963 if (err) { 3964 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3965 return err; 3966 } 3967 3968 return 0; 3969 } 3970 3971 static int at91ether_init(struct platform_device *pdev) 3972 { 3973 struct net_device *dev = platform_get_drvdata(pdev); 3974 struct macb *bp = netdev_priv(dev); 3975 int err; 3976 u32 reg; 3977 3978 bp->queues[0].bp = bp; 3979 3980 dev->netdev_ops = &at91ether_netdev_ops; 3981 dev->ethtool_ops = &macb_ethtool_ops; 3982 3983 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3984 0, dev->name, dev); 3985 if (err) 3986 return err; 3987 3988 macb_writel(bp, NCR, 0); 3989 3990 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3991 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3992 reg |= MACB_BIT(RM9200_RMII); 3993 3994 macb_writel(bp, NCFGR, reg); 3995 3996 return 0; 3997 } 3998 3999 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 4000 unsigned long parent_rate) 4001 { 4002 return mgmt->rate; 4003 } 4004 4005 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, 4006 unsigned long *parent_rate) 4007 { 4008 if (WARN_ON(rate < 2500000)) 4009 return 2500000; 4010 else if (rate == 2500000) 4011 return 2500000; 4012 else if (WARN_ON(rate < 13750000)) 4013 return 2500000; 4014 else if (WARN_ON(rate < 25000000)) 4015 return 25000000; 4016 else if (rate == 25000000) 4017 return 25000000; 4018 else if (WARN_ON(rate < 75000000)) 4019 return 25000000; 4020 else if (WARN_ON(rate < 125000000)) 4021 return 125000000; 4022 else if (rate == 125000000) 4023 return 125000000; 4024 4025 WARN_ON(rate > 125000000); 4026 4027 return 125000000; 4028 } 4029 4030 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 4031 unsigned long parent_rate) 4032 { 4033 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); 4034 if (rate != 125000000) 4035 iowrite32(1, mgmt->reg); 4036 else 4037 iowrite32(0, mgmt->reg); 4038 mgmt->rate = rate; 4039 4040 return 0; 4041 } 4042 4043 static const struct clk_ops fu540_c000_ops = { 4044 .recalc_rate = fu540_macb_tx_recalc_rate, 4045 .round_rate = fu540_macb_tx_round_rate, 4046 .set_rate = fu540_macb_tx_set_rate, 4047 }; 4048 4049 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 4050 struct clk **hclk, struct clk **tx_clk, 4051 struct clk **rx_clk, struct clk **tsu_clk) 4052 { 4053 struct clk_init_data init; 4054 int err = 0; 4055 4056 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 4057 if (err) 4058 return err; 4059 4060 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 4061 if (!mgmt) 4062 return -ENOMEM; 4063 4064 init.name = "sifive-gemgxl-mgmt"; 4065 init.ops = &fu540_c000_ops; 4066 init.flags = 0; 4067 init.num_parents = 0; 4068 4069 mgmt->rate = 0; 4070 mgmt->hw.init = &init; 4071 4072 *tx_clk = clk_register(NULL, &mgmt->hw); 4073 if (IS_ERR(*tx_clk)) 4074 return PTR_ERR(*tx_clk); 4075 4076 err = clk_prepare_enable(*tx_clk); 4077 if (err) 4078 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 4079 else 4080 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 4081 4082 return 0; 4083 } 4084 4085 static int fu540_c000_init(struct platform_device *pdev) 4086 { 4087 struct resource *res; 4088 4089 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 4090 if (!res) 4091 return -ENODEV; 4092 4093 mgmt->reg = ioremap(res->start, resource_size(res)); 4094 if (!mgmt->reg) 4095 return -ENOMEM; 4096 4097 return macb_init(pdev); 4098 } 4099 4100 static const struct macb_config fu540_c000_config = { 4101 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 4102 MACB_CAPS_GEM_HAS_PTP, 4103 .dma_burst_length = 16, 4104 .clk_init = fu540_c000_clk_init, 4105 .init = fu540_c000_init, 4106 .jumbo_max_len = 10240, 4107 }; 4108 4109 static const struct macb_config at91sam9260_config = { 4110 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4111 .clk_init = macb_clk_init, 4112 .init = macb_init, 4113 }; 4114 4115 static const struct macb_config sama5d3macb_config = { 4116 .caps = MACB_CAPS_SG_DISABLED 4117 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4118 .clk_init = macb_clk_init, 4119 .init = macb_init, 4120 }; 4121 4122 static const struct macb_config pc302gem_config = { 4123 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 4124 .dma_burst_length = 16, 4125 .clk_init = macb_clk_init, 4126 .init = macb_init, 4127 }; 4128 4129 static const struct macb_config sama5d2_config = { 4130 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4131 .dma_burst_length = 16, 4132 .clk_init = macb_clk_init, 4133 .init = macb_init, 4134 }; 4135 4136 static const struct macb_config sama5d3_config = { 4137 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 4138 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 4139 .dma_burst_length = 16, 4140 .clk_init = macb_clk_init, 4141 .init = macb_init, 4142 .jumbo_max_len = 10240, 4143 }; 4144 4145 static const struct macb_config sama5d4_config = { 4146 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4147 .dma_burst_length = 4, 4148 .clk_init = macb_clk_init, 4149 .init = macb_init, 4150 }; 4151 4152 static const struct macb_config emac_config = { 4153 .caps = MACB_CAPS_NEEDS_RSTONUBR, 4154 .clk_init = at91ether_clk_init, 4155 .init = at91ether_init, 4156 }; 4157 4158 static const struct macb_config np4_config = { 4159 .caps = MACB_CAPS_USRIO_DISABLED, 4160 .clk_init = macb_clk_init, 4161 .init = macb_init, 4162 }; 4163 4164 static const struct macb_config zynqmp_config = { 4165 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4166 MACB_CAPS_JUMBO | 4167 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 4168 .dma_burst_length = 16, 4169 .clk_init = macb_clk_init, 4170 .init = macb_init, 4171 .jumbo_max_len = 10240, 4172 }; 4173 4174 static const struct macb_config zynq_config = { 4175 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 4176 MACB_CAPS_NEEDS_RSTONUBR, 4177 .dma_burst_length = 16, 4178 .clk_init = macb_clk_init, 4179 .init = macb_init, 4180 }; 4181 4182 static const struct of_device_id macb_dt_ids[] = { 4183 { .compatible = "cdns,at32ap7000-macb" }, 4184 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 4185 { .compatible = "cdns,macb" }, 4186 { .compatible = "cdns,np4-macb", .data = &np4_config }, 4187 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 4188 { .compatible = "cdns,gem", .data = &pc302gem_config }, 4189 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 4190 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 4191 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 4192 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 4193 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 4194 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 4195 { .compatible = "cdns,emac", .data = &emac_config }, 4196 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 4197 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 4198 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 4199 { /* sentinel */ } 4200 }; 4201 MODULE_DEVICE_TABLE(of, macb_dt_ids); 4202 #endif /* CONFIG_OF */ 4203 4204 static const struct macb_config default_gem_config = { 4205 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4206 MACB_CAPS_JUMBO | 4207 MACB_CAPS_GEM_HAS_PTP, 4208 .dma_burst_length = 16, 4209 .clk_init = macb_clk_init, 4210 .init = macb_init, 4211 .jumbo_max_len = 10240, 4212 }; 4213 4214 static int macb_probe(struct platform_device *pdev) 4215 { 4216 const struct macb_config *macb_config = &default_gem_config; 4217 int (*clk_init)(struct platform_device *, struct clk **, 4218 struct clk **, struct clk **, struct clk **, 4219 struct clk **) = macb_config->clk_init; 4220 int (*init)(struct platform_device *) = macb_config->init; 4221 struct device_node *np = pdev->dev.of_node; 4222 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4223 struct clk *tsu_clk = NULL; 4224 unsigned int queue_mask, num_queues; 4225 bool native_io; 4226 phy_interface_t interface; 4227 struct net_device *dev; 4228 struct resource *regs; 4229 void __iomem *mem; 4230 const char *mac; 4231 struct macb *bp; 4232 int err, val; 4233 4234 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4235 mem = devm_ioremap_resource(&pdev->dev, regs); 4236 if (IS_ERR(mem)) 4237 return PTR_ERR(mem); 4238 4239 if (np) { 4240 const struct of_device_id *match; 4241 4242 match = of_match_node(macb_dt_ids, np); 4243 if (match && match->data) { 4244 macb_config = match->data; 4245 clk_init = macb_config->clk_init; 4246 init = macb_config->init; 4247 } 4248 } 4249 4250 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4251 if (err) 4252 return err; 4253 4254 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4255 pm_runtime_use_autosuspend(&pdev->dev); 4256 pm_runtime_get_noresume(&pdev->dev); 4257 pm_runtime_set_active(&pdev->dev); 4258 pm_runtime_enable(&pdev->dev); 4259 native_io = hw_is_native_io(mem); 4260 4261 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4262 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4263 if (!dev) { 4264 err = -ENOMEM; 4265 goto err_disable_clocks; 4266 } 4267 4268 dev->base_addr = regs->start; 4269 4270 SET_NETDEV_DEV(dev, &pdev->dev); 4271 4272 bp = netdev_priv(dev); 4273 bp->pdev = pdev; 4274 bp->dev = dev; 4275 bp->regs = mem; 4276 bp->native_io = native_io; 4277 if (native_io) { 4278 bp->macb_reg_readl = hw_readl_native; 4279 bp->macb_reg_writel = hw_writel_native; 4280 } else { 4281 bp->macb_reg_readl = hw_readl; 4282 bp->macb_reg_writel = hw_writel; 4283 } 4284 bp->num_queues = num_queues; 4285 bp->queue_mask = queue_mask; 4286 if (macb_config) 4287 bp->dma_burst_length = macb_config->dma_burst_length; 4288 bp->pclk = pclk; 4289 bp->hclk = hclk; 4290 bp->tx_clk = tx_clk; 4291 bp->rx_clk = rx_clk; 4292 bp->tsu_clk = tsu_clk; 4293 if (macb_config) 4294 bp->jumbo_max_len = macb_config->jumbo_max_len; 4295 4296 bp->wol = 0; 4297 if (of_get_property(np, "magic-packet", NULL)) 4298 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4299 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4300 4301 spin_lock_init(&bp->lock); 4302 4303 /* setup capabilities */ 4304 macb_configure_caps(bp, macb_config); 4305 4306 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4307 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4308 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4309 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4310 } 4311 #endif 4312 platform_set_drvdata(pdev, dev); 4313 4314 dev->irq = platform_get_irq(pdev, 0); 4315 if (dev->irq < 0) { 4316 err = dev->irq; 4317 goto err_out_free_netdev; 4318 } 4319 4320 /* MTU range: 68 - 1500 or 10240 */ 4321 dev->min_mtu = GEM_MTU_MIN_SIZE; 4322 if (bp->caps & MACB_CAPS_JUMBO) 4323 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4324 else 4325 dev->max_mtu = ETH_DATA_LEN; 4326 4327 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4328 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4329 if (val) 4330 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4331 macb_dma_desc_get_size(bp); 4332 4333 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4334 if (val) 4335 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4336 macb_dma_desc_get_size(bp); 4337 } 4338 4339 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4340 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4341 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4342 4343 mac = of_get_mac_address(np); 4344 if (PTR_ERR(mac) == -EPROBE_DEFER) { 4345 err = -EPROBE_DEFER; 4346 goto err_out_free_netdev; 4347 } else if (!IS_ERR_OR_NULL(mac)) { 4348 ether_addr_copy(bp->dev->dev_addr, mac); 4349 } else { 4350 macb_get_hwaddr(bp); 4351 } 4352 4353 err = of_get_phy_mode(np, &interface); 4354 if (err) 4355 /* not found in DT, MII by default */ 4356 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4357 else 4358 bp->phy_interface = interface; 4359 4360 bp->speed = SPEED_UNKNOWN; 4361 4362 /* IP specific init */ 4363 err = init(pdev); 4364 if (err) 4365 goto err_out_free_netdev; 4366 4367 err = macb_mii_init(bp); 4368 if (err) 4369 goto err_out_free_netdev; 4370 4371 netif_carrier_off(dev); 4372 4373 err = register_netdev(dev); 4374 if (err) { 4375 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4376 goto err_out_unregister_mdio; 4377 } 4378 4379 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4380 (unsigned long)bp); 4381 4382 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4383 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4384 dev->base_addr, dev->irq, dev->dev_addr); 4385 4386 pm_runtime_mark_last_busy(&bp->pdev->dev); 4387 pm_runtime_put_autosuspend(&bp->pdev->dev); 4388 4389 return 0; 4390 4391 err_out_unregister_mdio: 4392 mdiobus_unregister(bp->mii_bus); 4393 mdiobus_free(bp->mii_bus); 4394 4395 err_out_free_netdev: 4396 free_netdev(dev); 4397 4398 err_disable_clocks: 4399 clk_disable_unprepare(tx_clk); 4400 clk_unregister(tx_clk); 4401 clk_disable_unprepare(hclk); 4402 clk_disable_unprepare(pclk); 4403 clk_disable_unprepare(rx_clk); 4404 clk_disable_unprepare(tsu_clk); 4405 pm_runtime_disable(&pdev->dev); 4406 pm_runtime_set_suspended(&pdev->dev); 4407 pm_runtime_dont_use_autosuspend(&pdev->dev); 4408 4409 return err; 4410 } 4411 4412 static int macb_remove(struct platform_device *pdev) 4413 { 4414 struct net_device *dev; 4415 struct macb *bp; 4416 4417 dev = platform_get_drvdata(pdev); 4418 4419 if (dev) { 4420 bp = netdev_priv(dev); 4421 mdiobus_unregister(bp->mii_bus); 4422 mdiobus_free(bp->mii_bus); 4423 4424 unregister_netdev(dev); 4425 tasklet_kill(&bp->hresp_err_tasklet); 4426 pm_runtime_disable(&pdev->dev); 4427 pm_runtime_dont_use_autosuspend(&pdev->dev); 4428 if (!pm_runtime_suspended(&pdev->dev)) { 4429 clk_disable_unprepare(bp->tx_clk); 4430 clk_unregister(bp->tx_clk); 4431 clk_disable_unprepare(bp->hclk); 4432 clk_disable_unprepare(bp->pclk); 4433 clk_disable_unprepare(bp->rx_clk); 4434 clk_disable_unprepare(bp->tsu_clk); 4435 pm_runtime_set_suspended(&pdev->dev); 4436 } 4437 phylink_destroy(bp->phylink); 4438 free_netdev(dev); 4439 } 4440 4441 return 0; 4442 } 4443 4444 static int __maybe_unused macb_suspend(struct device *dev) 4445 { 4446 struct net_device *netdev = dev_get_drvdata(dev); 4447 struct macb *bp = netdev_priv(netdev); 4448 struct macb_queue *queue = bp->queues; 4449 unsigned long flags; 4450 unsigned int q; 4451 4452 if (!netif_running(netdev)) 4453 return 0; 4454 4455 if (bp->wol & MACB_WOL_ENABLED) { 4456 macb_writel(bp, IER, MACB_BIT(WOL)); 4457 macb_writel(bp, WOL, MACB_BIT(MAG)); 4458 enable_irq_wake(bp->queues[0].irq); 4459 netif_device_detach(netdev); 4460 } else { 4461 netif_device_detach(netdev); 4462 for (q = 0, queue = bp->queues; q < bp->num_queues; 4463 ++q, ++queue) 4464 napi_disable(&queue->napi); 4465 rtnl_lock(); 4466 phylink_stop(bp->phylink); 4467 rtnl_unlock(); 4468 spin_lock_irqsave(&bp->lock, flags); 4469 macb_reset_hw(bp); 4470 spin_unlock_irqrestore(&bp->lock, flags); 4471 4472 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4473 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 4474 4475 if (netdev->hw_features & NETIF_F_NTUPLE) 4476 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 4477 } 4478 4479 netif_carrier_off(netdev); 4480 if (bp->ptp_info) 4481 bp->ptp_info->ptp_remove(netdev); 4482 pm_runtime_force_suspend(dev); 4483 4484 return 0; 4485 } 4486 4487 static int __maybe_unused macb_resume(struct device *dev) 4488 { 4489 struct net_device *netdev = dev_get_drvdata(dev); 4490 struct macb *bp = netdev_priv(netdev); 4491 struct macb_queue *queue = bp->queues; 4492 unsigned int q; 4493 4494 if (!netif_running(netdev)) 4495 return 0; 4496 4497 pm_runtime_force_resume(dev); 4498 4499 if (bp->wol & MACB_WOL_ENABLED) { 4500 macb_writel(bp, IDR, MACB_BIT(WOL)); 4501 macb_writel(bp, WOL, 0); 4502 disable_irq_wake(bp->queues[0].irq); 4503 } else { 4504 macb_writel(bp, NCR, MACB_BIT(MPE)); 4505 4506 if (netdev->hw_features & NETIF_F_NTUPLE) 4507 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 4508 4509 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4510 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 4511 4512 for (q = 0, queue = bp->queues; q < bp->num_queues; 4513 ++q, ++queue) 4514 napi_enable(&queue->napi); 4515 rtnl_lock(); 4516 phylink_start(bp->phylink); 4517 rtnl_unlock(); 4518 } 4519 4520 macb_init_hw(bp); 4521 macb_set_rx_mode(netdev); 4522 macb_restore_features(bp); 4523 netif_device_attach(netdev); 4524 if (bp->ptp_info) 4525 bp->ptp_info->ptp_init(netdev); 4526 4527 return 0; 4528 } 4529 4530 static int __maybe_unused macb_runtime_suspend(struct device *dev) 4531 { 4532 struct net_device *netdev = dev_get_drvdata(dev); 4533 struct macb *bp = netdev_priv(netdev); 4534 4535 if (!(device_may_wakeup(&bp->dev->dev))) { 4536 clk_disable_unprepare(bp->tx_clk); 4537 clk_disable_unprepare(bp->hclk); 4538 clk_disable_unprepare(bp->pclk); 4539 clk_disable_unprepare(bp->rx_clk); 4540 } 4541 clk_disable_unprepare(bp->tsu_clk); 4542 4543 return 0; 4544 } 4545 4546 static int __maybe_unused macb_runtime_resume(struct device *dev) 4547 { 4548 struct net_device *netdev = dev_get_drvdata(dev); 4549 struct macb *bp = netdev_priv(netdev); 4550 4551 if (!(device_may_wakeup(&bp->dev->dev))) { 4552 clk_prepare_enable(bp->pclk); 4553 clk_prepare_enable(bp->hclk); 4554 clk_prepare_enable(bp->tx_clk); 4555 clk_prepare_enable(bp->rx_clk); 4556 } 4557 clk_prepare_enable(bp->tsu_clk); 4558 4559 return 0; 4560 } 4561 4562 static const struct dev_pm_ops macb_pm_ops = { 4563 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 4564 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 4565 }; 4566 4567 static struct platform_driver macb_driver = { 4568 .probe = macb_probe, 4569 .remove = macb_remove, 4570 .driver = { 4571 .name = "macb", 4572 .of_match_table = of_match_ptr(macb_dt_ids), 4573 .pm = &macb_pm_ops, 4574 }, 4575 }; 4576 4577 module_platform_driver(macb_driver); 4578 4579 MODULE_LICENSE("GPL"); 4580 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4581 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4582 MODULE_ALIAS("platform:macb"); 4583