1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/clk.h> 10 #include <linux/clk-provider.h> 11 #include <linux/crc32.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/kernel.h> 15 #include <linux/types.h> 16 #include <linux/circ_buf.h> 17 #include <linux/slab.h> 18 #include <linux/init.h> 19 #include <linux/io.h> 20 #include <linux/gpio.h> 21 #include <linux/gpio/consumer.h> 22 #include <linux/interrupt.h> 23 #include <linux/netdevice.h> 24 #include <linux/etherdevice.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/platform_data/macb.h> 27 #include <linux/platform_device.h> 28 #include <linux/phylink.h> 29 #include <linux/of.h> 30 #include <linux/of_device.h> 31 #include <linux/of_gpio.h> 32 #include <linux/of_mdio.h> 33 #include <linux/of_net.h> 34 #include <linux/ip.h> 35 #include <linux/udp.h> 36 #include <linux/tcp.h> 37 #include <linux/iopoll.h> 38 #include <linux/pm_runtime.h> 39 #include "macb.h" 40 41 /* This structure is only used for MACB on SiFive FU540 devices */ 42 struct sifive_fu540_macb_mgmt { 43 void __iomem *reg; 44 unsigned long rate; 45 struct clk_hw hw; 46 }; 47 48 #define MACB_RX_BUFFER_SIZE 128 49 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 50 51 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 52 #define MIN_RX_RING_SIZE 64 53 #define MAX_RX_RING_SIZE 8192 54 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 55 * (bp)->rx_ring_size) 56 57 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 58 #define MIN_TX_RING_SIZE 64 59 #define MAX_TX_RING_SIZE 4096 60 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 61 * (bp)->tx_ring_size) 62 63 /* level of occupied TX descriptors under which we wake up TX process */ 64 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 65 66 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 67 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 68 | MACB_BIT(ISR_RLE) \ 69 | MACB_BIT(TXERR)) 70 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 71 | MACB_BIT(TXUBR)) 72 73 /* Max length of transmit frame must be a multiple of 8 bytes */ 74 #define MACB_TX_LEN_ALIGN 8 75 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 76 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 77 78 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 79 #define MACB_NETIF_LSO NETIF_F_TSO 80 81 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 82 #define MACB_WOL_ENABLED (0x1 << 1) 83 84 /* Graceful stop timeouts in us. We should allow up to 85 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 86 */ 87 #define MACB_HALT_TIMEOUT 1230 88 89 #define MACB_PM_TIMEOUT 100 /* ms */ 90 91 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 92 93 /* DMA buffer descriptor might be different size 94 * depends on hardware configuration: 95 * 96 * 1. dma address width 32 bits: 97 * word 1: 32 bit address of Data Buffer 98 * word 2: control 99 * 100 * 2. dma address width 64 bits: 101 * word 1: 32 bit address of Data Buffer 102 * word 2: control 103 * word 3: upper 32 bit address of Data Buffer 104 * word 4: unused 105 * 106 * 3. dma address width 32 bits with hardware timestamping: 107 * word 1: 32 bit address of Data Buffer 108 * word 2: control 109 * word 3: timestamp word 1 110 * word 4: timestamp word 2 111 * 112 * 4. dma address width 64 bits with hardware timestamping: 113 * word 1: 32 bit address of Data Buffer 114 * word 2: control 115 * word 3: upper 32 bit address of Data Buffer 116 * word 4: unused 117 * word 5: timestamp word 1 118 * word 6: timestamp word 2 119 */ 120 static unsigned int macb_dma_desc_get_size(struct macb *bp) 121 { 122 #ifdef MACB_EXT_DESC 123 unsigned int desc_size; 124 125 switch (bp->hw_dma_cap) { 126 case HW_DMA_CAP_64B: 127 desc_size = sizeof(struct macb_dma_desc) 128 + sizeof(struct macb_dma_desc_64); 129 break; 130 case HW_DMA_CAP_PTP: 131 desc_size = sizeof(struct macb_dma_desc) 132 + sizeof(struct macb_dma_desc_ptp); 133 break; 134 case HW_DMA_CAP_64B_PTP: 135 desc_size = sizeof(struct macb_dma_desc) 136 + sizeof(struct macb_dma_desc_64) 137 + sizeof(struct macb_dma_desc_ptp); 138 break; 139 default: 140 desc_size = sizeof(struct macb_dma_desc); 141 } 142 return desc_size; 143 #endif 144 return sizeof(struct macb_dma_desc); 145 } 146 147 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 148 { 149 #ifdef MACB_EXT_DESC 150 switch (bp->hw_dma_cap) { 151 case HW_DMA_CAP_64B: 152 case HW_DMA_CAP_PTP: 153 desc_idx <<= 1; 154 break; 155 case HW_DMA_CAP_64B_PTP: 156 desc_idx *= 3; 157 break; 158 default: 159 break; 160 } 161 #endif 162 return desc_idx; 163 } 164 165 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 166 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 167 { 168 return (struct macb_dma_desc_64 *)((void *)desc 169 + sizeof(struct macb_dma_desc)); 170 } 171 #endif 172 173 /* Ring buffer accessors */ 174 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 175 { 176 return index & (bp->tx_ring_size - 1); 177 } 178 179 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 180 unsigned int index) 181 { 182 index = macb_tx_ring_wrap(queue->bp, index); 183 index = macb_adj_dma_desc_idx(queue->bp, index); 184 return &queue->tx_ring[index]; 185 } 186 187 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 188 unsigned int index) 189 { 190 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 191 } 192 193 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 194 { 195 dma_addr_t offset; 196 197 offset = macb_tx_ring_wrap(queue->bp, index) * 198 macb_dma_desc_get_size(queue->bp); 199 200 return queue->tx_ring_dma + offset; 201 } 202 203 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 204 { 205 return index & (bp->rx_ring_size - 1); 206 } 207 208 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 209 { 210 index = macb_rx_ring_wrap(queue->bp, index); 211 index = macb_adj_dma_desc_idx(queue->bp, index); 212 return &queue->rx_ring[index]; 213 } 214 215 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 216 { 217 return queue->rx_buffers + queue->bp->rx_buffer_size * 218 macb_rx_ring_wrap(queue->bp, index); 219 } 220 221 /* I/O accessors */ 222 static u32 hw_readl_native(struct macb *bp, int offset) 223 { 224 return __raw_readl(bp->regs + offset); 225 } 226 227 static void hw_writel_native(struct macb *bp, int offset, u32 value) 228 { 229 __raw_writel(value, bp->regs + offset); 230 } 231 232 static u32 hw_readl(struct macb *bp, int offset) 233 { 234 return readl_relaxed(bp->regs + offset); 235 } 236 237 static void hw_writel(struct macb *bp, int offset, u32 value) 238 { 239 writel_relaxed(value, bp->regs + offset); 240 } 241 242 /* Find the CPU endianness by using the loopback bit of NCR register. When the 243 * CPU is in big endian we need to program swapped mode for management 244 * descriptor access. 245 */ 246 static bool hw_is_native_io(void __iomem *addr) 247 { 248 u32 value = MACB_BIT(LLB); 249 250 __raw_writel(value, addr + MACB_NCR); 251 value = __raw_readl(addr + MACB_NCR); 252 253 /* Write 0 back to disable everything */ 254 __raw_writel(0, addr + MACB_NCR); 255 256 return value == MACB_BIT(LLB); 257 } 258 259 static bool hw_is_gem(void __iomem *addr, bool native_io) 260 { 261 u32 id; 262 263 if (native_io) 264 id = __raw_readl(addr + MACB_MID); 265 else 266 id = readl_relaxed(addr + MACB_MID); 267 268 return MACB_BFEXT(IDNUM, id) >= 0x2; 269 } 270 271 static void macb_set_hwaddr(struct macb *bp) 272 { 273 u32 bottom; 274 u16 top; 275 276 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 277 macb_or_gem_writel(bp, SA1B, bottom); 278 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 279 macb_or_gem_writel(bp, SA1T, top); 280 281 /* Clear unused address register sets */ 282 macb_or_gem_writel(bp, SA2B, 0); 283 macb_or_gem_writel(bp, SA2T, 0); 284 macb_or_gem_writel(bp, SA3B, 0); 285 macb_or_gem_writel(bp, SA3T, 0); 286 macb_or_gem_writel(bp, SA4B, 0); 287 macb_or_gem_writel(bp, SA4T, 0); 288 } 289 290 static void macb_get_hwaddr(struct macb *bp) 291 { 292 u32 bottom; 293 u16 top; 294 u8 addr[6]; 295 int i; 296 297 /* Check all 4 address register for valid address */ 298 for (i = 0; i < 4; i++) { 299 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 300 top = macb_or_gem_readl(bp, SA1T + i * 8); 301 302 addr[0] = bottom & 0xff; 303 addr[1] = (bottom >> 8) & 0xff; 304 addr[2] = (bottom >> 16) & 0xff; 305 addr[3] = (bottom >> 24) & 0xff; 306 addr[4] = top & 0xff; 307 addr[5] = (top >> 8) & 0xff; 308 309 if (is_valid_ether_addr(addr)) { 310 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 311 return; 312 } 313 } 314 315 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 316 eth_hw_addr_random(bp->dev); 317 } 318 319 static int macb_mdio_wait_for_idle(struct macb *bp) 320 { 321 u32 val; 322 323 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 324 1, MACB_MDIO_TIMEOUT); 325 } 326 327 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 328 { 329 struct macb *bp = bus->priv; 330 int status; 331 332 status = pm_runtime_get_sync(&bp->pdev->dev); 333 if (status < 0) 334 goto mdio_pm_exit; 335 336 status = macb_mdio_wait_for_idle(bp); 337 if (status < 0) 338 goto mdio_read_exit; 339 340 if (regnum & MII_ADDR_C45) { 341 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 342 | MACB_BF(RW, MACB_MAN_C45_ADDR) 343 | MACB_BF(PHYA, mii_id) 344 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 345 | MACB_BF(DATA, regnum & 0xFFFF) 346 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 347 348 status = macb_mdio_wait_for_idle(bp); 349 if (status < 0) 350 goto mdio_read_exit; 351 352 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 353 | MACB_BF(RW, MACB_MAN_C45_READ) 354 | MACB_BF(PHYA, mii_id) 355 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 356 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 357 } else { 358 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 359 | MACB_BF(RW, MACB_MAN_C22_READ) 360 | MACB_BF(PHYA, mii_id) 361 | MACB_BF(REGA, regnum) 362 | MACB_BF(CODE, MACB_MAN_C22_CODE))); 363 } 364 365 status = macb_mdio_wait_for_idle(bp); 366 if (status < 0) 367 goto mdio_read_exit; 368 369 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 370 371 mdio_read_exit: 372 pm_runtime_mark_last_busy(&bp->pdev->dev); 373 pm_runtime_put_autosuspend(&bp->pdev->dev); 374 mdio_pm_exit: 375 return status; 376 } 377 378 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 379 u16 value) 380 { 381 struct macb *bp = bus->priv; 382 int status; 383 384 status = pm_runtime_get_sync(&bp->pdev->dev); 385 if (status < 0) 386 goto mdio_pm_exit; 387 388 status = macb_mdio_wait_for_idle(bp); 389 if (status < 0) 390 goto mdio_write_exit; 391 392 if (regnum & MII_ADDR_C45) { 393 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 394 | MACB_BF(RW, MACB_MAN_C45_ADDR) 395 | MACB_BF(PHYA, mii_id) 396 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 397 | MACB_BF(DATA, regnum & 0xFFFF) 398 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 399 400 status = macb_mdio_wait_for_idle(bp); 401 if (status < 0) 402 goto mdio_write_exit; 403 404 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 405 | MACB_BF(RW, MACB_MAN_C45_WRITE) 406 | MACB_BF(PHYA, mii_id) 407 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 408 | MACB_BF(CODE, MACB_MAN_C45_CODE) 409 | MACB_BF(DATA, value))); 410 } else { 411 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 412 | MACB_BF(RW, MACB_MAN_C22_WRITE) 413 | MACB_BF(PHYA, mii_id) 414 | MACB_BF(REGA, regnum) 415 | MACB_BF(CODE, MACB_MAN_C22_CODE) 416 | MACB_BF(DATA, value))); 417 } 418 419 status = macb_mdio_wait_for_idle(bp); 420 if (status < 0) 421 goto mdio_write_exit; 422 423 mdio_write_exit: 424 pm_runtime_mark_last_busy(&bp->pdev->dev); 425 pm_runtime_put_autosuspend(&bp->pdev->dev); 426 mdio_pm_exit: 427 return status; 428 } 429 430 static void macb_init_buffers(struct macb *bp) 431 { 432 struct macb_queue *queue; 433 unsigned int q; 434 435 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 436 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 437 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 438 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 439 queue_writel(queue, RBQPH, 440 upper_32_bits(queue->rx_ring_dma)); 441 #endif 442 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 443 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 444 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 445 queue_writel(queue, TBQPH, 446 upper_32_bits(queue->tx_ring_dma)); 447 #endif 448 } 449 } 450 451 /** 452 * macb_set_tx_clk() - Set a clock to a new frequency 453 * @clk Pointer to the clock to change 454 * @rate New frequency in Hz 455 * @dev Pointer to the struct net_device 456 */ 457 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 458 { 459 long ferr, rate, rate_rounded; 460 461 if (!clk) 462 return; 463 464 switch (speed) { 465 case SPEED_10: 466 rate = 2500000; 467 break; 468 case SPEED_100: 469 rate = 25000000; 470 break; 471 case SPEED_1000: 472 rate = 125000000; 473 break; 474 default: 475 return; 476 } 477 478 rate_rounded = clk_round_rate(clk, rate); 479 if (rate_rounded < 0) 480 return; 481 482 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 483 * is not satisfied. 484 */ 485 ferr = abs(rate_rounded - rate); 486 ferr = DIV_ROUND_UP(ferr, rate / 100000); 487 if (ferr > 5) 488 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 489 rate); 490 491 if (clk_set_rate(clk, rate_rounded)) 492 netdev_err(dev, "adjusting tx_clk failed.\n"); 493 } 494 495 static void macb_validate(struct phylink_config *config, 496 unsigned long *supported, 497 struct phylink_link_state *state) 498 { 499 struct net_device *ndev = to_net_dev(config->dev); 500 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; 501 struct macb *bp = netdev_priv(ndev); 502 503 /* We only support MII, RMII, GMII, RGMII & SGMII. */ 504 if (state->interface != PHY_INTERFACE_MODE_NA && 505 state->interface != PHY_INTERFACE_MODE_MII && 506 state->interface != PHY_INTERFACE_MODE_RMII && 507 state->interface != PHY_INTERFACE_MODE_GMII && 508 state->interface != PHY_INTERFACE_MODE_SGMII && 509 !phy_interface_mode_is_rgmii(state->interface)) { 510 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); 511 return; 512 } 513 514 if (!macb_is_gem(bp) && 515 (state->interface == PHY_INTERFACE_MODE_GMII || 516 phy_interface_mode_is_rgmii(state->interface))) { 517 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); 518 return; 519 } 520 521 phylink_set_port_modes(mask); 522 phylink_set(mask, Autoneg); 523 phylink_set(mask, Asym_Pause); 524 525 phylink_set(mask, 10baseT_Half); 526 phylink_set(mask, 10baseT_Full); 527 phylink_set(mask, 100baseT_Half); 528 phylink_set(mask, 100baseT_Full); 529 530 if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && 531 (state->interface == PHY_INTERFACE_MODE_NA || 532 state->interface == PHY_INTERFACE_MODE_GMII || 533 state->interface == PHY_INTERFACE_MODE_SGMII || 534 phy_interface_mode_is_rgmii(state->interface))) { 535 phylink_set(mask, 1000baseT_Full); 536 phylink_set(mask, 1000baseX_Full); 537 538 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) 539 phylink_set(mask, 1000baseT_Half); 540 } 541 542 bitmap_and(supported, supported, mask, __ETHTOOL_LINK_MODE_MASK_NBITS); 543 bitmap_and(state->advertising, state->advertising, mask, 544 __ETHTOOL_LINK_MODE_MASK_NBITS); 545 } 546 547 static void macb_mac_pcs_get_state(struct phylink_config *config, 548 struct phylink_link_state *state) 549 { 550 state->link = 0; 551 } 552 553 static void macb_mac_an_restart(struct phylink_config *config) 554 { 555 /* Not supported */ 556 } 557 558 static void macb_mac_config(struct phylink_config *config, unsigned int mode, 559 const struct phylink_link_state *state) 560 { 561 struct net_device *ndev = to_net_dev(config->dev); 562 struct macb *bp = netdev_priv(ndev); 563 unsigned long flags; 564 u32 old_ctrl, ctrl; 565 566 spin_lock_irqsave(&bp->lock, flags); 567 568 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); 569 570 /* Clear all the bits we might set later */ 571 ctrl &= ~(GEM_BIT(GBE) | MACB_BIT(SPD) | MACB_BIT(FD) | MACB_BIT(PAE) | 572 GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); 573 574 if (state->speed == SPEED_1000) 575 ctrl |= GEM_BIT(GBE); 576 else if (state->speed == SPEED_100) 577 ctrl |= MACB_BIT(SPD); 578 579 if (state->duplex) 580 ctrl |= MACB_BIT(FD); 581 582 /* We do not support MLO_PAUSE_RX yet */ 583 if (state->pause & MLO_PAUSE_TX) 584 ctrl |= MACB_BIT(PAE); 585 586 if (state->interface == PHY_INTERFACE_MODE_SGMII) 587 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 588 589 /* Apply the new configuration, if any */ 590 if (old_ctrl ^ ctrl) 591 macb_or_gem_writel(bp, NCFGR, ctrl); 592 593 bp->speed = state->speed; 594 595 spin_unlock_irqrestore(&bp->lock, flags); 596 } 597 598 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, 599 phy_interface_t interface) 600 { 601 struct net_device *ndev = to_net_dev(config->dev); 602 struct macb *bp = netdev_priv(ndev); 603 struct macb_queue *queue; 604 unsigned int q; 605 u32 ctrl; 606 607 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 608 queue_writel(queue, IDR, 609 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 610 611 /* Disable Rx and Tx */ 612 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); 613 macb_writel(bp, NCR, ctrl); 614 615 netif_tx_stop_all_queues(ndev); 616 } 617 618 static void macb_mac_link_up(struct phylink_config *config, unsigned int mode, 619 phy_interface_t interface, struct phy_device *phy) 620 { 621 struct net_device *ndev = to_net_dev(config->dev); 622 struct macb *bp = netdev_priv(ndev); 623 struct macb_queue *queue; 624 unsigned int q; 625 626 macb_set_tx_clk(bp->tx_clk, bp->speed, ndev); 627 628 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down 629 * cleared the pipeline and control registers. 630 */ 631 bp->macbgem_ops.mog_init_rings(bp); 632 macb_init_buffers(bp); 633 634 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 635 queue_writel(queue, IER, 636 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 637 638 /* Enable Rx and Tx */ 639 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 640 641 netif_tx_wake_all_queues(ndev); 642 } 643 644 static const struct phylink_mac_ops macb_phylink_ops = { 645 .validate = macb_validate, 646 .mac_pcs_get_state = macb_mac_pcs_get_state, 647 .mac_an_restart = macb_mac_an_restart, 648 .mac_config = macb_mac_config, 649 .mac_link_down = macb_mac_link_down, 650 .mac_link_up = macb_mac_link_up, 651 }; 652 653 static bool macb_phy_handle_exists(struct device_node *dn) 654 { 655 dn = of_parse_phandle(dn, "phy-handle", 0); 656 of_node_put(dn); 657 return dn != NULL; 658 } 659 660 static int macb_phylink_connect(struct macb *bp) 661 { 662 struct device_node *dn = bp->pdev->dev.of_node; 663 struct net_device *dev = bp->dev; 664 struct phy_device *phydev; 665 int ret; 666 667 if (dn) 668 ret = phylink_of_phy_connect(bp->phylink, dn, 0); 669 670 if (!dn || (ret && !macb_phy_handle_exists(dn))) { 671 phydev = phy_find_first(bp->mii_bus); 672 if (!phydev) { 673 netdev_err(dev, "no PHY found\n"); 674 return -ENXIO; 675 } 676 677 /* attach the mac to the phy */ 678 ret = phylink_connect_phy(bp->phylink, phydev); 679 } 680 681 if (ret) { 682 netdev_err(dev, "Could not attach PHY (%d)\n", ret); 683 return ret; 684 } 685 686 phylink_start(bp->phylink); 687 688 return 0; 689 } 690 691 /* based on au1000_eth. c*/ 692 static int macb_mii_probe(struct net_device *dev) 693 { 694 struct macb *bp = netdev_priv(dev); 695 696 bp->phylink_config.dev = &dev->dev; 697 bp->phylink_config.type = PHYLINK_NETDEV; 698 699 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, 700 bp->phy_interface, &macb_phylink_ops); 701 if (IS_ERR(bp->phylink)) { 702 netdev_err(dev, "Could not create a phylink instance (%ld)\n", 703 PTR_ERR(bp->phylink)); 704 return PTR_ERR(bp->phylink); 705 } 706 707 return 0; 708 } 709 710 static int macb_mdiobus_register(struct macb *bp) 711 { 712 struct device_node *child, *np = bp->pdev->dev.of_node; 713 714 /* Only create the PHY from the device tree if at least one PHY is 715 * described. Otherwise scan the entire MDIO bus. We do this to support 716 * old device tree that did not follow the best practices and did not 717 * describe their network PHYs. 718 */ 719 for_each_available_child_of_node(np, child) 720 if (of_mdiobus_child_is_phy(child)) { 721 /* The loop increments the child refcount, 722 * decrement it before returning. 723 */ 724 of_node_put(child); 725 726 return of_mdiobus_register(bp->mii_bus, np); 727 } 728 729 return mdiobus_register(bp->mii_bus); 730 } 731 732 static int macb_mii_init(struct macb *bp) 733 { 734 int err = -ENXIO; 735 736 /* Enable management port */ 737 macb_writel(bp, NCR, MACB_BIT(MPE)); 738 739 bp->mii_bus = mdiobus_alloc(); 740 if (!bp->mii_bus) { 741 err = -ENOMEM; 742 goto err_out; 743 } 744 745 bp->mii_bus->name = "MACB_mii_bus"; 746 bp->mii_bus->read = &macb_mdio_read; 747 bp->mii_bus->write = &macb_mdio_write; 748 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 749 bp->pdev->name, bp->pdev->id); 750 bp->mii_bus->priv = bp; 751 bp->mii_bus->parent = &bp->pdev->dev; 752 753 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 754 755 err = macb_mdiobus_register(bp); 756 if (err) 757 goto err_out_free_mdiobus; 758 759 err = macb_mii_probe(bp->dev); 760 if (err) 761 goto err_out_unregister_bus; 762 763 return 0; 764 765 err_out_unregister_bus: 766 mdiobus_unregister(bp->mii_bus); 767 err_out_free_mdiobus: 768 mdiobus_free(bp->mii_bus); 769 err_out: 770 return err; 771 } 772 773 static void macb_update_stats(struct macb *bp) 774 { 775 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 776 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 777 int offset = MACB_PFR; 778 779 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 780 781 for (; p < end; p++, offset += 4) 782 *p += bp->macb_reg_readl(bp, offset); 783 } 784 785 static int macb_halt_tx(struct macb *bp) 786 { 787 unsigned long halt_time, timeout; 788 u32 status; 789 790 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 791 792 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 793 do { 794 halt_time = jiffies; 795 status = macb_readl(bp, TSR); 796 if (!(status & MACB_BIT(TGO))) 797 return 0; 798 799 udelay(250); 800 } while (time_before(halt_time, timeout)); 801 802 return -ETIMEDOUT; 803 } 804 805 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 806 { 807 if (tx_skb->mapping) { 808 if (tx_skb->mapped_as_page) 809 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 810 tx_skb->size, DMA_TO_DEVICE); 811 else 812 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 813 tx_skb->size, DMA_TO_DEVICE); 814 tx_skb->mapping = 0; 815 } 816 817 if (tx_skb->skb) { 818 dev_kfree_skb_any(tx_skb->skb); 819 tx_skb->skb = NULL; 820 } 821 } 822 823 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 824 { 825 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 826 struct macb_dma_desc_64 *desc_64; 827 828 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 829 desc_64 = macb_64b_desc(bp, desc); 830 desc_64->addrh = upper_32_bits(addr); 831 /* The low bits of RX address contain the RX_USED bit, clearing 832 * of which allows packet RX. Make sure the high bits are also 833 * visible to HW at that point. 834 */ 835 dma_wmb(); 836 } 837 #endif 838 desc->addr = lower_32_bits(addr); 839 } 840 841 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 842 { 843 dma_addr_t addr = 0; 844 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 845 struct macb_dma_desc_64 *desc_64; 846 847 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 848 desc_64 = macb_64b_desc(bp, desc); 849 addr = ((u64)(desc_64->addrh) << 32); 850 } 851 #endif 852 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 853 return addr; 854 } 855 856 static void macb_tx_error_task(struct work_struct *work) 857 { 858 struct macb_queue *queue = container_of(work, struct macb_queue, 859 tx_error_task); 860 struct macb *bp = queue->bp; 861 struct macb_tx_skb *tx_skb; 862 struct macb_dma_desc *desc; 863 struct sk_buff *skb; 864 unsigned int tail; 865 unsigned long flags; 866 867 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 868 (unsigned int)(queue - bp->queues), 869 queue->tx_tail, queue->tx_head); 870 871 /* Prevent the queue IRQ handlers from running: each of them may call 872 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 873 * As explained below, we have to halt the transmission before updating 874 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 875 * network engine about the macb/gem being halted. 876 */ 877 spin_lock_irqsave(&bp->lock, flags); 878 879 /* Make sure nobody is trying to queue up new packets */ 880 netif_tx_stop_all_queues(bp->dev); 881 882 /* Stop transmission now 883 * (in case we have just queued new packets) 884 * macb/gem must be halted to write TBQP register 885 */ 886 if (macb_halt_tx(bp)) 887 /* Just complain for now, reinitializing TX path can be good */ 888 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 889 890 /* Treat frames in TX queue including the ones that caused the error. 891 * Free transmit buffers in upper layer. 892 */ 893 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 894 u32 ctrl; 895 896 desc = macb_tx_desc(queue, tail); 897 ctrl = desc->ctrl; 898 tx_skb = macb_tx_skb(queue, tail); 899 skb = tx_skb->skb; 900 901 if (ctrl & MACB_BIT(TX_USED)) { 902 /* skb is set for the last buffer of the frame */ 903 while (!skb) { 904 macb_tx_unmap(bp, tx_skb); 905 tail++; 906 tx_skb = macb_tx_skb(queue, tail); 907 skb = tx_skb->skb; 908 } 909 910 /* ctrl still refers to the first buffer descriptor 911 * since it's the only one written back by the hardware 912 */ 913 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 914 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 915 macb_tx_ring_wrap(bp, tail), 916 skb->data); 917 bp->dev->stats.tx_packets++; 918 queue->stats.tx_packets++; 919 bp->dev->stats.tx_bytes += skb->len; 920 queue->stats.tx_bytes += skb->len; 921 } 922 } else { 923 /* "Buffers exhausted mid-frame" errors may only happen 924 * if the driver is buggy, so complain loudly about 925 * those. Statistics are updated by hardware. 926 */ 927 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 928 netdev_err(bp->dev, 929 "BUG: TX buffers exhausted mid-frame\n"); 930 931 desc->ctrl = ctrl | MACB_BIT(TX_USED); 932 } 933 934 macb_tx_unmap(bp, tx_skb); 935 } 936 937 /* Set end of TX queue */ 938 desc = macb_tx_desc(queue, 0); 939 macb_set_addr(bp, desc, 0); 940 desc->ctrl = MACB_BIT(TX_USED); 941 942 /* Make descriptor updates visible to hardware */ 943 wmb(); 944 945 /* Reinitialize the TX desc queue */ 946 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 947 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 948 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 949 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 950 #endif 951 /* Make TX ring reflect state of hardware */ 952 queue->tx_head = 0; 953 queue->tx_tail = 0; 954 955 /* Housework before enabling TX IRQ */ 956 macb_writel(bp, TSR, macb_readl(bp, TSR)); 957 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 958 959 /* Now we are ready to start transmission again */ 960 netif_tx_start_all_queues(bp->dev); 961 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 962 963 spin_unlock_irqrestore(&bp->lock, flags); 964 } 965 966 static void macb_tx_interrupt(struct macb_queue *queue) 967 { 968 unsigned int tail; 969 unsigned int head; 970 u32 status; 971 struct macb *bp = queue->bp; 972 u16 queue_index = queue - bp->queues; 973 974 status = macb_readl(bp, TSR); 975 macb_writel(bp, TSR, status); 976 977 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 978 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 979 980 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 981 (unsigned long)status); 982 983 head = queue->tx_head; 984 for (tail = queue->tx_tail; tail != head; tail++) { 985 struct macb_tx_skb *tx_skb; 986 struct sk_buff *skb; 987 struct macb_dma_desc *desc; 988 u32 ctrl; 989 990 desc = macb_tx_desc(queue, tail); 991 992 /* Make hw descriptor updates visible to CPU */ 993 rmb(); 994 995 ctrl = desc->ctrl; 996 997 /* TX_USED bit is only set by hardware on the very first buffer 998 * descriptor of the transmitted frame. 999 */ 1000 if (!(ctrl & MACB_BIT(TX_USED))) 1001 break; 1002 1003 /* Process all buffers of the current transmitted frame */ 1004 for (;; tail++) { 1005 tx_skb = macb_tx_skb(queue, tail); 1006 skb = tx_skb->skb; 1007 1008 /* First, update TX stats if needed */ 1009 if (skb) { 1010 if (unlikely(skb_shinfo(skb)->tx_flags & 1011 SKBTX_HW_TSTAMP) && 1012 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 1013 /* skb now belongs to timestamp buffer 1014 * and will be removed later 1015 */ 1016 tx_skb->skb = NULL; 1017 } 1018 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 1019 macb_tx_ring_wrap(bp, tail), 1020 skb->data); 1021 bp->dev->stats.tx_packets++; 1022 queue->stats.tx_packets++; 1023 bp->dev->stats.tx_bytes += skb->len; 1024 queue->stats.tx_bytes += skb->len; 1025 } 1026 1027 /* Now we can safely release resources */ 1028 macb_tx_unmap(bp, tx_skb); 1029 1030 /* skb is set only for the last buffer of the frame. 1031 * WARNING: at this point skb has been freed by 1032 * macb_tx_unmap(). 1033 */ 1034 if (skb) 1035 break; 1036 } 1037 } 1038 1039 queue->tx_tail = tail; 1040 if (__netif_subqueue_stopped(bp->dev, queue_index) && 1041 CIRC_CNT(queue->tx_head, queue->tx_tail, 1042 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 1043 netif_wake_subqueue(bp->dev, queue_index); 1044 } 1045 1046 static void gem_rx_refill(struct macb_queue *queue) 1047 { 1048 unsigned int entry; 1049 struct sk_buff *skb; 1050 dma_addr_t paddr; 1051 struct macb *bp = queue->bp; 1052 struct macb_dma_desc *desc; 1053 1054 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 1055 bp->rx_ring_size) > 0) { 1056 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 1057 1058 /* Make hw descriptor updates visible to CPU */ 1059 rmb(); 1060 1061 queue->rx_prepared_head++; 1062 desc = macb_rx_desc(queue, entry); 1063 1064 if (!queue->rx_skbuff[entry]) { 1065 /* allocate sk_buff for this free entry in ring */ 1066 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 1067 if (unlikely(!skb)) { 1068 netdev_err(bp->dev, 1069 "Unable to allocate sk_buff\n"); 1070 break; 1071 } 1072 1073 /* now fill corresponding descriptor entry */ 1074 paddr = dma_map_single(&bp->pdev->dev, skb->data, 1075 bp->rx_buffer_size, 1076 DMA_FROM_DEVICE); 1077 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 1078 dev_kfree_skb(skb); 1079 break; 1080 } 1081 1082 queue->rx_skbuff[entry] = skb; 1083 1084 if (entry == bp->rx_ring_size - 1) 1085 paddr |= MACB_BIT(RX_WRAP); 1086 desc->ctrl = 0; 1087 /* Setting addr clears RX_USED and allows reception, 1088 * make sure ctrl is cleared first to avoid a race. 1089 */ 1090 dma_wmb(); 1091 macb_set_addr(bp, desc, paddr); 1092 1093 /* properly align Ethernet header */ 1094 skb_reserve(skb, NET_IP_ALIGN); 1095 } else { 1096 desc->ctrl = 0; 1097 dma_wmb(); 1098 desc->addr &= ~MACB_BIT(RX_USED); 1099 } 1100 } 1101 1102 /* Make descriptor updates visible to hardware */ 1103 wmb(); 1104 1105 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 1106 queue, queue->rx_prepared_head, queue->rx_tail); 1107 } 1108 1109 /* Mark DMA descriptors from begin up to and not including end as unused */ 1110 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 1111 unsigned int end) 1112 { 1113 unsigned int frag; 1114 1115 for (frag = begin; frag != end; frag++) { 1116 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 1117 1118 desc->addr &= ~MACB_BIT(RX_USED); 1119 } 1120 1121 /* Make descriptor updates visible to hardware */ 1122 wmb(); 1123 1124 /* When this happens, the hardware stats registers for 1125 * whatever caused this is updated, so we don't have to record 1126 * anything. 1127 */ 1128 } 1129 1130 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 1131 int budget) 1132 { 1133 struct macb *bp = queue->bp; 1134 unsigned int len; 1135 unsigned int entry; 1136 struct sk_buff *skb; 1137 struct macb_dma_desc *desc; 1138 int count = 0; 1139 1140 while (count < budget) { 1141 u32 ctrl; 1142 dma_addr_t addr; 1143 bool rxused; 1144 1145 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1146 desc = macb_rx_desc(queue, entry); 1147 1148 /* Make hw descriptor updates visible to CPU */ 1149 rmb(); 1150 1151 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1152 addr = macb_get_addr(bp, desc); 1153 1154 if (!rxused) 1155 break; 1156 1157 /* Ensure ctrl is at least as up-to-date as rxused */ 1158 dma_rmb(); 1159 1160 ctrl = desc->ctrl; 1161 1162 queue->rx_tail++; 1163 count++; 1164 1165 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1166 netdev_err(bp->dev, 1167 "not whole frame pointed by descriptor\n"); 1168 bp->dev->stats.rx_dropped++; 1169 queue->stats.rx_dropped++; 1170 break; 1171 } 1172 skb = queue->rx_skbuff[entry]; 1173 if (unlikely(!skb)) { 1174 netdev_err(bp->dev, 1175 "inconsistent Rx descriptor chain\n"); 1176 bp->dev->stats.rx_dropped++; 1177 queue->stats.rx_dropped++; 1178 break; 1179 } 1180 /* now everything is ready for receiving packet */ 1181 queue->rx_skbuff[entry] = NULL; 1182 len = ctrl & bp->rx_frm_len_mask; 1183 1184 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1185 1186 skb_put(skb, len); 1187 dma_unmap_single(&bp->pdev->dev, addr, 1188 bp->rx_buffer_size, DMA_FROM_DEVICE); 1189 1190 skb->protocol = eth_type_trans(skb, bp->dev); 1191 skb_checksum_none_assert(skb); 1192 if (bp->dev->features & NETIF_F_RXCSUM && 1193 !(bp->dev->flags & IFF_PROMISC) && 1194 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1195 skb->ip_summed = CHECKSUM_UNNECESSARY; 1196 1197 bp->dev->stats.rx_packets++; 1198 queue->stats.rx_packets++; 1199 bp->dev->stats.rx_bytes += skb->len; 1200 queue->stats.rx_bytes += skb->len; 1201 1202 gem_ptp_do_rxstamp(bp, skb, desc); 1203 1204 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1205 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1206 skb->len, skb->csum); 1207 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1208 skb_mac_header(skb), 16, true); 1209 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1210 skb->data, 32, true); 1211 #endif 1212 1213 napi_gro_receive(napi, skb); 1214 } 1215 1216 gem_rx_refill(queue); 1217 1218 return count; 1219 } 1220 1221 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1222 unsigned int first_frag, unsigned int last_frag) 1223 { 1224 unsigned int len; 1225 unsigned int frag; 1226 unsigned int offset; 1227 struct sk_buff *skb; 1228 struct macb_dma_desc *desc; 1229 struct macb *bp = queue->bp; 1230 1231 desc = macb_rx_desc(queue, last_frag); 1232 len = desc->ctrl & bp->rx_frm_len_mask; 1233 1234 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1235 macb_rx_ring_wrap(bp, first_frag), 1236 macb_rx_ring_wrap(bp, last_frag), len); 1237 1238 /* The ethernet header starts NET_IP_ALIGN bytes into the 1239 * first buffer. Since the header is 14 bytes, this makes the 1240 * payload word-aligned. 1241 * 1242 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1243 * the two padding bytes into the skb so that we avoid hitting 1244 * the slowpath in memcpy(), and pull them off afterwards. 1245 */ 1246 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1247 if (!skb) { 1248 bp->dev->stats.rx_dropped++; 1249 for (frag = first_frag; ; frag++) { 1250 desc = macb_rx_desc(queue, frag); 1251 desc->addr &= ~MACB_BIT(RX_USED); 1252 if (frag == last_frag) 1253 break; 1254 } 1255 1256 /* Make descriptor updates visible to hardware */ 1257 wmb(); 1258 1259 return 1; 1260 } 1261 1262 offset = 0; 1263 len += NET_IP_ALIGN; 1264 skb_checksum_none_assert(skb); 1265 skb_put(skb, len); 1266 1267 for (frag = first_frag; ; frag++) { 1268 unsigned int frag_len = bp->rx_buffer_size; 1269 1270 if (offset + frag_len > len) { 1271 if (unlikely(frag != last_frag)) { 1272 dev_kfree_skb_any(skb); 1273 return -1; 1274 } 1275 frag_len = len - offset; 1276 } 1277 skb_copy_to_linear_data_offset(skb, offset, 1278 macb_rx_buffer(queue, frag), 1279 frag_len); 1280 offset += bp->rx_buffer_size; 1281 desc = macb_rx_desc(queue, frag); 1282 desc->addr &= ~MACB_BIT(RX_USED); 1283 1284 if (frag == last_frag) 1285 break; 1286 } 1287 1288 /* Make descriptor updates visible to hardware */ 1289 wmb(); 1290 1291 __skb_pull(skb, NET_IP_ALIGN); 1292 skb->protocol = eth_type_trans(skb, bp->dev); 1293 1294 bp->dev->stats.rx_packets++; 1295 bp->dev->stats.rx_bytes += skb->len; 1296 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1297 skb->len, skb->csum); 1298 napi_gro_receive(napi, skb); 1299 1300 return 0; 1301 } 1302 1303 static inline void macb_init_rx_ring(struct macb_queue *queue) 1304 { 1305 struct macb *bp = queue->bp; 1306 dma_addr_t addr; 1307 struct macb_dma_desc *desc = NULL; 1308 int i; 1309 1310 addr = queue->rx_buffers_dma; 1311 for (i = 0; i < bp->rx_ring_size; i++) { 1312 desc = macb_rx_desc(queue, i); 1313 macb_set_addr(bp, desc, addr); 1314 desc->ctrl = 0; 1315 addr += bp->rx_buffer_size; 1316 } 1317 desc->addr |= MACB_BIT(RX_WRAP); 1318 queue->rx_tail = 0; 1319 } 1320 1321 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1322 int budget) 1323 { 1324 struct macb *bp = queue->bp; 1325 bool reset_rx_queue = false; 1326 int received = 0; 1327 unsigned int tail; 1328 int first_frag = -1; 1329 1330 for (tail = queue->rx_tail; budget > 0; tail++) { 1331 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1332 u32 ctrl; 1333 1334 /* Make hw descriptor updates visible to CPU */ 1335 rmb(); 1336 1337 if (!(desc->addr & MACB_BIT(RX_USED))) 1338 break; 1339 1340 /* Ensure ctrl is at least as up-to-date as addr */ 1341 dma_rmb(); 1342 1343 ctrl = desc->ctrl; 1344 1345 if (ctrl & MACB_BIT(RX_SOF)) { 1346 if (first_frag != -1) 1347 discard_partial_frame(queue, first_frag, tail); 1348 first_frag = tail; 1349 } 1350 1351 if (ctrl & MACB_BIT(RX_EOF)) { 1352 int dropped; 1353 1354 if (unlikely(first_frag == -1)) { 1355 reset_rx_queue = true; 1356 continue; 1357 } 1358 1359 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1360 first_frag = -1; 1361 if (unlikely(dropped < 0)) { 1362 reset_rx_queue = true; 1363 continue; 1364 } 1365 if (!dropped) { 1366 received++; 1367 budget--; 1368 } 1369 } 1370 } 1371 1372 if (unlikely(reset_rx_queue)) { 1373 unsigned long flags; 1374 u32 ctrl; 1375 1376 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1377 1378 spin_lock_irqsave(&bp->lock, flags); 1379 1380 ctrl = macb_readl(bp, NCR); 1381 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1382 1383 macb_init_rx_ring(queue); 1384 queue_writel(queue, RBQP, queue->rx_ring_dma); 1385 1386 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1387 1388 spin_unlock_irqrestore(&bp->lock, flags); 1389 return received; 1390 } 1391 1392 if (first_frag != -1) 1393 queue->rx_tail = first_frag; 1394 else 1395 queue->rx_tail = tail; 1396 1397 return received; 1398 } 1399 1400 static int macb_poll(struct napi_struct *napi, int budget) 1401 { 1402 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1403 struct macb *bp = queue->bp; 1404 int work_done; 1405 u32 status; 1406 1407 status = macb_readl(bp, RSR); 1408 macb_writel(bp, RSR, status); 1409 1410 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1411 (unsigned long)status, budget); 1412 1413 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 1414 if (work_done < budget) { 1415 napi_complete_done(napi, work_done); 1416 1417 /* Packets received while interrupts were disabled */ 1418 status = macb_readl(bp, RSR); 1419 if (status) { 1420 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1421 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1422 napi_reschedule(napi); 1423 } else { 1424 queue_writel(queue, IER, bp->rx_intr_mask); 1425 } 1426 } 1427 1428 /* TODO: Handle errors */ 1429 1430 return work_done; 1431 } 1432 1433 static void macb_hresp_error_task(unsigned long data) 1434 { 1435 struct macb *bp = (struct macb *)data; 1436 struct net_device *dev = bp->dev; 1437 struct macb_queue *queue = bp->queues; 1438 unsigned int q; 1439 u32 ctrl; 1440 1441 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1442 queue_writel(queue, IDR, bp->rx_intr_mask | 1443 MACB_TX_INT_FLAGS | 1444 MACB_BIT(HRESP)); 1445 } 1446 ctrl = macb_readl(bp, NCR); 1447 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1448 macb_writel(bp, NCR, ctrl); 1449 1450 netif_tx_stop_all_queues(dev); 1451 netif_carrier_off(dev); 1452 1453 bp->macbgem_ops.mog_init_rings(bp); 1454 1455 /* Initialize TX and RX buffers */ 1456 macb_init_buffers(bp); 1457 1458 /* Enable interrupts */ 1459 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1460 queue_writel(queue, IER, 1461 bp->rx_intr_mask | 1462 MACB_TX_INT_FLAGS | 1463 MACB_BIT(HRESP)); 1464 1465 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1466 macb_writel(bp, NCR, ctrl); 1467 1468 netif_carrier_on(dev); 1469 netif_tx_start_all_queues(dev); 1470 } 1471 1472 static void macb_tx_restart(struct macb_queue *queue) 1473 { 1474 unsigned int head = queue->tx_head; 1475 unsigned int tail = queue->tx_tail; 1476 struct macb *bp = queue->bp; 1477 1478 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1479 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1480 1481 if (head == tail) 1482 return; 1483 1484 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1485 } 1486 1487 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1488 { 1489 struct macb_queue *queue = dev_id; 1490 struct macb *bp = queue->bp; 1491 struct net_device *dev = bp->dev; 1492 u32 status, ctrl; 1493 1494 status = queue_readl(queue, ISR); 1495 1496 if (unlikely(!status)) 1497 return IRQ_NONE; 1498 1499 spin_lock(&bp->lock); 1500 1501 while (status) { 1502 /* close possible race with dev_close */ 1503 if (unlikely(!netif_running(dev))) { 1504 queue_writel(queue, IDR, -1); 1505 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1506 queue_writel(queue, ISR, -1); 1507 break; 1508 } 1509 1510 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1511 (unsigned int)(queue - bp->queues), 1512 (unsigned long)status); 1513 1514 if (status & bp->rx_intr_mask) { 1515 /* There's no point taking any more interrupts 1516 * until we have processed the buffers. The 1517 * scheduling call may fail if the poll routine 1518 * is already scheduled, so disable interrupts 1519 * now. 1520 */ 1521 queue_writel(queue, IDR, bp->rx_intr_mask); 1522 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1523 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1524 1525 if (napi_schedule_prep(&queue->napi)) { 1526 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1527 __napi_schedule(&queue->napi); 1528 } 1529 } 1530 1531 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1532 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1533 schedule_work(&queue->tx_error_task); 1534 1535 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1536 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1537 1538 break; 1539 } 1540 1541 if (status & MACB_BIT(TCOMP)) 1542 macb_tx_interrupt(queue); 1543 1544 if (status & MACB_BIT(TXUBR)) 1545 macb_tx_restart(queue); 1546 1547 /* Link change detection isn't possible with RMII, so we'll 1548 * add that if/when we get our hands on a full-blown MII PHY. 1549 */ 1550 1551 /* There is a hardware issue under heavy load where DMA can 1552 * stop, this causes endless "used buffer descriptor read" 1553 * interrupts but it can be cleared by re-enabling RX. See 1554 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1555 * section 16.7.4 for details. RXUBR is only enabled for 1556 * these two versions. 1557 */ 1558 if (status & MACB_BIT(RXUBR)) { 1559 ctrl = macb_readl(bp, NCR); 1560 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1561 wmb(); 1562 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1563 1564 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1565 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1566 } 1567 1568 if (status & MACB_BIT(ISR_ROVR)) { 1569 /* We missed at least one packet */ 1570 if (macb_is_gem(bp)) 1571 bp->hw_stats.gem.rx_overruns++; 1572 else 1573 bp->hw_stats.macb.rx_overruns++; 1574 1575 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1576 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1577 } 1578 1579 if (status & MACB_BIT(HRESP)) { 1580 tasklet_schedule(&bp->hresp_err_tasklet); 1581 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1582 1583 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1584 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1585 } 1586 status = queue_readl(queue, ISR); 1587 } 1588 1589 spin_unlock(&bp->lock); 1590 1591 return IRQ_HANDLED; 1592 } 1593 1594 #ifdef CONFIG_NET_POLL_CONTROLLER 1595 /* Polling receive - used by netconsole and other diagnostic tools 1596 * to allow network i/o with interrupts disabled. 1597 */ 1598 static void macb_poll_controller(struct net_device *dev) 1599 { 1600 struct macb *bp = netdev_priv(dev); 1601 struct macb_queue *queue; 1602 unsigned long flags; 1603 unsigned int q; 1604 1605 local_irq_save(flags); 1606 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1607 macb_interrupt(dev->irq, queue); 1608 local_irq_restore(flags); 1609 } 1610 #endif 1611 1612 static unsigned int macb_tx_map(struct macb *bp, 1613 struct macb_queue *queue, 1614 struct sk_buff *skb, 1615 unsigned int hdrlen) 1616 { 1617 dma_addr_t mapping; 1618 unsigned int len, entry, i, tx_head = queue->tx_head; 1619 struct macb_tx_skb *tx_skb = NULL; 1620 struct macb_dma_desc *desc; 1621 unsigned int offset, size, count = 0; 1622 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1623 unsigned int eof = 1, mss_mfs = 0; 1624 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1625 1626 /* LSO */ 1627 if (skb_shinfo(skb)->gso_size != 0) { 1628 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1629 /* UDP - UFO */ 1630 lso_ctrl = MACB_LSO_UFO_ENABLE; 1631 else 1632 /* TCP - TSO */ 1633 lso_ctrl = MACB_LSO_TSO_ENABLE; 1634 } 1635 1636 /* First, map non-paged data */ 1637 len = skb_headlen(skb); 1638 1639 /* first buffer length */ 1640 size = hdrlen; 1641 1642 offset = 0; 1643 while (len) { 1644 entry = macb_tx_ring_wrap(bp, tx_head); 1645 tx_skb = &queue->tx_skb[entry]; 1646 1647 mapping = dma_map_single(&bp->pdev->dev, 1648 skb->data + offset, 1649 size, DMA_TO_DEVICE); 1650 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1651 goto dma_error; 1652 1653 /* Save info to properly release resources */ 1654 tx_skb->skb = NULL; 1655 tx_skb->mapping = mapping; 1656 tx_skb->size = size; 1657 tx_skb->mapped_as_page = false; 1658 1659 len -= size; 1660 offset += size; 1661 count++; 1662 tx_head++; 1663 1664 size = min(len, bp->max_tx_length); 1665 } 1666 1667 /* Then, map paged data from fragments */ 1668 for (f = 0; f < nr_frags; f++) { 1669 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1670 1671 len = skb_frag_size(frag); 1672 offset = 0; 1673 while (len) { 1674 size = min(len, bp->max_tx_length); 1675 entry = macb_tx_ring_wrap(bp, tx_head); 1676 tx_skb = &queue->tx_skb[entry]; 1677 1678 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1679 offset, size, DMA_TO_DEVICE); 1680 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1681 goto dma_error; 1682 1683 /* Save info to properly release resources */ 1684 tx_skb->skb = NULL; 1685 tx_skb->mapping = mapping; 1686 tx_skb->size = size; 1687 tx_skb->mapped_as_page = true; 1688 1689 len -= size; 1690 offset += size; 1691 count++; 1692 tx_head++; 1693 } 1694 } 1695 1696 /* Should never happen */ 1697 if (unlikely(!tx_skb)) { 1698 netdev_err(bp->dev, "BUG! empty skb!\n"); 1699 return 0; 1700 } 1701 1702 /* This is the last buffer of the frame: save socket buffer */ 1703 tx_skb->skb = skb; 1704 1705 /* Update TX ring: update buffer descriptors in reverse order 1706 * to avoid race condition 1707 */ 1708 1709 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1710 * to set the end of TX queue 1711 */ 1712 i = tx_head; 1713 entry = macb_tx_ring_wrap(bp, i); 1714 ctrl = MACB_BIT(TX_USED); 1715 desc = macb_tx_desc(queue, entry); 1716 desc->ctrl = ctrl; 1717 1718 if (lso_ctrl) { 1719 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1720 /* include header and FCS in value given to h/w */ 1721 mss_mfs = skb_shinfo(skb)->gso_size + 1722 skb_transport_offset(skb) + 1723 ETH_FCS_LEN; 1724 else /* TSO */ { 1725 mss_mfs = skb_shinfo(skb)->gso_size; 1726 /* TCP Sequence Number Source Select 1727 * can be set only for TSO 1728 */ 1729 seq_ctrl = 0; 1730 } 1731 } 1732 1733 do { 1734 i--; 1735 entry = macb_tx_ring_wrap(bp, i); 1736 tx_skb = &queue->tx_skb[entry]; 1737 desc = macb_tx_desc(queue, entry); 1738 1739 ctrl = (u32)tx_skb->size; 1740 if (eof) { 1741 ctrl |= MACB_BIT(TX_LAST); 1742 eof = 0; 1743 } 1744 if (unlikely(entry == (bp->tx_ring_size - 1))) 1745 ctrl |= MACB_BIT(TX_WRAP); 1746 1747 /* First descriptor is header descriptor */ 1748 if (i == queue->tx_head) { 1749 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1750 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1751 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1752 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1753 ctrl |= MACB_BIT(TX_NOCRC); 1754 } else 1755 /* Only set MSS/MFS on payload descriptors 1756 * (second or later descriptor) 1757 */ 1758 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1759 1760 /* Set TX buffer descriptor */ 1761 macb_set_addr(bp, desc, tx_skb->mapping); 1762 /* desc->addr must be visible to hardware before clearing 1763 * 'TX_USED' bit in desc->ctrl. 1764 */ 1765 wmb(); 1766 desc->ctrl = ctrl; 1767 } while (i != queue->tx_head); 1768 1769 queue->tx_head = tx_head; 1770 1771 return count; 1772 1773 dma_error: 1774 netdev_err(bp->dev, "TX DMA map failed\n"); 1775 1776 for (i = queue->tx_head; i != tx_head; i++) { 1777 tx_skb = macb_tx_skb(queue, i); 1778 1779 macb_tx_unmap(bp, tx_skb); 1780 } 1781 1782 return 0; 1783 } 1784 1785 static netdev_features_t macb_features_check(struct sk_buff *skb, 1786 struct net_device *dev, 1787 netdev_features_t features) 1788 { 1789 unsigned int nr_frags, f; 1790 unsigned int hdrlen; 1791 1792 /* Validate LSO compatibility */ 1793 1794 /* there is only one buffer */ 1795 if (!skb_is_nonlinear(skb)) 1796 return features; 1797 1798 /* length of header */ 1799 hdrlen = skb_transport_offset(skb); 1800 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1801 hdrlen += tcp_hdrlen(skb); 1802 1803 /* For LSO: 1804 * When software supplies two or more payload buffers all payload buffers 1805 * apart from the last must be a multiple of 8 bytes in size. 1806 */ 1807 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1808 return features & ~MACB_NETIF_LSO; 1809 1810 nr_frags = skb_shinfo(skb)->nr_frags; 1811 /* No need to check last fragment */ 1812 nr_frags--; 1813 for (f = 0; f < nr_frags; f++) { 1814 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1815 1816 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1817 return features & ~MACB_NETIF_LSO; 1818 } 1819 return features; 1820 } 1821 1822 static inline int macb_clear_csum(struct sk_buff *skb) 1823 { 1824 /* no change for packets without checksum offloading */ 1825 if (skb->ip_summed != CHECKSUM_PARTIAL) 1826 return 0; 1827 1828 /* make sure we can modify the header */ 1829 if (unlikely(skb_cow_head(skb, 0))) 1830 return -1; 1831 1832 /* initialize checksum field 1833 * This is required - at least for Zynq, which otherwise calculates 1834 * wrong UDP header checksums for UDP packets with UDP data len <=2 1835 */ 1836 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1837 return 0; 1838 } 1839 1840 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1841 { 1842 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1843 int padlen = ETH_ZLEN - (*skb)->len; 1844 int headroom = skb_headroom(*skb); 1845 int tailroom = skb_tailroom(*skb); 1846 struct sk_buff *nskb; 1847 u32 fcs; 1848 1849 if (!(ndev->features & NETIF_F_HW_CSUM) || 1850 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1851 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1852 return 0; 1853 1854 if (padlen <= 0) { 1855 /* FCS could be appeded to tailroom. */ 1856 if (tailroom >= ETH_FCS_LEN) 1857 goto add_fcs; 1858 /* FCS could be appeded by moving data to headroom. */ 1859 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1860 padlen = 0; 1861 /* No room for FCS, need to reallocate skb. */ 1862 else 1863 padlen = ETH_FCS_LEN; 1864 } else { 1865 /* Add room for FCS. */ 1866 padlen += ETH_FCS_LEN; 1867 } 1868 1869 if (!cloned && headroom + tailroom >= padlen) { 1870 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1871 skb_set_tail_pointer(*skb, (*skb)->len); 1872 } else { 1873 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1874 if (!nskb) 1875 return -ENOMEM; 1876 1877 dev_consume_skb_any(*skb); 1878 *skb = nskb; 1879 } 1880 1881 if (padlen > ETH_FCS_LEN) 1882 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1883 1884 add_fcs: 1885 /* set FCS to packet */ 1886 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1887 fcs = ~fcs; 1888 1889 skb_put_u8(*skb, fcs & 0xff); 1890 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1891 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1892 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1893 1894 return 0; 1895 } 1896 1897 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1898 { 1899 u16 queue_index = skb_get_queue_mapping(skb); 1900 struct macb *bp = netdev_priv(dev); 1901 struct macb_queue *queue = &bp->queues[queue_index]; 1902 unsigned long flags; 1903 unsigned int desc_cnt, nr_frags, frag_size, f; 1904 unsigned int hdrlen; 1905 bool is_lso, is_udp = 0; 1906 netdev_tx_t ret = NETDEV_TX_OK; 1907 1908 if (macb_clear_csum(skb)) { 1909 dev_kfree_skb_any(skb); 1910 return ret; 1911 } 1912 1913 if (macb_pad_and_fcs(&skb, dev)) { 1914 dev_kfree_skb_any(skb); 1915 return ret; 1916 } 1917 1918 is_lso = (skb_shinfo(skb)->gso_size != 0); 1919 1920 if (is_lso) { 1921 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1922 1923 /* length of headers */ 1924 if (is_udp) 1925 /* only queue eth + ip headers separately for UDP */ 1926 hdrlen = skb_transport_offset(skb); 1927 else 1928 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1929 if (skb_headlen(skb) < hdrlen) { 1930 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1931 /* if this is required, would need to copy to single buffer */ 1932 return NETDEV_TX_BUSY; 1933 } 1934 } else 1935 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1936 1937 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1938 netdev_vdbg(bp->dev, 1939 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1940 queue_index, skb->len, skb->head, skb->data, 1941 skb_tail_pointer(skb), skb_end_pointer(skb)); 1942 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1943 skb->data, 16, true); 1944 #endif 1945 1946 /* Count how many TX buffer descriptors are needed to send this 1947 * socket buffer: skb fragments of jumbo frames may need to be 1948 * split into many buffer descriptors. 1949 */ 1950 if (is_lso && (skb_headlen(skb) > hdrlen)) 1951 /* extra header descriptor if also payload in first buffer */ 1952 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1953 else 1954 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1955 nr_frags = skb_shinfo(skb)->nr_frags; 1956 for (f = 0; f < nr_frags; f++) { 1957 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1958 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1959 } 1960 1961 spin_lock_irqsave(&bp->lock, flags); 1962 1963 /* This is a hard error, log it. */ 1964 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1965 bp->tx_ring_size) < desc_cnt) { 1966 netif_stop_subqueue(dev, queue_index); 1967 spin_unlock_irqrestore(&bp->lock, flags); 1968 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1969 queue->tx_head, queue->tx_tail); 1970 return NETDEV_TX_BUSY; 1971 } 1972 1973 /* Map socket buffer for DMA transfer */ 1974 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1975 dev_kfree_skb_any(skb); 1976 goto unlock; 1977 } 1978 1979 /* Make newly initialized descriptor visible to hardware */ 1980 wmb(); 1981 skb_tx_timestamp(skb); 1982 1983 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1984 1985 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1986 netif_stop_subqueue(dev, queue_index); 1987 1988 unlock: 1989 spin_unlock_irqrestore(&bp->lock, flags); 1990 1991 return ret; 1992 } 1993 1994 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1995 { 1996 if (!macb_is_gem(bp)) { 1997 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1998 } else { 1999 bp->rx_buffer_size = size; 2000 2001 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 2002 netdev_dbg(bp->dev, 2003 "RX buffer must be multiple of %d bytes, expanding\n", 2004 RX_BUFFER_MULTIPLE); 2005 bp->rx_buffer_size = 2006 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 2007 } 2008 } 2009 2010 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 2011 bp->dev->mtu, bp->rx_buffer_size); 2012 } 2013 2014 static void gem_free_rx_buffers(struct macb *bp) 2015 { 2016 struct sk_buff *skb; 2017 struct macb_dma_desc *desc; 2018 struct macb_queue *queue; 2019 dma_addr_t addr; 2020 unsigned int q; 2021 int i; 2022 2023 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2024 if (!queue->rx_skbuff) 2025 continue; 2026 2027 for (i = 0; i < bp->rx_ring_size; i++) { 2028 skb = queue->rx_skbuff[i]; 2029 2030 if (!skb) 2031 continue; 2032 2033 desc = macb_rx_desc(queue, i); 2034 addr = macb_get_addr(bp, desc); 2035 2036 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 2037 DMA_FROM_DEVICE); 2038 dev_kfree_skb_any(skb); 2039 skb = NULL; 2040 } 2041 2042 kfree(queue->rx_skbuff); 2043 queue->rx_skbuff = NULL; 2044 } 2045 } 2046 2047 static void macb_free_rx_buffers(struct macb *bp) 2048 { 2049 struct macb_queue *queue = &bp->queues[0]; 2050 2051 if (queue->rx_buffers) { 2052 dma_free_coherent(&bp->pdev->dev, 2053 bp->rx_ring_size * bp->rx_buffer_size, 2054 queue->rx_buffers, queue->rx_buffers_dma); 2055 queue->rx_buffers = NULL; 2056 } 2057 } 2058 2059 static void macb_free_consistent(struct macb *bp) 2060 { 2061 struct macb_queue *queue; 2062 unsigned int q; 2063 int size; 2064 2065 bp->macbgem_ops.mog_free_rx_buffers(bp); 2066 2067 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2068 kfree(queue->tx_skb); 2069 queue->tx_skb = NULL; 2070 if (queue->tx_ring) { 2071 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2072 dma_free_coherent(&bp->pdev->dev, size, 2073 queue->tx_ring, queue->tx_ring_dma); 2074 queue->tx_ring = NULL; 2075 } 2076 if (queue->rx_ring) { 2077 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2078 dma_free_coherent(&bp->pdev->dev, size, 2079 queue->rx_ring, queue->rx_ring_dma); 2080 queue->rx_ring = NULL; 2081 } 2082 } 2083 } 2084 2085 static int gem_alloc_rx_buffers(struct macb *bp) 2086 { 2087 struct macb_queue *queue; 2088 unsigned int q; 2089 int size; 2090 2091 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2092 size = bp->rx_ring_size * sizeof(struct sk_buff *); 2093 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 2094 if (!queue->rx_skbuff) 2095 return -ENOMEM; 2096 else 2097 netdev_dbg(bp->dev, 2098 "Allocated %d RX struct sk_buff entries at %p\n", 2099 bp->rx_ring_size, queue->rx_skbuff); 2100 } 2101 return 0; 2102 } 2103 2104 static int macb_alloc_rx_buffers(struct macb *bp) 2105 { 2106 struct macb_queue *queue = &bp->queues[0]; 2107 int size; 2108 2109 size = bp->rx_ring_size * bp->rx_buffer_size; 2110 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 2111 &queue->rx_buffers_dma, GFP_KERNEL); 2112 if (!queue->rx_buffers) 2113 return -ENOMEM; 2114 2115 netdev_dbg(bp->dev, 2116 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 2117 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 2118 return 0; 2119 } 2120 2121 static int macb_alloc_consistent(struct macb *bp) 2122 { 2123 struct macb_queue *queue; 2124 unsigned int q; 2125 int size; 2126 2127 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2128 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2129 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2130 &queue->tx_ring_dma, 2131 GFP_KERNEL); 2132 if (!queue->tx_ring) 2133 goto out_err; 2134 netdev_dbg(bp->dev, 2135 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2136 q, size, (unsigned long)queue->tx_ring_dma, 2137 queue->tx_ring); 2138 2139 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2140 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2141 if (!queue->tx_skb) 2142 goto out_err; 2143 2144 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2145 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2146 &queue->rx_ring_dma, GFP_KERNEL); 2147 if (!queue->rx_ring) 2148 goto out_err; 2149 netdev_dbg(bp->dev, 2150 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2151 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2152 } 2153 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2154 goto out_err; 2155 2156 return 0; 2157 2158 out_err: 2159 macb_free_consistent(bp); 2160 return -ENOMEM; 2161 } 2162 2163 static void gem_init_rings(struct macb *bp) 2164 { 2165 struct macb_queue *queue; 2166 struct macb_dma_desc *desc = NULL; 2167 unsigned int q; 2168 int i; 2169 2170 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2171 for (i = 0; i < bp->tx_ring_size; i++) { 2172 desc = macb_tx_desc(queue, i); 2173 macb_set_addr(bp, desc, 0); 2174 desc->ctrl = MACB_BIT(TX_USED); 2175 } 2176 desc->ctrl |= MACB_BIT(TX_WRAP); 2177 queue->tx_head = 0; 2178 queue->tx_tail = 0; 2179 2180 queue->rx_tail = 0; 2181 queue->rx_prepared_head = 0; 2182 2183 gem_rx_refill(queue); 2184 } 2185 2186 } 2187 2188 static void macb_init_rings(struct macb *bp) 2189 { 2190 int i; 2191 struct macb_dma_desc *desc = NULL; 2192 2193 macb_init_rx_ring(&bp->queues[0]); 2194 2195 for (i = 0; i < bp->tx_ring_size; i++) { 2196 desc = macb_tx_desc(&bp->queues[0], i); 2197 macb_set_addr(bp, desc, 0); 2198 desc->ctrl = MACB_BIT(TX_USED); 2199 } 2200 bp->queues[0].tx_head = 0; 2201 bp->queues[0].tx_tail = 0; 2202 desc->ctrl |= MACB_BIT(TX_WRAP); 2203 } 2204 2205 static void macb_reset_hw(struct macb *bp) 2206 { 2207 struct macb_queue *queue; 2208 unsigned int q; 2209 u32 ctrl = macb_readl(bp, NCR); 2210 2211 /* Disable RX and TX (XXX: Should we halt the transmission 2212 * more gracefully?) 2213 */ 2214 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2215 2216 /* Clear the stats registers (XXX: Update stats first?) */ 2217 ctrl |= MACB_BIT(CLRSTAT); 2218 2219 macb_writel(bp, NCR, ctrl); 2220 2221 /* Clear all status flags */ 2222 macb_writel(bp, TSR, -1); 2223 macb_writel(bp, RSR, -1); 2224 2225 /* Disable all interrupts */ 2226 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2227 queue_writel(queue, IDR, -1); 2228 queue_readl(queue, ISR); 2229 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2230 queue_writel(queue, ISR, -1); 2231 } 2232 } 2233 2234 static u32 gem_mdc_clk_div(struct macb *bp) 2235 { 2236 u32 config; 2237 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2238 2239 if (pclk_hz <= 20000000) 2240 config = GEM_BF(CLK, GEM_CLK_DIV8); 2241 else if (pclk_hz <= 40000000) 2242 config = GEM_BF(CLK, GEM_CLK_DIV16); 2243 else if (pclk_hz <= 80000000) 2244 config = GEM_BF(CLK, GEM_CLK_DIV32); 2245 else if (pclk_hz <= 120000000) 2246 config = GEM_BF(CLK, GEM_CLK_DIV48); 2247 else if (pclk_hz <= 160000000) 2248 config = GEM_BF(CLK, GEM_CLK_DIV64); 2249 else 2250 config = GEM_BF(CLK, GEM_CLK_DIV96); 2251 2252 return config; 2253 } 2254 2255 static u32 macb_mdc_clk_div(struct macb *bp) 2256 { 2257 u32 config; 2258 unsigned long pclk_hz; 2259 2260 if (macb_is_gem(bp)) 2261 return gem_mdc_clk_div(bp); 2262 2263 pclk_hz = clk_get_rate(bp->pclk); 2264 if (pclk_hz <= 20000000) 2265 config = MACB_BF(CLK, MACB_CLK_DIV8); 2266 else if (pclk_hz <= 40000000) 2267 config = MACB_BF(CLK, MACB_CLK_DIV16); 2268 else if (pclk_hz <= 80000000) 2269 config = MACB_BF(CLK, MACB_CLK_DIV32); 2270 else 2271 config = MACB_BF(CLK, MACB_CLK_DIV64); 2272 2273 return config; 2274 } 2275 2276 /* Get the DMA bus width field of the network configuration register that we 2277 * should program. We find the width from decoding the design configuration 2278 * register to find the maximum supported data bus width. 2279 */ 2280 static u32 macb_dbw(struct macb *bp) 2281 { 2282 if (!macb_is_gem(bp)) 2283 return 0; 2284 2285 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2286 case 4: 2287 return GEM_BF(DBW, GEM_DBW128); 2288 case 2: 2289 return GEM_BF(DBW, GEM_DBW64); 2290 case 1: 2291 default: 2292 return GEM_BF(DBW, GEM_DBW32); 2293 } 2294 } 2295 2296 /* Configure the receive DMA engine 2297 * - use the correct receive buffer size 2298 * - set best burst length for DMA operations 2299 * (if not supported by FIFO, it will fallback to default) 2300 * - set both rx/tx packet buffers to full memory size 2301 * These are configurable parameters for GEM. 2302 */ 2303 static void macb_configure_dma(struct macb *bp) 2304 { 2305 struct macb_queue *queue; 2306 u32 buffer_size; 2307 unsigned int q; 2308 u32 dmacfg; 2309 2310 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2311 if (macb_is_gem(bp)) { 2312 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2313 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2314 if (q) 2315 queue_writel(queue, RBQS, buffer_size); 2316 else 2317 dmacfg |= GEM_BF(RXBS, buffer_size); 2318 } 2319 if (bp->dma_burst_length) 2320 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2321 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2322 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2323 2324 if (bp->native_io) 2325 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2326 else 2327 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2328 2329 if (bp->dev->features & NETIF_F_HW_CSUM) 2330 dmacfg |= GEM_BIT(TXCOEN); 2331 else 2332 dmacfg &= ~GEM_BIT(TXCOEN); 2333 2334 dmacfg &= ~GEM_BIT(ADDR64); 2335 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2336 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2337 dmacfg |= GEM_BIT(ADDR64); 2338 #endif 2339 #ifdef CONFIG_MACB_USE_HWSTAMP 2340 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2341 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2342 #endif 2343 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2344 dmacfg); 2345 gem_writel(bp, DMACFG, dmacfg); 2346 } 2347 } 2348 2349 static void macb_init_hw(struct macb *bp) 2350 { 2351 u32 config; 2352 2353 macb_reset_hw(bp); 2354 macb_set_hwaddr(bp); 2355 2356 config = macb_mdc_clk_div(bp); 2357 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2358 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2359 if (bp->caps & MACB_CAPS_JUMBO) 2360 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2361 else 2362 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2363 if (bp->dev->flags & IFF_PROMISC) 2364 config |= MACB_BIT(CAF); /* Copy All Frames */ 2365 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2366 config |= GEM_BIT(RXCOEN); 2367 if (!(bp->dev->flags & IFF_BROADCAST)) 2368 config |= MACB_BIT(NBC); /* No BroadCast */ 2369 config |= macb_dbw(bp); 2370 macb_writel(bp, NCFGR, config); 2371 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2372 gem_writel(bp, JML, bp->jumbo_max_len); 2373 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2374 if (bp->caps & MACB_CAPS_JUMBO) 2375 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2376 2377 macb_configure_dma(bp); 2378 } 2379 2380 /* The hash address register is 64 bits long and takes up two 2381 * locations in the memory map. The least significant bits are stored 2382 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2383 * 2384 * The unicast hash enable and the multicast hash enable bits in the 2385 * network configuration register enable the reception of hash matched 2386 * frames. The destination address is reduced to a 6 bit index into 2387 * the 64 bit hash register using the following hash function. The 2388 * hash function is an exclusive or of every sixth bit of the 2389 * destination address. 2390 * 2391 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2392 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2393 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2394 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2395 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2396 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2397 * 2398 * da[0] represents the least significant bit of the first byte 2399 * received, that is, the multicast/unicast indicator, and da[47] 2400 * represents the most significant bit of the last byte received. If 2401 * the hash index, hi[n], points to a bit that is set in the hash 2402 * register then the frame will be matched according to whether the 2403 * frame is multicast or unicast. A multicast match will be signalled 2404 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2405 * index points to a bit set in the hash register. A unicast match 2406 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2407 * and the hash index points to a bit set in the hash register. To 2408 * receive all multicast frames, the hash register should be set with 2409 * all ones and the multicast hash enable bit should be set in the 2410 * network configuration register. 2411 */ 2412 2413 static inline int hash_bit_value(int bitnr, __u8 *addr) 2414 { 2415 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2416 return 1; 2417 return 0; 2418 } 2419 2420 /* Return the hash index value for the specified address. */ 2421 static int hash_get_index(__u8 *addr) 2422 { 2423 int i, j, bitval; 2424 int hash_index = 0; 2425 2426 for (j = 0; j < 6; j++) { 2427 for (i = 0, bitval = 0; i < 8; i++) 2428 bitval ^= hash_bit_value(i * 6 + j, addr); 2429 2430 hash_index |= (bitval << j); 2431 } 2432 2433 return hash_index; 2434 } 2435 2436 /* Add multicast addresses to the internal multicast-hash table. */ 2437 static void macb_sethashtable(struct net_device *dev) 2438 { 2439 struct netdev_hw_addr *ha; 2440 unsigned long mc_filter[2]; 2441 unsigned int bitnr; 2442 struct macb *bp = netdev_priv(dev); 2443 2444 mc_filter[0] = 0; 2445 mc_filter[1] = 0; 2446 2447 netdev_for_each_mc_addr(ha, dev) { 2448 bitnr = hash_get_index(ha->addr); 2449 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2450 } 2451 2452 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2453 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2454 } 2455 2456 /* Enable/Disable promiscuous and multicast modes. */ 2457 static void macb_set_rx_mode(struct net_device *dev) 2458 { 2459 unsigned long cfg; 2460 struct macb *bp = netdev_priv(dev); 2461 2462 cfg = macb_readl(bp, NCFGR); 2463 2464 if (dev->flags & IFF_PROMISC) { 2465 /* Enable promiscuous mode */ 2466 cfg |= MACB_BIT(CAF); 2467 2468 /* Disable RX checksum offload */ 2469 if (macb_is_gem(bp)) 2470 cfg &= ~GEM_BIT(RXCOEN); 2471 } else { 2472 /* Disable promiscuous mode */ 2473 cfg &= ~MACB_BIT(CAF); 2474 2475 /* Enable RX checksum offload only if requested */ 2476 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2477 cfg |= GEM_BIT(RXCOEN); 2478 } 2479 2480 if (dev->flags & IFF_ALLMULTI) { 2481 /* Enable all multicast mode */ 2482 macb_or_gem_writel(bp, HRB, -1); 2483 macb_or_gem_writel(bp, HRT, -1); 2484 cfg |= MACB_BIT(NCFGR_MTI); 2485 } else if (!netdev_mc_empty(dev)) { 2486 /* Enable specific multicasts */ 2487 macb_sethashtable(dev); 2488 cfg |= MACB_BIT(NCFGR_MTI); 2489 } else if (dev->flags & (~IFF_ALLMULTI)) { 2490 /* Disable all multicast mode */ 2491 macb_or_gem_writel(bp, HRB, 0); 2492 macb_or_gem_writel(bp, HRT, 0); 2493 cfg &= ~MACB_BIT(NCFGR_MTI); 2494 } 2495 2496 macb_writel(bp, NCFGR, cfg); 2497 } 2498 2499 static int macb_open(struct net_device *dev) 2500 { 2501 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2502 struct macb *bp = netdev_priv(dev); 2503 struct macb_queue *queue; 2504 unsigned int q; 2505 int err; 2506 2507 netdev_dbg(bp->dev, "open\n"); 2508 2509 err = pm_runtime_get_sync(&bp->pdev->dev); 2510 if (err < 0) 2511 goto pm_exit; 2512 2513 /* RX buffers initialization */ 2514 macb_init_rx_buffer_size(bp, bufsz); 2515 2516 err = macb_alloc_consistent(bp); 2517 if (err) { 2518 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2519 err); 2520 goto pm_exit; 2521 } 2522 2523 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2524 napi_enable(&queue->napi); 2525 2526 macb_init_hw(bp); 2527 2528 err = macb_phylink_connect(bp); 2529 if (err) 2530 goto pm_exit; 2531 2532 netif_tx_start_all_queues(dev); 2533 2534 if (bp->ptp_info) 2535 bp->ptp_info->ptp_init(dev); 2536 2537 pm_exit: 2538 if (err) { 2539 pm_runtime_put_sync(&bp->pdev->dev); 2540 return err; 2541 } 2542 return 0; 2543 } 2544 2545 static int macb_close(struct net_device *dev) 2546 { 2547 struct macb *bp = netdev_priv(dev); 2548 struct macb_queue *queue; 2549 unsigned long flags; 2550 unsigned int q; 2551 2552 netif_tx_stop_all_queues(dev); 2553 2554 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2555 napi_disable(&queue->napi); 2556 2557 phylink_stop(bp->phylink); 2558 phylink_disconnect_phy(bp->phylink); 2559 2560 spin_lock_irqsave(&bp->lock, flags); 2561 macb_reset_hw(bp); 2562 netif_carrier_off(dev); 2563 spin_unlock_irqrestore(&bp->lock, flags); 2564 2565 macb_free_consistent(bp); 2566 2567 if (bp->ptp_info) 2568 bp->ptp_info->ptp_remove(dev); 2569 2570 pm_runtime_put(&bp->pdev->dev); 2571 2572 return 0; 2573 } 2574 2575 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2576 { 2577 if (netif_running(dev)) 2578 return -EBUSY; 2579 2580 dev->mtu = new_mtu; 2581 2582 return 0; 2583 } 2584 2585 static void gem_update_stats(struct macb *bp) 2586 { 2587 struct macb_queue *queue; 2588 unsigned int i, q, idx; 2589 unsigned long *stat; 2590 2591 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2592 2593 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2594 u32 offset = gem_statistics[i].offset; 2595 u64 val = bp->macb_reg_readl(bp, offset); 2596 2597 bp->ethtool_stats[i] += val; 2598 *p += val; 2599 2600 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2601 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2602 val = bp->macb_reg_readl(bp, offset + 4); 2603 bp->ethtool_stats[i] += ((u64)val) << 32; 2604 *(++p) += val; 2605 } 2606 } 2607 2608 idx = GEM_STATS_LEN; 2609 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2610 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2611 bp->ethtool_stats[idx++] = *stat; 2612 } 2613 2614 static struct net_device_stats *gem_get_stats(struct macb *bp) 2615 { 2616 struct gem_stats *hwstat = &bp->hw_stats.gem; 2617 struct net_device_stats *nstat = &bp->dev->stats; 2618 2619 gem_update_stats(bp); 2620 2621 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2622 hwstat->rx_alignment_errors + 2623 hwstat->rx_resource_errors + 2624 hwstat->rx_overruns + 2625 hwstat->rx_oversize_frames + 2626 hwstat->rx_jabbers + 2627 hwstat->rx_undersized_frames + 2628 hwstat->rx_length_field_frame_errors); 2629 nstat->tx_errors = (hwstat->tx_late_collisions + 2630 hwstat->tx_excessive_collisions + 2631 hwstat->tx_underrun + 2632 hwstat->tx_carrier_sense_errors); 2633 nstat->multicast = hwstat->rx_multicast_frames; 2634 nstat->collisions = (hwstat->tx_single_collision_frames + 2635 hwstat->tx_multiple_collision_frames + 2636 hwstat->tx_excessive_collisions); 2637 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2638 hwstat->rx_jabbers + 2639 hwstat->rx_undersized_frames + 2640 hwstat->rx_length_field_frame_errors); 2641 nstat->rx_over_errors = hwstat->rx_resource_errors; 2642 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2643 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2644 nstat->rx_fifo_errors = hwstat->rx_overruns; 2645 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2646 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2647 nstat->tx_fifo_errors = hwstat->tx_underrun; 2648 2649 return nstat; 2650 } 2651 2652 static void gem_get_ethtool_stats(struct net_device *dev, 2653 struct ethtool_stats *stats, u64 *data) 2654 { 2655 struct macb *bp; 2656 2657 bp = netdev_priv(dev); 2658 gem_update_stats(bp); 2659 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2660 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2661 } 2662 2663 static int gem_get_sset_count(struct net_device *dev, int sset) 2664 { 2665 struct macb *bp = netdev_priv(dev); 2666 2667 switch (sset) { 2668 case ETH_SS_STATS: 2669 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2670 default: 2671 return -EOPNOTSUPP; 2672 } 2673 } 2674 2675 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2676 { 2677 char stat_string[ETH_GSTRING_LEN]; 2678 struct macb *bp = netdev_priv(dev); 2679 struct macb_queue *queue; 2680 unsigned int i; 2681 unsigned int q; 2682 2683 switch (sset) { 2684 case ETH_SS_STATS: 2685 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2686 memcpy(p, gem_statistics[i].stat_string, 2687 ETH_GSTRING_LEN); 2688 2689 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2690 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2691 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2692 q, queue_statistics[i].stat_string); 2693 memcpy(p, stat_string, ETH_GSTRING_LEN); 2694 } 2695 } 2696 break; 2697 } 2698 } 2699 2700 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2701 { 2702 struct macb *bp = netdev_priv(dev); 2703 struct net_device_stats *nstat = &bp->dev->stats; 2704 struct macb_stats *hwstat = &bp->hw_stats.macb; 2705 2706 if (macb_is_gem(bp)) 2707 return gem_get_stats(bp); 2708 2709 /* read stats from hardware */ 2710 macb_update_stats(bp); 2711 2712 /* Convert HW stats into netdevice stats */ 2713 nstat->rx_errors = (hwstat->rx_fcs_errors + 2714 hwstat->rx_align_errors + 2715 hwstat->rx_resource_errors + 2716 hwstat->rx_overruns + 2717 hwstat->rx_oversize_pkts + 2718 hwstat->rx_jabbers + 2719 hwstat->rx_undersize_pkts + 2720 hwstat->rx_length_mismatch); 2721 nstat->tx_errors = (hwstat->tx_late_cols + 2722 hwstat->tx_excessive_cols + 2723 hwstat->tx_underruns + 2724 hwstat->tx_carrier_errors + 2725 hwstat->sqe_test_errors); 2726 nstat->collisions = (hwstat->tx_single_cols + 2727 hwstat->tx_multiple_cols + 2728 hwstat->tx_excessive_cols); 2729 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2730 hwstat->rx_jabbers + 2731 hwstat->rx_undersize_pkts + 2732 hwstat->rx_length_mismatch); 2733 nstat->rx_over_errors = hwstat->rx_resource_errors + 2734 hwstat->rx_overruns; 2735 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2736 nstat->rx_frame_errors = hwstat->rx_align_errors; 2737 nstat->rx_fifo_errors = hwstat->rx_overruns; 2738 /* XXX: What does "missed" mean? */ 2739 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2740 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2741 nstat->tx_fifo_errors = hwstat->tx_underruns; 2742 /* Don't know about heartbeat or window errors... */ 2743 2744 return nstat; 2745 } 2746 2747 static int macb_get_regs_len(struct net_device *netdev) 2748 { 2749 return MACB_GREGS_NBR * sizeof(u32); 2750 } 2751 2752 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2753 void *p) 2754 { 2755 struct macb *bp = netdev_priv(dev); 2756 unsigned int tail, head; 2757 u32 *regs_buff = p; 2758 2759 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2760 | MACB_GREGS_VERSION; 2761 2762 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2763 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2764 2765 regs_buff[0] = macb_readl(bp, NCR); 2766 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2767 regs_buff[2] = macb_readl(bp, NSR); 2768 regs_buff[3] = macb_readl(bp, TSR); 2769 regs_buff[4] = macb_readl(bp, RBQP); 2770 regs_buff[5] = macb_readl(bp, TBQP); 2771 regs_buff[6] = macb_readl(bp, RSR); 2772 regs_buff[7] = macb_readl(bp, IMR); 2773 2774 regs_buff[8] = tail; 2775 regs_buff[9] = head; 2776 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2777 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2778 2779 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2780 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2781 if (macb_is_gem(bp)) 2782 regs_buff[13] = gem_readl(bp, DMACFG); 2783 } 2784 2785 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2786 { 2787 struct macb *bp = netdev_priv(netdev); 2788 2789 wol->supported = 0; 2790 wol->wolopts = 0; 2791 2792 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) 2793 phylink_ethtool_get_wol(bp->phylink, wol); 2794 } 2795 2796 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2797 { 2798 struct macb *bp = netdev_priv(netdev); 2799 int ret; 2800 2801 ret = phylink_ethtool_set_wol(bp->phylink, wol); 2802 if (!ret) 2803 return 0; 2804 2805 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2806 (wol->wolopts & ~WAKE_MAGIC)) 2807 return -EOPNOTSUPP; 2808 2809 if (wol->wolopts & WAKE_MAGIC) 2810 bp->wol |= MACB_WOL_ENABLED; 2811 else 2812 bp->wol &= ~MACB_WOL_ENABLED; 2813 2814 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2815 2816 return 0; 2817 } 2818 2819 static int macb_get_link_ksettings(struct net_device *netdev, 2820 struct ethtool_link_ksettings *kset) 2821 { 2822 struct macb *bp = netdev_priv(netdev); 2823 2824 return phylink_ethtool_ksettings_get(bp->phylink, kset); 2825 } 2826 2827 static int macb_set_link_ksettings(struct net_device *netdev, 2828 const struct ethtool_link_ksettings *kset) 2829 { 2830 struct macb *bp = netdev_priv(netdev); 2831 2832 return phylink_ethtool_ksettings_set(bp->phylink, kset); 2833 } 2834 2835 static void macb_get_ringparam(struct net_device *netdev, 2836 struct ethtool_ringparam *ring) 2837 { 2838 struct macb *bp = netdev_priv(netdev); 2839 2840 ring->rx_max_pending = MAX_RX_RING_SIZE; 2841 ring->tx_max_pending = MAX_TX_RING_SIZE; 2842 2843 ring->rx_pending = bp->rx_ring_size; 2844 ring->tx_pending = bp->tx_ring_size; 2845 } 2846 2847 static int macb_set_ringparam(struct net_device *netdev, 2848 struct ethtool_ringparam *ring) 2849 { 2850 struct macb *bp = netdev_priv(netdev); 2851 u32 new_rx_size, new_tx_size; 2852 unsigned int reset = 0; 2853 2854 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2855 return -EINVAL; 2856 2857 new_rx_size = clamp_t(u32, ring->rx_pending, 2858 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2859 new_rx_size = roundup_pow_of_two(new_rx_size); 2860 2861 new_tx_size = clamp_t(u32, ring->tx_pending, 2862 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2863 new_tx_size = roundup_pow_of_two(new_tx_size); 2864 2865 if ((new_tx_size == bp->tx_ring_size) && 2866 (new_rx_size == bp->rx_ring_size)) { 2867 /* nothing to do */ 2868 return 0; 2869 } 2870 2871 if (netif_running(bp->dev)) { 2872 reset = 1; 2873 macb_close(bp->dev); 2874 } 2875 2876 bp->rx_ring_size = new_rx_size; 2877 bp->tx_ring_size = new_tx_size; 2878 2879 if (reset) 2880 macb_open(bp->dev); 2881 2882 return 0; 2883 } 2884 2885 #ifdef CONFIG_MACB_USE_HWSTAMP 2886 static unsigned int gem_get_tsu_rate(struct macb *bp) 2887 { 2888 struct clk *tsu_clk; 2889 unsigned int tsu_rate; 2890 2891 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2892 if (!IS_ERR(tsu_clk)) 2893 tsu_rate = clk_get_rate(tsu_clk); 2894 /* try pclk instead */ 2895 else if (!IS_ERR(bp->pclk)) { 2896 tsu_clk = bp->pclk; 2897 tsu_rate = clk_get_rate(tsu_clk); 2898 } else 2899 return -ENOTSUPP; 2900 return tsu_rate; 2901 } 2902 2903 static s32 gem_get_ptp_max_adj(void) 2904 { 2905 return 64000000; 2906 } 2907 2908 static int gem_get_ts_info(struct net_device *dev, 2909 struct ethtool_ts_info *info) 2910 { 2911 struct macb *bp = netdev_priv(dev); 2912 2913 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2914 ethtool_op_get_ts_info(dev, info); 2915 return 0; 2916 } 2917 2918 info->so_timestamping = 2919 SOF_TIMESTAMPING_TX_SOFTWARE | 2920 SOF_TIMESTAMPING_RX_SOFTWARE | 2921 SOF_TIMESTAMPING_SOFTWARE | 2922 SOF_TIMESTAMPING_TX_HARDWARE | 2923 SOF_TIMESTAMPING_RX_HARDWARE | 2924 SOF_TIMESTAMPING_RAW_HARDWARE; 2925 info->tx_types = 2926 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2927 (1 << HWTSTAMP_TX_OFF) | 2928 (1 << HWTSTAMP_TX_ON); 2929 info->rx_filters = 2930 (1 << HWTSTAMP_FILTER_NONE) | 2931 (1 << HWTSTAMP_FILTER_ALL); 2932 2933 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2934 2935 return 0; 2936 } 2937 2938 static struct macb_ptp_info gem_ptp_info = { 2939 .ptp_init = gem_ptp_init, 2940 .ptp_remove = gem_ptp_remove, 2941 .get_ptp_max_adj = gem_get_ptp_max_adj, 2942 .get_tsu_rate = gem_get_tsu_rate, 2943 .get_ts_info = gem_get_ts_info, 2944 .get_hwtst = gem_get_hwtst, 2945 .set_hwtst = gem_set_hwtst, 2946 }; 2947 #endif 2948 2949 static int macb_get_ts_info(struct net_device *netdev, 2950 struct ethtool_ts_info *info) 2951 { 2952 struct macb *bp = netdev_priv(netdev); 2953 2954 if (bp->ptp_info) 2955 return bp->ptp_info->get_ts_info(netdev, info); 2956 2957 return ethtool_op_get_ts_info(netdev, info); 2958 } 2959 2960 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2961 { 2962 struct net_device *netdev = bp->dev; 2963 struct ethtool_rx_fs_item *item; 2964 u32 t2_scr; 2965 int num_t2_scr; 2966 2967 if (!(netdev->features & NETIF_F_NTUPLE)) 2968 return; 2969 2970 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2971 2972 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2973 struct ethtool_rx_flow_spec *fs = &item->fs; 2974 struct ethtool_tcpip4_spec *tp4sp_m; 2975 2976 if (fs->location >= num_t2_scr) 2977 continue; 2978 2979 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2980 2981 /* enable/disable screener regs for the flow entry */ 2982 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2983 2984 /* only enable fields with no masking */ 2985 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2986 2987 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2988 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2989 else 2990 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2991 2992 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2993 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2994 else 2995 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2996 2997 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2998 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2999 else 3000 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 3001 3002 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 3003 } 3004 } 3005 3006 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 3007 { 3008 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 3009 uint16_t index = fs->location; 3010 u32 w0, w1, t2_scr; 3011 bool cmp_a = false; 3012 bool cmp_b = false; 3013 bool cmp_c = false; 3014 3015 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 3016 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3017 3018 /* ignore field if any masking set */ 3019 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 3020 /* 1st compare reg - IP source address */ 3021 w0 = 0; 3022 w1 = 0; 3023 w0 = tp4sp_v->ip4src; 3024 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3025 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3026 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 3027 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 3028 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 3029 cmp_a = true; 3030 } 3031 3032 /* ignore field if any masking set */ 3033 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 3034 /* 2nd compare reg - IP destination address */ 3035 w0 = 0; 3036 w1 = 0; 3037 w0 = tp4sp_v->ip4dst; 3038 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3039 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3040 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 3041 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 3042 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 3043 cmp_b = true; 3044 } 3045 3046 /* ignore both port fields if masking set in both */ 3047 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 3048 /* 3rd compare reg - source port, destination port */ 3049 w0 = 0; 3050 w1 = 0; 3051 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 3052 if (tp4sp_m->psrc == tp4sp_m->pdst) { 3053 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 3054 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3055 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3056 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3057 } else { 3058 /* only one port definition */ 3059 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 3060 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 3061 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 3062 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 3063 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3064 } else { /* dst port */ 3065 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3066 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 3067 } 3068 } 3069 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 3070 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 3071 cmp_c = true; 3072 } 3073 3074 t2_scr = 0; 3075 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 3076 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 3077 if (cmp_a) 3078 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 3079 if (cmp_b) 3080 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 3081 if (cmp_c) 3082 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 3083 gem_writel_n(bp, SCRT2, index, t2_scr); 3084 } 3085 3086 static int gem_add_flow_filter(struct net_device *netdev, 3087 struct ethtool_rxnfc *cmd) 3088 { 3089 struct macb *bp = netdev_priv(netdev); 3090 struct ethtool_rx_flow_spec *fs = &cmd->fs; 3091 struct ethtool_rx_fs_item *item, *newfs; 3092 unsigned long flags; 3093 int ret = -EINVAL; 3094 bool added = false; 3095 3096 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 3097 if (newfs == NULL) 3098 return -ENOMEM; 3099 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 3100 3101 netdev_dbg(netdev, 3102 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3103 fs->flow_type, (int)fs->ring_cookie, fs->location, 3104 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3105 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3106 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 3107 3108 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3109 3110 /* find correct place to add in list */ 3111 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3112 if (item->fs.location > newfs->fs.location) { 3113 list_add_tail(&newfs->list, &item->list); 3114 added = true; 3115 break; 3116 } else if (item->fs.location == fs->location) { 3117 netdev_err(netdev, "Rule not added: location %d not free!\n", 3118 fs->location); 3119 ret = -EBUSY; 3120 goto err; 3121 } 3122 } 3123 if (!added) 3124 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3125 3126 gem_prog_cmp_regs(bp, fs); 3127 bp->rx_fs_list.count++; 3128 /* enable filtering if NTUPLE on */ 3129 gem_enable_flow_filters(bp, 1); 3130 3131 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3132 return 0; 3133 3134 err: 3135 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3136 kfree(newfs); 3137 return ret; 3138 } 3139 3140 static int gem_del_flow_filter(struct net_device *netdev, 3141 struct ethtool_rxnfc *cmd) 3142 { 3143 struct macb *bp = netdev_priv(netdev); 3144 struct ethtool_rx_fs_item *item; 3145 struct ethtool_rx_flow_spec *fs; 3146 unsigned long flags; 3147 3148 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3149 3150 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3151 if (item->fs.location == cmd->fs.location) { 3152 /* disable screener regs for the flow entry */ 3153 fs = &(item->fs); 3154 netdev_dbg(netdev, 3155 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3156 fs->flow_type, (int)fs->ring_cookie, fs->location, 3157 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3158 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3159 htons(fs->h_u.tcp_ip4_spec.psrc), 3160 htons(fs->h_u.tcp_ip4_spec.pdst)); 3161 3162 gem_writel_n(bp, SCRT2, fs->location, 0); 3163 3164 list_del(&item->list); 3165 bp->rx_fs_list.count--; 3166 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3167 kfree(item); 3168 return 0; 3169 } 3170 } 3171 3172 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3173 return -EINVAL; 3174 } 3175 3176 static int gem_get_flow_entry(struct net_device *netdev, 3177 struct ethtool_rxnfc *cmd) 3178 { 3179 struct macb *bp = netdev_priv(netdev); 3180 struct ethtool_rx_fs_item *item; 3181 3182 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3183 if (item->fs.location == cmd->fs.location) { 3184 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3185 return 0; 3186 } 3187 } 3188 return -EINVAL; 3189 } 3190 3191 static int gem_get_all_flow_entries(struct net_device *netdev, 3192 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3193 { 3194 struct macb *bp = netdev_priv(netdev); 3195 struct ethtool_rx_fs_item *item; 3196 uint32_t cnt = 0; 3197 3198 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3199 if (cnt == cmd->rule_cnt) 3200 return -EMSGSIZE; 3201 rule_locs[cnt] = item->fs.location; 3202 cnt++; 3203 } 3204 cmd->data = bp->max_tuples; 3205 cmd->rule_cnt = cnt; 3206 3207 return 0; 3208 } 3209 3210 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3211 u32 *rule_locs) 3212 { 3213 struct macb *bp = netdev_priv(netdev); 3214 int ret = 0; 3215 3216 switch (cmd->cmd) { 3217 case ETHTOOL_GRXRINGS: 3218 cmd->data = bp->num_queues; 3219 break; 3220 case ETHTOOL_GRXCLSRLCNT: 3221 cmd->rule_cnt = bp->rx_fs_list.count; 3222 break; 3223 case ETHTOOL_GRXCLSRULE: 3224 ret = gem_get_flow_entry(netdev, cmd); 3225 break; 3226 case ETHTOOL_GRXCLSRLALL: 3227 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3228 break; 3229 default: 3230 netdev_err(netdev, 3231 "Command parameter %d is not supported\n", cmd->cmd); 3232 ret = -EOPNOTSUPP; 3233 } 3234 3235 return ret; 3236 } 3237 3238 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3239 { 3240 struct macb *bp = netdev_priv(netdev); 3241 int ret; 3242 3243 switch (cmd->cmd) { 3244 case ETHTOOL_SRXCLSRLINS: 3245 if ((cmd->fs.location >= bp->max_tuples) 3246 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3247 ret = -EINVAL; 3248 break; 3249 } 3250 ret = gem_add_flow_filter(netdev, cmd); 3251 break; 3252 case ETHTOOL_SRXCLSRLDEL: 3253 ret = gem_del_flow_filter(netdev, cmd); 3254 break; 3255 default: 3256 netdev_err(netdev, 3257 "Command parameter %d is not supported\n", cmd->cmd); 3258 ret = -EOPNOTSUPP; 3259 } 3260 3261 return ret; 3262 } 3263 3264 static const struct ethtool_ops macb_ethtool_ops = { 3265 .get_regs_len = macb_get_regs_len, 3266 .get_regs = macb_get_regs, 3267 .get_link = ethtool_op_get_link, 3268 .get_ts_info = ethtool_op_get_ts_info, 3269 .get_wol = macb_get_wol, 3270 .set_wol = macb_set_wol, 3271 .get_link_ksettings = macb_get_link_ksettings, 3272 .set_link_ksettings = macb_set_link_ksettings, 3273 .get_ringparam = macb_get_ringparam, 3274 .set_ringparam = macb_set_ringparam, 3275 }; 3276 3277 static const struct ethtool_ops gem_ethtool_ops = { 3278 .get_regs_len = macb_get_regs_len, 3279 .get_regs = macb_get_regs, 3280 .get_link = ethtool_op_get_link, 3281 .get_ts_info = macb_get_ts_info, 3282 .get_ethtool_stats = gem_get_ethtool_stats, 3283 .get_strings = gem_get_ethtool_strings, 3284 .get_sset_count = gem_get_sset_count, 3285 .get_link_ksettings = macb_get_link_ksettings, 3286 .set_link_ksettings = macb_set_link_ksettings, 3287 .get_ringparam = macb_get_ringparam, 3288 .set_ringparam = macb_set_ringparam, 3289 .get_rxnfc = gem_get_rxnfc, 3290 .set_rxnfc = gem_set_rxnfc, 3291 }; 3292 3293 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3294 { 3295 struct macb *bp = netdev_priv(dev); 3296 3297 if (!netif_running(dev)) 3298 return -EINVAL; 3299 3300 if (bp->ptp_info) { 3301 switch (cmd) { 3302 case SIOCSHWTSTAMP: 3303 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3304 case SIOCGHWTSTAMP: 3305 return bp->ptp_info->get_hwtst(dev, rq); 3306 } 3307 } 3308 3309 return phylink_mii_ioctl(bp->phylink, rq, cmd); 3310 } 3311 3312 static inline void macb_set_txcsum_feature(struct macb *bp, 3313 netdev_features_t features) 3314 { 3315 u32 val; 3316 3317 if (!macb_is_gem(bp)) 3318 return; 3319 3320 val = gem_readl(bp, DMACFG); 3321 if (features & NETIF_F_HW_CSUM) 3322 val |= GEM_BIT(TXCOEN); 3323 else 3324 val &= ~GEM_BIT(TXCOEN); 3325 3326 gem_writel(bp, DMACFG, val); 3327 } 3328 3329 static inline void macb_set_rxcsum_feature(struct macb *bp, 3330 netdev_features_t features) 3331 { 3332 struct net_device *netdev = bp->dev; 3333 u32 val; 3334 3335 if (!macb_is_gem(bp)) 3336 return; 3337 3338 val = gem_readl(bp, NCFGR); 3339 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 3340 val |= GEM_BIT(RXCOEN); 3341 else 3342 val &= ~GEM_BIT(RXCOEN); 3343 3344 gem_writel(bp, NCFGR, val); 3345 } 3346 3347 static inline void macb_set_rxflow_feature(struct macb *bp, 3348 netdev_features_t features) 3349 { 3350 if (!macb_is_gem(bp)) 3351 return; 3352 3353 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 3354 } 3355 3356 static int macb_set_features(struct net_device *netdev, 3357 netdev_features_t features) 3358 { 3359 struct macb *bp = netdev_priv(netdev); 3360 netdev_features_t changed = features ^ netdev->features; 3361 3362 /* TX checksum offload */ 3363 if (changed & NETIF_F_HW_CSUM) 3364 macb_set_txcsum_feature(bp, features); 3365 3366 /* RX checksum offload */ 3367 if (changed & NETIF_F_RXCSUM) 3368 macb_set_rxcsum_feature(bp, features); 3369 3370 /* RX Flow Filters */ 3371 if (changed & NETIF_F_NTUPLE) 3372 macb_set_rxflow_feature(bp, features); 3373 3374 return 0; 3375 } 3376 3377 static void macb_restore_features(struct macb *bp) 3378 { 3379 struct net_device *netdev = bp->dev; 3380 netdev_features_t features = netdev->features; 3381 3382 /* TX checksum offload */ 3383 macb_set_txcsum_feature(bp, features); 3384 3385 /* RX checksum offload */ 3386 macb_set_rxcsum_feature(bp, features); 3387 3388 /* RX Flow Filters */ 3389 macb_set_rxflow_feature(bp, features); 3390 } 3391 3392 static const struct net_device_ops macb_netdev_ops = { 3393 .ndo_open = macb_open, 3394 .ndo_stop = macb_close, 3395 .ndo_start_xmit = macb_start_xmit, 3396 .ndo_set_rx_mode = macb_set_rx_mode, 3397 .ndo_get_stats = macb_get_stats, 3398 .ndo_do_ioctl = macb_ioctl, 3399 .ndo_validate_addr = eth_validate_addr, 3400 .ndo_change_mtu = macb_change_mtu, 3401 .ndo_set_mac_address = eth_mac_addr, 3402 #ifdef CONFIG_NET_POLL_CONTROLLER 3403 .ndo_poll_controller = macb_poll_controller, 3404 #endif 3405 .ndo_set_features = macb_set_features, 3406 .ndo_features_check = macb_features_check, 3407 }; 3408 3409 /* Configure peripheral capabilities according to device tree 3410 * and integration options used 3411 */ 3412 static void macb_configure_caps(struct macb *bp, 3413 const struct macb_config *dt_conf) 3414 { 3415 u32 dcfg; 3416 3417 if (dt_conf) 3418 bp->caps = dt_conf->caps; 3419 3420 if (hw_is_gem(bp->regs, bp->native_io)) { 3421 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3422 3423 dcfg = gem_readl(bp, DCFG1); 3424 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3425 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3426 dcfg = gem_readl(bp, DCFG2); 3427 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3428 bp->caps |= MACB_CAPS_FIFO_MODE; 3429 #ifdef CONFIG_MACB_USE_HWSTAMP 3430 if (gem_has_ptp(bp)) { 3431 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3432 dev_err(&bp->pdev->dev, 3433 "GEM doesn't support hardware ptp.\n"); 3434 else { 3435 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3436 bp->ptp_info = &gem_ptp_info; 3437 } 3438 } 3439 #endif 3440 } 3441 3442 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3443 } 3444 3445 static void macb_probe_queues(void __iomem *mem, 3446 bool native_io, 3447 unsigned int *queue_mask, 3448 unsigned int *num_queues) 3449 { 3450 unsigned int hw_q; 3451 3452 *queue_mask = 0x1; 3453 *num_queues = 1; 3454 3455 /* is it macb or gem ? 3456 * 3457 * We need to read directly from the hardware here because 3458 * we are early in the probe process and don't have the 3459 * MACB_CAPS_MACB_IS_GEM flag positioned 3460 */ 3461 if (!hw_is_gem(mem, native_io)) 3462 return; 3463 3464 /* bit 0 is never set but queue 0 always exists */ 3465 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3466 3467 *queue_mask |= 0x1; 3468 3469 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3470 if (*queue_mask & (1 << hw_q)) 3471 (*num_queues)++; 3472 } 3473 3474 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3475 struct clk **hclk, struct clk **tx_clk, 3476 struct clk **rx_clk, struct clk **tsu_clk) 3477 { 3478 struct macb_platform_data *pdata; 3479 int err; 3480 3481 pdata = dev_get_platdata(&pdev->dev); 3482 if (pdata) { 3483 *pclk = pdata->pclk; 3484 *hclk = pdata->hclk; 3485 } else { 3486 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3487 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3488 } 3489 3490 if (IS_ERR_OR_NULL(*pclk)) { 3491 err = PTR_ERR(*pclk); 3492 if (!err) 3493 err = -ENODEV; 3494 3495 dev_err(&pdev->dev, "failed to get macb_clk (%d)\n", err); 3496 return err; 3497 } 3498 3499 if (IS_ERR_OR_NULL(*hclk)) { 3500 err = PTR_ERR(*hclk); 3501 if (!err) 3502 err = -ENODEV; 3503 3504 dev_err(&pdev->dev, "failed to get hclk (%d)\n", err); 3505 return err; 3506 } 3507 3508 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); 3509 if (IS_ERR(*tx_clk)) 3510 return PTR_ERR(*tx_clk); 3511 3512 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); 3513 if (IS_ERR(*rx_clk)) 3514 return PTR_ERR(*rx_clk); 3515 3516 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); 3517 if (IS_ERR(*tsu_clk)) 3518 return PTR_ERR(*tsu_clk); 3519 3520 err = clk_prepare_enable(*pclk); 3521 if (err) { 3522 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3523 return err; 3524 } 3525 3526 err = clk_prepare_enable(*hclk); 3527 if (err) { 3528 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 3529 goto err_disable_pclk; 3530 } 3531 3532 err = clk_prepare_enable(*tx_clk); 3533 if (err) { 3534 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 3535 goto err_disable_hclk; 3536 } 3537 3538 err = clk_prepare_enable(*rx_clk); 3539 if (err) { 3540 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 3541 goto err_disable_txclk; 3542 } 3543 3544 err = clk_prepare_enable(*tsu_clk); 3545 if (err) { 3546 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 3547 goto err_disable_rxclk; 3548 } 3549 3550 return 0; 3551 3552 err_disable_rxclk: 3553 clk_disable_unprepare(*rx_clk); 3554 3555 err_disable_txclk: 3556 clk_disable_unprepare(*tx_clk); 3557 3558 err_disable_hclk: 3559 clk_disable_unprepare(*hclk); 3560 3561 err_disable_pclk: 3562 clk_disable_unprepare(*pclk); 3563 3564 return err; 3565 } 3566 3567 static int macb_init(struct platform_device *pdev) 3568 { 3569 struct net_device *dev = platform_get_drvdata(pdev); 3570 unsigned int hw_q, q; 3571 struct macb *bp = netdev_priv(dev); 3572 struct macb_queue *queue; 3573 int err; 3574 u32 val, reg; 3575 3576 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3577 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3578 3579 /* set the queue register mapping once for all: queue0 has a special 3580 * register mapping but we don't want to test the queue index then 3581 * compute the corresponding register offset at run time. 3582 */ 3583 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3584 if (!(bp->queue_mask & (1 << hw_q))) 3585 continue; 3586 3587 queue = &bp->queues[q]; 3588 queue->bp = bp; 3589 netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT); 3590 if (hw_q) { 3591 queue->ISR = GEM_ISR(hw_q - 1); 3592 queue->IER = GEM_IER(hw_q - 1); 3593 queue->IDR = GEM_IDR(hw_q - 1); 3594 queue->IMR = GEM_IMR(hw_q - 1); 3595 queue->TBQP = GEM_TBQP(hw_q - 1); 3596 queue->RBQP = GEM_RBQP(hw_q - 1); 3597 queue->RBQS = GEM_RBQS(hw_q - 1); 3598 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3599 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3600 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3601 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3602 } 3603 #endif 3604 } else { 3605 /* queue0 uses legacy registers */ 3606 queue->ISR = MACB_ISR; 3607 queue->IER = MACB_IER; 3608 queue->IDR = MACB_IDR; 3609 queue->IMR = MACB_IMR; 3610 queue->TBQP = MACB_TBQP; 3611 queue->RBQP = MACB_RBQP; 3612 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3613 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3614 queue->TBQPH = MACB_TBQPH; 3615 queue->RBQPH = MACB_RBQPH; 3616 } 3617 #endif 3618 } 3619 3620 /* get irq: here we use the linux queue index, not the hardware 3621 * queue index. the queue irq definitions in the device tree 3622 * must remove the optional gaps that could exist in the 3623 * hardware queue mask. 3624 */ 3625 queue->irq = platform_get_irq(pdev, q); 3626 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3627 IRQF_SHARED, dev->name, queue); 3628 if (err) { 3629 dev_err(&pdev->dev, 3630 "Unable to request IRQ %d (error %d)\n", 3631 queue->irq, err); 3632 return err; 3633 } 3634 3635 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3636 q++; 3637 } 3638 3639 dev->netdev_ops = &macb_netdev_ops; 3640 3641 /* setup appropriated routines according to adapter type */ 3642 if (macb_is_gem(bp)) { 3643 bp->max_tx_length = GEM_MAX_TX_LEN; 3644 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3645 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3646 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3647 bp->macbgem_ops.mog_rx = gem_rx; 3648 dev->ethtool_ops = &gem_ethtool_ops; 3649 } else { 3650 bp->max_tx_length = MACB_MAX_TX_LEN; 3651 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3652 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3653 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3654 bp->macbgem_ops.mog_rx = macb_rx; 3655 dev->ethtool_ops = &macb_ethtool_ops; 3656 } 3657 3658 /* Set features */ 3659 dev->hw_features = NETIF_F_SG; 3660 3661 /* Check LSO capability */ 3662 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3663 dev->hw_features |= MACB_NETIF_LSO; 3664 3665 /* Checksum offload is only available on gem with packet buffer */ 3666 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3667 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3668 if (bp->caps & MACB_CAPS_SG_DISABLED) 3669 dev->hw_features &= ~NETIF_F_SG; 3670 dev->features = dev->hw_features; 3671 3672 /* Check RX Flow Filters support. 3673 * Max Rx flows set by availability of screeners & compare regs: 3674 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3675 */ 3676 reg = gem_readl(bp, DCFG8); 3677 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3678 GEM_BFEXT(T2SCR, reg)); 3679 if (bp->max_tuples > 0) { 3680 /* also needs one ethtype match to check IPv4 */ 3681 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3682 /* program this reg now */ 3683 reg = 0; 3684 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3685 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3686 /* Filtering is supported in hw but don't enable it in kernel now */ 3687 dev->hw_features |= NETIF_F_NTUPLE; 3688 /* init Rx flow definitions */ 3689 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3690 bp->rx_fs_list.count = 0; 3691 spin_lock_init(&bp->rx_fs_lock); 3692 } else 3693 bp->max_tuples = 0; 3694 } 3695 3696 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3697 val = 0; 3698 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3699 val = GEM_BIT(RGMII); 3700 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3701 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3702 val = MACB_BIT(RMII); 3703 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3704 val = MACB_BIT(MII); 3705 3706 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3707 val |= MACB_BIT(CLKEN); 3708 3709 macb_or_gem_writel(bp, USRIO, val); 3710 } 3711 3712 /* Set MII management clock divider */ 3713 val = macb_mdc_clk_div(bp); 3714 val |= macb_dbw(bp); 3715 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3716 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3717 macb_writel(bp, NCFGR, val); 3718 3719 return 0; 3720 } 3721 3722 #if defined(CONFIG_OF) 3723 /* 1518 rounded up */ 3724 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3725 /* max number of receive buffers */ 3726 #define AT91ETHER_MAX_RX_DESCR 9 3727 3728 static struct sifive_fu540_macb_mgmt *mgmt; 3729 3730 /* Initialize and start the Receiver and Transmit subsystems */ 3731 static int at91ether_start(struct net_device *dev) 3732 { 3733 struct macb *lp = netdev_priv(dev); 3734 struct macb_queue *q = &lp->queues[0]; 3735 struct macb_dma_desc *desc; 3736 dma_addr_t addr; 3737 u32 ctl; 3738 int i; 3739 3740 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3741 (AT91ETHER_MAX_RX_DESCR * 3742 macb_dma_desc_get_size(lp)), 3743 &q->rx_ring_dma, GFP_KERNEL); 3744 if (!q->rx_ring) 3745 return -ENOMEM; 3746 3747 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3748 AT91ETHER_MAX_RX_DESCR * 3749 AT91ETHER_MAX_RBUFF_SZ, 3750 &q->rx_buffers_dma, GFP_KERNEL); 3751 if (!q->rx_buffers) { 3752 dma_free_coherent(&lp->pdev->dev, 3753 AT91ETHER_MAX_RX_DESCR * 3754 macb_dma_desc_get_size(lp), 3755 q->rx_ring, q->rx_ring_dma); 3756 q->rx_ring = NULL; 3757 return -ENOMEM; 3758 } 3759 3760 addr = q->rx_buffers_dma; 3761 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3762 desc = macb_rx_desc(q, i); 3763 macb_set_addr(lp, desc, addr); 3764 desc->ctrl = 0; 3765 addr += AT91ETHER_MAX_RBUFF_SZ; 3766 } 3767 3768 /* Set the Wrap bit on the last descriptor */ 3769 desc->addr |= MACB_BIT(RX_WRAP); 3770 3771 /* Reset buffer index */ 3772 q->rx_tail = 0; 3773 3774 /* Program address of descriptor list in Rx Buffer Queue register */ 3775 macb_writel(lp, RBQP, q->rx_ring_dma); 3776 3777 /* Enable Receive and Transmit */ 3778 ctl = macb_readl(lp, NCR); 3779 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3780 3781 return 0; 3782 } 3783 3784 /* Open the ethernet interface */ 3785 static int at91ether_open(struct net_device *dev) 3786 { 3787 struct macb *lp = netdev_priv(dev); 3788 u32 ctl; 3789 int ret; 3790 3791 /* Clear internal statistics */ 3792 ctl = macb_readl(lp, NCR); 3793 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3794 3795 macb_set_hwaddr(lp); 3796 3797 ret = at91ether_start(dev); 3798 if (ret) 3799 return ret; 3800 3801 /* Enable MAC interrupts */ 3802 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3803 MACB_BIT(RXUBR) | 3804 MACB_BIT(ISR_TUND) | 3805 MACB_BIT(ISR_RLE) | 3806 MACB_BIT(TCOMP) | 3807 MACB_BIT(ISR_ROVR) | 3808 MACB_BIT(HRESP)); 3809 3810 ret = macb_phylink_connect(lp); 3811 if (ret) 3812 return ret; 3813 3814 netif_start_queue(dev); 3815 3816 return 0; 3817 } 3818 3819 /* Close the interface */ 3820 static int at91ether_close(struct net_device *dev) 3821 { 3822 struct macb *lp = netdev_priv(dev); 3823 struct macb_queue *q = &lp->queues[0]; 3824 u32 ctl; 3825 3826 /* Disable Receiver and Transmitter */ 3827 ctl = macb_readl(lp, NCR); 3828 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3829 3830 /* Disable MAC interrupts */ 3831 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3832 MACB_BIT(RXUBR) | 3833 MACB_BIT(ISR_TUND) | 3834 MACB_BIT(ISR_RLE) | 3835 MACB_BIT(TCOMP) | 3836 MACB_BIT(ISR_ROVR) | 3837 MACB_BIT(HRESP)); 3838 3839 netif_stop_queue(dev); 3840 3841 phylink_stop(lp->phylink); 3842 phylink_disconnect_phy(lp->phylink); 3843 3844 dma_free_coherent(&lp->pdev->dev, 3845 AT91ETHER_MAX_RX_DESCR * 3846 macb_dma_desc_get_size(lp), 3847 q->rx_ring, q->rx_ring_dma); 3848 q->rx_ring = NULL; 3849 3850 dma_free_coherent(&lp->pdev->dev, 3851 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3852 q->rx_buffers, q->rx_buffers_dma); 3853 q->rx_buffers = NULL; 3854 3855 return 0; 3856 } 3857 3858 /* Transmit packet */ 3859 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3860 struct net_device *dev) 3861 { 3862 struct macb *lp = netdev_priv(dev); 3863 3864 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3865 netif_stop_queue(dev); 3866 3867 /* Store packet information (to free when Tx completed) */ 3868 lp->skb = skb; 3869 lp->skb_length = skb->len; 3870 lp->skb_physaddr = dma_map_single(&lp->pdev->dev, skb->data, 3871 skb->len, DMA_TO_DEVICE); 3872 if (dma_mapping_error(&lp->pdev->dev, lp->skb_physaddr)) { 3873 dev_kfree_skb_any(skb); 3874 dev->stats.tx_dropped++; 3875 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3876 return NETDEV_TX_OK; 3877 } 3878 3879 /* Set address of the data in the Transmit Address register */ 3880 macb_writel(lp, TAR, lp->skb_physaddr); 3881 /* Set length of the packet in the Transmit Control register */ 3882 macb_writel(lp, TCR, skb->len); 3883 3884 } else { 3885 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3886 return NETDEV_TX_BUSY; 3887 } 3888 3889 return NETDEV_TX_OK; 3890 } 3891 3892 /* Extract received frame from buffer descriptors and sent to upper layers. 3893 * (Called from interrupt context) 3894 */ 3895 static void at91ether_rx(struct net_device *dev) 3896 { 3897 struct macb *lp = netdev_priv(dev); 3898 struct macb_queue *q = &lp->queues[0]; 3899 struct macb_dma_desc *desc; 3900 unsigned char *p_recv; 3901 struct sk_buff *skb; 3902 unsigned int pktlen; 3903 3904 desc = macb_rx_desc(q, q->rx_tail); 3905 while (desc->addr & MACB_BIT(RX_USED)) { 3906 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3907 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3908 skb = netdev_alloc_skb(dev, pktlen + 2); 3909 if (skb) { 3910 skb_reserve(skb, 2); 3911 skb_put_data(skb, p_recv, pktlen); 3912 3913 skb->protocol = eth_type_trans(skb, dev); 3914 dev->stats.rx_packets++; 3915 dev->stats.rx_bytes += pktlen; 3916 netif_rx(skb); 3917 } else { 3918 dev->stats.rx_dropped++; 3919 } 3920 3921 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3922 dev->stats.multicast++; 3923 3924 /* reset ownership bit */ 3925 desc->addr &= ~MACB_BIT(RX_USED); 3926 3927 /* wrap after last buffer */ 3928 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3929 q->rx_tail = 0; 3930 else 3931 q->rx_tail++; 3932 3933 desc = macb_rx_desc(q, q->rx_tail); 3934 } 3935 } 3936 3937 /* MAC interrupt handler */ 3938 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3939 { 3940 struct net_device *dev = dev_id; 3941 struct macb *lp = netdev_priv(dev); 3942 u32 intstatus, ctl; 3943 3944 /* MAC Interrupt Status register indicates what interrupts are pending. 3945 * It is automatically cleared once read. 3946 */ 3947 intstatus = macb_readl(lp, ISR); 3948 3949 /* Receive complete */ 3950 if (intstatus & MACB_BIT(RCOMP)) 3951 at91ether_rx(dev); 3952 3953 /* Transmit complete */ 3954 if (intstatus & MACB_BIT(TCOMP)) { 3955 /* The TCOM bit is set even if the transmission failed */ 3956 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3957 dev->stats.tx_errors++; 3958 3959 if (lp->skb) { 3960 dev_consume_skb_irq(lp->skb); 3961 lp->skb = NULL; 3962 dma_unmap_single(&lp->pdev->dev, lp->skb_physaddr, 3963 lp->skb_length, DMA_TO_DEVICE); 3964 dev->stats.tx_packets++; 3965 dev->stats.tx_bytes += lp->skb_length; 3966 } 3967 netif_wake_queue(dev); 3968 } 3969 3970 /* Work-around for EMAC Errata section 41.3.1 */ 3971 if (intstatus & MACB_BIT(RXUBR)) { 3972 ctl = macb_readl(lp, NCR); 3973 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3974 wmb(); 3975 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3976 } 3977 3978 if (intstatus & MACB_BIT(ISR_ROVR)) 3979 netdev_err(dev, "ROVR error\n"); 3980 3981 return IRQ_HANDLED; 3982 } 3983 3984 #ifdef CONFIG_NET_POLL_CONTROLLER 3985 static void at91ether_poll_controller(struct net_device *dev) 3986 { 3987 unsigned long flags; 3988 3989 local_irq_save(flags); 3990 at91ether_interrupt(dev->irq, dev); 3991 local_irq_restore(flags); 3992 } 3993 #endif 3994 3995 static const struct net_device_ops at91ether_netdev_ops = { 3996 .ndo_open = at91ether_open, 3997 .ndo_stop = at91ether_close, 3998 .ndo_start_xmit = at91ether_start_xmit, 3999 .ndo_get_stats = macb_get_stats, 4000 .ndo_set_rx_mode = macb_set_rx_mode, 4001 .ndo_set_mac_address = eth_mac_addr, 4002 .ndo_do_ioctl = macb_ioctl, 4003 .ndo_validate_addr = eth_validate_addr, 4004 #ifdef CONFIG_NET_POLL_CONTROLLER 4005 .ndo_poll_controller = at91ether_poll_controller, 4006 #endif 4007 }; 4008 4009 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 4010 struct clk **hclk, struct clk **tx_clk, 4011 struct clk **rx_clk, struct clk **tsu_clk) 4012 { 4013 int err; 4014 4015 *hclk = NULL; 4016 *tx_clk = NULL; 4017 *rx_clk = NULL; 4018 *tsu_clk = NULL; 4019 4020 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 4021 if (IS_ERR(*pclk)) 4022 return PTR_ERR(*pclk); 4023 4024 err = clk_prepare_enable(*pclk); 4025 if (err) { 4026 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 4027 return err; 4028 } 4029 4030 return 0; 4031 } 4032 4033 static int at91ether_init(struct platform_device *pdev) 4034 { 4035 struct net_device *dev = platform_get_drvdata(pdev); 4036 struct macb *bp = netdev_priv(dev); 4037 int err; 4038 u32 reg; 4039 4040 bp->queues[0].bp = bp; 4041 4042 dev->netdev_ops = &at91ether_netdev_ops; 4043 dev->ethtool_ops = &macb_ethtool_ops; 4044 4045 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 4046 0, dev->name, dev); 4047 if (err) 4048 return err; 4049 4050 macb_writel(bp, NCR, 0); 4051 4052 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 4053 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 4054 reg |= MACB_BIT(RM9200_RMII); 4055 4056 macb_writel(bp, NCFGR, reg); 4057 4058 return 0; 4059 } 4060 4061 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 4062 unsigned long parent_rate) 4063 { 4064 return mgmt->rate; 4065 } 4066 4067 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, 4068 unsigned long *parent_rate) 4069 { 4070 if (WARN_ON(rate < 2500000)) 4071 return 2500000; 4072 else if (rate == 2500000) 4073 return 2500000; 4074 else if (WARN_ON(rate < 13750000)) 4075 return 2500000; 4076 else if (WARN_ON(rate < 25000000)) 4077 return 25000000; 4078 else if (rate == 25000000) 4079 return 25000000; 4080 else if (WARN_ON(rate < 75000000)) 4081 return 25000000; 4082 else if (WARN_ON(rate < 125000000)) 4083 return 125000000; 4084 else if (rate == 125000000) 4085 return 125000000; 4086 4087 WARN_ON(rate > 125000000); 4088 4089 return 125000000; 4090 } 4091 4092 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 4093 unsigned long parent_rate) 4094 { 4095 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); 4096 if (rate != 125000000) 4097 iowrite32(1, mgmt->reg); 4098 else 4099 iowrite32(0, mgmt->reg); 4100 mgmt->rate = rate; 4101 4102 return 0; 4103 } 4104 4105 static const struct clk_ops fu540_c000_ops = { 4106 .recalc_rate = fu540_macb_tx_recalc_rate, 4107 .round_rate = fu540_macb_tx_round_rate, 4108 .set_rate = fu540_macb_tx_set_rate, 4109 }; 4110 4111 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 4112 struct clk **hclk, struct clk **tx_clk, 4113 struct clk **rx_clk, struct clk **tsu_clk) 4114 { 4115 struct clk_init_data init; 4116 int err = 0; 4117 4118 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 4119 if (err) 4120 return err; 4121 4122 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 4123 if (!mgmt) 4124 return -ENOMEM; 4125 4126 init.name = "sifive-gemgxl-mgmt"; 4127 init.ops = &fu540_c000_ops; 4128 init.flags = 0; 4129 init.num_parents = 0; 4130 4131 mgmt->rate = 0; 4132 mgmt->hw.init = &init; 4133 4134 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); 4135 if (IS_ERR(*tx_clk)) 4136 return PTR_ERR(*tx_clk); 4137 4138 err = clk_prepare_enable(*tx_clk); 4139 if (err) 4140 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 4141 else 4142 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 4143 4144 return 0; 4145 } 4146 4147 static int fu540_c000_init(struct platform_device *pdev) 4148 { 4149 struct resource *res; 4150 4151 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 4152 if (!res) 4153 return -ENODEV; 4154 4155 mgmt->reg = ioremap(res->start, resource_size(res)); 4156 if (!mgmt->reg) 4157 return -ENOMEM; 4158 4159 return macb_init(pdev); 4160 } 4161 4162 static const struct macb_config fu540_c000_config = { 4163 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 4164 MACB_CAPS_GEM_HAS_PTP, 4165 .dma_burst_length = 16, 4166 .clk_init = fu540_c000_clk_init, 4167 .init = fu540_c000_init, 4168 .jumbo_max_len = 10240, 4169 }; 4170 4171 static const struct macb_config at91sam9260_config = { 4172 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4173 .clk_init = macb_clk_init, 4174 .init = macb_init, 4175 }; 4176 4177 static const struct macb_config sama5d3macb_config = { 4178 .caps = MACB_CAPS_SG_DISABLED 4179 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4180 .clk_init = macb_clk_init, 4181 .init = macb_init, 4182 }; 4183 4184 static const struct macb_config pc302gem_config = { 4185 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 4186 .dma_burst_length = 16, 4187 .clk_init = macb_clk_init, 4188 .init = macb_init, 4189 }; 4190 4191 static const struct macb_config sama5d2_config = { 4192 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4193 .dma_burst_length = 16, 4194 .clk_init = macb_clk_init, 4195 .init = macb_init, 4196 }; 4197 4198 static const struct macb_config sama5d3_config = { 4199 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 4200 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 4201 .dma_burst_length = 16, 4202 .clk_init = macb_clk_init, 4203 .init = macb_init, 4204 .jumbo_max_len = 10240, 4205 }; 4206 4207 static const struct macb_config sama5d4_config = { 4208 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4209 .dma_burst_length = 4, 4210 .clk_init = macb_clk_init, 4211 .init = macb_init, 4212 }; 4213 4214 static const struct macb_config emac_config = { 4215 .caps = MACB_CAPS_NEEDS_RSTONUBR, 4216 .clk_init = at91ether_clk_init, 4217 .init = at91ether_init, 4218 }; 4219 4220 static const struct macb_config np4_config = { 4221 .caps = MACB_CAPS_USRIO_DISABLED, 4222 .clk_init = macb_clk_init, 4223 .init = macb_init, 4224 }; 4225 4226 static const struct macb_config zynqmp_config = { 4227 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4228 MACB_CAPS_JUMBO | 4229 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 4230 .dma_burst_length = 16, 4231 .clk_init = macb_clk_init, 4232 .init = macb_init, 4233 .jumbo_max_len = 10240, 4234 }; 4235 4236 static const struct macb_config zynq_config = { 4237 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 4238 MACB_CAPS_NEEDS_RSTONUBR, 4239 .dma_burst_length = 16, 4240 .clk_init = macb_clk_init, 4241 .init = macb_init, 4242 }; 4243 4244 static const struct of_device_id macb_dt_ids[] = { 4245 { .compatible = "cdns,at32ap7000-macb" }, 4246 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 4247 { .compatible = "cdns,macb" }, 4248 { .compatible = "cdns,np4-macb", .data = &np4_config }, 4249 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 4250 { .compatible = "cdns,gem", .data = &pc302gem_config }, 4251 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 4252 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 4253 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 4254 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 4255 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 4256 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 4257 { .compatible = "cdns,emac", .data = &emac_config }, 4258 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 4259 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 4260 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 4261 { /* sentinel */ } 4262 }; 4263 MODULE_DEVICE_TABLE(of, macb_dt_ids); 4264 #endif /* CONFIG_OF */ 4265 4266 static const struct macb_config default_gem_config = { 4267 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4268 MACB_CAPS_JUMBO | 4269 MACB_CAPS_GEM_HAS_PTP, 4270 .dma_burst_length = 16, 4271 .clk_init = macb_clk_init, 4272 .init = macb_init, 4273 .jumbo_max_len = 10240, 4274 }; 4275 4276 static int macb_probe(struct platform_device *pdev) 4277 { 4278 const struct macb_config *macb_config = &default_gem_config; 4279 int (*clk_init)(struct platform_device *, struct clk **, 4280 struct clk **, struct clk **, struct clk **, 4281 struct clk **) = macb_config->clk_init; 4282 int (*init)(struct platform_device *) = macb_config->init; 4283 struct device_node *np = pdev->dev.of_node; 4284 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4285 struct clk *tsu_clk = NULL; 4286 unsigned int queue_mask, num_queues; 4287 bool native_io; 4288 phy_interface_t interface; 4289 struct net_device *dev; 4290 struct resource *regs; 4291 void __iomem *mem; 4292 const char *mac; 4293 struct macb *bp; 4294 int err, val; 4295 4296 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4297 mem = devm_ioremap_resource(&pdev->dev, regs); 4298 if (IS_ERR(mem)) 4299 return PTR_ERR(mem); 4300 4301 if (np) { 4302 const struct of_device_id *match; 4303 4304 match = of_match_node(macb_dt_ids, np); 4305 if (match && match->data) { 4306 macb_config = match->data; 4307 clk_init = macb_config->clk_init; 4308 init = macb_config->init; 4309 } 4310 } 4311 4312 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4313 if (err) 4314 return err; 4315 4316 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4317 pm_runtime_use_autosuspend(&pdev->dev); 4318 pm_runtime_get_noresume(&pdev->dev); 4319 pm_runtime_set_active(&pdev->dev); 4320 pm_runtime_enable(&pdev->dev); 4321 native_io = hw_is_native_io(mem); 4322 4323 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4324 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4325 if (!dev) { 4326 err = -ENOMEM; 4327 goto err_disable_clocks; 4328 } 4329 4330 dev->base_addr = regs->start; 4331 4332 SET_NETDEV_DEV(dev, &pdev->dev); 4333 4334 bp = netdev_priv(dev); 4335 bp->pdev = pdev; 4336 bp->dev = dev; 4337 bp->regs = mem; 4338 bp->native_io = native_io; 4339 if (native_io) { 4340 bp->macb_reg_readl = hw_readl_native; 4341 bp->macb_reg_writel = hw_writel_native; 4342 } else { 4343 bp->macb_reg_readl = hw_readl; 4344 bp->macb_reg_writel = hw_writel; 4345 } 4346 bp->num_queues = num_queues; 4347 bp->queue_mask = queue_mask; 4348 if (macb_config) 4349 bp->dma_burst_length = macb_config->dma_burst_length; 4350 bp->pclk = pclk; 4351 bp->hclk = hclk; 4352 bp->tx_clk = tx_clk; 4353 bp->rx_clk = rx_clk; 4354 bp->tsu_clk = tsu_clk; 4355 if (macb_config) 4356 bp->jumbo_max_len = macb_config->jumbo_max_len; 4357 4358 bp->wol = 0; 4359 if (of_get_property(np, "magic-packet", NULL)) 4360 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4361 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4362 4363 spin_lock_init(&bp->lock); 4364 4365 /* setup capabilities */ 4366 macb_configure_caps(bp, macb_config); 4367 4368 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4369 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4370 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4371 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4372 } 4373 #endif 4374 platform_set_drvdata(pdev, dev); 4375 4376 dev->irq = platform_get_irq(pdev, 0); 4377 if (dev->irq < 0) { 4378 err = dev->irq; 4379 goto err_out_free_netdev; 4380 } 4381 4382 /* MTU range: 68 - 1500 or 10240 */ 4383 dev->min_mtu = GEM_MTU_MIN_SIZE; 4384 if (bp->caps & MACB_CAPS_JUMBO) 4385 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4386 else 4387 dev->max_mtu = ETH_DATA_LEN; 4388 4389 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4390 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4391 if (val) 4392 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4393 macb_dma_desc_get_size(bp); 4394 4395 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4396 if (val) 4397 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4398 macb_dma_desc_get_size(bp); 4399 } 4400 4401 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4402 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4403 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4404 4405 mac = of_get_mac_address(np); 4406 if (PTR_ERR(mac) == -EPROBE_DEFER) { 4407 err = -EPROBE_DEFER; 4408 goto err_out_free_netdev; 4409 } else if (!IS_ERR_OR_NULL(mac)) { 4410 ether_addr_copy(bp->dev->dev_addr, mac); 4411 } else { 4412 macb_get_hwaddr(bp); 4413 } 4414 4415 err = of_get_phy_mode(np, &interface); 4416 if (err) 4417 /* not found in DT, MII by default */ 4418 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4419 else 4420 bp->phy_interface = interface; 4421 4422 bp->speed = SPEED_UNKNOWN; 4423 4424 /* IP specific init */ 4425 err = init(pdev); 4426 if (err) 4427 goto err_out_free_netdev; 4428 4429 err = macb_mii_init(bp); 4430 if (err) 4431 goto err_out_free_netdev; 4432 4433 netif_carrier_off(dev); 4434 4435 err = register_netdev(dev); 4436 if (err) { 4437 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4438 goto err_out_unregister_mdio; 4439 } 4440 4441 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4442 (unsigned long)bp); 4443 4444 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4445 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4446 dev->base_addr, dev->irq, dev->dev_addr); 4447 4448 pm_runtime_mark_last_busy(&bp->pdev->dev); 4449 pm_runtime_put_autosuspend(&bp->pdev->dev); 4450 4451 return 0; 4452 4453 err_out_unregister_mdio: 4454 mdiobus_unregister(bp->mii_bus); 4455 mdiobus_free(bp->mii_bus); 4456 4457 err_out_free_netdev: 4458 free_netdev(dev); 4459 4460 err_disable_clocks: 4461 clk_disable_unprepare(tx_clk); 4462 clk_disable_unprepare(hclk); 4463 clk_disable_unprepare(pclk); 4464 clk_disable_unprepare(rx_clk); 4465 clk_disable_unprepare(tsu_clk); 4466 pm_runtime_disable(&pdev->dev); 4467 pm_runtime_set_suspended(&pdev->dev); 4468 pm_runtime_dont_use_autosuspend(&pdev->dev); 4469 4470 return err; 4471 } 4472 4473 static int macb_remove(struct platform_device *pdev) 4474 { 4475 struct net_device *dev; 4476 struct macb *bp; 4477 4478 dev = platform_get_drvdata(pdev); 4479 4480 if (dev) { 4481 bp = netdev_priv(dev); 4482 mdiobus_unregister(bp->mii_bus); 4483 mdiobus_free(bp->mii_bus); 4484 4485 unregister_netdev(dev); 4486 tasklet_kill(&bp->hresp_err_tasklet); 4487 pm_runtime_disable(&pdev->dev); 4488 pm_runtime_dont_use_autosuspend(&pdev->dev); 4489 if (!pm_runtime_suspended(&pdev->dev)) { 4490 clk_disable_unprepare(bp->tx_clk); 4491 clk_disable_unprepare(bp->hclk); 4492 clk_disable_unprepare(bp->pclk); 4493 clk_disable_unprepare(bp->rx_clk); 4494 clk_disable_unprepare(bp->tsu_clk); 4495 pm_runtime_set_suspended(&pdev->dev); 4496 } 4497 phylink_destroy(bp->phylink); 4498 free_netdev(dev); 4499 } 4500 4501 return 0; 4502 } 4503 4504 static int __maybe_unused macb_suspend(struct device *dev) 4505 { 4506 struct net_device *netdev = dev_get_drvdata(dev); 4507 struct macb *bp = netdev_priv(netdev); 4508 struct macb_queue *queue = bp->queues; 4509 unsigned long flags; 4510 unsigned int q; 4511 4512 if (!netif_running(netdev)) 4513 return 0; 4514 4515 if (bp->wol & MACB_WOL_ENABLED) { 4516 macb_writel(bp, IER, MACB_BIT(WOL)); 4517 macb_writel(bp, WOL, MACB_BIT(MAG)); 4518 enable_irq_wake(bp->queues[0].irq); 4519 netif_device_detach(netdev); 4520 } else { 4521 netif_device_detach(netdev); 4522 for (q = 0, queue = bp->queues; q < bp->num_queues; 4523 ++q, ++queue) 4524 napi_disable(&queue->napi); 4525 rtnl_lock(); 4526 phylink_stop(bp->phylink); 4527 rtnl_unlock(); 4528 spin_lock_irqsave(&bp->lock, flags); 4529 macb_reset_hw(bp); 4530 spin_unlock_irqrestore(&bp->lock, flags); 4531 4532 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4533 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 4534 4535 if (netdev->hw_features & NETIF_F_NTUPLE) 4536 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 4537 } 4538 4539 netif_carrier_off(netdev); 4540 if (bp->ptp_info) 4541 bp->ptp_info->ptp_remove(netdev); 4542 pm_runtime_force_suspend(dev); 4543 4544 return 0; 4545 } 4546 4547 static int __maybe_unused macb_resume(struct device *dev) 4548 { 4549 struct net_device *netdev = dev_get_drvdata(dev); 4550 struct macb *bp = netdev_priv(netdev); 4551 struct macb_queue *queue = bp->queues; 4552 unsigned int q; 4553 4554 if (!netif_running(netdev)) 4555 return 0; 4556 4557 pm_runtime_force_resume(dev); 4558 4559 if (bp->wol & MACB_WOL_ENABLED) { 4560 macb_writel(bp, IDR, MACB_BIT(WOL)); 4561 macb_writel(bp, WOL, 0); 4562 disable_irq_wake(bp->queues[0].irq); 4563 } else { 4564 macb_writel(bp, NCR, MACB_BIT(MPE)); 4565 4566 if (netdev->hw_features & NETIF_F_NTUPLE) 4567 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 4568 4569 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4570 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 4571 4572 for (q = 0, queue = bp->queues; q < bp->num_queues; 4573 ++q, ++queue) 4574 napi_enable(&queue->napi); 4575 rtnl_lock(); 4576 phylink_start(bp->phylink); 4577 rtnl_unlock(); 4578 } 4579 4580 macb_init_hw(bp); 4581 macb_set_rx_mode(netdev); 4582 macb_restore_features(bp); 4583 netif_device_attach(netdev); 4584 if (bp->ptp_info) 4585 bp->ptp_info->ptp_init(netdev); 4586 4587 return 0; 4588 } 4589 4590 static int __maybe_unused macb_runtime_suspend(struct device *dev) 4591 { 4592 struct net_device *netdev = dev_get_drvdata(dev); 4593 struct macb *bp = netdev_priv(netdev); 4594 4595 if (!(device_may_wakeup(&bp->dev->dev))) { 4596 clk_disable_unprepare(bp->tx_clk); 4597 clk_disable_unprepare(bp->hclk); 4598 clk_disable_unprepare(bp->pclk); 4599 clk_disable_unprepare(bp->rx_clk); 4600 } 4601 clk_disable_unprepare(bp->tsu_clk); 4602 4603 return 0; 4604 } 4605 4606 static int __maybe_unused macb_runtime_resume(struct device *dev) 4607 { 4608 struct net_device *netdev = dev_get_drvdata(dev); 4609 struct macb *bp = netdev_priv(netdev); 4610 4611 if (!(device_may_wakeup(&bp->dev->dev))) { 4612 clk_prepare_enable(bp->pclk); 4613 clk_prepare_enable(bp->hclk); 4614 clk_prepare_enable(bp->tx_clk); 4615 clk_prepare_enable(bp->rx_clk); 4616 } 4617 clk_prepare_enable(bp->tsu_clk); 4618 4619 return 0; 4620 } 4621 4622 static const struct dev_pm_ops macb_pm_ops = { 4623 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 4624 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 4625 }; 4626 4627 static struct platform_driver macb_driver = { 4628 .probe = macb_probe, 4629 .remove = macb_remove, 4630 .driver = { 4631 .name = "macb", 4632 .of_match_table = of_match_ptr(macb_dt_ids), 4633 .pm = &macb_pm_ops, 4634 }, 4635 }; 4636 4637 module_platform_driver(macb_driver); 4638 4639 MODULE_LICENSE("GPL"); 4640 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4641 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4642 MODULE_ALIAS("platform:macb"); 4643