1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/clk.h> 10 #include <linux/clk-provider.h> 11 #include <linux/crc32.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/kernel.h> 15 #include <linux/types.h> 16 #include <linux/circ_buf.h> 17 #include <linux/slab.h> 18 #include <linux/init.h> 19 #include <linux/io.h> 20 #include <linux/gpio.h> 21 #include <linux/gpio/consumer.h> 22 #include <linux/interrupt.h> 23 #include <linux/netdevice.h> 24 #include <linux/etherdevice.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/platform_device.h> 27 #include <linux/phylink.h> 28 #include <linux/of.h> 29 #include <linux/of_device.h> 30 #include <linux/of_gpio.h> 31 #include <linux/of_mdio.h> 32 #include <linux/of_net.h> 33 #include <linux/ip.h> 34 #include <linux/udp.h> 35 #include <linux/tcp.h> 36 #include <linux/iopoll.h> 37 #include <linux/pm_runtime.h> 38 #include "macb.h" 39 40 /* This structure is only used for MACB on SiFive FU540 devices */ 41 struct sifive_fu540_macb_mgmt { 42 void __iomem *reg; 43 unsigned long rate; 44 struct clk_hw hw; 45 }; 46 47 #define MACB_RX_BUFFER_SIZE 128 48 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 49 50 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 51 #define MIN_RX_RING_SIZE 64 52 #define MAX_RX_RING_SIZE 8192 53 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 54 * (bp)->rx_ring_size) 55 56 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 57 #define MIN_TX_RING_SIZE 64 58 #define MAX_TX_RING_SIZE 4096 59 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 60 * (bp)->tx_ring_size) 61 62 /* level of occupied TX descriptors under which we wake up TX process */ 63 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 64 65 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 66 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 67 | MACB_BIT(ISR_RLE) \ 68 | MACB_BIT(TXERR)) 69 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 70 | MACB_BIT(TXUBR)) 71 72 /* Max length of transmit frame must be a multiple of 8 bytes */ 73 #define MACB_TX_LEN_ALIGN 8 74 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 75 /* Limit maximum TX length as per Cadence TSO errata. This is to avoid a 76 * false amba_error in TX path from the DMA assuming there is not enough 77 * space in the SRAM (16KB) even when there is. 78 */ 79 #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) 80 81 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 82 #define MACB_NETIF_LSO NETIF_F_TSO 83 84 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 85 #define MACB_WOL_ENABLED (0x1 << 1) 86 87 #define HS_SPEED_10000M 4 88 #define MACB_SERDES_RATE_10G 1 89 90 /* Graceful stop timeouts in us. We should allow up to 91 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 92 */ 93 #define MACB_HALT_TIMEOUT 1230 94 95 #define MACB_PM_TIMEOUT 100 /* ms */ 96 97 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 98 99 /* DMA buffer descriptor might be different size 100 * depends on hardware configuration: 101 * 102 * 1. dma address width 32 bits: 103 * word 1: 32 bit address of Data Buffer 104 * word 2: control 105 * 106 * 2. dma address width 64 bits: 107 * word 1: 32 bit address of Data Buffer 108 * word 2: control 109 * word 3: upper 32 bit address of Data Buffer 110 * word 4: unused 111 * 112 * 3. dma address width 32 bits with hardware timestamping: 113 * word 1: 32 bit address of Data Buffer 114 * word 2: control 115 * word 3: timestamp word 1 116 * word 4: timestamp word 2 117 * 118 * 4. dma address width 64 bits with hardware timestamping: 119 * word 1: 32 bit address of Data Buffer 120 * word 2: control 121 * word 3: upper 32 bit address of Data Buffer 122 * word 4: unused 123 * word 5: timestamp word 1 124 * word 6: timestamp word 2 125 */ 126 static unsigned int macb_dma_desc_get_size(struct macb *bp) 127 { 128 #ifdef MACB_EXT_DESC 129 unsigned int desc_size; 130 131 switch (bp->hw_dma_cap) { 132 case HW_DMA_CAP_64B: 133 desc_size = sizeof(struct macb_dma_desc) 134 + sizeof(struct macb_dma_desc_64); 135 break; 136 case HW_DMA_CAP_PTP: 137 desc_size = sizeof(struct macb_dma_desc) 138 + sizeof(struct macb_dma_desc_ptp); 139 break; 140 case HW_DMA_CAP_64B_PTP: 141 desc_size = sizeof(struct macb_dma_desc) 142 + sizeof(struct macb_dma_desc_64) 143 + sizeof(struct macb_dma_desc_ptp); 144 break; 145 default: 146 desc_size = sizeof(struct macb_dma_desc); 147 } 148 return desc_size; 149 #endif 150 return sizeof(struct macb_dma_desc); 151 } 152 153 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 154 { 155 #ifdef MACB_EXT_DESC 156 switch (bp->hw_dma_cap) { 157 case HW_DMA_CAP_64B: 158 case HW_DMA_CAP_PTP: 159 desc_idx <<= 1; 160 break; 161 case HW_DMA_CAP_64B_PTP: 162 desc_idx *= 3; 163 break; 164 default: 165 break; 166 } 167 #endif 168 return desc_idx; 169 } 170 171 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 172 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 173 { 174 return (struct macb_dma_desc_64 *)((void *)desc 175 + sizeof(struct macb_dma_desc)); 176 } 177 #endif 178 179 /* Ring buffer accessors */ 180 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 181 { 182 return index & (bp->tx_ring_size - 1); 183 } 184 185 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 186 unsigned int index) 187 { 188 index = macb_tx_ring_wrap(queue->bp, index); 189 index = macb_adj_dma_desc_idx(queue->bp, index); 190 return &queue->tx_ring[index]; 191 } 192 193 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 194 unsigned int index) 195 { 196 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 197 } 198 199 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 200 { 201 dma_addr_t offset; 202 203 offset = macb_tx_ring_wrap(queue->bp, index) * 204 macb_dma_desc_get_size(queue->bp); 205 206 return queue->tx_ring_dma + offset; 207 } 208 209 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 210 { 211 return index & (bp->rx_ring_size - 1); 212 } 213 214 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 215 { 216 index = macb_rx_ring_wrap(queue->bp, index); 217 index = macb_adj_dma_desc_idx(queue->bp, index); 218 return &queue->rx_ring[index]; 219 } 220 221 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 222 { 223 return queue->rx_buffers + queue->bp->rx_buffer_size * 224 macb_rx_ring_wrap(queue->bp, index); 225 } 226 227 /* I/O accessors */ 228 static u32 hw_readl_native(struct macb *bp, int offset) 229 { 230 return __raw_readl(bp->regs + offset); 231 } 232 233 static void hw_writel_native(struct macb *bp, int offset, u32 value) 234 { 235 __raw_writel(value, bp->regs + offset); 236 } 237 238 static u32 hw_readl(struct macb *bp, int offset) 239 { 240 return readl_relaxed(bp->regs + offset); 241 } 242 243 static void hw_writel(struct macb *bp, int offset, u32 value) 244 { 245 writel_relaxed(value, bp->regs + offset); 246 } 247 248 /* Find the CPU endianness by using the loopback bit of NCR register. When the 249 * CPU is in big endian we need to program swapped mode for management 250 * descriptor access. 251 */ 252 static bool hw_is_native_io(void __iomem *addr) 253 { 254 u32 value = MACB_BIT(LLB); 255 256 __raw_writel(value, addr + MACB_NCR); 257 value = __raw_readl(addr + MACB_NCR); 258 259 /* Write 0 back to disable everything */ 260 __raw_writel(0, addr + MACB_NCR); 261 262 return value == MACB_BIT(LLB); 263 } 264 265 static bool hw_is_gem(void __iomem *addr, bool native_io) 266 { 267 u32 id; 268 269 if (native_io) 270 id = __raw_readl(addr + MACB_MID); 271 else 272 id = readl_relaxed(addr + MACB_MID); 273 274 return MACB_BFEXT(IDNUM, id) >= 0x2; 275 } 276 277 static void macb_set_hwaddr(struct macb *bp) 278 { 279 u32 bottom; 280 u16 top; 281 282 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 283 macb_or_gem_writel(bp, SA1B, bottom); 284 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 285 macb_or_gem_writel(bp, SA1T, top); 286 287 /* Clear unused address register sets */ 288 macb_or_gem_writel(bp, SA2B, 0); 289 macb_or_gem_writel(bp, SA2T, 0); 290 macb_or_gem_writel(bp, SA3B, 0); 291 macb_or_gem_writel(bp, SA3T, 0); 292 macb_or_gem_writel(bp, SA4B, 0); 293 macb_or_gem_writel(bp, SA4T, 0); 294 } 295 296 static void macb_get_hwaddr(struct macb *bp) 297 { 298 u32 bottom; 299 u16 top; 300 u8 addr[6]; 301 int i; 302 303 /* Check all 4 address register for valid address */ 304 for (i = 0; i < 4; i++) { 305 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 306 top = macb_or_gem_readl(bp, SA1T + i * 8); 307 308 addr[0] = bottom & 0xff; 309 addr[1] = (bottom >> 8) & 0xff; 310 addr[2] = (bottom >> 16) & 0xff; 311 addr[3] = (bottom >> 24) & 0xff; 312 addr[4] = top & 0xff; 313 addr[5] = (top >> 8) & 0xff; 314 315 if (is_valid_ether_addr(addr)) { 316 eth_hw_addr_set(bp->dev, addr); 317 return; 318 } 319 } 320 321 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 322 eth_hw_addr_random(bp->dev); 323 } 324 325 static int macb_mdio_wait_for_idle(struct macb *bp) 326 { 327 u32 val; 328 329 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 330 1, MACB_MDIO_TIMEOUT); 331 } 332 333 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 334 { 335 struct macb *bp = bus->priv; 336 int status; 337 338 status = pm_runtime_get_sync(&bp->pdev->dev); 339 if (status < 0) { 340 pm_runtime_put_noidle(&bp->pdev->dev); 341 goto mdio_pm_exit; 342 } 343 344 status = macb_mdio_wait_for_idle(bp); 345 if (status < 0) 346 goto mdio_read_exit; 347 348 if (regnum & MII_ADDR_C45) { 349 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 350 | MACB_BF(RW, MACB_MAN_C45_ADDR) 351 | MACB_BF(PHYA, mii_id) 352 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 353 | MACB_BF(DATA, regnum & 0xFFFF) 354 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 355 356 status = macb_mdio_wait_for_idle(bp); 357 if (status < 0) 358 goto mdio_read_exit; 359 360 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 361 | MACB_BF(RW, MACB_MAN_C45_READ) 362 | MACB_BF(PHYA, mii_id) 363 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 364 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 365 } else { 366 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 367 | MACB_BF(RW, MACB_MAN_C22_READ) 368 | MACB_BF(PHYA, mii_id) 369 | MACB_BF(REGA, regnum) 370 | MACB_BF(CODE, MACB_MAN_C22_CODE))); 371 } 372 373 status = macb_mdio_wait_for_idle(bp); 374 if (status < 0) 375 goto mdio_read_exit; 376 377 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 378 379 mdio_read_exit: 380 pm_runtime_mark_last_busy(&bp->pdev->dev); 381 pm_runtime_put_autosuspend(&bp->pdev->dev); 382 mdio_pm_exit: 383 return status; 384 } 385 386 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 387 u16 value) 388 { 389 struct macb *bp = bus->priv; 390 int status; 391 392 status = pm_runtime_get_sync(&bp->pdev->dev); 393 if (status < 0) { 394 pm_runtime_put_noidle(&bp->pdev->dev); 395 goto mdio_pm_exit; 396 } 397 398 status = macb_mdio_wait_for_idle(bp); 399 if (status < 0) 400 goto mdio_write_exit; 401 402 if (regnum & MII_ADDR_C45) { 403 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 404 | MACB_BF(RW, MACB_MAN_C45_ADDR) 405 | MACB_BF(PHYA, mii_id) 406 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 407 | MACB_BF(DATA, regnum & 0xFFFF) 408 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 409 410 status = macb_mdio_wait_for_idle(bp); 411 if (status < 0) 412 goto mdio_write_exit; 413 414 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 415 | MACB_BF(RW, MACB_MAN_C45_WRITE) 416 | MACB_BF(PHYA, mii_id) 417 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 418 | MACB_BF(CODE, MACB_MAN_C45_CODE) 419 | MACB_BF(DATA, value))); 420 } else { 421 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 422 | MACB_BF(RW, MACB_MAN_C22_WRITE) 423 | MACB_BF(PHYA, mii_id) 424 | MACB_BF(REGA, regnum) 425 | MACB_BF(CODE, MACB_MAN_C22_CODE) 426 | MACB_BF(DATA, value))); 427 } 428 429 status = macb_mdio_wait_for_idle(bp); 430 if (status < 0) 431 goto mdio_write_exit; 432 433 mdio_write_exit: 434 pm_runtime_mark_last_busy(&bp->pdev->dev); 435 pm_runtime_put_autosuspend(&bp->pdev->dev); 436 mdio_pm_exit: 437 return status; 438 } 439 440 static void macb_init_buffers(struct macb *bp) 441 { 442 struct macb_queue *queue; 443 unsigned int q; 444 445 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 446 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 447 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 448 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 449 queue_writel(queue, RBQPH, 450 upper_32_bits(queue->rx_ring_dma)); 451 #endif 452 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 453 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 454 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 455 queue_writel(queue, TBQPH, 456 upper_32_bits(queue->tx_ring_dma)); 457 #endif 458 } 459 } 460 461 /** 462 * macb_set_tx_clk() - Set a clock to a new frequency 463 * @bp: pointer to struct macb 464 * @speed: New frequency in Hz 465 */ 466 static void macb_set_tx_clk(struct macb *bp, int speed) 467 { 468 long ferr, rate, rate_rounded; 469 470 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) 471 return; 472 473 /* In case of MII the PHY is the clock master */ 474 if (bp->phy_interface == PHY_INTERFACE_MODE_MII) 475 return; 476 477 switch (speed) { 478 case SPEED_10: 479 rate = 2500000; 480 break; 481 case SPEED_100: 482 rate = 25000000; 483 break; 484 case SPEED_1000: 485 rate = 125000000; 486 break; 487 default: 488 return; 489 } 490 491 rate_rounded = clk_round_rate(bp->tx_clk, rate); 492 if (rate_rounded < 0) 493 return; 494 495 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 496 * is not satisfied. 497 */ 498 ferr = abs(rate_rounded - rate); 499 ferr = DIV_ROUND_UP(ferr, rate / 100000); 500 if (ferr > 5) 501 netdev_warn(bp->dev, 502 "unable to generate target frequency: %ld Hz\n", 503 rate); 504 505 if (clk_set_rate(bp->tx_clk, rate_rounded)) 506 netdev_err(bp->dev, "adjusting tx_clk failed.\n"); 507 } 508 509 static void macb_validate(struct phylink_config *config, 510 unsigned long *supported, 511 struct phylink_link_state *state) 512 { 513 struct net_device *ndev = to_net_dev(config->dev); 514 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; 515 struct macb *bp = netdev_priv(ndev); 516 bool have_1g=false, have_sgmii=false, have_10g=false; 517 518 /* Determine what modes are supported */ 519 if (macb_is_gem(bp) && 520 (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { 521 have_1g = true; 522 if (bp->caps & MACB_CAPS_PCS) 523 have_sgmii = true; 524 if (bp->caps & MACB_CAPS_HIGH_SPEED) 525 have_10g = true; 526 } 527 528 /* Eliminate unsupported modes */ 529 switch (state->interface) { 530 case PHY_INTERFACE_MODE_NA: 531 case PHY_INTERFACE_MODE_MII: 532 case PHY_INTERFACE_MODE_RMII: 533 break; 534 535 case PHY_INTERFACE_MODE_GMII: 536 case PHY_INTERFACE_MODE_RGMII: 537 case PHY_INTERFACE_MODE_RGMII_ID: 538 case PHY_INTERFACE_MODE_RGMII_RXID: 539 case PHY_INTERFACE_MODE_RGMII_TXID: 540 if (have_1g) 541 break; 542 linkmode_zero(supported); 543 return; 544 545 case PHY_INTERFACE_MODE_SGMII: 546 if (have_sgmii) 547 break; 548 linkmode_zero(supported); 549 return; 550 551 case PHY_INTERFACE_MODE_10GBASER: 552 if (have_10g) 553 break; 554 fallthrough; 555 556 default: 557 linkmode_zero(supported); 558 return; 559 } 560 561 phylink_set_port_modes(mask); 562 phylink_set(mask, Autoneg); 563 phylink_set(mask, Asym_Pause); 564 565 /* And set the appropriate mask */ 566 switch (state->interface) { 567 case PHY_INTERFACE_MODE_NA: 568 case PHY_INTERFACE_MODE_10GBASER: 569 if (have_10g) { 570 phylink_set_10g_modes(mask); 571 phylink_set(mask, 10000baseKR_Full); 572 } 573 if (state->interface != PHY_INTERFACE_MODE_NA) 574 break; 575 fallthrough; 576 577 /* FIXME: Do we actually support 10/100 for SGMII? Half duplex? */ 578 case PHY_INTERFACE_MODE_SGMII: 579 if (!have_sgmii && state->interface != PHY_INTERFACE_MODE_NA) 580 break; 581 fallthrough; 582 583 case PHY_INTERFACE_MODE_GMII: 584 case PHY_INTERFACE_MODE_RGMII: 585 case PHY_INTERFACE_MODE_RGMII_ID: 586 case PHY_INTERFACE_MODE_RGMII_RXID: 587 case PHY_INTERFACE_MODE_RGMII_TXID: 588 if (have_1g) { 589 phylink_set(mask, 1000baseT_Full); 590 phylink_set(mask, 1000baseX_Full); 591 592 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) 593 phylink_set(mask, 1000baseT_Half); 594 } else if (state->interface != PHY_INTERFACE_MODE_NA) { 595 break; 596 } 597 fallthrough; 598 599 default: 600 phylink_set(mask, 10baseT_Half); 601 phylink_set(mask, 10baseT_Full); 602 phylink_set(mask, 100baseT_Half); 603 phylink_set(mask, 100baseT_Full); 604 break; 605 } 606 607 linkmode_and(supported, supported, mask); 608 linkmode_and(state->advertising, state->advertising, mask); 609 } 610 611 static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, 612 phy_interface_t interface, int speed, 613 int duplex) 614 { 615 struct macb *bp = container_of(pcs, struct macb, phylink_pcs); 616 u32 config; 617 618 config = gem_readl(bp, USX_CONTROL); 619 config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config); 620 config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config); 621 config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS)); 622 config |= GEM_BIT(TX_EN); 623 gem_writel(bp, USX_CONTROL, config); 624 } 625 626 static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, 627 struct phylink_link_state *state) 628 { 629 struct macb *bp = container_of(pcs, struct macb, phylink_pcs); 630 u32 val; 631 632 state->speed = SPEED_10000; 633 state->duplex = 1; 634 state->an_complete = 1; 635 636 val = gem_readl(bp, USX_STATUS); 637 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); 638 val = gem_readl(bp, NCFGR); 639 if (val & GEM_BIT(PAE)) 640 state->pause = MLO_PAUSE_RX; 641 } 642 643 static int macb_usx_pcs_config(struct phylink_pcs *pcs, 644 unsigned int mode, 645 phy_interface_t interface, 646 const unsigned long *advertising, 647 bool permit_pause_to_mac) 648 { 649 struct macb *bp = container_of(pcs, struct macb, phylink_pcs); 650 651 gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | 652 GEM_BIT(SIGNAL_OK)); 653 654 return 0; 655 } 656 657 static void macb_pcs_get_state(struct phylink_pcs *pcs, 658 struct phylink_link_state *state) 659 { 660 state->link = 0; 661 } 662 663 static void macb_pcs_an_restart(struct phylink_pcs *pcs) 664 { 665 /* Not supported */ 666 } 667 668 static int macb_pcs_config(struct phylink_pcs *pcs, 669 unsigned int mode, 670 phy_interface_t interface, 671 const unsigned long *advertising, 672 bool permit_pause_to_mac) 673 { 674 return 0; 675 } 676 677 static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { 678 .pcs_get_state = macb_usx_pcs_get_state, 679 .pcs_config = macb_usx_pcs_config, 680 .pcs_link_up = macb_usx_pcs_link_up, 681 }; 682 683 static const struct phylink_pcs_ops macb_phylink_pcs_ops = { 684 .pcs_get_state = macb_pcs_get_state, 685 .pcs_an_restart = macb_pcs_an_restart, 686 .pcs_config = macb_pcs_config, 687 }; 688 689 static void macb_mac_config(struct phylink_config *config, unsigned int mode, 690 const struct phylink_link_state *state) 691 { 692 struct net_device *ndev = to_net_dev(config->dev); 693 struct macb *bp = netdev_priv(ndev); 694 unsigned long flags; 695 u32 old_ctrl, ctrl; 696 u32 old_ncr, ncr; 697 698 spin_lock_irqsave(&bp->lock, flags); 699 700 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); 701 old_ncr = ncr = macb_or_gem_readl(bp, NCR); 702 703 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { 704 if (state->interface == PHY_INTERFACE_MODE_RMII) 705 ctrl |= MACB_BIT(RM9200_RMII); 706 } else if (macb_is_gem(bp)) { 707 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); 708 ncr &= ~GEM_BIT(ENABLE_HS_MAC); 709 710 if (state->interface == PHY_INTERFACE_MODE_SGMII) { 711 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 712 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) { 713 ctrl |= GEM_BIT(PCSSEL); 714 ncr |= GEM_BIT(ENABLE_HS_MAC); 715 } else if (bp->caps & MACB_CAPS_MIIONRGMII && 716 bp->phy_interface == PHY_INTERFACE_MODE_MII) { 717 ncr |= MACB_BIT(MIIONRGMII); 718 } 719 } 720 721 /* Apply the new configuration, if any */ 722 if (old_ctrl ^ ctrl) 723 macb_or_gem_writel(bp, NCFGR, ctrl); 724 725 if (old_ncr ^ ncr) 726 macb_or_gem_writel(bp, NCR, ncr); 727 728 /* Disable AN for SGMII fixed link configuration, enable otherwise. 729 * Must be written after PCSSEL is set in NCFGR, 730 * otherwise writes will not take effect. 731 */ 732 if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) { 733 u32 pcsctrl, old_pcsctrl; 734 735 old_pcsctrl = gem_readl(bp, PCSCNTRL); 736 if (mode == MLO_AN_FIXED) 737 pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); 738 else 739 pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG); 740 if (old_pcsctrl != pcsctrl) 741 gem_writel(bp, PCSCNTRL, pcsctrl); 742 } 743 744 spin_unlock_irqrestore(&bp->lock, flags); 745 } 746 747 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, 748 phy_interface_t interface) 749 { 750 struct net_device *ndev = to_net_dev(config->dev); 751 struct macb *bp = netdev_priv(ndev); 752 struct macb_queue *queue; 753 unsigned int q; 754 u32 ctrl; 755 756 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 757 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 758 queue_writel(queue, IDR, 759 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 760 761 /* Disable Rx and Tx */ 762 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); 763 macb_writel(bp, NCR, ctrl); 764 765 netif_tx_stop_all_queues(ndev); 766 } 767 768 static void macb_mac_link_up(struct phylink_config *config, 769 struct phy_device *phy, 770 unsigned int mode, phy_interface_t interface, 771 int speed, int duplex, 772 bool tx_pause, bool rx_pause) 773 { 774 struct net_device *ndev = to_net_dev(config->dev); 775 struct macb *bp = netdev_priv(ndev); 776 struct macb_queue *queue; 777 unsigned long flags; 778 unsigned int q; 779 u32 ctrl; 780 781 spin_lock_irqsave(&bp->lock, flags); 782 783 ctrl = macb_or_gem_readl(bp, NCFGR); 784 785 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 786 787 if (speed == SPEED_100) 788 ctrl |= MACB_BIT(SPD); 789 790 if (duplex) 791 ctrl |= MACB_BIT(FD); 792 793 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 794 ctrl &= ~MACB_BIT(PAE); 795 if (macb_is_gem(bp)) { 796 ctrl &= ~GEM_BIT(GBE); 797 798 if (speed == SPEED_1000) 799 ctrl |= GEM_BIT(GBE); 800 } 801 802 if (rx_pause) 803 ctrl |= MACB_BIT(PAE); 804 805 macb_set_tx_clk(bp, speed); 806 807 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down 808 * cleared the pipeline and control registers. 809 */ 810 bp->macbgem_ops.mog_init_rings(bp); 811 macb_init_buffers(bp); 812 813 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 814 queue_writel(queue, IER, 815 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 816 } 817 818 macb_or_gem_writel(bp, NCFGR, ctrl); 819 820 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER) 821 gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, 822 gem_readl(bp, HS_MAC_CONFIG))); 823 824 spin_unlock_irqrestore(&bp->lock, flags); 825 826 /* Enable Rx and Tx */ 827 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 828 829 netif_tx_wake_all_queues(ndev); 830 } 831 832 static int macb_mac_prepare(struct phylink_config *config, unsigned int mode, 833 phy_interface_t interface) 834 { 835 struct net_device *ndev = to_net_dev(config->dev); 836 struct macb *bp = netdev_priv(ndev); 837 838 if (interface == PHY_INTERFACE_MODE_10GBASER) 839 bp->phylink_pcs.ops = &macb_phylink_usx_pcs_ops; 840 else if (interface == PHY_INTERFACE_MODE_SGMII) 841 bp->phylink_pcs.ops = &macb_phylink_pcs_ops; 842 else 843 bp->phylink_pcs.ops = NULL; 844 845 if (bp->phylink_pcs.ops) 846 phylink_set_pcs(bp->phylink, &bp->phylink_pcs); 847 848 return 0; 849 } 850 851 static const struct phylink_mac_ops macb_phylink_ops = { 852 .validate = macb_validate, 853 .mac_prepare = macb_mac_prepare, 854 .mac_config = macb_mac_config, 855 .mac_link_down = macb_mac_link_down, 856 .mac_link_up = macb_mac_link_up, 857 }; 858 859 static bool macb_phy_handle_exists(struct device_node *dn) 860 { 861 dn = of_parse_phandle(dn, "phy-handle", 0); 862 of_node_put(dn); 863 return dn != NULL; 864 } 865 866 static int macb_phylink_connect(struct macb *bp) 867 { 868 struct device_node *dn = bp->pdev->dev.of_node; 869 struct net_device *dev = bp->dev; 870 struct phy_device *phydev; 871 int ret; 872 873 if (dn) 874 ret = phylink_of_phy_connect(bp->phylink, dn, 0); 875 876 if (!dn || (ret && !macb_phy_handle_exists(dn))) { 877 phydev = phy_find_first(bp->mii_bus); 878 if (!phydev) { 879 netdev_err(dev, "no PHY found\n"); 880 return -ENXIO; 881 } 882 883 /* attach the mac to the phy */ 884 ret = phylink_connect_phy(bp->phylink, phydev); 885 } 886 887 if (ret) { 888 netdev_err(dev, "Could not attach PHY (%d)\n", ret); 889 return ret; 890 } 891 892 phylink_start(bp->phylink); 893 894 return 0; 895 } 896 897 static void macb_get_pcs_fixed_state(struct phylink_config *config, 898 struct phylink_link_state *state) 899 { 900 struct net_device *ndev = to_net_dev(config->dev); 901 struct macb *bp = netdev_priv(ndev); 902 903 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; 904 } 905 906 /* based on au1000_eth. c*/ 907 static int macb_mii_probe(struct net_device *dev) 908 { 909 struct macb *bp = netdev_priv(dev); 910 911 bp->phylink_config.dev = &dev->dev; 912 bp->phylink_config.type = PHYLINK_NETDEV; 913 914 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { 915 bp->phylink_config.poll_fixed_state = true; 916 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; 917 } 918 919 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, 920 bp->phy_interface, &macb_phylink_ops); 921 if (IS_ERR(bp->phylink)) { 922 netdev_err(dev, "Could not create a phylink instance (%ld)\n", 923 PTR_ERR(bp->phylink)); 924 return PTR_ERR(bp->phylink); 925 } 926 927 return 0; 928 } 929 930 static int macb_mdiobus_register(struct macb *bp) 931 { 932 struct device_node *child, *np = bp->pdev->dev.of_node; 933 934 /* If we have a child named mdio, probe it instead of looking for PHYs 935 * directly under the MAC node 936 */ 937 child = of_get_child_by_name(np, "mdio"); 938 if (child) { 939 int ret = of_mdiobus_register(bp->mii_bus, child); 940 941 of_node_put(child); 942 return ret; 943 } 944 945 if (of_phy_is_fixed_link(np)) 946 return mdiobus_register(bp->mii_bus); 947 948 /* Only create the PHY from the device tree if at least one PHY is 949 * described. Otherwise scan the entire MDIO bus. We do this to support 950 * old device tree that did not follow the best practices and did not 951 * describe their network PHYs. 952 */ 953 for_each_available_child_of_node(np, child) 954 if (of_mdiobus_child_is_phy(child)) { 955 /* The loop increments the child refcount, 956 * decrement it before returning. 957 */ 958 of_node_put(child); 959 960 return of_mdiobus_register(bp->mii_bus, np); 961 } 962 963 return mdiobus_register(bp->mii_bus); 964 } 965 966 static int macb_mii_init(struct macb *bp) 967 { 968 int err = -ENXIO; 969 970 /* Enable management port */ 971 macb_writel(bp, NCR, MACB_BIT(MPE)); 972 973 bp->mii_bus = mdiobus_alloc(); 974 if (!bp->mii_bus) { 975 err = -ENOMEM; 976 goto err_out; 977 } 978 979 bp->mii_bus->name = "MACB_mii_bus"; 980 bp->mii_bus->read = &macb_mdio_read; 981 bp->mii_bus->write = &macb_mdio_write; 982 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 983 bp->pdev->name, bp->pdev->id); 984 bp->mii_bus->priv = bp; 985 bp->mii_bus->parent = &bp->pdev->dev; 986 987 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 988 989 err = macb_mdiobus_register(bp); 990 if (err) 991 goto err_out_free_mdiobus; 992 993 err = macb_mii_probe(bp->dev); 994 if (err) 995 goto err_out_unregister_bus; 996 997 return 0; 998 999 err_out_unregister_bus: 1000 mdiobus_unregister(bp->mii_bus); 1001 err_out_free_mdiobus: 1002 mdiobus_free(bp->mii_bus); 1003 err_out: 1004 return err; 1005 } 1006 1007 static void macb_update_stats(struct macb *bp) 1008 { 1009 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 1010 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 1011 int offset = MACB_PFR; 1012 1013 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 1014 1015 for (; p < end; p++, offset += 4) 1016 *p += bp->macb_reg_readl(bp, offset); 1017 } 1018 1019 static int macb_halt_tx(struct macb *bp) 1020 { 1021 unsigned long halt_time, timeout; 1022 u32 status; 1023 1024 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 1025 1026 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 1027 do { 1028 halt_time = jiffies; 1029 status = macb_readl(bp, TSR); 1030 if (!(status & MACB_BIT(TGO))) 1031 return 0; 1032 1033 udelay(250); 1034 } while (time_before(halt_time, timeout)); 1035 1036 return -ETIMEDOUT; 1037 } 1038 1039 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 1040 { 1041 if (tx_skb->mapping) { 1042 if (tx_skb->mapped_as_page) 1043 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 1044 tx_skb->size, DMA_TO_DEVICE); 1045 else 1046 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 1047 tx_skb->size, DMA_TO_DEVICE); 1048 tx_skb->mapping = 0; 1049 } 1050 1051 if (tx_skb->skb) { 1052 dev_kfree_skb_any(tx_skb->skb); 1053 tx_skb->skb = NULL; 1054 } 1055 } 1056 1057 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 1058 { 1059 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1060 struct macb_dma_desc_64 *desc_64; 1061 1062 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 1063 desc_64 = macb_64b_desc(bp, desc); 1064 desc_64->addrh = upper_32_bits(addr); 1065 /* The low bits of RX address contain the RX_USED bit, clearing 1066 * of which allows packet RX. Make sure the high bits are also 1067 * visible to HW at that point. 1068 */ 1069 dma_wmb(); 1070 } 1071 #endif 1072 desc->addr = lower_32_bits(addr); 1073 } 1074 1075 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 1076 { 1077 dma_addr_t addr = 0; 1078 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1079 struct macb_dma_desc_64 *desc_64; 1080 1081 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 1082 desc_64 = macb_64b_desc(bp, desc); 1083 addr = ((u64)(desc_64->addrh) << 32); 1084 } 1085 #endif 1086 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 1087 return addr; 1088 } 1089 1090 static void macb_tx_error_task(struct work_struct *work) 1091 { 1092 struct macb_queue *queue = container_of(work, struct macb_queue, 1093 tx_error_task); 1094 struct macb *bp = queue->bp; 1095 struct macb_tx_skb *tx_skb; 1096 struct macb_dma_desc *desc; 1097 struct sk_buff *skb; 1098 unsigned int tail; 1099 unsigned long flags; 1100 1101 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 1102 (unsigned int)(queue - bp->queues), 1103 queue->tx_tail, queue->tx_head); 1104 1105 /* Prevent the queue IRQ handlers from running: each of them may call 1106 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 1107 * As explained below, we have to halt the transmission before updating 1108 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 1109 * network engine about the macb/gem being halted. 1110 */ 1111 spin_lock_irqsave(&bp->lock, flags); 1112 1113 /* Make sure nobody is trying to queue up new packets */ 1114 netif_tx_stop_all_queues(bp->dev); 1115 1116 /* Stop transmission now 1117 * (in case we have just queued new packets) 1118 * macb/gem must be halted to write TBQP register 1119 */ 1120 if (macb_halt_tx(bp)) 1121 /* Just complain for now, reinitializing TX path can be good */ 1122 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 1123 1124 /* Treat frames in TX queue including the ones that caused the error. 1125 * Free transmit buffers in upper layer. 1126 */ 1127 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 1128 u32 ctrl; 1129 1130 desc = macb_tx_desc(queue, tail); 1131 ctrl = desc->ctrl; 1132 tx_skb = macb_tx_skb(queue, tail); 1133 skb = tx_skb->skb; 1134 1135 if (ctrl & MACB_BIT(TX_USED)) { 1136 /* skb is set for the last buffer of the frame */ 1137 while (!skb) { 1138 macb_tx_unmap(bp, tx_skb); 1139 tail++; 1140 tx_skb = macb_tx_skb(queue, tail); 1141 skb = tx_skb->skb; 1142 } 1143 1144 /* ctrl still refers to the first buffer descriptor 1145 * since it's the only one written back by the hardware 1146 */ 1147 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 1148 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 1149 macb_tx_ring_wrap(bp, tail), 1150 skb->data); 1151 bp->dev->stats.tx_packets++; 1152 queue->stats.tx_packets++; 1153 bp->dev->stats.tx_bytes += skb->len; 1154 queue->stats.tx_bytes += skb->len; 1155 } 1156 } else { 1157 /* "Buffers exhausted mid-frame" errors may only happen 1158 * if the driver is buggy, so complain loudly about 1159 * those. Statistics are updated by hardware. 1160 */ 1161 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 1162 netdev_err(bp->dev, 1163 "BUG: TX buffers exhausted mid-frame\n"); 1164 1165 desc->ctrl = ctrl | MACB_BIT(TX_USED); 1166 } 1167 1168 macb_tx_unmap(bp, tx_skb); 1169 } 1170 1171 /* Set end of TX queue */ 1172 desc = macb_tx_desc(queue, 0); 1173 macb_set_addr(bp, desc, 0); 1174 desc->ctrl = MACB_BIT(TX_USED); 1175 1176 /* Make descriptor updates visible to hardware */ 1177 wmb(); 1178 1179 /* Reinitialize the TX desc queue */ 1180 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1181 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1182 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1183 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 1184 #endif 1185 /* Make TX ring reflect state of hardware */ 1186 queue->tx_head = 0; 1187 queue->tx_tail = 0; 1188 1189 /* Housework before enabling TX IRQ */ 1190 macb_writel(bp, TSR, macb_readl(bp, TSR)); 1191 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 1192 1193 /* Now we are ready to start transmission again */ 1194 netif_tx_start_all_queues(bp->dev); 1195 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1196 1197 spin_unlock_irqrestore(&bp->lock, flags); 1198 } 1199 1200 static void macb_tx_interrupt(struct macb_queue *queue) 1201 { 1202 unsigned int tail; 1203 unsigned int head; 1204 u32 status; 1205 struct macb *bp = queue->bp; 1206 u16 queue_index = queue - bp->queues; 1207 1208 status = macb_readl(bp, TSR); 1209 macb_writel(bp, TSR, status); 1210 1211 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1212 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 1213 1214 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 1215 (unsigned long)status); 1216 1217 head = queue->tx_head; 1218 for (tail = queue->tx_tail; tail != head; tail++) { 1219 struct macb_tx_skb *tx_skb; 1220 struct sk_buff *skb; 1221 struct macb_dma_desc *desc; 1222 u32 ctrl; 1223 1224 desc = macb_tx_desc(queue, tail); 1225 1226 /* Make hw descriptor updates visible to CPU */ 1227 rmb(); 1228 1229 ctrl = desc->ctrl; 1230 1231 /* TX_USED bit is only set by hardware on the very first buffer 1232 * descriptor of the transmitted frame. 1233 */ 1234 if (!(ctrl & MACB_BIT(TX_USED))) 1235 break; 1236 1237 /* Process all buffers of the current transmitted frame */ 1238 for (;; tail++) { 1239 tx_skb = macb_tx_skb(queue, tail); 1240 skb = tx_skb->skb; 1241 1242 /* First, update TX stats if needed */ 1243 if (skb) { 1244 if (unlikely(skb_shinfo(skb)->tx_flags & 1245 SKBTX_HW_TSTAMP) && 1246 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 1247 /* skb now belongs to timestamp buffer 1248 * and will be removed later 1249 */ 1250 tx_skb->skb = NULL; 1251 } 1252 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 1253 macb_tx_ring_wrap(bp, tail), 1254 skb->data); 1255 bp->dev->stats.tx_packets++; 1256 queue->stats.tx_packets++; 1257 bp->dev->stats.tx_bytes += skb->len; 1258 queue->stats.tx_bytes += skb->len; 1259 } 1260 1261 /* Now we can safely release resources */ 1262 macb_tx_unmap(bp, tx_skb); 1263 1264 /* skb is set only for the last buffer of the frame. 1265 * WARNING: at this point skb has been freed by 1266 * macb_tx_unmap(). 1267 */ 1268 if (skb) 1269 break; 1270 } 1271 } 1272 1273 queue->tx_tail = tail; 1274 if (__netif_subqueue_stopped(bp->dev, queue_index) && 1275 CIRC_CNT(queue->tx_head, queue->tx_tail, 1276 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 1277 netif_wake_subqueue(bp->dev, queue_index); 1278 } 1279 1280 static void gem_rx_refill(struct macb_queue *queue) 1281 { 1282 unsigned int entry; 1283 struct sk_buff *skb; 1284 dma_addr_t paddr; 1285 struct macb *bp = queue->bp; 1286 struct macb_dma_desc *desc; 1287 1288 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 1289 bp->rx_ring_size) > 0) { 1290 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 1291 1292 /* Make hw descriptor updates visible to CPU */ 1293 rmb(); 1294 1295 queue->rx_prepared_head++; 1296 desc = macb_rx_desc(queue, entry); 1297 1298 if (!queue->rx_skbuff[entry]) { 1299 /* allocate sk_buff for this free entry in ring */ 1300 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 1301 if (unlikely(!skb)) { 1302 netdev_err(bp->dev, 1303 "Unable to allocate sk_buff\n"); 1304 break; 1305 } 1306 1307 /* now fill corresponding descriptor entry */ 1308 paddr = dma_map_single(&bp->pdev->dev, skb->data, 1309 bp->rx_buffer_size, 1310 DMA_FROM_DEVICE); 1311 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 1312 dev_kfree_skb(skb); 1313 break; 1314 } 1315 1316 queue->rx_skbuff[entry] = skb; 1317 1318 if (entry == bp->rx_ring_size - 1) 1319 paddr |= MACB_BIT(RX_WRAP); 1320 desc->ctrl = 0; 1321 /* Setting addr clears RX_USED and allows reception, 1322 * make sure ctrl is cleared first to avoid a race. 1323 */ 1324 dma_wmb(); 1325 macb_set_addr(bp, desc, paddr); 1326 1327 /* properly align Ethernet header */ 1328 skb_reserve(skb, NET_IP_ALIGN); 1329 } else { 1330 desc->ctrl = 0; 1331 dma_wmb(); 1332 desc->addr &= ~MACB_BIT(RX_USED); 1333 } 1334 } 1335 1336 /* Make descriptor updates visible to hardware */ 1337 wmb(); 1338 1339 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 1340 queue, queue->rx_prepared_head, queue->rx_tail); 1341 } 1342 1343 /* Mark DMA descriptors from begin up to and not including end as unused */ 1344 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 1345 unsigned int end) 1346 { 1347 unsigned int frag; 1348 1349 for (frag = begin; frag != end; frag++) { 1350 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 1351 1352 desc->addr &= ~MACB_BIT(RX_USED); 1353 } 1354 1355 /* Make descriptor updates visible to hardware */ 1356 wmb(); 1357 1358 /* When this happens, the hardware stats registers for 1359 * whatever caused this is updated, so we don't have to record 1360 * anything. 1361 */ 1362 } 1363 1364 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 1365 int budget) 1366 { 1367 struct macb *bp = queue->bp; 1368 unsigned int len; 1369 unsigned int entry; 1370 struct sk_buff *skb; 1371 struct macb_dma_desc *desc; 1372 int count = 0; 1373 1374 while (count < budget) { 1375 u32 ctrl; 1376 dma_addr_t addr; 1377 bool rxused; 1378 1379 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1380 desc = macb_rx_desc(queue, entry); 1381 1382 /* Make hw descriptor updates visible to CPU */ 1383 rmb(); 1384 1385 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1386 addr = macb_get_addr(bp, desc); 1387 1388 if (!rxused) 1389 break; 1390 1391 /* Ensure ctrl is at least as up-to-date as rxused */ 1392 dma_rmb(); 1393 1394 ctrl = desc->ctrl; 1395 1396 queue->rx_tail++; 1397 count++; 1398 1399 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1400 netdev_err(bp->dev, 1401 "not whole frame pointed by descriptor\n"); 1402 bp->dev->stats.rx_dropped++; 1403 queue->stats.rx_dropped++; 1404 break; 1405 } 1406 skb = queue->rx_skbuff[entry]; 1407 if (unlikely(!skb)) { 1408 netdev_err(bp->dev, 1409 "inconsistent Rx descriptor chain\n"); 1410 bp->dev->stats.rx_dropped++; 1411 queue->stats.rx_dropped++; 1412 break; 1413 } 1414 /* now everything is ready for receiving packet */ 1415 queue->rx_skbuff[entry] = NULL; 1416 len = ctrl & bp->rx_frm_len_mask; 1417 1418 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1419 1420 skb_put(skb, len); 1421 dma_unmap_single(&bp->pdev->dev, addr, 1422 bp->rx_buffer_size, DMA_FROM_DEVICE); 1423 1424 skb->protocol = eth_type_trans(skb, bp->dev); 1425 skb_checksum_none_assert(skb); 1426 if (bp->dev->features & NETIF_F_RXCSUM && 1427 !(bp->dev->flags & IFF_PROMISC) && 1428 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1429 skb->ip_summed = CHECKSUM_UNNECESSARY; 1430 1431 bp->dev->stats.rx_packets++; 1432 queue->stats.rx_packets++; 1433 bp->dev->stats.rx_bytes += skb->len; 1434 queue->stats.rx_bytes += skb->len; 1435 1436 gem_ptp_do_rxstamp(bp, skb, desc); 1437 1438 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1439 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1440 skb->len, skb->csum); 1441 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1442 skb_mac_header(skb), 16, true); 1443 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1444 skb->data, 32, true); 1445 #endif 1446 1447 napi_gro_receive(napi, skb); 1448 } 1449 1450 gem_rx_refill(queue); 1451 1452 return count; 1453 } 1454 1455 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1456 unsigned int first_frag, unsigned int last_frag) 1457 { 1458 unsigned int len; 1459 unsigned int frag; 1460 unsigned int offset; 1461 struct sk_buff *skb; 1462 struct macb_dma_desc *desc; 1463 struct macb *bp = queue->bp; 1464 1465 desc = macb_rx_desc(queue, last_frag); 1466 len = desc->ctrl & bp->rx_frm_len_mask; 1467 1468 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1469 macb_rx_ring_wrap(bp, first_frag), 1470 macb_rx_ring_wrap(bp, last_frag), len); 1471 1472 /* The ethernet header starts NET_IP_ALIGN bytes into the 1473 * first buffer. Since the header is 14 bytes, this makes the 1474 * payload word-aligned. 1475 * 1476 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1477 * the two padding bytes into the skb so that we avoid hitting 1478 * the slowpath in memcpy(), and pull them off afterwards. 1479 */ 1480 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1481 if (!skb) { 1482 bp->dev->stats.rx_dropped++; 1483 for (frag = first_frag; ; frag++) { 1484 desc = macb_rx_desc(queue, frag); 1485 desc->addr &= ~MACB_BIT(RX_USED); 1486 if (frag == last_frag) 1487 break; 1488 } 1489 1490 /* Make descriptor updates visible to hardware */ 1491 wmb(); 1492 1493 return 1; 1494 } 1495 1496 offset = 0; 1497 len += NET_IP_ALIGN; 1498 skb_checksum_none_assert(skb); 1499 skb_put(skb, len); 1500 1501 for (frag = first_frag; ; frag++) { 1502 unsigned int frag_len = bp->rx_buffer_size; 1503 1504 if (offset + frag_len > len) { 1505 if (unlikely(frag != last_frag)) { 1506 dev_kfree_skb_any(skb); 1507 return -1; 1508 } 1509 frag_len = len - offset; 1510 } 1511 skb_copy_to_linear_data_offset(skb, offset, 1512 macb_rx_buffer(queue, frag), 1513 frag_len); 1514 offset += bp->rx_buffer_size; 1515 desc = macb_rx_desc(queue, frag); 1516 desc->addr &= ~MACB_BIT(RX_USED); 1517 1518 if (frag == last_frag) 1519 break; 1520 } 1521 1522 /* Make descriptor updates visible to hardware */ 1523 wmb(); 1524 1525 __skb_pull(skb, NET_IP_ALIGN); 1526 skb->protocol = eth_type_trans(skb, bp->dev); 1527 1528 bp->dev->stats.rx_packets++; 1529 bp->dev->stats.rx_bytes += skb->len; 1530 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1531 skb->len, skb->csum); 1532 napi_gro_receive(napi, skb); 1533 1534 return 0; 1535 } 1536 1537 static inline void macb_init_rx_ring(struct macb_queue *queue) 1538 { 1539 struct macb *bp = queue->bp; 1540 dma_addr_t addr; 1541 struct macb_dma_desc *desc = NULL; 1542 int i; 1543 1544 addr = queue->rx_buffers_dma; 1545 for (i = 0; i < bp->rx_ring_size; i++) { 1546 desc = macb_rx_desc(queue, i); 1547 macb_set_addr(bp, desc, addr); 1548 desc->ctrl = 0; 1549 addr += bp->rx_buffer_size; 1550 } 1551 desc->addr |= MACB_BIT(RX_WRAP); 1552 queue->rx_tail = 0; 1553 } 1554 1555 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1556 int budget) 1557 { 1558 struct macb *bp = queue->bp; 1559 bool reset_rx_queue = false; 1560 int received = 0; 1561 unsigned int tail; 1562 int first_frag = -1; 1563 1564 for (tail = queue->rx_tail; budget > 0; tail++) { 1565 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1566 u32 ctrl; 1567 1568 /* Make hw descriptor updates visible to CPU */ 1569 rmb(); 1570 1571 if (!(desc->addr & MACB_BIT(RX_USED))) 1572 break; 1573 1574 /* Ensure ctrl is at least as up-to-date as addr */ 1575 dma_rmb(); 1576 1577 ctrl = desc->ctrl; 1578 1579 if (ctrl & MACB_BIT(RX_SOF)) { 1580 if (first_frag != -1) 1581 discard_partial_frame(queue, first_frag, tail); 1582 first_frag = tail; 1583 } 1584 1585 if (ctrl & MACB_BIT(RX_EOF)) { 1586 int dropped; 1587 1588 if (unlikely(first_frag == -1)) { 1589 reset_rx_queue = true; 1590 continue; 1591 } 1592 1593 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1594 first_frag = -1; 1595 if (unlikely(dropped < 0)) { 1596 reset_rx_queue = true; 1597 continue; 1598 } 1599 if (!dropped) { 1600 received++; 1601 budget--; 1602 } 1603 } 1604 } 1605 1606 if (unlikely(reset_rx_queue)) { 1607 unsigned long flags; 1608 u32 ctrl; 1609 1610 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1611 1612 spin_lock_irqsave(&bp->lock, flags); 1613 1614 ctrl = macb_readl(bp, NCR); 1615 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1616 1617 macb_init_rx_ring(queue); 1618 queue_writel(queue, RBQP, queue->rx_ring_dma); 1619 1620 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1621 1622 spin_unlock_irqrestore(&bp->lock, flags); 1623 return received; 1624 } 1625 1626 if (first_frag != -1) 1627 queue->rx_tail = first_frag; 1628 else 1629 queue->rx_tail = tail; 1630 1631 return received; 1632 } 1633 1634 static int macb_poll(struct napi_struct *napi, int budget) 1635 { 1636 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1637 struct macb *bp = queue->bp; 1638 int work_done; 1639 u32 status; 1640 1641 status = macb_readl(bp, RSR); 1642 macb_writel(bp, RSR, status); 1643 1644 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1645 (unsigned long)status, budget); 1646 1647 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 1648 if (work_done < budget) { 1649 napi_complete_done(napi, work_done); 1650 1651 /* Packets received while interrupts were disabled */ 1652 status = macb_readl(bp, RSR); 1653 if (status) { 1654 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1655 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1656 napi_reschedule(napi); 1657 } else { 1658 queue_writel(queue, IER, bp->rx_intr_mask); 1659 } 1660 } 1661 1662 /* TODO: Handle errors */ 1663 1664 return work_done; 1665 } 1666 1667 static void macb_hresp_error_task(struct tasklet_struct *t) 1668 { 1669 struct macb *bp = from_tasklet(bp, t, hresp_err_tasklet); 1670 struct net_device *dev = bp->dev; 1671 struct macb_queue *queue; 1672 unsigned int q; 1673 u32 ctrl; 1674 1675 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1676 queue_writel(queue, IDR, bp->rx_intr_mask | 1677 MACB_TX_INT_FLAGS | 1678 MACB_BIT(HRESP)); 1679 } 1680 ctrl = macb_readl(bp, NCR); 1681 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1682 macb_writel(bp, NCR, ctrl); 1683 1684 netif_tx_stop_all_queues(dev); 1685 netif_carrier_off(dev); 1686 1687 bp->macbgem_ops.mog_init_rings(bp); 1688 1689 /* Initialize TX and RX buffers */ 1690 macb_init_buffers(bp); 1691 1692 /* Enable interrupts */ 1693 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1694 queue_writel(queue, IER, 1695 bp->rx_intr_mask | 1696 MACB_TX_INT_FLAGS | 1697 MACB_BIT(HRESP)); 1698 1699 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1700 macb_writel(bp, NCR, ctrl); 1701 1702 netif_carrier_on(dev); 1703 netif_tx_start_all_queues(dev); 1704 } 1705 1706 static void macb_tx_restart(struct macb_queue *queue) 1707 { 1708 unsigned int head = queue->tx_head; 1709 unsigned int tail = queue->tx_tail; 1710 struct macb *bp = queue->bp; 1711 1712 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1713 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1714 1715 if (head == tail) 1716 return; 1717 1718 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1719 } 1720 1721 static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) 1722 { 1723 struct macb_queue *queue = dev_id; 1724 struct macb *bp = queue->bp; 1725 u32 status; 1726 1727 status = queue_readl(queue, ISR); 1728 1729 if (unlikely(!status)) 1730 return IRQ_NONE; 1731 1732 spin_lock(&bp->lock); 1733 1734 if (status & MACB_BIT(WOL)) { 1735 queue_writel(queue, IDR, MACB_BIT(WOL)); 1736 macb_writel(bp, WOL, 0); 1737 netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", 1738 (unsigned int)(queue - bp->queues), 1739 (unsigned long)status); 1740 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1741 queue_writel(queue, ISR, MACB_BIT(WOL)); 1742 pm_wakeup_event(&bp->pdev->dev, 0); 1743 } 1744 1745 spin_unlock(&bp->lock); 1746 1747 return IRQ_HANDLED; 1748 } 1749 1750 static irqreturn_t gem_wol_interrupt(int irq, void *dev_id) 1751 { 1752 struct macb_queue *queue = dev_id; 1753 struct macb *bp = queue->bp; 1754 u32 status; 1755 1756 status = queue_readl(queue, ISR); 1757 1758 if (unlikely(!status)) 1759 return IRQ_NONE; 1760 1761 spin_lock(&bp->lock); 1762 1763 if (status & GEM_BIT(WOL)) { 1764 queue_writel(queue, IDR, GEM_BIT(WOL)); 1765 gem_writel(bp, WOL, 0); 1766 netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", 1767 (unsigned int)(queue - bp->queues), 1768 (unsigned long)status); 1769 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1770 queue_writel(queue, ISR, GEM_BIT(WOL)); 1771 pm_wakeup_event(&bp->pdev->dev, 0); 1772 } 1773 1774 spin_unlock(&bp->lock); 1775 1776 return IRQ_HANDLED; 1777 } 1778 1779 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1780 { 1781 struct macb_queue *queue = dev_id; 1782 struct macb *bp = queue->bp; 1783 struct net_device *dev = bp->dev; 1784 u32 status, ctrl; 1785 1786 status = queue_readl(queue, ISR); 1787 1788 if (unlikely(!status)) 1789 return IRQ_NONE; 1790 1791 spin_lock(&bp->lock); 1792 1793 while (status) { 1794 /* close possible race with dev_close */ 1795 if (unlikely(!netif_running(dev))) { 1796 queue_writel(queue, IDR, -1); 1797 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1798 queue_writel(queue, ISR, -1); 1799 break; 1800 } 1801 1802 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1803 (unsigned int)(queue - bp->queues), 1804 (unsigned long)status); 1805 1806 if (status & bp->rx_intr_mask) { 1807 /* There's no point taking any more interrupts 1808 * until we have processed the buffers. The 1809 * scheduling call may fail if the poll routine 1810 * is already scheduled, so disable interrupts 1811 * now. 1812 */ 1813 queue_writel(queue, IDR, bp->rx_intr_mask); 1814 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1815 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1816 1817 if (napi_schedule_prep(&queue->napi)) { 1818 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1819 __napi_schedule(&queue->napi); 1820 } 1821 } 1822 1823 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1824 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1825 schedule_work(&queue->tx_error_task); 1826 1827 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1828 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1829 1830 break; 1831 } 1832 1833 if (status & MACB_BIT(TCOMP)) 1834 macb_tx_interrupt(queue); 1835 1836 if (status & MACB_BIT(TXUBR)) 1837 macb_tx_restart(queue); 1838 1839 /* Link change detection isn't possible with RMII, so we'll 1840 * add that if/when we get our hands on a full-blown MII PHY. 1841 */ 1842 1843 /* There is a hardware issue under heavy load where DMA can 1844 * stop, this causes endless "used buffer descriptor read" 1845 * interrupts but it can be cleared by re-enabling RX. See 1846 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1847 * section 16.7.4 for details. RXUBR is only enabled for 1848 * these two versions. 1849 */ 1850 if (status & MACB_BIT(RXUBR)) { 1851 ctrl = macb_readl(bp, NCR); 1852 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1853 wmb(); 1854 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1855 1856 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1857 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1858 } 1859 1860 if (status & MACB_BIT(ISR_ROVR)) { 1861 /* We missed at least one packet */ 1862 if (macb_is_gem(bp)) 1863 bp->hw_stats.gem.rx_overruns++; 1864 else 1865 bp->hw_stats.macb.rx_overruns++; 1866 1867 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1868 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1869 } 1870 1871 if (status & MACB_BIT(HRESP)) { 1872 tasklet_schedule(&bp->hresp_err_tasklet); 1873 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1874 1875 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1876 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1877 } 1878 status = queue_readl(queue, ISR); 1879 } 1880 1881 spin_unlock(&bp->lock); 1882 1883 return IRQ_HANDLED; 1884 } 1885 1886 #ifdef CONFIG_NET_POLL_CONTROLLER 1887 /* Polling receive - used by netconsole and other diagnostic tools 1888 * to allow network i/o with interrupts disabled. 1889 */ 1890 static void macb_poll_controller(struct net_device *dev) 1891 { 1892 struct macb *bp = netdev_priv(dev); 1893 struct macb_queue *queue; 1894 unsigned long flags; 1895 unsigned int q; 1896 1897 local_irq_save(flags); 1898 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1899 macb_interrupt(dev->irq, queue); 1900 local_irq_restore(flags); 1901 } 1902 #endif 1903 1904 static unsigned int macb_tx_map(struct macb *bp, 1905 struct macb_queue *queue, 1906 struct sk_buff *skb, 1907 unsigned int hdrlen) 1908 { 1909 dma_addr_t mapping; 1910 unsigned int len, entry, i, tx_head = queue->tx_head; 1911 struct macb_tx_skb *tx_skb = NULL; 1912 struct macb_dma_desc *desc; 1913 unsigned int offset, size, count = 0; 1914 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1915 unsigned int eof = 1, mss_mfs = 0; 1916 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1917 1918 /* LSO */ 1919 if (skb_shinfo(skb)->gso_size != 0) { 1920 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1921 /* UDP - UFO */ 1922 lso_ctrl = MACB_LSO_UFO_ENABLE; 1923 else 1924 /* TCP - TSO */ 1925 lso_ctrl = MACB_LSO_TSO_ENABLE; 1926 } 1927 1928 /* First, map non-paged data */ 1929 len = skb_headlen(skb); 1930 1931 /* first buffer length */ 1932 size = hdrlen; 1933 1934 offset = 0; 1935 while (len) { 1936 entry = macb_tx_ring_wrap(bp, tx_head); 1937 tx_skb = &queue->tx_skb[entry]; 1938 1939 mapping = dma_map_single(&bp->pdev->dev, 1940 skb->data + offset, 1941 size, DMA_TO_DEVICE); 1942 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1943 goto dma_error; 1944 1945 /* Save info to properly release resources */ 1946 tx_skb->skb = NULL; 1947 tx_skb->mapping = mapping; 1948 tx_skb->size = size; 1949 tx_skb->mapped_as_page = false; 1950 1951 len -= size; 1952 offset += size; 1953 count++; 1954 tx_head++; 1955 1956 size = min(len, bp->max_tx_length); 1957 } 1958 1959 /* Then, map paged data from fragments */ 1960 for (f = 0; f < nr_frags; f++) { 1961 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1962 1963 len = skb_frag_size(frag); 1964 offset = 0; 1965 while (len) { 1966 size = min(len, bp->max_tx_length); 1967 entry = macb_tx_ring_wrap(bp, tx_head); 1968 tx_skb = &queue->tx_skb[entry]; 1969 1970 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1971 offset, size, DMA_TO_DEVICE); 1972 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1973 goto dma_error; 1974 1975 /* Save info to properly release resources */ 1976 tx_skb->skb = NULL; 1977 tx_skb->mapping = mapping; 1978 tx_skb->size = size; 1979 tx_skb->mapped_as_page = true; 1980 1981 len -= size; 1982 offset += size; 1983 count++; 1984 tx_head++; 1985 } 1986 } 1987 1988 /* Should never happen */ 1989 if (unlikely(!tx_skb)) { 1990 netdev_err(bp->dev, "BUG! empty skb!\n"); 1991 return 0; 1992 } 1993 1994 /* This is the last buffer of the frame: save socket buffer */ 1995 tx_skb->skb = skb; 1996 1997 /* Update TX ring: update buffer descriptors in reverse order 1998 * to avoid race condition 1999 */ 2000 2001 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 2002 * to set the end of TX queue 2003 */ 2004 i = tx_head; 2005 entry = macb_tx_ring_wrap(bp, i); 2006 ctrl = MACB_BIT(TX_USED); 2007 desc = macb_tx_desc(queue, entry); 2008 desc->ctrl = ctrl; 2009 2010 if (lso_ctrl) { 2011 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 2012 /* include header and FCS in value given to h/w */ 2013 mss_mfs = skb_shinfo(skb)->gso_size + 2014 skb_transport_offset(skb) + 2015 ETH_FCS_LEN; 2016 else /* TSO */ { 2017 mss_mfs = skb_shinfo(skb)->gso_size; 2018 /* TCP Sequence Number Source Select 2019 * can be set only for TSO 2020 */ 2021 seq_ctrl = 0; 2022 } 2023 } 2024 2025 do { 2026 i--; 2027 entry = macb_tx_ring_wrap(bp, i); 2028 tx_skb = &queue->tx_skb[entry]; 2029 desc = macb_tx_desc(queue, entry); 2030 2031 ctrl = (u32)tx_skb->size; 2032 if (eof) { 2033 ctrl |= MACB_BIT(TX_LAST); 2034 eof = 0; 2035 } 2036 if (unlikely(entry == (bp->tx_ring_size - 1))) 2037 ctrl |= MACB_BIT(TX_WRAP); 2038 2039 /* First descriptor is header descriptor */ 2040 if (i == queue->tx_head) { 2041 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 2042 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 2043 if ((bp->dev->features & NETIF_F_HW_CSUM) && 2044 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 2045 ctrl |= MACB_BIT(TX_NOCRC); 2046 } else 2047 /* Only set MSS/MFS on payload descriptors 2048 * (second or later descriptor) 2049 */ 2050 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 2051 2052 /* Set TX buffer descriptor */ 2053 macb_set_addr(bp, desc, tx_skb->mapping); 2054 /* desc->addr must be visible to hardware before clearing 2055 * 'TX_USED' bit in desc->ctrl. 2056 */ 2057 wmb(); 2058 desc->ctrl = ctrl; 2059 } while (i != queue->tx_head); 2060 2061 queue->tx_head = tx_head; 2062 2063 return count; 2064 2065 dma_error: 2066 netdev_err(bp->dev, "TX DMA map failed\n"); 2067 2068 for (i = queue->tx_head; i != tx_head; i++) { 2069 tx_skb = macb_tx_skb(queue, i); 2070 2071 macb_tx_unmap(bp, tx_skb); 2072 } 2073 2074 return 0; 2075 } 2076 2077 static netdev_features_t macb_features_check(struct sk_buff *skb, 2078 struct net_device *dev, 2079 netdev_features_t features) 2080 { 2081 unsigned int nr_frags, f; 2082 unsigned int hdrlen; 2083 2084 /* Validate LSO compatibility */ 2085 2086 /* there is only one buffer or protocol is not UDP */ 2087 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) 2088 return features; 2089 2090 /* length of header */ 2091 hdrlen = skb_transport_offset(skb); 2092 2093 /* For UFO only: 2094 * When software supplies two or more payload buffers all payload buffers 2095 * apart from the last must be a multiple of 8 bytes in size. 2096 */ 2097 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 2098 return features & ~MACB_NETIF_LSO; 2099 2100 nr_frags = skb_shinfo(skb)->nr_frags; 2101 /* No need to check last fragment */ 2102 nr_frags--; 2103 for (f = 0; f < nr_frags; f++) { 2104 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2105 2106 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 2107 return features & ~MACB_NETIF_LSO; 2108 } 2109 return features; 2110 } 2111 2112 static inline int macb_clear_csum(struct sk_buff *skb) 2113 { 2114 /* no change for packets without checksum offloading */ 2115 if (skb->ip_summed != CHECKSUM_PARTIAL) 2116 return 0; 2117 2118 /* make sure we can modify the header */ 2119 if (unlikely(skb_cow_head(skb, 0))) 2120 return -1; 2121 2122 /* initialize checksum field 2123 * This is required - at least for Zynq, which otherwise calculates 2124 * wrong UDP header checksums for UDP packets with UDP data len <=2 2125 */ 2126 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 2127 return 0; 2128 } 2129 2130 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 2131 { 2132 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || 2133 skb_is_nonlinear(*skb); 2134 int padlen = ETH_ZLEN - (*skb)->len; 2135 int headroom = skb_headroom(*skb); 2136 int tailroom = skb_tailroom(*skb); 2137 struct sk_buff *nskb; 2138 u32 fcs; 2139 2140 if (!(ndev->features & NETIF_F_HW_CSUM) || 2141 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 2142 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 2143 return 0; 2144 2145 if (padlen <= 0) { 2146 /* FCS could be appeded to tailroom. */ 2147 if (tailroom >= ETH_FCS_LEN) 2148 goto add_fcs; 2149 /* FCS could be appeded by moving data to headroom. */ 2150 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 2151 padlen = 0; 2152 /* No room for FCS, need to reallocate skb. */ 2153 else 2154 padlen = ETH_FCS_LEN; 2155 } else { 2156 /* Add room for FCS. */ 2157 padlen += ETH_FCS_LEN; 2158 } 2159 2160 if (!cloned && headroom + tailroom >= padlen) { 2161 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 2162 skb_set_tail_pointer(*skb, (*skb)->len); 2163 } else { 2164 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 2165 if (!nskb) 2166 return -ENOMEM; 2167 2168 dev_consume_skb_any(*skb); 2169 *skb = nskb; 2170 } 2171 2172 if (padlen > ETH_FCS_LEN) 2173 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 2174 2175 add_fcs: 2176 /* set FCS to packet */ 2177 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 2178 fcs = ~fcs; 2179 2180 skb_put_u8(*skb, fcs & 0xff); 2181 skb_put_u8(*skb, (fcs >> 8) & 0xff); 2182 skb_put_u8(*skb, (fcs >> 16) & 0xff); 2183 skb_put_u8(*skb, (fcs >> 24) & 0xff); 2184 2185 return 0; 2186 } 2187 2188 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 2189 { 2190 u16 queue_index = skb_get_queue_mapping(skb); 2191 struct macb *bp = netdev_priv(dev); 2192 struct macb_queue *queue = &bp->queues[queue_index]; 2193 unsigned long flags; 2194 unsigned int desc_cnt, nr_frags, frag_size, f; 2195 unsigned int hdrlen; 2196 bool is_lso; 2197 netdev_tx_t ret = NETDEV_TX_OK; 2198 2199 if (macb_clear_csum(skb)) { 2200 dev_kfree_skb_any(skb); 2201 return ret; 2202 } 2203 2204 if (macb_pad_and_fcs(&skb, dev)) { 2205 dev_kfree_skb_any(skb); 2206 return ret; 2207 } 2208 2209 is_lso = (skb_shinfo(skb)->gso_size != 0); 2210 2211 if (is_lso) { 2212 /* length of headers */ 2213 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2214 /* only queue eth + ip headers separately for UDP */ 2215 hdrlen = skb_transport_offset(skb); 2216 else 2217 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 2218 if (skb_headlen(skb) < hdrlen) { 2219 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 2220 /* if this is required, would need to copy to single buffer */ 2221 return NETDEV_TX_BUSY; 2222 } 2223 } else 2224 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 2225 2226 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 2227 netdev_vdbg(bp->dev, 2228 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 2229 queue_index, skb->len, skb->head, skb->data, 2230 skb_tail_pointer(skb), skb_end_pointer(skb)); 2231 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 2232 skb->data, 16, true); 2233 #endif 2234 2235 /* Count how many TX buffer descriptors are needed to send this 2236 * socket buffer: skb fragments of jumbo frames may need to be 2237 * split into many buffer descriptors. 2238 */ 2239 if (is_lso && (skb_headlen(skb) > hdrlen)) 2240 /* extra header descriptor if also payload in first buffer */ 2241 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 2242 else 2243 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 2244 nr_frags = skb_shinfo(skb)->nr_frags; 2245 for (f = 0; f < nr_frags; f++) { 2246 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 2247 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 2248 } 2249 2250 spin_lock_irqsave(&bp->lock, flags); 2251 2252 /* This is a hard error, log it. */ 2253 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 2254 bp->tx_ring_size) < desc_cnt) { 2255 netif_stop_subqueue(dev, queue_index); 2256 spin_unlock_irqrestore(&bp->lock, flags); 2257 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 2258 queue->tx_head, queue->tx_tail); 2259 return NETDEV_TX_BUSY; 2260 } 2261 2262 /* Map socket buffer for DMA transfer */ 2263 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 2264 dev_kfree_skb_any(skb); 2265 goto unlock; 2266 } 2267 2268 /* Make newly initialized descriptor visible to hardware */ 2269 wmb(); 2270 skb_tx_timestamp(skb); 2271 2272 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 2273 2274 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 2275 netif_stop_subqueue(dev, queue_index); 2276 2277 unlock: 2278 spin_unlock_irqrestore(&bp->lock, flags); 2279 2280 return ret; 2281 } 2282 2283 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 2284 { 2285 if (!macb_is_gem(bp)) { 2286 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 2287 } else { 2288 bp->rx_buffer_size = size; 2289 2290 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 2291 netdev_dbg(bp->dev, 2292 "RX buffer must be multiple of %d bytes, expanding\n", 2293 RX_BUFFER_MULTIPLE); 2294 bp->rx_buffer_size = 2295 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 2296 } 2297 } 2298 2299 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 2300 bp->dev->mtu, bp->rx_buffer_size); 2301 } 2302 2303 static void gem_free_rx_buffers(struct macb *bp) 2304 { 2305 struct sk_buff *skb; 2306 struct macb_dma_desc *desc; 2307 struct macb_queue *queue; 2308 dma_addr_t addr; 2309 unsigned int q; 2310 int i; 2311 2312 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2313 if (!queue->rx_skbuff) 2314 continue; 2315 2316 for (i = 0; i < bp->rx_ring_size; i++) { 2317 skb = queue->rx_skbuff[i]; 2318 2319 if (!skb) 2320 continue; 2321 2322 desc = macb_rx_desc(queue, i); 2323 addr = macb_get_addr(bp, desc); 2324 2325 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 2326 DMA_FROM_DEVICE); 2327 dev_kfree_skb_any(skb); 2328 skb = NULL; 2329 } 2330 2331 kfree(queue->rx_skbuff); 2332 queue->rx_skbuff = NULL; 2333 } 2334 } 2335 2336 static void macb_free_rx_buffers(struct macb *bp) 2337 { 2338 struct macb_queue *queue = &bp->queues[0]; 2339 2340 if (queue->rx_buffers) { 2341 dma_free_coherent(&bp->pdev->dev, 2342 bp->rx_ring_size * bp->rx_buffer_size, 2343 queue->rx_buffers, queue->rx_buffers_dma); 2344 queue->rx_buffers = NULL; 2345 } 2346 } 2347 2348 static void macb_free_consistent(struct macb *bp) 2349 { 2350 struct macb_queue *queue; 2351 unsigned int q; 2352 int size; 2353 2354 bp->macbgem_ops.mog_free_rx_buffers(bp); 2355 2356 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2357 kfree(queue->tx_skb); 2358 queue->tx_skb = NULL; 2359 if (queue->tx_ring) { 2360 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2361 dma_free_coherent(&bp->pdev->dev, size, 2362 queue->tx_ring, queue->tx_ring_dma); 2363 queue->tx_ring = NULL; 2364 } 2365 if (queue->rx_ring) { 2366 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2367 dma_free_coherent(&bp->pdev->dev, size, 2368 queue->rx_ring, queue->rx_ring_dma); 2369 queue->rx_ring = NULL; 2370 } 2371 } 2372 } 2373 2374 static int gem_alloc_rx_buffers(struct macb *bp) 2375 { 2376 struct macb_queue *queue; 2377 unsigned int q; 2378 int size; 2379 2380 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2381 size = bp->rx_ring_size * sizeof(struct sk_buff *); 2382 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 2383 if (!queue->rx_skbuff) 2384 return -ENOMEM; 2385 else 2386 netdev_dbg(bp->dev, 2387 "Allocated %d RX struct sk_buff entries at %p\n", 2388 bp->rx_ring_size, queue->rx_skbuff); 2389 } 2390 return 0; 2391 } 2392 2393 static int macb_alloc_rx_buffers(struct macb *bp) 2394 { 2395 struct macb_queue *queue = &bp->queues[0]; 2396 int size; 2397 2398 size = bp->rx_ring_size * bp->rx_buffer_size; 2399 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 2400 &queue->rx_buffers_dma, GFP_KERNEL); 2401 if (!queue->rx_buffers) 2402 return -ENOMEM; 2403 2404 netdev_dbg(bp->dev, 2405 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 2406 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 2407 return 0; 2408 } 2409 2410 static int macb_alloc_consistent(struct macb *bp) 2411 { 2412 struct macb_queue *queue; 2413 unsigned int q; 2414 int size; 2415 2416 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2417 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2418 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2419 &queue->tx_ring_dma, 2420 GFP_KERNEL); 2421 if (!queue->tx_ring) 2422 goto out_err; 2423 netdev_dbg(bp->dev, 2424 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2425 q, size, (unsigned long)queue->tx_ring_dma, 2426 queue->tx_ring); 2427 2428 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2429 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2430 if (!queue->tx_skb) 2431 goto out_err; 2432 2433 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2434 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2435 &queue->rx_ring_dma, GFP_KERNEL); 2436 if (!queue->rx_ring) 2437 goto out_err; 2438 netdev_dbg(bp->dev, 2439 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2440 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2441 } 2442 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2443 goto out_err; 2444 2445 return 0; 2446 2447 out_err: 2448 macb_free_consistent(bp); 2449 return -ENOMEM; 2450 } 2451 2452 static void gem_init_rings(struct macb *bp) 2453 { 2454 struct macb_queue *queue; 2455 struct macb_dma_desc *desc = NULL; 2456 unsigned int q; 2457 int i; 2458 2459 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2460 for (i = 0; i < bp->tx_ring_size; i++) { 2461 desc = macb_tx_desc(queue, i); 2462 macb_set_addr(bp, desc, 0); 2463 desc->ctrl = MACB_BIT(TX_USED); 2464 } 2465 desc->ctrl |= MACB_BIT(TX_WRAP); 2466 queue->tx_head = 0; 2467 queue->tx_tail = 0; 2468 2469 queue->rx_tail = 0; 2470 queue->rx_prepared_head = 0; 2471 2472 gem_rx_refill(queue); 2473 } 2474 2475 } 2476 2477 static void macb_init_rings(struct macb *bp) 2478 { 2479 int i; 2480 struct macb_dma_desc *desc = NULL; 2481 2482 macb_init_rx_ring(&bp->queues[0]); 2483 2484 for (i = 0; i < bp->tx_ring_size; i++) { 2485 desc = macb_tx_desc(&bp->queues[0], i); 2486 macb_set_addr(bp, desc, 0); 2487 desc->ctrl = MACB_BIT(TX_USED); 2488 } 2489 bp->queues[0].tx_head = 0; 2490 bp->queues[0].tx_tail = 0; 2491 desc->ctrl |= MACB_BIT(TX_WRAP); 2492 } 2493 2494 static void macb_reset_hw(struct macb *bp) 2495 { 2496 struct macb_queue *queue; 2497 unsigned int q; 2498 u32 ctrl = macb_readl(bp, NCR); 2499 2500 /* Disable RX and TX (XXX: Should we halt the transmission 2501 * more gracefully?) 2502 */ 2503 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2504 2505 /* Clear the stats registers (XXX: Update stats first?) */ 2506 ctrl |= MACB_BIT(CLRSTAT); 2507 2508 macb_writel(bp, NCR, ctrl); 2509 2510 /* Clear all status flags */ 2511 macb_writel(bp, TSR, -1); 2512 macb_writel(bp, RSR, -1); 2513 2514 /* Disable all interrupts */ 2515 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2516 queue_writel(queue, IDR, -1); 2517 queue_readl(queue, ISR); 2518 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2519 queue_writel(queue, ISR, -1); 2520 } 2521 } 2522 2523 static u32 gem_mdc_clk_div(struct macb *bp) 2524 { 2525 u32 config; 2526 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2527 2528 if (pclk_hz <= 20000000) 2529 config = GEM_BF(CLK, GEM_CLK_DIV8); 2530 else if (pclk_hz <= 40000000) 2531 config = GEM_BF(CLK, GEM_CLK_DIV16); 2532 else if (pclk_hz <= 80000000) 2533 config = GEM_BF(CLK, GEM_CLK_DIV32); 2534 else if (pclk_hz <= 120000000) 2535 config = GEM_BF(CLK, GEM_CLK_DIV48); 2536 else if (pclk_hz <= 160000000) 2537 config = GEM_BF(CLK, GEM_CLK_DIV64); 2538 else 2539 config = GEM_BF(CLK, GEM_CLK_DIV96); 2540 2541 return config; 2542 } 2543 2544 static u32 macb_mdc_clk_div(struct macb *bp) 2545 { 2546 u32 config; 2547 unsigned long pclk_hz; 2548 2549 if (macb_is_gem(bp)) 2550 return gem_mdc_clk_div(bp); 2551 2552 pclk_hz = clk_get_rate(bp->pclk); 2553 if (pclk_hz <= 20000000) 2554 config = MACB_BF(CLK, MACB_CLK_DIV8); 2555 else if (pclk_hz <= 40000000) 2556 config = MACB_BF(CLK, MACB_CLK_DIV16); 2557 else if (pclk_hz <= 80000000) 2558 config = MACB_BF(CLK, MACB_CLK_DIV32); 2559 else 2560 config = MACB_BF(CLK, MACB_CLK_DIV64); 2561 2562 return config; 2563 } 2564 2565 /* Get the DMA bus width field of the network configuration register that we 2566 * should program. We find the width from decoding the design configuration 2567 * register to find the maximum supported data bus width. 2568 */ 2569 static u32 macb_dbw(struct macb *bp) 2570 { 2571 if (!macb_is_gem(bp)) 2572 return 0; 2573 2574 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2575 case 4: 2576 return GEM_BF(DBW, GEM_DBW128); 2577 case 2: 2578 return GEM_BF(DBW, GEM_DBW64); 2579 case 1: 2580 default: 2581 return GEM_BF(DBW, GEM_DBW32); 2582 } 2583 } 2584 2585 /* Configure the receive DMA engine 2586 * - use the correct receive buffer size 2587 * - set best burst length for DMA operations 2588 * (if not supported by FIFO, it will fallback to default) 2589 * - set both rx/tx packet buffers to full memory size 2590 * These are configurable parameters for GEM. 2591 */ 2592 static void macb_configure_dma(struct macb *bp) 2593 { 2594 struct macb_queue *queue; 2595 u32 buffer_size; 2596 unsigned int q; 2597 u32 dmacfg; 2598 2599 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2600 if (macb_is_gem(bp)) { 2601 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2602 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2603 if (q) 2604 queue_writel(queue, RBQS, buffer_size); 2605 else 2606 dmacfg |= GEM_BF(RXBS, buffer_size); 2607 } 2608 if (bp->dma_burst_length) 2609 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2610 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2611 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2612 2613 if (bp->native_io) 2614 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2615 else 2616 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2617 2618 if (bp->dev->features & NETIF_F_HW_CSUM) 2619 dmacfg |= GEM_BIT(TXCOEN); 2620 else 2621 dmacfg &= ~GEM_BIT(TXCOEN); 2622 2623 dmacfg &= ~GEM_BIT(ADDR64); 2624 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2625 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2626 dmacfg |= GEM_BIT(ADDR64); 2627 #endif 2628 #ifdef CONFIG_MACB_USE_HWSTAMP 2629 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2630 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2631 #endif 2632 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2633 dmacfg); 2634 gem_writel(bp, DMACFG, dmacfg); 2635 } 2636 } 2637 2638 static void macb_init_hw(struct macb *bp) 2639 { 2640 u32 config; 2641 2642 macb_reset_hw(bp); 2643 macb_set_hwaddr(bp); 2644 2645 config = macb_mdc_clk_div(bp); 2646 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2647 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2648 if (bp->caps & MACB_CAPS_JUMBO) 2649 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2650 else 2651 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2652 if (bp->dev->flags & IFF_PROMISC) 2653 config |= MACB_BIT(CAF); /* Copy All Frames */ 2654 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2655 config |= GEM_BIT(RXCOEN); 2656 if (!(bp->dev->flags & IFF_BROADCAST)) 2657 config |= MACB_BIT(NBC); /* No BroadCast */ 2658 config |= macb_dbw(bp); 2659 macb_writel(bp, NCFGR, config); 2660 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2661 gem_writel(bp, JML, bp->jumbo_max_len); 2662 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2663 if (bp->caps & MACB_CAPS_JUMBO) 2664 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2665 2666 macb_configure_dma(bp); 2667 } 2668 2669 /* The hash address register is 64 bits long and takes up two 2670 * locations in the memory map. The least significant bits are stored 2671 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2672 * 2673 * The unicast hash enable and the multicast hash enable bits in the 2674 * network configuration register enable the reception of hash matched 2675 * frames. The destination address is reduced to a 6 bit index into 2676 * the 64 bit hash register using the following hash function. The 2677 * hash function is an exclusive or of every sixth bit of the 2678 * destination address. 2679 * 2680 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2681 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2682 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2683 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2684 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2685 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2686 * 2687 * da[0] represents the least significant bit of the first byte 2688 * received, that is, the multicast/unicast indicator, and da[47] 2689 * represents the most significant bit of the last byte received. If 2690 * the hash index, hi[n], points to a bit that is set in the hash 2691 * register then the frame will be matched according to whether the 2692 * frame is multicast or unicast. A multicast match will be signalled 2693 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2694 * index points to a bit set in the hash register. A unicast match 2695 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2696 * and the hash index points to a bit set in the hash register. To 2697 * receive all multicast frames, the hash register should be set with 2698 * all ones and the multicast hash enable bit should be set in the 2699 * network configuration register. 2700 */ 2701 2702 static inline int hash_bit_value(int bitnr, __u8 *addr) 2703 { 2704 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2705 return 1; 2706 return 0; 2707 } 2708 2709 /* Return the hash index value for the specified address. */ 2710 static int hash_get_index(__u8 *addr) 2711 { 2712 int i, j, bitval; 2713 int hash_index = 0; 2714 2715 for (j = 0; j < 6; j++) { 2716 for (i = 0, bitval = 0; i < 8; i++) 2717 bitval ^= hash_bit_value(i * 6 + j, addr); 2718 2719 hash_index |= (bitval << j); 2720 } 2721 2722 return hash_index; 2723 } 2724 2725 /* Add multicast addresses to the internal multicast-hash table. */ 2726 static void macb_sethashtable(struct net_device *dev) 2727 { 2728 struct netdev_hw_addr *ha; 2729 unsigned long mc_filter[2]; 2730 unsigned int bitnr; 2731 struct macb *bp = netdev_priv(dev); 2732 2733 mc_filter[0] = 0; 2734 mc_filter[1] = 0; 2735 2736 netdev_for_each_mc_addr(ha, dev) { 2737 bitnr = hash_get_index(ha->addr); 2738 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2739 } 2740 2741 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2742 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2743 } 2744 2745 /* Enable/Disable promiscuous and multicast modes. */ 2746 static void macb_set_rx_mode(struct net_device *dev) 2747 { 2748 unsigned long cfg; 2749 struct macb *bp = netdev_priv(dev); 2750 2751 cfg = macb_readl(bp, NCFGR); 2752 2753 if (dev->flags & IFF_PROMISC) { 2754 /* Enable promiscuous mode */ 2755 cfg |= MACB_BIT(CAF); 2756 2757 /* Disable RX checksum offload */ 2758 if (macb_is_gem(bp)) 2759 cfg &= ~GEM_BIT(RXCOEN); 2760 } else { 2761 /* Disable promiscuous mode */ 2762 cfg &= ~MACB_BIT(CAF); 2763 2764 /* Enable RX checksum offload only if requested */ 2765 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2766 cfg |= GEM_BIT(RXCOEN); 2767 } 2768 2769 if (dev->flags & IFF_ALLMULTI) { 2770 /* Enable all multicast mode */ 2771 macb_or_gem_writel(bp, HRB, -1); 2772 macb_or_gem_writel(bp, HRT, -1); 2773 cfg |= MACB_BIT(NCFGR_MTI); 2774 } else if (!netdev_mc_empty(dev)) { 2775 /* Enable specific multicasts */ 2776 macb_sethashtable(dev); 2777 cfg |= MACB_BIT(NCFGR_MTI); 2778 } else if (dev->flags & (~IFF_ALLMULTI)) { 2779 /* Disable all multicast mode */ 2780 macb_or_gem_writel(bp, HRB, 0); 2781 macb_or_gem_writel(bp, HRT, 0); 2782 cfg &= ~MACB_BIT(NCFGR_MTI); 2783 } 2784 2785 macb_writel(bp, NCFGR, cfg); 2786 } 2787 2788 static int macb_open(struct net_device *dev) 2789 { 2790 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2791 struct macb *bp = netdev_priv(dev); 2792 struct macb_queue *queue; 2793 unsigned int q; 2794 int err; 2795 2796 netdev_dbg(bp->dev, "open\n"); 2797 2798 err = pm_runtime_get_sync(&bp->pdev->dev); 2799 if (err < 0) 2800 goto pm_exit; 2801 2802 /* RX buffers initialization */ 2803 macb_init_rx_buffer_size(bp, bufsz); 2804 2805 err = macb_alloc_consistent(bp); 2806 if (err) { 2807 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2808 err); 2809 goto pm_exit; 2810 } 2811 2812 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2813 napi_enable(&queue->napi); 2814 2815 macb_init_hw(bp); 2816 2817 err = macb_phylink_connect(bp); 2818 if (err) 2819 goto reset_hw; 2820 2821 netif_tx_start_all_queues(dev); 2822 2823 if (bp->ptp_info) 2824 bp->ptp_info->ptp_init(dev); 2825 2826 return 0; 2827 2828 reset_hw: 2829 macb_reset_hw(bp); 2830 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2831 napi_disable(&queue->napi); 2832 macb_free_consistent(bp); 2833 pm_exit: 2834 pm_runtime_put_sync(&bp->pdev->dev); 2835 return err; 2836 } 2837 2838 static int macb_close(struct net_device *dev) 2839 { 2840 struct macb *bp = netdev_priv(dev); 2841 struct macb_queue *queue; 2842 unsigned long flags; 2843 unsigned int q; 2844 2845 netif_tx_stop_all_queues(dev); 2846 2847 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2848 napi_disable(&queue->napi); 2849 2850 phylink_stop(bp->phylink); 2851 phylink_disconnect_phy(bp->phylink); 2852 2853 spin_lock_irqsave(&bp->lock, flags); 2854 macb_reset_hw(bp); 2855 netif_carrier_off(dev); 2856 spin_unlock_irqrestore(&bp->lock, flags); 2857 2858 macb_free_consistent(bp); 2859 2860 if (bp->ptp_info) 2861 bp->ptp_info->ptp_remove(dev); 2862 2863 pm_runtime_put(&bp->pdev->dev); 2864 2865 return 0; 2866 } 2867 2868 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2869 { 2870 if (netif_running(dev)) 2871 return -EBUSY; 2872 2873 dev->mtu = new_mtu; 2874 2875 return 0; 2876 } 2877 2878 static void gem_update_stats(struct macb *bp) 2879 { 2880 struct macb_queue *queue; 2881 unsigned int i, q, idx; 2882 unsigned long *stat; 2883 2884 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2885 2886 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2887 u32 offset = gem_statistics[i].offset; 2888 u64 val = bp->macb_reg_readl(bp, offset); 2889 2890 bp->ethtool_stats[i] += val; 2891 *p += val; 2892 2893 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2894 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2895 val = bp->macb_reg_readl(bp, offset + 4); 2896 bp->ethtool_stats[i] += ((u64)val) << 32; 2897 *(++p) += val; 2898 } 2899 } 2900 2901 idx = GEM_STATS_LEN; 2902 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2903 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2904 bp->ethtool_stats[idx++] = *stat; 2905 } 2906 2907 static struct net_device_stats *gem_get_stats(struct macb *bp) 2908 { 2909 struct gem_stats *hwstat = &bp->hw_stats.gem; 2910 struct net_device_stats *nstat = &bp->dev->stats; 2911 2912 if (!netif_running(bp->dev)) 2913 return nstat; 2914 2915 gem_update_stats(bp); 2916 2917 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2918 hwstat->rx_alignment_errors + 2919 hwstat->rx_resource_errors + 2920 hwstat->rx_overruns + 2921 hwstat->rx_oversize_frames + 2922 hwstat->rx_jabbers + 2923 hwstat->rx_undersized_frames + 2924 hwstat->rx_length_field_frame_errors); 2925 nstat->tx_errors = (hwstat->tx_late_collisions + 2926 hwstat->tx_excessive_collisions + 2927 hwstat->tx_underrun + 2928 hwstat->tx_carrier_sense_errors); 2929 nstat->multicast = hwstat->rx_multicast_frames; 2930 nstat->collisions = (hwstat->tx_single_collision_frames + 2931 hwstat->tx_multiple_collision_frames + 2932 hwstat->tx_excessive_collisions); 2933 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2934 hwstat->rx_jabbers + 2935 hwstat->rx_undersized_frames + 2936 hwstat->rx_length_field_frame_errors); 2937 nstat->rx_over_errors = hwstat->rx_resource_errors; 2938 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2939 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2940 nstat->rx_fifo_errors = hwstat->rx_overruns; 2941 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2942 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2943 nstat->tx_fifo_errors = hwstat->tx_underrun; 2944 2945 return nstat; 2946 } 2947 2948 static void gem_get_ethtool_stats(struct net_device *dev, 2949 struct ethtool_stats *stats, u64 *data) 2950 { 2951 struct macb *bp; 2952 2953 bp = netdev_priv(dev); 2954 gem_update_stats(bp); 2955 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2956 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2957 } 2958 2959 static int gem_get_sset_count(struct net_device *dev, int sset) 2960 { 2961 struct macb *bp = netdev_priv(dev); 2962 2963 switch (sset) { 2964 case ETH_SS_STATS: 2965 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2966 default: 2967 return -EOPNOTSUPP; 2968 } 2969 } 2970 2971 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2972 { 2973 char stat_string[ETH_GSTRING_LEN]; 2974 struct macb *bp = netdev_priv(dev); 2975 struct macb_queue *queue; 2976 unsigned int i; 2977 unsigned int q; 2978 2979 switch (sset) { 2980 case ETH_SS_STATS: 2981 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2982 memcpy(p, gem_statistics[i].stat_string, 2983 ETH_GSTRING_LEN); 2984 2985 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2986 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2987 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2988 q, queue_statistics[i].stat_string); 2989 memcpy(p, stat_string, ETH_GSTRING_LEN); 2990 } 2991 } 2992 break; 2993 } 2994 } 2995 2996 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2997 { 2998 struct macb *bp = netdev_priv(dev); 2999 struct net_device_stats *nstat = &bp->dev->stats; 3000 struct macb_stats *hwstat = &bp->hw_stats.macb; 3001 3002 if (macb_is_gem(bp)) 3003 return gem_get_stats(bp); 3004 3005 /* read stats from hardware */ 3006 macb_update_stats(bp); 3007 3008 /* Convert HW stats into netdevice stats */ 3009 nstat->rx_errors = (hwstat->rx_fcs_errors + 3010 hwstat->rx_align_errors + 3011 hwstat->rx_resource_errors + 3012 hwstat->rx_overruns + 3013 hwstat->rx_oversize_pkts + 3014 hwstat->rx_jabbers + 3015 hwstat->rx_undersize_pkts + 3016 hwstat->rx_length_mismatch); 3017 nstat->tx_errors = (hwstat->tx_late_cols + 3018 hwstat->tx_excessive_cols + 3019 hwstat->tx_underruns + 3020 hwstat->tx_carrier_errors + 3021 hwstat->sqe_test_errors); 3022 nstat->collisions = (hwstat->tx_single_cols + 3023 hwstat->tx_multiple_cols + 3024 hwstat->tx_excessive_cols); 3025 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 3026 hwstat->rx_jabbers + 3027 hwstat->rx_undersize_pkts + 3028 hwstat->rx_length_mismatch); 3029 nstat->rx_over_errors = hwstat->rx_resource_errors + 3030 hwstat->rx_overruns; 3031 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 3032 nstat->rx_frame_errors = hwstat->rx_align_errors; 3033 nstat->rx_fifo_errors = hwstat->rx_overruns; 3034 /* XXX: What does "missed" mean? */ 3035 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 3036 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 3037 nstat->tx_fifo_errors = hwstat->tx_underruns; 3038 /* Don't know about heartbeat or window errors... */ 3039 3040 return nstat; 3041 } 3042 3043 static int macb_get_regs_len(struct net_device *netdev) 3044 { 3045 return MACB_GREGS_NBR * sizeof(u32); 3046 } 3047 3048 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 3049 void *p) 3050 { 3051 struct macb *bp = netdev_priv(dev); 3052 unsigned int tail, head; 3053 u32 *regs_buff = p; 3054 3055 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 3056 | MACB_GREGS_VERSION; 3057 3058 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 3059 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 3060 3061 regs_buff[0] = macb_readl(bp, NCR); 3062 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 3063 regs_buff[2] = macb_readl(bp, NSR); 3064 regs_buff[3] = macb_readl(bp, TSR); 3065 regs_buff[4] = macb_readl(bp, RBQP); 3066 regs_buff[5] = macb_readl(bp, TBQP); 3067 regs_buff[6] = macb_readl(bp, RSR); 3068 regs_buff[7] = macb_readl(bp, IMR); 3069 3070 regs_buff[8] = tail; 3071 regs_buff[9] = head; 3072 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 3073 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 3074 3075 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 3076 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 3077 if (macb_is_gem(bp)) 3078 regs_buff[13] = gem_readl(bp, DMACFG); 3079 } 3080 3081 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 3082 { 3083 struct macb *bp = netdev_priv(netdev); 3084 3085 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 3086 phylink_ethtool_get_wol(bp->phylink, wol); 3087 wol->supported |= WAKE_MAGIC; 3088 3089 if (bp->wol & MACB_WOL_ENABLED) 3090 wol->wolopts |= WAKE_MAGIC; 3091 } 3092 } 3093 3094 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 3095 { 3096 struct macb *bp = netdev_priv(netdev); 3097 int ret; 3098 3099 /* Pass the order to phylink layer */ 3100 ret = phylink_ethtool_set_wol(bp->phylink, wol); 3101 /* Don't manage WoL on MAC if handled by the PHY 3102 * or if there's a failure in talking to the PHY 3103 */ 3104 if (!ret || ret != -EOPNOTSUPP) 3105 return ret; 3106 3107 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 3108 (wol->wolopts & ~WAKE_MAGIC)) 3109 return -EOPNOTSUPP; 3110 3111 if (wol->wolopts & WAKE_MAGIC) 3112 bp->wol |= MACB_WOL_ENABLED; 3113 else 3114 bp->wol &= ~MACB_WOL_ENABLED; 3115 3116 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 3117 3118 return 0; 3119 } 3120 3121 static int macb_get_link_ksettings(struct net_device *netdev, 3122 struct ethtool_link_ksettings *kset) 3123 { 3124 struct macb *bp = netdev_priv(netdev); 3125 3126 return phylink_ethtool_ksettings_get(bp->phylink, kset); 3127 } 3128 3129 static int macb_set_link_ksettings(struct net_device *netdev, 3130 const struct ethtool_link_ksettings *kset) 3131 { 3132 struct macb *bp = netdev_priv(netdev); 3133 3134 return phylink_ethtool_ksettings_set(bp->phylink, kset); 3135 } 3136 3137 static void macb_get_ringparam(struct net_device *netdev, 3138 struct ethtool_ringparam *ring, 3139 struct kernel_ethtool_ringparam *kernel_ring, 3140 struct netlink_ext_ack *extack) 3141 { 3142 struct macb *bp = netdev_priv(netdev); 3143 3144 ring->rx_max_pending = MAX_RX_RING_SIZE; 3145 ring->tx_max_pending = MAX_TX_RING_SIZE; 3146 3147 ring->rx_pending = bp->rx_ring_size; 3148 ring->tx_pending = bp->tx_ring_size; 3149 } 3150 3151 static int macb_set_ringparam(struct net_device *netdev, 3152 struct ethtool_ringparam *ring, 3153 struct kernel_ethtool_ringparam *kernel_ring, 3154 struct netlink_ext_ack *extack) 3155 { 3156 struct macb *bp = netdev_priv(netdev); 3157 u32 new_rx_size, new_tx_size; 3158 unsigned int reset = 0; 3159 3160 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 3161 return -EINVAL; 3162 3163 new_rx_size = clamp_t(u32, ring->rx_pending, 3164 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 3165 new_rx_size = roundup_pow_of_two(new_rx_size); 3166 3167 new_tx_size = clamp_t(u32, ring->tx_pending, 3168 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 3169 new_tx_size = roundup_pow_of_two(new_tx_size); 3170 3171 if ((new_tx_size == bp->tx_ring_size) && 3172 (new_rx_size == bp->rx_ring_size)) { 3173 /* nothing to do */ 3174 return 0; 3175 } 3176 3177 if (netif_running(bp->dev)) { 3178 reset = 1; 3179 macb_close(bp->dev); 3180 } 3181 3182 bp->rx_ring_size = new_rx_size; 3183 bp->tx_ring_size = new_tx_size; 3184 3185 if (reset) 3186 macb_open(bp->dev); 3187 3188 return 0; 3189 } 3190 3191 #ifdef CONFIG_MACB_USE_HWSTAMP 3192 static unsigned int gem_get_tsu_rate(struct macb *bp) 3193 { 3194 struct clk *tsu_clk; 3195 unsigned int tsu_rate; 3196 3197 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 3198 if (!IS_ERR(tsu_clk)) 3199 tsu_rate = clk_get_rate(tsu_clk); 3200 /* try pclk instead */ 3201 else if (!IS_ERR(bp->pclk)) { 3202 tsu_clk = bp->pclk; 3203 tsu_rate = clk_get_rate(tsu_clk); 3204 } else 3205 return -ENOTSUPP; 3206 return tsu_rate; 3207 } 3208 3209 static s32 gem_get_ptp_max_adj(void) 3210 { 3211 return 64000000; 3212 } 3213 3214 static int gem_get_ts_info(struct net_device *dev, 3215 struct ethtool_ts_info *info) 3216 { 3217 struct macb *bp = netdev_priv(dev); 3218 3219 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 3220 ethtool_op_get_ts_info(dev, info); 3221 return 0; 3222 } 3223 3224 info->so_timestamping = 3225 SOF_TIMESTAMPING_TX_SOFTWARE | 3226 SOF_TIMESTAMPING_RX_SOFTWARE | 3227 SOF_TIMESTAMPING_SOFTWARE | 3228 SOF_TIMESTAMPING_TX_HARDWARE | 3229 SOF_TIMESTAMPING_RX_HARDWARE | 3230 SOF_TIMESTAMPING_RAW_HARDWARE; 3231 info->tx_types = 3232 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 3233 (1 << HWTSTAMP_TX_OFF) | 3234 (1 << HWTSTAMP_TX_ON); 3235 info->rx_filters = 3236 (1 << HWTSTAMP_FILTER_NONE) | 3237 (1 << HWTSTAMP_FILTER_ALL); 3238 3239 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 3240 3241 return 0; 3242 } 3243 3244 static struct macb_ptp_info gem_ptp_info = { 3245 .ptp_init = gem_ptp_init, 3246 .ptp_remove = gem_ptp_remove, 3247 .get_ptp_max_adj = gem_get_ptp_max_adj, 3248 .get_tsu_rate = gem_get_tsu_rate, 3249 .get_ts_info = gem_get_ts_info, 3250 .get_hwtst = gem_get_hwtst, 3251 .set_hwtst = gem_set_hwtst, 3252 }; 3253 #endif 3254 3255 static int macb_get_ts_info(struct net_device *netdev, 3256 struct ethtool_ts_info *info) 3257 { 3258 struct macb *bp = netdev_priv(netdev); 3259 3260 if (bp->ptp_info) 3261 return bp->ptp_info->get_ts_info(netdev, info); 3262 3263 return ethtool_op_get_ts_info(netdev, info); 3264 } 3265 3266 static void gem_enable_flow_filters(struct macb *bp, bool enable) 3267 { 3268 struct net_device *netdev = bp->dev; 3269 struct ethtool_rx_fs_item *item; 3270 u32 t2_scr; 3271 int num_t2_scr; 3272 3273 if (!(netdev->features & NETIF_F_NTUPLE)) 3274 return; 3275 3276 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 3277 3278 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3279 struct ethtool_rx_flow_spec *fs = &item->fs; 3280 struct ethtool_tcpip4_spec *tp4sp_m; 3281 3282 if (fs->location >= num_t2_scr) 3283 continue; 3284 3285 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 3286 3287 /* enable/disable screener regs for the flow entry */ 3288 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 3289 3290 /* only enable fields with no masking */ 3291 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3292 3293 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 3294 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 3295 else 3296 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 3297 3298 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 3299 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 3300 else 3301 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 3302 3303 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 3304 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 3305 else 3306 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 3307 3308 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 3309 } 3310 } 3311 3312 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 3313 { 3314 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 3315 uint16_t index = fs->location; 3316 u32 w0, w1, t2_scr; 3317 bool cmp_a = false; 3318 bool cmp_b = false; 3319 bool cmp_c = false; 3320 3321 if (!macb_is_gem(bp)) 3322 return; 3323 3324 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 3325 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3326 3327 /* ignore field if any masking set */ 3328 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 3329 /* 1st compare reg - IP source address */ 3330 w0 = 0; 3331 w1 = 0; 3332 w0 = tp4sp_v->ip4src; 3333 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3334 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3335 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 3336 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 3337 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 3338 cmp_a = true; 3339 } 3340 3341 /* ignore field if any masking set */ 3342 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 3343 /* 2nd compare reg - IP destination address */ 3344 w0 = 0; 3345 w1 = 0; 3346 w0 = tp4sp_v->ip4dst; 3347 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3348 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3349 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 3350 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 3351 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 3352 cmp_b = true; 3353 } 3354 3355 /* ignore both port fields if masking set in both */ 3356 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 3357 /* 3rd compare reg - source port, destination port */ 3358 w0 = 0; 3359 w1 = 0; 3360 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 3361 if (tp4sp_m->psrc == tp4sp_m->pdst) { 3362 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 3363 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3364 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3365 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3366 } else { 3367 /* only one port definition */ 3368 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 3369 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 3370 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 3371 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 3372 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3373 } else { /* dst port */ 3374 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3375 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 3376 } 3377 } 3378 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 3379 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 3380 cmp_c = true; 3381 } 3382 3383 t2_scr = 0; 3384 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 3385 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 3386 if (cmp_a) 3387 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 3388 if (cmp_b) 3389 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 3390 if (cmp_c) 3391 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 3392 gem_writel_n(bp, SCRT2, index, t2_scr); 3393 } 3394 3395 static int gem_add_flow_filter(struct net_device *netdev, 3396 struct ethtool_rxnfc *cmd) 3397 { 3398 struct macb *bp = netdev_priv(netdev); 3399 struct ethtool_rx_flow_spec *fs = &cmd->fs; 3400 struct ethtool_rx_fs_item *item, *newfs; 3401 unsigned long flags; 3402 int ret = -EINVAL; 3403 bool added = false; 3404 3405 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 3406 if (newfs == NULL) 3407 return -ENOMEM; 3408 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 3409 3410 netdev_dbg(netdev, 3411 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3412 fs->flow_type, (int)fs->ring_cookie, fs->location, 3413 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3414 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3415 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 3416 3417 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3418 3419 /* find correct place to add in list */ 3420 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3421 if (item->fs.location > newfs->fs.location) { 3422 list_add_tail(&newfs->list, &item->list); 3423 added = true; 3424 break; 3425 } else if (item->fs.location == fs->location) { 3426 netdev_err(netdev, "Rule not added: location %d not free!\n", 3427 fs->location); 3428 ret = -EBUSY; 3429 goto err; 3430 } 3431 } 3432 if (!added) 3433 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3434 3435 gem_prog_cmp_regs(bp, fs); 3436 bp->rx_fs_list.count++; 3437 /* enable filtering if NTUPLE on */ 3438 gem_enable_flow_filters(bp, 1); 3439 3440 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3441 return 0; 3442 3443 err: 3444 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3445 kfree(newfs); 3446 return ret; 3447 } 3448 3449 static int gem_del_flow_filter(struct net_device *netdev, 3450 struct ethtool_rxnfc *cmd) 3451 { 3452 struct macb *bp = netdev_priv(netdev); 3453 struct ethtool_rx_fs_item *item; 3454 struct ethtool_rx_flow_spec *fs; 3455 unsigned long flags; 3456 3457 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3458 3459 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3460 if (item->fs.location == cmd->fs.location) { 3461 /* disable screener regs for the flow entry */ 3462 fs = &(item->fs); 3463 netdev_dbg(netdev, 3464 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3465 fs->flow_type, (int)fs->ring_cookie, fs->location, 3466 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3467 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3468 htons(fs->h_u.tcp_ip4_spec.psrc), 3469 htons(fs->h_u.tcp_ip4_spec.pdst)); 3470 3471 gem_writel_n(bp, SCRT2, fs->location, 0); 3472 3473 list_del(&item->list); 3474 bp->rx_fs_list.count--; 3475 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3476 kfree(item); 3477 return 0; 3478 } 3479 } 3480 3481 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3482 return -EINVAL; 3483 } 3484 3485 static int gem_get_flow_entry(struct net_device *netdev, 3486 struct ethtool_rxnfc *cmd) 3487 { 3488 struct macb *bp = netdev_priv(netdev); 3489 struct ethtool_rx_fs_item *item; 3490 3491 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3492 if (item->fs.location == cmd->fs.location) { 3493 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3494 return 0; 3495 } 3496 } 3497 return -EINVAL; 3498 } 3499 3500 static int gem_get_all_flow_entries(struct net_device *netdev, 3501 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3502 { 3503 struct macb *bp = netdev_priv(netdev); 3504 struct ethtool_rx_fs_item *item; 3505 uint32_t cnt = 0; 3506 3507 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3508 if (cnt == cmd->rule_cnt) 3509 return -EMSGSIZE; 3510 rule_locs[cnt] = item->fs.location; 3511 cnt++; 3512 } 3513 cmd->data = bp->max_tuples; 3514 cmd->rule_cnt = cnt; 3515 3516 return 0; 3517 } 3518 3519 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3520 u32 *rule_locs) 3521 { 3522 struct macb *bp = netdev_priv(netdev); 3523 int ret = 0; 3524 3525 switch (cmd->cmd) { 3526 case ETHTOOL_GRXRINGS: 3527 cmd->data = bp->num_queues; 3528 break; 3529 case ETHTOOL_GRXCLSRLCNT: 3530 cmd->rule_cnt = bp->rx_fs_list.count; 3531 break; 3532 case ETHTOOL_GRXCLSRULE: 3533 ret = gem_get_flow_entry(netdev, cmd); 3534 break; 3535 case ETHTOOL_GRXCLSRLALL: 3536 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3537 break; 3538 default: 3539 netdev_err(netdev, 3540 "Command parameter %d is not supported\n", cmd->cmd); 3541 ret = -EOPNOTSUPP; 3542 } 3543 3544 return ret; 3545 } 3546 3547 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3548 { 3549 struct macb *bp = netdev_priv(netdev); 3550 int ret; 3551 3552 switch (cmd->cmd) { 3553 case ETHTOOL_SRXCLSRLINS: 3554 if ((cmd->fs.location >= bp->max_tuples) 3555 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3556 ret = -EINVAL; 3557 break; 3558 } 3559 ret = gem_add_flow_filter(netdev, cmd); 3560 break; 3561 case ETHTOOL_SRXCLSRLDEL: 3562 ret = gem_del_flow_filter(netdev, cmd); 3563 break; 3564 default: 3565 netdev_err(netdev, 3566 "Command parameter %d is not supported\n", cmd->cmd); 3567 ret = -EOPNOTSUPP; 3568 } 3569 3570 return ret; 3571 } 3572 3573 static const struct ethtool_ops macb_ethtool_ops = { 3574 .get_regs_len = macb_get_regs_len, 3575 .get_regs = macb_get_regs, 3576 .get_link = ethtool_op_get_link, 3577 .get_ts_info = ethtool_op_get_ts_info, 3578 .get_wol = macb_get_wol, 3579 .set_wol = macb_set_wol, 3580 .get_link_ksettings = macb_get_link_ksettings, 3581 .set_link_ksettings = macb_set_link_ksettings, 3582 .get_ringparam = macb_get_ringparam, 3583 .set_ringparam = macb_set_ringparam, 3584 }; 3585 3586 static const struct ethtool_ops gem_ethtool_ops = { 3587 .get_regs_len = macb_get_regs_len, 3588 .get_regs = macb_get_regs, 3589 .get_wol = macb_get_wol, 3590 .set_wol = macb_set_wol, 3591 .get_link = ethtool_op_get_link, 3592 .get_ts_info = macb_get_ts_info, 3593 .get_ethtool_stats = gem_get_ethtool_stats, 3594 .get_strings = gem_get_ethtool_strings, 3595 .get_sset_count = gem_get_sset_count, 3596 .get_link_ksettings = macb_get_link_ksettings, 3597 .set_link_ksettings = macb_set_link_ksettings, 3598 .get_ringparam = macb_get_ringparam, 3599 .set_ringparam = macb_set_ringparam, 3600 .get_rxnfc = gem_get_rxnfc, 3601 .set_rxnfc = gem_set_rxnfc, 3602 }; 3603 3604 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3605 { 3606 struct macb *bp = netdev_priv(dev); 3607 3608 if (!netif_running(dev)) 3609 return -EINVAL; 3610 3611 if (bp->ptp_info) { 3612 switch (cmd) { 3613 case SIOCSHWTSTAMP: 3614 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3615 case SIOCGHWTSTAMP: 3616 return bp->ptp_info->get_hwtst(dev, rq); 3617 } 3618 } 3619 3620 return phylink_mii_ioctl(bp->phylink, rq, cmd); 3621 } 3622 3623 static inline void macb_set_txcsum_feature(struct macb *bp, 3624 netdev_features_t features) 3625 { 3626 u32 val; 3627 3628 if (!macb_is_gem(bp)) 3629 return; 3630 3631 val = gem_readl(bp, DMACFG); 3632 if (features & NETIF_F_HW_CSUM) 3633 val |= GEM_BIT(TXCOEN); 3634 else 3635 val &= ~GEM_BIT(TXCOEN); 3636 3637 gem_writel(bp, DMACFG, val); 3638 } 3639 3640 static inline void macb_set_rxcsum_feature(struct macb *bp, 3641 netdev_features_t features) 3642 { 3643 struct net_device *netdev = bp->dev; 3644 u32 val; 3645 3646 if (!macb_is_gem(bp)) 3647 return; 3648 3649 val = gem_readl(bp, NCFGR); 3650 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 3651 val |= GEM_BIT(RXCOEN); 3652 else 3653 val &= ~GEM_BIT(RXCOEN); 3654 3655 gem_writel(bp, NCFGR, val); 3656 } 3657 3658 static inline void macb_set_rxflow_feature(struct macb *bp, 3659 netdev_features_t features) 3660 { 3661 if (!macb_is_gem(bp)) 3662 return; 3663 3664 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 3665 } 3666 3667 static int macb_set_features(struct net_device *netdev, 3668 netdev_features_t features) 3669 { 3670 struct macb *bp = netdev_priv(netdev); 3671 netdev_features_t changed = features ^ netdev->features; 3672 3673 /* TX checksum offload */ 3674 if (changed & NETIF_F_HW_CSUM) 3675 macb_set_txcsum_feature(bp, features); 3676 3677 /* RX checksum offload */ 3678 if (changed & NETIF_F_RXCSUM) 3679 macb_set_rxcsum_feature(bp, features); 3680 3681 /* RX Flow Filters */ 3682 if (changed & NETIF_F_NTUPLE) 3683 macb_set_rxflow_feature(bp, features); 3684 3685 return 0; 3686 } 3687 3688 static void macb_restore_features(struct macb *bp) 3689 { 3690 struct net_device *netdev = bp->dev; 3691 netdev_features_t features = netdev->features; 3692 struct ethtool_rx_fs_item *item; 3693 3694 /* TX checksum offload */ 3695 macb_set_txcsum_feature(bp, features); 3696 3697 /* RX checksum offload */ 3698 macb_set_rxcsum_feature(bp, features); 3699 3700 /* RX Flow Filters */ 3701 list_for_each_entry(item, &bp->rx_fs_list.list, list) 3702 gem_prog_cmp_regs(bp, &item->fs); 3703 3704 macb_set_rxflow_feature(bp, features); 3705 } 3706 3707 static const struct net_device_ops macb_netdev_ops = { 3708 .ndo_open = macb_open, 3709 .ndo_stop = macb_close, 3710 .ndo_start_xmit = macb_start_xmit, 3711 .ndo_set_rx_mode = macb_set_rx_mode, 3712 .ndo_get_stats = macb_get_stats, 3713 .ndo_eth_ioctl = macb_ioctl, 3714 .ndo_validate_addr = eth_validate_addr, 3715 .ndo_change_mtu = macb_change_mtu, 3716 .ndo_set_mac_address = eth_mac_addr, 3717 #ifdef CONFIG_NET_POLL_CONTROLLER 3718 .ndo_poll_controller = macb_poll_controller, 3719 #endif 3720 .ndo_set_features = macb_set_features, 3721 .ndo_features_check = macb_features_check, 3722 }; 3723 3724 /* Configure peripheral capabilities according to device tree 3725 * and integration options used 3726 */ 3727 static void macb_configure_caps(struct macb *bp, 3728 const struct macb_config *dt_conf) 3729 { 3730 u32 dcfg; 3731 3732 if (dt_conf) 3733 bp->caps = dt_conf->caps; 3734 3735 if (hw_is_gem(bp->regs, bp->native_io)) { 3736 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3737 3738 dcfg = gem_readl(bp, DCFG1); 3739 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3740 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3741 if (GEM_BFEXT(NO_PCS, dcfg) == 0) 3742 bp->caps |= MACB_CAPS_PCS; 3743 dcfg = gem_readl(bp, DCFG12); 3744 if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) 3745 bp->caps |= MACB_CAPS_HIGH_SPEED; 3746 dcfg = gem_readl(bp, DCFG2); 3747 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3748 bp->caps |= MACB_CAPS_FIFO_MODE; 3749 #ifdef CONFIG_MACB_USE_HWSTAMP 3750 if (gem_has_ptp(bp)) { 3751 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3752 dev_err(&bp->pdev->dev, 3753 "GEM doesn't support hardware ptp.\n"); 3754 else { 3755 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3756 bp->ptp_info = &gem_ptp_info; 3757 } 3758 } 3759 #endif 3760 } 3761 3762 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3763 } 3764 3765 static void macb_probe_queues(void __iomem *mem, 3766 bool native_io, 3767 unsigned int *queue_mask, 3768 unsigned int *num_queues) 3769 { 3770 *queue_mask = 0x1; 3771 *num_queues = 1; 3772 3773 /* is it macb or gem ? 3774 * 3775 * We need to read directly from the hardware here because 3776 * we are early in the probe process and don't have the 3777 * MACB_CAPS_MACB_IS_GEM flag positioned 3778 */ 3779 if (!hw_is_gem(mem, native_io)) 3780 return; 3781 3782 /* bit 0 is never set but queue 0 always exists */ 3783 *queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff; 3784 *num_queues = hweight32(*queue_mask); 3785 } 3786 3787 static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk, 3788 struct clk *rx_clk, struct clk *tsu_clk) 3789 { 3790 struct clk_bulk_data clks[] = { 3791 { .clk = tsu_clk, }, 3792 { .clk = rx_clk, }, 3793 { .clk = pclk, }, 3794 { .clk = hclk, }, 3795 { .clk = tx_clk }, 3796 }; 3797 3798 clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); 3799 } 3800 3801 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3802 struct clk **hclk, struct clk **tx_clk, 3803 struct clk **rx_clk, struct clk **tsu_clk) 3804 { 3805 struct macb_platform_data *pdata; 3806 int err; 3807 3808 pdata = dev_get_platdata(&pdev->dev); 3809 if (pdata) { 3810 *pclk = pdata->pclk; 3811 *hclk = pdata->hclk; 3812 } else { 3813 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3814 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3815 } 3816 3817 if (IS_ERR_OR_NULL(*pclk)) 3818 return dev_err_probe(&pdev->dev, 3819 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, 3820 "failed to get pclk\n"); 3821 3822 if (IS_ERR_OR_NULL(*hclk)) 3823 return dev_err_probe(&pdev->dev, 3824 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, 3825 "failed to get hclk\n"); 3826 3827 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); 3828 if (IS_ERR(*tx_clk)) 3829 return PTR_ERR(*tx_clk); 3830 3831 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); 3832 if (IS_ERR(*rx_clk)) 3833 return PTR_ERR(*rx_clk); 3834 3835 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); 3836 if (IS_ERR(*tsu_clk)) 3837 return PTR_ERR(*tsu_clk); 3838 3839 err = clk_prepare_enable(*pclk); 3840 if (err) { 3841 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3842 return err; 3843 } 3844 3845 err = clk_prepare_enable(*hclk); 3846 if (err) { 3847 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 3848 goto err_disable_pclk; 3849 } 3850 3851 err = clk_prepare_enable(*tx_clk); 3852 if (err) { 3853 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 3854 goto err_disable_hclk; 3855 } 3856 3857 err = clk_prepare_enable(*rx_clk); 3858 if (err) { 3859 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 3860 goto err_disable_txclk; 3861 } 3862 3863 err = clk_prepare_enable(*tsu_clk); 3864 if (err) { 3865 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 3866 goto err_disable_rxclk; 3867 } 3868 3869 return 0; 3870 3871 err_disable_rxclk: 3872 clk_disable_unprepare(*rx_clk); 3873 3874 err_disable_txclk: 3875 clk_disable_unprepare(*tx_clk); 3876 3877 err_disable_hclk: 3878 clk_disable_unprepare(*hclk); 3879 3880 err_disable_pclk: 3881 clk_disable_unprepare(*pclk); 3882 3883 return err; 3884 } 3885 3886 static int macb_init(struct platform_device *pdev) 3887 { 3888 struct net_device *dev = platform_get_drvdata(pdev); 3889 unsigned int hw_q, q; 3890 struct macb *bp = netdev_priv(dev); 3891 struct macb_queue *queue; 3892 int err; 3893 u32 val, reg; 3894 3895 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3896 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3897 3898 /* set the queue register mapping once for all: queue0 has a special 3899 * register mapping but we don't want to test the queue index then 3900 * compute the corresponding register offset at run time. 3901 */ 3902 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3903 if (!(bp->queue_mask & (1 << hw_q))) 3904 continue; 3905 3906 queue = &bp->queues[q]; 3907 queue->bp = bp; 3908 netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT); 3909 if (hw_q) { 3910 queue->ISR = GEM_ISR(hw_q - 1); 3911 queue->IER = GEM_IER(hw_q - 1); 3912 queue->IDR = GEM_IDR(hw_q - 1); 3913 queue->IMR = GEM_IMR(hw_q - 1); 3914 queue->TBQP = GEM_TBQP(hw_q - 1); 3915 queue->RBQP = GEM_RBQP(hw_q - 1); 3916 queue->RBQS = GEM_RBQS(hw_q - 1); 3917 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3918 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3919 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3920 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3921 } 3922 #endif 3923 } else { 3924 /* queue0 uses legacy registers */ 3925 queue->ISR = MACB_ISR; 3926 queue->IER = MACB_IER; 3927 queue->IDR = MACB_IDR; 3928 queue->IMR = MACB_IMR; 3929 queue->TBQP = MACB_TBQP; 3930 queue->RBQP = MACB_RBQP; 3931 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3932 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3933 queue->TBQPH = MACB_TBQPH; 3934 queue->RBQPH = MACB_RBQPH; 3935 } 3936 #endif 3937 } 3938 3939 /* get irq: here we use the linux queue index, not the hardware 3940 * queue index. the queue irq definitions in the device tree 3941 * must remove the optional gaps that could exist in the 3942 * hardware queue mask. 3943 */ 3944 queue->irq = platform_get_irq(pdev, q); 3945 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3946 IRQF_SHARED, dev->name, queue); 3947 if (err) { 3948 dev_err(&pdev->dev, 3949 "Unable to request IRQ %d (error %d)\n", 3950 queue->irq, err); 3951 return err; 3952 } 3953 3954 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3955 q++; 3956 } 3957 3958 dev->netdev_ops = &macb_netdev_ops; 3959 3960 /* setup appropriated routines according to adapter type */ 3961 if (macb_is_gem(bp)) { 3962 bp->max_tx_length = GEM_MAX_TX_LEN; 3963 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3964 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3965 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3966 bp->macbgem_ops.mog_rx = gem_rx; 3967 dev->ethtool_ops = &gem_ethtool_ops; 3968 } else { 3969 bp->max_tx_length = MACB_MAX_TX_LEN; 3970 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3971 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3972 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3973 bp->macbgem_ops.mog_rx = macb_rx; 3974 dev->ethtool_ops = &macb_ethtool_ops; 3975 } 3976 3977 /* Set features */ 3978 dev->hw_features = NETIF_F_SG; 3979 3980 /* Check LSO capability */ 3981 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3982 dev->hw_features |= MACB_NETIF_LSO; 3983 3984 /* Checksum offload is only available on gem with packet buffer */ 3985 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3986 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3987 if (bp->caps & MACB_CAPS_SG_DISABLED) 3988 dev->hw_features &= ~NETIF_F_SG; 3989 dev->features = dev->hw_features; 3990 3991 /* Check RX Flow Filters support. 3992 * Max Rx flows set by availability of screeners & compare regs: 3993 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3994 */ 3995 reg = gem_readl(bp, DCFG8); 3996 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3997 GEM_BFEXT(T2SCR, reg)); 3998 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3999 if (bp->max_tuples > 0) { 4000 /* also needs one ethtype match to check IPv4 */ 4001 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 4002 /* program this reg now */ 4003 reg = 0; 4004 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 4005 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 4006 /* Filtering is supported in hw but don't enable it in kernel now */ 4007 dev->hw_features |= NETIF_F_NTUPLE; 4008 /* init Rx flow definitions */ 4009 bp->rx_fs_list.count = 0; 4010 spin_lock_init(&bp->rx_fs_lock); 4011 } else 4012 bp->max_tuples = 0; 4013 } 4014 4015 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 4016 val = 0; 4017 if (phy_interface_mode_is_rgmii(bp->phy_interface)) 4018 val = bp->usrio->rgmii; 4019 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 4020 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 4021 val = bp->usrio->rmii; 4022 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 4023 val = bp->usrio->mii; 4024 4025 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 4026 val |= bp->usrio->refclk; 4027 4028 macb_or_gem_writel(bp, USRIO, val); 4029 } 4030 4031 /* Set MII management clock divider */ 4032 val = macb_mdc_clk_div(bp); 4033 val |= macb_dbw(bp); 4034 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 4035 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 4036 macb_writel(bp, NCFGR, val); 4037 4038 return 0; 4039 } 4040 4041 static const struct macb_usrio_config macb_default_usrio = { 4042 .mii = MACB_BIT(MII), 4043 .rmii = MACB_BIT(RMII), 4044 .rgmii = GEM_BIT(RGMII), 4045 .refclk = MACB_BIT(CLKEN), 4046 }; 4047 4048 #if defined(CONFIG_OF) 4049 /* 1518 rounded up */ 4050 #define AT91ETHER_MAX_RBUFF_SZ 0x600 4051 /* max number of receive buffers */ 4052 #define AT91ETHER_MAX_RX_DESCR 9 4053 4054 static struct sifive_fu540_macb_mgmt *mgmt; 4055 4056 static int at91ether_alloc_coherent(struct macb *lp) 4057 { 4058 struct macb_queue *q = &lp->queues[0]; 4059 4060 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 4061 (AT91ETHER_MAX_RX_DESCR * 4062 macb_dma_desc_get_size(lp)), 4063 &q->rx_ring_dma, GFP_KERNEL); 4064 if (!q->rx_ring) 4065 return -ENOMEM; 4066 4067 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 4068 AT91ETHER_MAX_RX_DESCR * 4069 AT91ETHER_MAX_RBUFF_SZ, 4070 &q->rx_buffers_dma, GFP_KERNEL); 4071 if (!q->rx_buffers) { 4072 dma_free_coherent(&lp->pdev->dev, 4073 AT91ETHER_MAX_RX_DESCR * 4074 macb_dma_desc_get_size(lp), 4075 q->rx_ring, q->rx_ring_dma); 4076 q->rx_ring = NULL; 4077 return -ENOMEM; 4078 } 4079 4080 return 0; 4081 } 4082 4083 static void at91ether_free_coherent(struct macb *lp) 4084 { 4085 struct macb_queue *q = &lp->queues[0]; 4086 4087 if (q->rx_ring) { 4088 dma_free_coherent(&lp->pdev->dev, 4089 AT91ETHER_MAX_RX_DESCR * 4090 macb_dma_desc_get_size(lp), 4091 q->rx_ring, q->rx_ring_dma); 4092 q->rx_ring = NULL; 4093 } 4094 4095 if (q->rx_buffers) { 4096 dma_free_coherent(&lp->pdev->dev, 4097 AT91ETHER_MAX_RX_DESCR * 4098 AT91ETHER_MAX_RBUFF_SZ, 4099 q->rx_buffers, q->rx_buffers_dma); 4100 q->rx_buffers = NULL; 4101 } 4102 } 4103 4104 /* Initialize and start the Receiver and Transmit subsystems */ 4105 static int at91ether_start(struct macb *lp) 4106 { 4107 struct macb_queue *q = &lp->queues[0]; 4108 struct macb_dma_desc *desc; 4109 dma_addr_t addr; 4110 u32 ctl; 4111 int i, ret; 4112 4113 ret = at91ether_alloc_coherent(lp); 4114 if (ret) 4115 return ret; 4116 4117 addr = q->rx_buffers_dma; 4118 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 4119 desc = macb_rx_desc(q, i); 4120 macb_set_addr(lp, desc, addr); 4121 desc->ctrl = 0; 4122 addr += AT91ETHER_MAX_RBUFF_SZ; 4123 } 4124 4125 /* Set the Wrap bit on the last descriptor */ 4126 desc->addr |= MACB_BIT(RX_WRAP); 4127 4128 /* Reset buffer index */ 4129 q->rx_tail = 0; 4130 4131 /* Program address of descriptor list in Rx Buffer Queue register */ 4132 macb_writel(lp, RBQP, q->rx_ring_dma); 4133 4134 /* Enable Receive and Transmit */ 4135 ctl = macb_readl(lp, NCR); 4136 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 4137 4138 /* Enable MAC interrupts */ 4139 macb_writel(lp, IER, MACB_BIT(RCOMP) | 4140 MACB_BIT(RXUBR) | 4141 MACB_BIT(ISR_TUND) | 4142 MACB_BIT(ISR_RLE) | 4143 MACB_BIT(TCOMP) | 4144 MACB_BIT(ISR_ROVR) | 4145 MACB_BIT(HRESP)); 4146 4147 return 0; 4148 } 4149 4150 static void at91ether_stop(struct macb *lp) 4151 { 4152 u32 ctl; 4153 4154 /* Disable MAC interrupts */ 4155 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 4156 MACB_BIT(RXUBR) | 4157 MACB_BIT(ISR_TUND) | 4158 MACB_BIT(ISR_RLE) | 4159 MACB_BIT(TCOMP) | 4160 MACB_BIT(ISR_ROVR) | 4161 MACB_BIT(HRESP)); 4162 4163 /* Disable Receiver and Transmitter */ 4164 ctl = macb_readl(lp, NCR); 4165 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 4166 4167 /* Free resources. */ 4168 at91ether_free_coherent(lp); 4169 } 4170 4171 /* Open the ethernet interface */ 4172 static int at91ether_open(struct net_device *dev) 4173 { 4174 struct macb *lp = netdev_priv(dev); 4175 u32 ctl; 4176 int ret; 4177 4178 ret = pm_runtime_get_sync(&lp->pdev->dev); 4179 if (ret < 0) { 4180 pm_runtime_put_noidle(&lp->pdev->dev); 4181 return ret; 4182 } 4183 4184 /* Clear internal statistics */ 4185 ctl = macb_readl(lp, NCR); 4186 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 4187 4188 macb_set_hwaddr(lp); 4189 4190 ret = at91ether_start(lp); 4191 if (ret) 4192 goto pm_exit; 4193 4194 ret = macb_phylink_connect(lp); 4195 if (ret) 4196 goto stop; 4197 4198 netif_start_queue(dev); 4199 4200 return 0; 4201 4202 stop: 4203 at91ether_stop(lp); 4204 pm_exit: 4205 pm_runtime_put_sync(&lp->pdev->dev); 4206 return ret; 4207 } 4208 4209 /* Close the interface */ 4210 static int at91ether_close(struct net_device *dev) 4211 { 4212 struct macb *lp = netdev_priv(dev); 4213 4214 netif_stop_queue(dev); 4215 4216 phylink_stop(lp->phylink); 4217 phylink_disconnect_phy(lp->phylink); 4218 4219 at91ether_stop(lp); 4220 4221 return pm_runtime_put(&lp->pdev->dev); 4222 } 4223 4224 /* Transmit packet */ 4225 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 4226 struct net_device *dev) 4227 { 4228 struct macb *lp = netdev_priv(dev); 4229 4230 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 4231 int desc = 0; 4232 4233 netif_stop_queue(dev); 4234 4235 /* Store packet information (to free when Tx completed) */ 4236 lp->rm9200_txq[desc].skb = skb; 4237 lp->rm9200_txq[desc].size = skb->len; 4238 lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data, 4239 skb->len, DMA_TO_DEVICE); 4240 if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) { 4241 dev_kfree_skb_any(skb); 4242 dev->stats.tx_dropped++; 4243 netdev_err(dev, "%s: DMA mapping error\n", __func__); 4244 return NETDEV_TX_OK; 4245 } 4246 4247 /* Set address of the data in the Transmit Address register */ 4248 macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping); 4249 /* Set length of the packet in the Transmit Control register */ 4250 macb_writel(lp, TCR, skb->len); 4251 4252 } else { 4253 netdev_err(dev, "%s called, but device is busy!\n", __func__); 4254 return NETDEV_TX_BUSY; 4255 } 4256 4257 return NETDEV_TX_OK; 4258 } 4259 4260 /* Extract received frame from buffer descriptors and sent to upper layers. 4261 * (Called from interrupt context) 4262 */ 4263 static void at91ether_rx(struct net_device *dev) 4264 { 4265 struct macb *lp = netdev_priv(dev); 4266 struct macb_queue *q = &lp->queues[0]; 4267 struct macb_dma_desc *desc; 4268 unsigned char *p_recv; 4269 struct sk_buff *skb; 4270 unsigned int pktlen; 4271 4272 desc = macb_rx_desc(q, q->rx_tail); 4273 while (desc->addr & MACB_BIT(RX_USED)) { 4274 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 4275 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 4276 skb = netdev_alloc_skb(dev, pktlen + 2); 4277 if (skb) { 4278 skb_reserve(skb, 2); 4279 skb_put_data(skb, p_recv, pktlen); 4280 4281 skb->protocol = eth_type_trans(skb, dev); 4282 dev->stats.rx_packets++; 4283 dev->stats.rx_bytes += pktlen; 4284 netif_rx(skb); 4285 } else { 4286 dev->stats.rx_dropped++; 4287 } 4288 4289 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 4290 dev->stats.multicast++; 4291 4292 /* reset ownership bit */ 4293 desc->addr &= ~MACB_BIT(RX_USED); 4294 4295 /* wrap after last buffer */ 4296 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 4297 q->rx_tail = 0; 4298 else 4299 q->rx_tail++; 4300 4301 desc = macb_rx_desc(q, q->rx_tail); 4302 } 4303 } 4304 4305 /* MAC interrupt handler */ 4306 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 4307 { 4308 struct net_device *dev = dev_id; 4309 struct macb *lp = netdev_priv(dev); 4310 u32 intstatus, ctl; 4311 unsigned int desc; 4312 4313 /* MAC Interrupt Status register indicates what interrupts are pending. 4314 * It is automatically cleared once read. 4315 */ 4316 intstatus = macb_readl(lp, ISR); 4317 4318 /* Receive complete */ 4319 if (intstatus & MACB_BIT(RCOMP)) 4320 at91ether_rx(dev); 4321 4322 /* Transmit complete */ 4323 if (intstatus & MACB_BIT(TCOMP)) { 4324 /* The TCOM bit is set even if the transmission failed */ 4325 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 4326 dev->stats.tx_errors++; 4327 4328 desc = 0; 4329 if (lp->rm9200_txq[desc].skb) { 4330 dev_consume_skb_irq(lp->rm9200_txq[desc].skb); 4331 lp->rm9200_txq[desc].skb = NULL; 4332 dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, 4333 lp->rm9200_txq[desc].size, DMA_TO_DEVICE); 4334 dev->stats.tx_packets++; 4335 dev->stats.tx_bytes += lp->rm9200_txq[desc].size; 4336 } 4337 netif_wake_queue(dev); 4338 } 4339 4340 /* Work-around for EMAC Errata section 41.3.1 */ 4341 if (intstatus & MACB_BIT(RXUBR)) { 4342 ctl = macb_readl(lp, NCR); 4343 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 4344 wmb(); 4345 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 4346 } 4347 4348 if (intstatus & MACB_BIT(ISR_ROVR)) 4349 netdev_err(dev, "ROVR error\n"); 4350 4351 return IRQ_HANDLED; 4352 } 4353 4354 #ifdef CONFIG_NET_POLL_CONTROLLER 4355 static void at91ether_poll_controller(struct net_device *dev) 4356 { 4357 unsigned long flags; 4358 4359 local_irq_save(flags); 4360 at91ether_interrupt(dev->irq, dev); 4361 local_irq_restore(flags); 4362 } 4363 #endif 4364 4365 static const struct net_device_ops at91ether_netdev_ops = { 4366 .ndo_open = at91ether_open, 4367 .ndo_stop = at91ether_close, 4368 .ndo_start_xmit = at91ether_start_xmit, 4369 .ndo_get_stats = macb_get_stats, 4370 .ndo_set_rx_mode = macb_set_rx_mode, 4371 .ndo_set_mac_address = eth_mac_addr, 4372 .ndo_eth_ioctl = macb_ioctl, 4373 .ndo_validate_addr = eth_validate_addr, 4374 #ifdef CONFIG_NET_POLL_CONTROLLER 4375 .ndo_poll_controller = at91ether_poll_controller, 4376 #endif 4377 }; 4378 4379 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 4380 struct clk **hclk, struct clk **tx_clk, 4381 struct clk **rx_clk, struct clk **tsu_clk) 4382 { 4383 int err; 4384 4385 *hclk = NULL; 4386 *tx_clk = NULL; 4387 *rx_clk = NULL; 4388 *tsu_clk = NULL; 4389 4390 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 4391 if (IS_ERR(*pclk)) 4392 return PTR_ERR(*pclk); 4393 4394 err = clk_prepare_enable(*pclk); 4395 if (err) { 4396 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 4397 return err; 4398 } 4399 4400 return 0; 4401 } 4402 4403 static int at91ether_init(struct platform_device *pdev) 4404 { 4405 struct net_device *dev = platform_get_drvdata(pdev); 4406 struct macb *bp = netdev_priv(dev); 4407 int err; 4408 4409 bp->queues[0].bp = bp; 4410 4411 dev->netdev_ops = &at91ether_netdev_ops; 4412 dev->ethtool_ops = &macb_ethtool_ops; 4413 4414 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 4415 0, dev->name, dev); 4416 if (err) 4417 return err; 4418 4419 macb_writel(bp, NCR, 0); 4420 4421 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); 4422 4423 return 0; 4424 } 4425 4426 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 4427 unsigned long parent_rate) 4428 { 4429 return mgmt->rate; 4430 } 4431 4432 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, 4433 unsigned long *parent_rate) 4434 { 4435 if (WARN_ON(rate < 2500000)) 4436 return 2500000; 4437 else if (rate == 2500000) 4438 return 2500000; 4439 else if (WARN_ON(rate < 13750000)) 4440 return 2500000; 4441 else if (WARN_ON(rate < 25000000)) 4442 return 25000000; 4443 else if (rate == 25000000) 4444 return 25000000; 4445 else if (WARN_ON(rate < 75000000)) 4446 return 25000000; 4447 else if (WARN_ON(rate < 125000000)) 4448 return 125000000; 4449 else if (rate == 125000000) 4450 return 125000000; 4451 4452 WARN_ON(rate > 125000000); 4453 4454 return 125000000; 4455 } 4456 4457 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 4458 unsigned long parent_rate) 4459 { 4460 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); 4461 if (rate != 125000000) 4462 iowrite32(1, mgmt->reg); 4463 else 4464 iowrite32(0, mgmt->reg); 4465 mgmt->rate = rate; 4466 4467 return 0; 4468 } 4469 4470 static const struct clk_ops fu540_c000_ops = { 4471 .recalc_rate = fu540_macb_tx_recalc_rate, 4472 .round_rate = fu540_macb_tx_round_rate, 4473 .set_rate = fu540_macb_tx_set_rate, 4474 }; 4475 4476 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 4477 struct clk **hclk, struct clk **tx_clk, 4478 struct clk **rx_clk, struct clk **tsu_clk) 4479 { 4480 struct clk_init_data init; 4481 int err = 0; 4482 4483 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 4484 if (err) 4485 return err; 4486 4487 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 4488 if (!mgmt) { 4489 err = -ENOMEM; 4490 goto err_disable_clks; 4491 } 4492 4493 init.name = "sifive-gemgxl-mgmt"; 4494 init.ops = &fu540_c000_ops; 4495 init.flags = 0; 4496 init.num_parents = 0; 4497 4498 mgmt->rate = 0; 4499 mgmt->hw.init = &init; 4500 4501 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); 4502 if (IS_ERR(*tx_clk)) { 4503 err = PTR_ERR(*tx_clk); 4504 goto err_disable_clks; 4505 } 4506 4507 err = clk_prepare_enable(*tx_clk); 4508 if (err) { 4509 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 4510 *tx_clk = NULL; 4511 goto err_disable_clks; 4512 } else { 4513 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 4514 } 4515 4516 return 0; 4517 4518 err_disable_clks: 4519 macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk); 4520 4521 return err; 4522 } 4523 4524 static int fu540_c000_init(struct platform_device *pdev) 4525 { 4526 mgmt->reg = devm_platform_ioremap_resource(pdev, 1); 4527 if (IS_ERR(mgmt->reg)) 4528 return PTR_ERR(mgmt->reg); 4529 4530 return macb_init(pdev); 4531 } 4532 4533 static const struct macb_usrio_config sama7g5_usrio = { 4534 .mii = 0, 4535 .rmii = 1, 4536 .rgmii = 2, 4537 .refclk = BIT(2), 4538 .hdfctlen = BIT(6), 4539 }; 4540 4541 static const struct macb_config fu540_c000_config = { 4542 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 4543 MACB_CAPS_GEM_HAS_PTP, 4544 .dma_burst_length = 16, 4545 .clk_init = fu540_c000_clk_init, 4546 .init = fu540_c000_init, 4547 .jumbo_max_len = 10240, 4548 .usrio = &macb_default_usrio, 4549 }; 4550 4551 static const struct macb_config at91sam9260_config = { 4552 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4553 .clk_init = macb_clk_init, 4554 .init = macb_init, 4555 .usrio = &macb_default_usrio, 4556 }; 4557 4558 static const struct macb_config sama5d3macb_config = { 4559 .caps = MACB_CAPS_SG_DISABLED 4560 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4561 .clk_init = macb_clk_init, 4562 .init = macb_init, 4563 .usrio = &macb_default_usrio, 4564 }; 4565 4566 static const struct macb_config pc302gem_config = { 4567 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 4568 .dma_burst_length = 16, 4569 .clk_init = macb_clk_init, 4570 .init = macb_init, 4571 .usrio = &macb_default_usrio, 4572 }; 4573 4574 static const struct macb_config sama5d2_config = { 4575 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4576 .dma_burst_length = 16, 4577 .clk_init = macb_clk_init, 4578 .init = macb_init, 4579 .usrio = &macb_default_usrio, 4580 }; 4581 4582 static const struct macb_config sama5d29_config = { 4583 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP, 4584 .dma_burst_length = 16, 4585 .clk_init = macb_clk_init, 4586 .init = macb_init, 4587 .usrio = &macb_default_usrio, 4588 }; 4589 4590 static const struct macb_config sama5d3_config = { 4591 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 4592 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 4593 .dma_burst_length = 16, 4594 .clk_init = macb_clk_init, 4595 .init = macb_init, 4596 .jumbo_max_len = 10240, 4597 .usrio = &macb_default_usrio, 4598 }; 4599 4600 static const struct macb_config sama5d4_config = { 4601 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4602 .dma_burst_length = 4, 4603 .clk_init = macb_clk_init, 4604 .init = macb_init, 4605 .usrio = &macb_default_usrio, 4606 }; 4607 4608 static const struct macb_config emac_config = { 4609 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC, 4610 .clk_init = at91ether_clk_init, 4611 .init = at91ether_init, 4612 .usrio = &macb_default_usrio, 4613 }; 4614 4615 static const struct macb_config np4_config = { 4616 .caps = MACB_CAPS_USRIO_DISABLED, 4617 .clk_init = macb_clk_init, 4618 .init = macb_init, 4619 .usrio = &macb_default_usrio, 4620 }; 4621 4622 static const struct macb_config zynqmp_config = { 4623 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4624 MACB_CAPS_JUMBO | 4625 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 4626 .dma_burst_length = 16, 4627 .clk_init = macb_clk_init, 4628 .init = macb_init, 4629 .jumbo_max_len = 10240, 4630 .usrio = &macb_default_usrio, 4631 }; 4632 4633 static const struct macb_config zynq_config = { 4634 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 4635 MACB_CAPS_NEEDS_RSTONUBR, 4636 .dma_burst_length = 16, 4637 .clk_init = macb_clk_init, 4638 .init = macb_init, 4639 .usrio = &macb_default_usrio, 4640 }; 4641 4642 static const struct macb_config sama7g5_gem_config = { 4643 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | 4644 MACB_CAPS_MIIONRGMII, 4645 .dma_burst_length = 16, 4646 .clk_init = macb_clk_init, 4647 .init = macb_init, 4648 .usrio = &sama7g5_usrio, 4649 }; 4650 4651 static const struct macb_config sama7g5_emac_config = { 4652 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 4653 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII, 4654 .dma_burst_length = 16, 4655 .clk_init = macb_clk_init, 4656 .init = macb_init, 4657 .usrio = &sama7g5_usrio, 4658 }; 4659 4660 static const struct of_device_id macb_dt_ids[] = { 4661 { .compatible = "cdns,at32ap7000-macb" }, 4662 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 4663 { .compatible = "cdns,macb" }, 4664 { .compatible = "cdns,np4-macb", .data = &np4_config }, 4665 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 4666 { .compatible = "cdns,gem", .data = &pc302gem_config }, 4667 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 4668 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 4669 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config }, 4670 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 4671 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 4672 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 4673 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 4674 { .compatible = "cdns,emac", .data = &emac_config }, 4675 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 4676 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 4677 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 4678 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, 4679 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, 4680 { /* sentinel */ } 4681 }; 4682 MODULE_DEVICE_TABLE(of, macb_dt_ids); 4683 #endif /* CONFIG_OF */ 4684 4685 static const struct macb_config default_gem_config = { 4686 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4687 MACB_CAPS_JUMBO | 4688 MACB_CAPS_GEM_HAS_PTP, 4689 .dma_burst_length = 16, 4690 .clk_init = macb_clk_init, 4691 .init = macb_init, 4692 .usrio = &macb_default_usrio, 4693 .jumbo_max_len = 10240, 4694 }; 4695 4696 static int macb_probe(struct platform_device *pdev) 4697 { 4698 const struct macb_config *macb_config = &default_gem_config; 4699 int (*clk_init)(struct platform_device *, struct clk **, 4700 struct clk **, struct clk **, struct clk **, 4701 struct clk **) = macb_config->clk_init; 4702 int (*init)(struct platform_device *) = macb_config->init; 4703 struct device_node *np = pdev->dev.of_node; 4704 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4705 struct clk *tsu_clk = NULL; 4706 unsigned int queue_mask, num_queues; 4707 bool native_io; 4708 phy_interface_t interface; 4709 struct net_device *dev; 4710 struct resource *regs; 4711 void __iomem *mem; 4712 struct macb *bp; 4713 int err, val; 4714 4715 mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); 4716 if (IS_ERR(mem)) 4717 return PTR_ERR(mem); 4718 4719 if (np) { 4720 const struct of_device_id *match; 4721 4722 match = of_match_node(macb_dt_ids, np); 4723 if (match && match->data) { 4724 macb_config = match->data; 4725 clk_init = macb_config->clk_init; 4726 init = macb_config->init; 4727 } 4728 } 4729 4730 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4731 if (err) 4732 return err; 4733 4734 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4735 pm_runtime_use_autosuspend(&pdev->dev); 4736 pm_runtime_get_noresume(&pdev->dev); 4737 pm_runtime_set_active(&pdev->dev); 4738 pm_runtime_enable(&pdev->dev); 4739 native_io = hw_is_native_io(mem); 4740 4741 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4742 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4743 if (!dev) { 4744 err = -ENOMEM; 4745 goto err_disable_clocks; 4746 } 4747 4748 dev->base_addr = regs->start; 4749 4750 SET_NETDEV_DEV(dev, &pdev->dev); 4751 4752 bp = netdev_priv(dev); 4753 bp->pdev = pdev; 4754 bp->dev = dev; 4755 bp->regs = mem; 4756 bp->native_io = native_io; 4757 if (native_io) { 4758 bp->macb_reg_readl = hw_readl_native; 4759 bp->macb_reg_writel = hw_writel_native; 4760 } else { 4761 bp->macb_reg_readl = hw_readl; 4762 bp->macb_reg_writel = hw_writel; 4763 } 4764 bp->num_queues = num_queues; 4765 bp->queue_mask = queue_mask; 4766 if (macb_config) 4767 bp->dma_burst_length = macb_config->dma_burst_length; 4768 bp->pclk = pclk; 4769 bp->hclk = hclk; 4770 bp->tx_clk = tx_clk; 4771 bp->rx_clk = rx_clk; 4772 bp->tsu_clk = tsu_clk; 4773 if (macb_config) 4774 bp->jumbo_max_len = macb_config->jumbo_max_len; 4775 4776 bp->wol = 0; 4777 if (of_get_property(np, "magic-packet", NULL)) 4778 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4779 device_set_wakeup_capable(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4780 4781 bp->usrio = macb_config->usrio; 4782 4783 spin_lock_init(&bp->lock); 4784 4785 /* setup capabilities */ 4786 macb_configure_caps(bp, macb_config); 4787 4788 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4789 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4790 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4791 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4792 } 4793 #endif 4794 platform_set_drvdata(pdev, dev); 4795 4796 dev->irq = platform_get_irq(pdev, 0); 4797 if (dev->irq < 0) { 4798 err = dev->irq; 4799 goto err_out_free_netdev; 4800 } 4801 4802 /* MTU range: 68 - 1500 or 10240 */ 4803 dev->min_mtu = GEM_MTU_MIN_SIZE; 4804 if (bp->caps & MACB_CAPS_JUMBO) 4805 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4806 else 4807 dev->max_mtu = ETH_DATA_LEN; 4808 4809 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4810 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4811 if (val) 4812 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4813 macb_dma_desc_get_size(bp); 4814 4815 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4816 if (val) 4817 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4818 macb_dma_desc_get_size(bp); 4819 } 4820 4821 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4822 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4823 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4824 4825 err = of_get_ethdev_address(np, bp->dev); 4826 if (err == -EPROBE_DEFER) 4827 goto err_out_free_netdev; 4828 else if (err) 4829 macb_get_hwaddr(bp); 4830 4831 err = of_get_phy_mode(np, &interface); 4832 if (err) 4833 /* not found in DT, MII by default */ 4834 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4835 else 4836 bp->phy_interface = interface; 4837 4838 /* IP specific init */ 4839 err = init(pdev); 4840 if (err) 4841 goto err_out_free_netdev; 4842 4843 err = macb_mii_init(bp); 4844 if (err) 4845 goto err_out_free_netdev; 4846 4847 netif_carrier_off(dev); 4848 4849 err = register_netdev(dev); 4850 if (err) { 4851 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4852 goto err_out_unregister_mdio; 4853 } 4854 4855 tasklet_setup(&bp->hresp_err_tasklet, macb_hresp_error_task); 4856 4857 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4858 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4859 dev->base_addr, dev->irq, dev->dev_addr); 4860 4861 pm_runtime_mark_last_busy(&bp->pdev->dev); 4862 pm_runtime_put_autosuspend(&bp->pdev->dev); 4863 4864 return 0; 4865 4866 err_out_unregister_mdio: 4867 mdiobus_unregister(bp->mii_bus); 4868 mdiobus_free(bp->mii_bus); 4869 4870 err_out_free_netdev: 4871 free_netdev(dev); 4872 4873 err_disable_clocks: 4874 macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); 4875 pm_runtime_disable(&pdev->dev); 4876 pm_runtime_set_suspended(&pdev->dev); 4877 pm_runtime_dont_use_autosuspend(&pdev->dev); 4878 4879 return err; 4880 } 4881 4882 static int macb_remove(struct platform_device *pdev) 4883 { 4884 struct net_device *dev; 4885 struct macb *bp; 4886 4887 dev = platform_get_drvdata(pdev); 4888 4889 if (dev) { 4890 bp = netdev_priv(dev); 4891 mdiobus_unregister(bp->mii_bus); 4892 mdiobus_free(bp->mii_bus); 4893 4894 unregister_netdev(dev); 4895 tasklet_kill(&bp->hresp_err_tasklet); 4896 pm_runtime_disable(&pdev->dev); 4897 pm_runtime_dont_use_autosuspend(&pdev->dev); 4898 if (!pm_runtime_suspended(&pdev->dev)) { 4899 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, 4900 bp->rx_clk, bp->tsu_clk); 4901 pm_runtime_set_suspended(&pdev->dev); 4902 } 4903 phylink_destroy(bp->phylink); 4904 free_netdev(dev); 4905 } 4906 4907 return 0; 4908 } 4909 4910 static int __maybe_unused macb_suspend(struct device *dev) 4911 { 4912 struct net_device *netdev = dev_get_drvdata(dev); 4913 struct macb *bp = netdev_priv(netdev); 4914 struct macb_queue *queue; 4915 unsigned long flags; 4916 unsigned int q; 4917 int err; 4918 4919 if (!netif_running(netdev)) 4920 return 0; 4921 4922 if (bp->wol & MACB_WOL_ENABLED) { 4923 spin_lock_irqsave(&bp->lock, flags); 4924 /* Flush all status bits */ 4925 macb_writel(bp, TSR, -1); 4926 macb_writel(bp, RSR, -1); 4927 for (q = 0, queue = bp->queues; q < bp->num_queues; 4928 ++q, ++queue) { 4929 /* Disable all interrupts */ 4930 queue_writel(queue, IDR, -1); 4931 queue_readl(queue, ISR); 4932 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 4933 queue_writel(queue, ISR, -1); 4934 } 4935 /* Change interrupt handler and 4936 * Enable WoL IRQ on queue 0 4937 */ 4938 devm_free_irq(dev, bp->queues[0].irq, bp->queues); 4939 if (macb_is_gem(bp)) { 4940 err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, 4941 IRQF_SHARED, netdev->name, bp->queues); 4942 if (err) { 4943 dev_err(dev, 4944 "Unable to request IRQ %d (error %d)\n", 4945 bp->queues[0].irq, err); 4946 spin_unlock_irqrestore(&bp->lock, flags); 4947 return err; 4948 } 4949 queue_writel(bp->queues, IER, GEM_BIT(WOL)); 4950 gem_writel(bp, WOL, MACB_BIT(MAG)); 4951 } else { 4952 err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, 4953 IRQF_SHARED, netdev->name, bp->queues); 4954 if (err) { 4955 dev_err(dev, 4956 "Unable to request IRQ %d (error %d)\n", 4957 bp->queues[0].irq, err); 4958 spin_unlock_irqrestore(&bp->lock, flags); 4959 return err; 4960 } 4961 queue_writel(bp->queues, IER, MACB_BIT(WOL)); 4962 macb_writel(bp, WOL, MACB_BIT(MAG)); 4963 } 4964 spin_unlock_irqrestore(&bp->lock, flags); 4965 4966 enable_irq_wake(bp->queues[0].irq); 4967 } 4968 4969 netif_device_detach(netdev); 4970 for (q = 0, queue = bp->queues; q < bp->num_queues; 4971 ++q, ++queue) 4972 napi_disable(&queue->napi); 4973 4974 if (!(bp->wol & MACB_WOL_ENABLED)) { 4975 rtnl_lock(); 4976 phylink_stop(bp->phylink); 4977 rtnl_unlock(); 4978 spin_lock_irqsave(&bp->lock, flags); 4979 macb_reset_hw(bp); 4980 spin_unlock_irqrestore(&bp->lock, flags); 4981 } 4982 4983 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4984 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 4985 4986 if (netdev->hw_features & NETIF_F_NTUPLE) 4987 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 4988 4989 if (bp->ptp_info) 4990 bp->ptp_info->ptp_remove(netdev); 4991 if (!device_may_wakeup(dev)) 4992 pm_runtime_force_suspend(dev); 4993 4994 return 0; 4995 } 4996 4997 static int __maybe_unused macb_resume(struct device *dev) 4998 { 4999 struct net_device *netdev = dev_get_drvdata(dev); 5000 struct macb *bp = netdev_priv(netdev); 5001 struct macb_queue *queue; 5002 unsigned long flags; 5003 unsigned int q; 5004 int err; 5005 5006 if (!netif_running(netdev)) 5007 return 0; 5008 5009 if (!device_may_wakeup(dev)) 5010 pm_runtime_force_resume(dev); 5011 5012 if (bp->wol & MACB_WOL_ENABLED) { 5013 spin_lock_irqsave(&bp->lock, flags); 5014 /* Disable WoL */ 5015 if (macb_is_gem(bp)) { 5016 queue_writel(bp->queues, IDR, GEM_BIT(WOL)); 5017 gem_writel(bp, WOL, 0); 5018 } else { 5019 queue_writel(bp->queues, IDR, MACB_BIT(WOL)); 5020 macb_writel(bp, WOL, 0); 5021 } 5022 /* Clear ISR on queue 0 */ 5023 queue_readl(bp->queues, ISR); 5024 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 5025 queue_writel(bp->queues, ISR, -1); 5026 /* Replace interrupt handler on queue 0 */ 5027 devm_free_irq(dev, bp->queues[0].irq, bp->queues); 5028 err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, 5029 IRQF_SHARED, netdev->name, bp->queues); 5030 if (err) { 5031 dev_err(dev, 5032 "Unable to request IRQ %d (error %d)\n", 5033 bp->queues[0].irq, err); 5034 spin_unlock_irqrestore(&bp->lock, flags); 5035 return err; 5036 } 5037 spin_unlock_irqrestore(&bp->lock, flags); 5038 5039 disable_irq_wake(bp->queues[0].irq); 5040 5041 /* Now make sure we disable phy before moving 5042 * to common restore path 5043 */ 5044 rtnl_lock(); 5045 phylink_stop(bp->phylink); 5046 rtnl_unlock(); 5047 } 5048 5049 for (q = 0, queue = bp->queues; q < bp->num_queues; 5050 ++q, ++queue) 5051 napi_enable(&queue->napi); 5052 5053 if (netdev->hw_features & NETIF_F_NTUPLE) 5054 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 5055 5056 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 5057 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 5058 5059 macb_writel(bp, NCR, MACB_BIT(MPE)); 5060 macb_init_hw(bp); 5061 macb_set_rx_mode(netdev); 5062 macb_restore_features(bp); 5063 rtnl_lock(); 5064 phylink_start(bp->phylink); 5065 rtnl_unlock(); 5066 5067 netif_device_attach(netdev); 5068 if (bp->ptp_info) 5069 bp->ptp_info->ptp_init(netdev); 5070 5071 return 0; 5072 } 5073 5074 static int __maybe_unused macb_runtime_suspend(struct device *dev) 5075 { 5076 struct net_device *netdev = dev_get_drvdata(dev); 5077 struct macb *bp = netdev_priv(netdev); 5078 5079 if (!(device_may_wakeup(dev))) 5080 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); 5081 else 5082 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); 5083 5084 return 0; 5085 } 5086 5087 static int __maybe_unused macb_runtime_resume(struct device *dev) 5088 { 5089 struct net_device *netdev = dev_get_drvdata(dev); 5090 struct macb *bp = netdev_priv(netdev); 5091 5092 if (!(device_may_wakeup(dev))) { 5093 clk_prepare_enable(bp->pclk); 5094 clk_prepare_enable(bp->hclk); 5095 clk_prepare_enable(bp->tx_clk); 5096 clk_prepare_enable(bp->rx_clk); 5097 } 5098 clk_prepare_enable(bp->tsu_clk); 5099 5100 return 0; 5101 } 5102 5103 static const struct dev_pm_ops macb_pm_ops = { 5104 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 5105 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 5106 }; 5107 5108 static struct platform_driver macb_driver = { 5109 .probe = macb_probe, 5110 .remove = macb_remove, 5111 .driver = { 5112 .name = "macb", 5113 .of_match_table = of_match_ptr(macb_dt_ids), 5114 .pm = &macb_pm_ops, 5115 }, 5116 }; 5117 5118 module_platform_driver(macb_driver); 5119 5120 MODULE_LICENSE("GPL"); 5121 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 5122 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 5123 MODULE_ALIAS("platform:macb"); 5124