1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/circ_buf.h> 10 #include <linux/clk-provider.h> 11 #include <linux/clk.h> 12 #include <linux/crc32.h> 13 #include <linux/delay.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/etherdevice.h> 16 #include <linux/firmware/xlnx-zynqmp.h> 17 #include <linux/inetdevice.h> 18 #include <linux/init.h> 19 #include <linux/interrupt.h> 20 #include <linux/io.h> 21 #include <linux/iopoll.h> 22 #include <linux/ip.h> 23 #include <linux/kernel.h> 24 #include <linux/module.h> 25 #include <linux/moduleparam.h> 26 #include <linux/netdevice.h> 27 #include <linux/of.h> 28 #include <linux/of_mdio.h> 29 #include <linux/of_net.h> 30 #include <linux/phy/phy.h> 31 #include <linux/phylink.h> 32 #include <linux/platform_device.h> 33 #include <linux/pm_runtime.h> 34 #include <linux/ptp_classify.h> 35 #include <linux/reset.h> 36 #include <linux/slab.h> 37 #include <linux/tcp.h> 38 #include <linux/types.h> 39 #include <linux/udp.h> 40 #include <linux/gcd.h> 41 #include <net/pkt_sched.h> 42 #include "macb.h" 43 44 /* This structure is only used for MACB on SiFive FU540 devices */ 45 struct sifive_fu540_macb_mgmt { 46 void __iomem *reg; 47 unsigned long rate; 48 struct clk_hw hw; 49 }; 50 51 #define MACB_RX_BUFFER_SIZE 128 52 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 53 #define RX_BUFFER_MAX (0xFF * RX_BUFFER_MULTIPLE) /* 16320 bytes */ 54 55 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 56 #define MIN_RX_RING_SIZE 64 57 #define MAX_RX_RING_SIZE 8192 58 59 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 60 #define MIN_TX_RING_SIZE 64 61 #define MAX_TX_RING_SIZE 4096 62 63 /* level of occupied TX descriptors under which we wake up TX process */ 64 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 65 66 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 67 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 68 | MACB_BIT(ISR_RLE) \ 69 | MACB_BIT(TXERR)) 70 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 71 | MACB_BIT(TXUBR)) 72 73 #define MACB_INT_MISC_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(RXUBR) | \ 74 MACB_BIT(ISR_ROVR) | MACB_BIT(HRESP) | \ 75 GEM_BIT(WOL) | MACB_BIT(WOL)) 76 77 /* Max length of transmit frame must be a multiple of 8 bytes */ 78 #define MACB_TX_LEN_ALIGN 8 79 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 80 /* Limit maximum TX length as per Cadence TSO errata. This is to avoid a 81 * false amba_error in TX path from the DMA assuming there is not enough 82 * space in the SRAM (16KB) even when there is. 83 */ 84 #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) 85 86 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 87 #define MACB_NETIF_LSO NETIF_F_TSO 88 89 #define MACB_WOL_ENABLED BIT(0) 90 91 #define HS_SPEED_10000M 4 92 #define MACB_SERDES_RATE_10G 1 93 94 /* Graceful stop timeouts in us. We should allow up to 95 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 96 */ 97 #define MACB_HALT_TIMEOUT 14000 98 #define MACB_PM_TIMEOUT 100 /* ms */ 99 100 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 101 102 /* DMA buffer descriptor might be different size 103 * depends on hardware configuration: 104 * 105 * 1. dma address width 32 bits: 106 * word 1: 32 bit address of Data Buffer 107 * word 2: control 108 * 109 * 2. dma address width 64 bits: 110 * word 1: 32 bit address of Data Buffer 111 * word 2: control 112 * word 3: upper 32 bit address of Data Buffer 113 * word 4: unused 114 * 115 * 3. dma address width 32 bits with hardware timestamping: 116 * word 1: 32 bit address of Data Buffer 117 * word 2: control 118 * word 3: timestamp word 1 119 * word 4: timestamp word 2 120 * 121 * 4. dma address width 64 bits with hardware timestamping: 122 * word 1: 32 bit address of Data Buffer 123 * word 2: control 124 * word 3: upper 32 bit address of Data Buffer 125 * word 4: unused 126 * word 5: timestamp word 1 127 * word 6: timestamp word 2 128 */ 129 static unsigned int macb_dma_desc_get_size(struct macb *bp) 130 { 131 unsigned int desc_size = sizeof(struct macb_dma_desc); 132 133 if (macb_dma64(bp)) 134 desc_size += sizeof(struct macb_dma_desc_64); 135 if (macb_dma_ptp(bp)) 136 desc_size += sizeof(struct macb_dma_desc_ptp); 137 138 return desc_size; 139 } 140 141 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 142 { 143 return desc_idx * (1 + macb_dma64(bp) + macb_dma_ptp(bp)); 144 } 145 146 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 147 { 148 return (struct macb_dma_desc_64 *)((void *)desc 149 + sizeof(struct macb_dma_desc)); 150 } 151 152 /* Ring buffer accessors */ 153 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 154 { 155 return index & (bp->tx_ring_size - 1); 156 } 157 158 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 159 unsigned int index) 160 { 161 index = macb_tx_ring_wrap(queue->bp, index); 162 index = macb_adj_dma_desc_idx(queue->bp, index); 163 return &queue->tx_ring[index]; 164 } 165 166 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 167 unsigned int index) 168 { 169 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 170 } 171 172 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 173 { 174 dma_addr_t offset; 175 176 offset = macb_tx_ring_wrap(queue->bp, index) * 177 macb_dma_desc_get_size(queue->bp); 178 179 return queue->tx_ring_dma + offset; 180 } 181 182 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 183 { 184 return index & (bp->rx_ring_size - 1); 185 } 186 187 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 188 { 189 index = macb_rx_ring_wrap(queue->bp, index); 190 index = macb_adj_dma_desc_idx(queue->bp, index); 191 return &queue->rx_ring[index]; 192 } 193 194 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 195 { 196 return queue->rx_buffers + queue->bp->rx_buffer_size * 197 macb_rx_ring_wrap(queue->bp, index); 198 } 199 200 /* I/O accessors */ 201 static u32 hw_readl_native(struct macb *bp, int offset) 202 { 203 return __raw_readl(bp->regs + offset); 204 } 205 206 static void hw_writel_native(struct macb *bp, int offset, u32 value) 207 { 208 __raw_writel(value, bp->regs + offset); 209 } 210 211 static u32 hw_readl(struct macb *bp, int offset) 212 { 213 return readl_relaxed(bp->regs + offset); 214 } 215 216 static void hw_writel(struct macb *bp, int offset, u32 value) 217 { 218 writel_relaxed(value, bp->regs + offset); 219 } 220 221 /* Find the CPU endianness by using the loopback bit of NCR register. When the 222 * CPU is in big endian we need to program swapped mode for management 223 * descriptor access. 224 */ 225 static bool hw_is_native_io(void __iomem *addr) 226 { 227 u32 value = MACB_BIT(LLB); 228 229 __raw_writel(value, addr + MACB_NCR); 230 value = __raw_readl(addr + MACB_NCR); 231 232 /* Write 0 back to disable everything */ 233 __raw_writel(0, addr + MACB_NCR); 234 235 return value == MACB_BIT(LLB); 236 } 237 238 static bool hw_is_gem(void __iomem *addr, bool native_io) 239 { 240 u32 id; 241 242 if (native_io) 243 id = __raw_readl(addr + MACB_MID); 244 else 245 id = readl_relaxed(addr + MACB_MID); 246 247 return MACB_BFEXT(IDNUM, id) >= 0x2; 248 } 249 250 static void macb_set_hwaddr(struct macb *bp) 251 { 252 u32 bottom; 253 u16 top; 254 255 bottom = get_unaligned_le32(bp->netdev->dev_addr); 256 macb_or_gem_writel(bp, SA1B, bottom); 257 top = get_unaligned_le16(bp->netdev->dev_addr + 4); 258 macb_or_gem_writel(bp, SA1T, top); 259 260 if (gem_has_ptp(bp)) { 261 gem_writel(bp, RXPTPUNI, bottom); 262 gem_writel(bp, TXPTPUNI, bottom); 263 } 264 265 /* Clear unused address register sets */ 266 macb_or_gem_writel(bp, SA2B, 0); 267 macb_or_gem_writel(bp, SA2T, 0); 268 macb_or_gem_writel(bp, SA3B, 0); 269 macb_or_gem_writel(bp, SA3T, 0); 270 macb_or_gem_writel(bp, SA4B, 0); 271 macb_or_gem_writel(bp, SA4T, 0); 272 } 273 274 static void macb_get_hwaddr(struct macb *bp) 275 { 276 u32 bottom; 277 u16 top; 278 u8 addr[6]; 279 int i; 280 281 /* Check all 4 address register for valid address */ 282 for (i = 0; i < 4; i++) { 283 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 284 top = macb_or_gem_readl(bp, SA1T + i * 8); 285 286 addr[0] = bottom & 0xff; 287 addr[1] = (bottom >> 8) & 0xff; 288 addr[2] = (bottom >> 16) & 0xff; 289 addr[3] = (bottom >> 24) & 0xff; 290 addr[4] = top & 0xff; 291 addr[5] = (top >> 8) & 0xff; 292 293 if (is_valid_ether_addr(addr)) { 294 eth_hw_addr_set(bp->netdev, addr); 295 return; 296 } 297 } 298 299 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 300 eth_hw_addr_random(bp->netdev); 301 } 302 303 static int macb_mdio_wait_for_idle(struct macb *bp) 304 { 305 u32 val; 306 307 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 308 1, MACB_MDIO_TIMEOUT); 309 } 310 311 static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum) 312 { 313 struct macb *bp = bus->priv; 314 int status; 315 316 status = pm_runtime_resume_and_get(&bp->pdev->dev); 317 if (status < 0) 318 goto mdio_pm_exit; 319 320 status = macb_mdio_wait_for_idle(bp); 321 if (status < 0) 322 goto mdio_read_exit; 323 324 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 325 | MACB_BF(RW, MACB_MAN_C22_READ) 326 | MACB_BF(PHYA, mii_id) 327 | MACB_BF(REGA, regnum) 328 | MACB_BF(CODE, MACB_MAN_C22_CODE))); 329 330 status = macb_mdio_wait_for_idle(bp); 331 if (status < 0) 332 goto mdio_read_exit; 333 334 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 335 336 mdio_read_exit: 337 pm_runtime_put_autosuspend(&bp->pdev->dev); 338 mdio_pm_exit: 339 return status; 340 } 341 342 static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad, 343 int regnum) 344 { 345 struct macb *bp = bus->priv; 346 int status; 347 348 status = pm_runtime_get_sync(&bp->pdev->dev); 349 if (status < 0) { 350 pm_runtime_put_noidle(&bp->pdev->dev); 351 goto mdio_pm_exit; 352 } 353 354 status = macb_mdio_wait_for_idle(bp); 355 if (status < 0) 356 goto mdio_read_exit; 357 358 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 359 | MACB_BF(RW, MACB_MAN_C45_ADDR) 360 | MACB_BF(PHYA, mii_id) 361 | MACB_BF(REGA, devad & 0x1F) 362 | MACB_BF(DATA, regnum & 0xFFFF) 363 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 364 365 status = macb_mdio_wait_for_idle(bp); 366 if (status < 0) 367 goto mdio_read_exit; 368 369 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 370 | MACB_BF(RW, MACB_MAN_C45_READ) 371 | MACB_BF(PHYA, mii_id) 372 | MACB_BF(REGA, devad & 0x1F) 373 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 374 375 status = macb_mdio_wait_for_idle(bp); 376 if (status < 0) 377 goto mdio_read_exit; 378 379 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 380 381 mdio_read_exit: 382 pm_runtime_put_autosuspend(&bp->pdev->dev); 383 mdio_pm_exit: 384 return status; 385 } 386 387 static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum, 388 u16 value) 389 { 390 struct macb *bp = bus->priv; 391 int status; 392 393 status = pm_runtime_resume_and_get(&bp->pdev->dev); 394 if (status < 0) 395 goto mdio_pm_exit; 396 397 status = macb_mdio_wait_for_idle(bp); 398 if (status < 0) 399 goto mdio_write_exit; 400 401 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 402 | MACB_BF(RW, MACB_MAN_C22_WRITE) 403 | MACB_BF(PHYA, mii_id) 404 | MACB_BF(REGA, regnum) 405 | MACB_BF(CODE, MACB_MAN_C22_CODE) 406 | MACB_BF(DATA, value))); 407 408 status = macb_mdio_wait_for_idle(bp); 409 if (status < 0) 410 goto mdio_write_exit; 411 412 mdio_write_exit: 413 pm_runtime_put_autosuspend(&bp->pdev->dev); 414 mdio_pm_exit: 415 return status; 416 } 417 418 static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id, 419 int devad, int regnum, 420 u16 value) 421 { 422 struct macb *bp = bus->priv; 423 int status; 424 425 status = pm_runtime_get_sync(&bp->pdev->dev); 426 if (status < 0) { 427 pm_runtime_put_noidle(&bp->pdev->dev); 428 goto mdio_pm_exit; 429 } 430 431 status = macb_mdio_wait_for_idle(bp); 432 if (status < 0) 433 goto mdio_write_exit; 434 435 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 436 | MACB_BF(RW, MACB_MAN_C45_ADDR) 437 | MACB_BF(PHYA, mii_id) 438 | MACB_BF(REGA, devad & 0x1F) 439 | MACB_BF(DATA, regnum & 0xFFFF) 440 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 441 442 status = macb_mdio_wait_for_idle(bp); 443 if (status < 0) 444 goto mdio_write_exit; 445 446 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 447 | MACB_BF(RW, MACB_MAN_C45_WRITE) 448 | MACB_BF(PHYA, mii_id) 449 | MACB_BF(REGA, devad & 0x1F) 450 | MACB_BF(CODE, MACB_MAN_C45_CODE) 451 | MACB_BF(DATA, value))); 452 453 status = macb_mdio_wait_for_idle(bp); 454 if (status < 0) 455 goto mdio_write_exit; 456 457 mdio_write_exit: 458 pm_runtime_put_autosuspend(&bp->pdev->dev); 459 mdio_pm_exit: 460 return status; 461 } 462 463 static void macb_init_buffers(struct macb *bp) 464 { 465 struct macb_queue *queue; 466 unsigned int q; 467 468 /* Single register for all queues' high 32 bits. */ 469 if (macb_dma64(bp)) { 470 macb_writel(bp, RBQPH, 471 upper_32_bits(bp->queues[0].rx_ring_dma)); 472 macb_writel(bp, TBQPH, 473 upper_32_bits(bp->queues[0].tx_ring_dma)); 474 } 475 476 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 477 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 478 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 479 } 480 } 481 482 /** 483 * macb_set_tx_clk() - Set a clock to a new frequency 484 * @bp: pointer to struct macb 485 * @speed: New frequency in Hz 486 */ 487 static void macb_set_tx_clk(struct macb *bp, int speed) 488 { 489 long ferr, rate, rate_rounded; 490 491 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) 492 return; 493 494 /* In case of MII the PHY is the clock master */ 495 if (bp->phy_interface == PHY_INTERFACE_MODE_MII) 496 return; 497 498 rate = rgmii_clock(speed); 499 if (rate < 0) 500 return; 501 502 rate_rounded = clk_round_rate(bp->tx_clk, rate); 503 if (rate_rounded < 0) 504 return; 505 506 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 507 * is not satisfied. 508 */ 509 ferr = abs(rate_rounded - rate); 510 ferr = DIV_ROUND_UP(ferr, rate / 100000); 511 if (ferr > 5) 512 netdev_warn(bp->netdev, 513 "unable to generate target frequency: %ld Hz\n", 514 rate); 515 516 if (clk_set_rate(bp->tx_clk, rate_rounded)) 517 netdev_err(bp->netdev, "adjusting tx_clk failed.\n"); 518 } 519 520 static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode, 521 phy_interface_t interface, int speed, 522 int duplex) 523 { 524 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 525 u32 config; 526 527 config = gem_readl(bp, USX_CONTROL); 528 config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config); 529 config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config); 530 config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS)); 531 config |= GEM_BIT(TX_EN); 532 gem_writel(bp, USX_CONTROL, config); 533 } 534 535 static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, 536 unsigned int neg_mode, 537 struct phylink_link_state *state) 538 { 539 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 540 u32 val; 541 542 state->speed = SPEED_10000; 543 state->duplex = 1; 544 state->an_complete = 1; 545 546 val = gem_readl(bp, USX_STATUS); 547 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); 548 val = gem_readl(bp, NCFGR); 549 if (val & GEM_BIT(PAE)) 550 state->pause = MLO_PAUSE_RX; 551 } 552 553 static int macb_usx_pcs_config(struct phylink_pcs *pcs, 554 unsigned int neg_mode, 555 phy_interface_t interface, 556 const unsigned long *advertising, 557 bool permit_pause_to_mac) 558 { 559 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 560 561 gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | 562 GEM_BIT(SIGNAL_OK)); 563 564 return 0; 565 } 566 567 static unsigned int macb_pcs_inband_caps(struct phylink_pcs *pcs, 568 phy_interface_t interface) 569 { 570 return LINK_INBAND_DISABLE | LINK_INBAND_ENABLE; 571 } 572 573 static void macb_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode, 574 struct phylink_link_state *state) 575 { 576 struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs); 577 u16 bmsr, lpa; 578 579 bmsr = gem_readl(bp, PCSSTS); 580 lpa = gem_readl(bp, PCSANLPBASE); 581 phylink_mii_c22_pcs_decode_state(state, neg_mode, bmsr, lpa); 582 } 583 584 static void macb_pcs_an_restart(struct phylink_pcs *pcs) 585 { 586 /* Not supported */ 587 } 588 589 static int macb_pcs_config(struct phylink_pcs *pcs, 590 unsigned int neg_mode, 591 phy_interface_t interface, 592 const unsigned long *advertising, 593 bool permit_pause_to_mac) 594 { 595 struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs); 596 u32 old, new; 597 598 old = gem_readl(bp, PCSANADV); 599 new = phylink_mii_c22_pcs_encode_advertisement(interface, advertising); 600 if (new != -EINVAL && old != new) 601 gem_writel(bp, PCSANADV, new); 602 603 /* Disable AN if it's not to be used, enable otherwise. 604 * Must be written after PCSSEL is set in NCFGR which is done in 605 * macb_mac_config(), otherwise writes will not take effect. 606 */ 607 old = gem_readl(bp, PCSCNTRL); 608 if (neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED) 609 new = old | BMCR_ANENABLE; 610 else 611 new = old & ~BMCR_ANENABLE; 612 if (old != new) 613 gem_writel(bp, PCSCNTRL, new); 614 615 return 0; 616 } 617 618 static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { 619 .pcs_get_state = macb_usx_pcs_get_state, 620 .pcs_config = macb_usx_pcs_config, 621 .pcs_link_up = macb_usx_pcs_link_up, 622 }; 623 624 static const struct phylink_pcs_ops macb_phylink_pcs_ops = { 625 .pcs_inband_caps = macb_pcs_inband_caps, 626 .pcs_get_state = macb_pcs_get_state, 627 .pcs_an_restart = macb_pcs_an_restart, 628 .pcs_config = macb_pcs_config, 629 }; 630 631 static bool macb_tx_lpi_set(struct macb *bp, bool enable) 632 { 633 u32 old, ncr; 634 635 lockdep_assert_held(&bp->lock); 636 637 ncr = macb_readl(bp, NCR); 638 old = ncr; 639 if (enable) 640 ncr |= GEM_BIT(TXLPIEN); 641 else 642 ncr &= ~GEM_BIT(TXLPIEN); 643 if (old != ncr) 644 macb_writel(bp, NCR, ncr); 645 646 return old != ncr; 647 } 648 649 static bool macb_tx_all_queues_idle(struct macb *bp) 650 { 651 struct macb_queue *queue; 652 unsigned int q; 653 654 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 655 if (READ_ONCE(queue->tx_head) != READ_ONCE(queue->tx_tail)) 656 return false; 657 } 658 return true; 659 } 660 661 static void macb_tx_lpi_work_fn(struct work_struct *work) 662 { 663 struct macb *bp = container_of(work, struct macb, tx_lpi_work.work); 664 unsigned long flags; 665 666 spin_lock_irqsave(&bp->lock, flags); 667 if (bp->eee_active && macb_tx_all_queues_idle(bp)) 668 macb_tx_lpi_set(bp, true); 669 spin_unlock_irqrestore(&bp->lock, flags); 670 } 671 672 static void macb_tx_lpi_schedule(struct macb *bp) 673 { 674 if (bp->eee_active) 675 mod_delayed_work(system_wq, &bp->tx_lpi_work, 676 usecs_to_jiffies(bp->tx_lpi_timer)); 677 } 678 679 /* Wake from LPI before transmitting. The MAC must deassert TXLPIEN 680 * and wait for the PHY to exit LPI before any frame can be sent. 681 * IEEE 802.3az Tw_sys is ~17us for 1000BASE-T, ~30us for 100BASE-TX; 682 * we use a conservative 50us. 683 */ 684 static void macb_tx_lpi_wake(struct macb *bp) 685 { 686 lockdep_assert_held(&bp->lock); 687 688 if (!bp->eee_active) 689 return; 690 691 if (!macb_tx_lpi_set(bp, false)) 692 return; 693 694 cancel_delayed_work(&bp->tx_lpi_work); 695 udelay(50); 696 } 697 698 static void macb_mac_disable_tx_lpi(struct phylink_config *config) 699 { 700 struct net_device *netdev = to_net_dev(config->dev); 701 struct macb *bp = netdev_priv(netdev); 702 unsigned long flags; 703 704 cancel_delayed_work_sync(&bp->tx_lpi_work); 705 706 spin_lock_irqsave(&bp->lock, flags); 707 bp->eee_active = false; 708 macb_tx_lpi_set(bp, false); 709 spin_unlock_irqrestore(&bp->lock, flags); 710 } 711 712 static int macb_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, 713 bool tx_clk_stop) 714 { 715 struct net_device *netdev = to_net_dev(config->dev); 716 struct macb *bp = netdev_priv(netdev); 717 unsigned long flags; 718 719 spin_lock_irqsave(&bp->lock, flags); 720 bp->tx_lpi_timer = timer; 721 bp->eee_active = true; 722 spin_unlock_irqrestore(&bp->lock, flags); 723 724 /* Defer initial LPI entry by 1 second after link-up per 725 * IEEE 802.3az section 22.7a. 726 */ 727 mod_delayed_work(system_wq, &bp->tx_lpi_work, msecs_to_jiffies(1000)); 728 729 return 0; 730 } 731 732 static void macb_mac_config(struct phylink_config *config, unsigned int mode, 733 const struct phylink_link_state *state) 734 { 735 struct net_device *netdev = to_net_dev(config->dev); 736 struct macb *bp = netdev_priv(netdev); 737 unsigned long flags; 738 u32 old_ctrl, ctrl; 739 u32 old_ncr, ncr; 740 741 spin_lock_irqsave(&bp->lock, flags); 742 743 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); 744 old_ncr = ncr = macb_or_gem_readl(bp, NCR); 745 746 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { 747 if (state->interface == PHY_INTERFACE_MODE_RMII) 748 ctrl |= MACB_BIT(RM9200_RMII); 749 } else if (macb_is_gem(bp)) { 750 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); 751 ncr &= ~GEM_BIT(ENABLE_HS_MAC); 752 753 if (state->interface == PHY_INTERFACE_MODE_SGMII) { 754 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 755 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) { 756 ctrl |= GEM_BIT(PCSSEL); 757 ncr |= GEM_BIT(ENABLE_HS_MAC); 758 } else if (bp->caps & MACB_CAPS_MIIONRGMII && 759 bp->phy_interface == PHY_INTERFACE_MODE_MII) { 760 ncr |= MACB_BIT(MIIONRGMII); 761 } 762 } 763 764 /* Apply the new configuration, if any */ 765 if (old_ctrl ^ ctrl) 766 macb_or_gem_writel(bp, NCFGR, ctrl); 767 768 if (old_ncr ^ ncr) 769 macb_or_gem_writel(bp, NCR, ncr); 770 771 spin_unlock_irqrestore(&bp->lock, flags); 772 } 773 774 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, 775 phy_interface_t interface) 776 { 777 struct net_device *netdev = to_net_dev(config->dev); 778 struct macb *bp = netdev_priv(netdev); 779 struct macb_queue *queue; 780 unsigned int q; 781 u32 ctrl; 782 783 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 784 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 785 queue_writel(queue, IDR, 786 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 787 788 /* Disable Rx and Tx */ 789 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); 790 macb_writel(bp, NCR, ctrl); 791 792 netif_tx_stop_all_queues(netdev); 793 } 794 795 /* Use juggling algorithm to left rotate tx ring and tx skb array */ 796 static void gem_shuffle_tx_one_ring(struct macb_queue *queue) 797 { 798 unsigned int head, tail, count, ring_size, desc_size; 799 struct macb_tx_skb tx_skb, *skb_curr, *skb_next; 800 struct macb_dma_desc *desc_curr, *desc_next; 801 unsigned int i, cycles, shift, curr, next; 802 struct macb *bp = queue->bp; 803 unsigned char desc[24]; 804 unsigned long flags; 805 806 desc_size = macb_dma_desc_get_size(bp); 807 808 if (WARN_ON_ONCE(desc_size > ARRAY_SIZE(desc))) 809 return; 810 811 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 812 head = queue->tx_head; 813 tail = queue->tx_tail; 814 ring_size = bp->tx_ring_size; 815 count = CIRC_CNT(head, tail, ring_size); 816 817 if (!(tail % ring_size)) 818 goto unlock; 819 820 if (!count) { 821 queue->tx_head = 0; 822 queue->tx_tail = 0; 823 goto unlock; 824 } 825 826 shift = tail % ring_size; 827 cycles = gcd(ring_size, shift); 828 829 for (i = 0; i < cycles; i++) { 830 memcpy(&desc, macb_tx_desc(queue, i), desc_size); 831 memcpy(&tx_skb, macb_tx_skb(queue, i), 832 sizeof(struct macb_tx_skb)); 833 834 curr = i; 835 next = (curr + shift) % ring_size; 836 837 while (next != i) { 838 desc_curr = macb_tx_desc(queue, curr); 839 desc_next = macb_tx_desc(queue, next); 840 841 memcpy(desc_curr, desc_next, desc_size); 842 843 if (next == ring_size - 1) 844 desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); 845 if (curr == ring_size - 1) 846 desc_curr->ctrl |= MACB_BIT(TX_WRAP); 847 848 skb_curr = macb_tx_skb(queue, curr); 849 skb_next = macb_tx_skb(queue, next); 850 memcpy(skb_curr, skb_next, sizeof(struct macb_tx_skb)); 851 852 curr = next; 853 next = (curr + shift) % ring_size; 854 } 855 856 desc_curr = macb_tx_desc(queue, curr); 857 memcpy(desc_curr, &desc, desc_size); 858 if (i == ring_size - 1) 859 desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); 860 if (curr == ring_size - 1) 861 desc_curr->ctrl |= MACB_BIT(TX_WRAP); 862 memcpy(macb_tx_skb(queue, curr), &tx_skb, 863 sizeof(struct macb_tx_skb)); 864 } 865 866 queue->tx_head = count; 867 queue->tx_tail = 0; 868 869 /* Make descriptor updates visible to hardware */ 870 wmb(); 871 872 unlock: 873 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 874 } 875 876 /* Rotate the queue so that the tail is at index 0 */ 877 static void gem_shuffle_tx_rings(struct macb *bp) 878 { 879 struct macb_queue *queue; 880 unsigned int q; 881 882 for (q = 0, queue = bp->queues; q < bp->num_queues; q++, queue++) 883 gem_shuffle_tx_one_ring(queue); 884 } 885 886 static void macb_mac_link_up(struct phylink_config *config, 887 struct phy_device *phydev, 888 unsigned int mode, phy_interface_t interface, 889 int speed, int duplex, 890 bool tx_pause, bool rx_pause) 891 { 892 struct net_device *netdev = to_net_dev(config->dev); 893 struct macb *bp = netdev_priv(netdev); 894 struct macb_queue *queue; 895 unsigned long flags; 896 unsigned int q; 897 u32 ctrl; 898 899 spin_lock_irqsave(&bp->lock, flags); 900 901 ctrl = macb_or_gem_readl(bp, NCFGR); 902 903 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 904 905 if (speed == SPEED_100) 906 ctrl |= MACB_BIT(SPD); 907 908 if (duplex) 909 ctrl |= MACB_BIT(FD); 910 911 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 912 ctrl &= ~MACB_BIT(PAE); 913 if (macb_is_gem(bp)) { 914 ctrl &= ~GEM_BIT(GBE); 915 916 if (speed == SPEED_1000) 917 ctrl |= GEM_BIT(GBE); 918 } 919 920 if (rx_pause) 921 ctrl |= MACB_BIT(PAE); 922 923 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 924 queue_writel(queue, IER, 925 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 926 } 927 } 928 929 macb_or_gem_writel(bp, NCFGR, ctrl); 930 931 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER) 932 gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, 933 gem_readl(bp, HS_MAC_CONFIG))); 934 935 spin_unlock_irqrestore(&bp->lock, flags); 936 937 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 938 macb_set_tx_clk(bp, speed); 939 gem_shuffle_tx_rings(bp); 940 } 941 942 /* Enable Rx and Tx; Enable PTP unicast */ 943 ctrl = macb_readl(bp, NCR); 944 if (gem_has_ptp(bp)) 945 ctrl |= MACB_BIT(PTPUNI); 946 947 macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); 948 949 netif_tx_wake_all_queues(netdev); 950 } 951 952 static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config, 953 phy_interface_t interface) 954 { 955 struct net_device *netdev = to_net_dev(config->dev); 956 struct macb *bp = netdev_priv(netdev); 957 958 if (interface == PHY_INTERFACE_MODE_10GBASER) 959 return &bp->phylink_usx_pcs; 960 else if (interface == PHY_INTERFACE_MODE_SGMII) 961 return &bp->phylink_sgmii_pcs; 962 else 963 return NULL; 964 } 965 966 static const struct phylink_mac_ops macb_phylink_ops = { 967 .mac_select_pcs = macb_mac_select_pcs, 968 .mac_config = macb_mac_config, 969 .mac_link_down = macb_mac_link_down, 970 .mac_link_up = macb_mac_link_up, 971 .mac_disable_tx_lpi = macb_mac_disable_tx_lpi, 972 .mac_enable_tx_lpi = macb_mac_enable_tx_lpi, 973 }; 974 975 static bool macb_phy_handle_exists(struct device_node *dn) 976 { 977 dn = of_parse_phandle(dn, "phy-handle", 0); 978 of_node_put(dn); 979 return dn != NULL; 980 } 981 982 static int macb_phylink_connect(struct macb *bp) 983 { 984 struct device_node *dn = bp->pdev->dev.of_node; 985 struct net_device *netdev = bp->netdev; 986 struct phy_device *phydev; 987 int ret; 988 989 if (dn) 990 ret = phylink_of_phy_connect(bp->phylink, dn, 0); 991 992 if (!dn || (ret && !macb_phy_handle_exists(dn))) { 993 phydev = phy_find_first(bp->mii_bus); 994 if (!phydev) { 995 netdev_err(netdev, "no PHY found\n"); 996 return -ENXIO; 997 } 998 999 /* attach the mac to the phy */ 1000 ret = phylink_connect_phy(bp->phylink, phydev); 1001 } 1002 1003 if (ret) { 1004 netdev_err(netdev, "Could not attach PHY (%d)\n", ret); 1005 return ret; 1006 } 1007 1008 phylink_start(bp->phylink); 1009 1010 return 0; 1011 } 1012 1013 static void macb_get_pcs_fixed_state(struct phylink_config *config, 1014 struct phylink_link_state *state) 1015 { 1016 struct net_device *netdev = to_net_dev(config->dev); 1017 struct macb *bp = netdev_priv(netdev); 1018 1019 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; 1020 } 1021 1022 /* based on au1000_eth. c*/ 1023 static int macb_mii_probe(struct net_device *netdev) 1024 { 1025 struct macb *bp = netdev_priv(netdev); 1026 1027 bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; 1028 bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; 1029 1030 bp->phylink_config.dev = &netdev->dev; 1031 bp->phylink_config.type = PHYLINK_NETDEV; 1032 bp->phylink_config.mac_managed_pm = true; 1033 1034 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { 1035 bp->phylink_config.poll_fixed_state = true; 1036 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; 1037 /* The PCSAUTONEG bit in PCSCNTRL is on out of reset. Setting 1038 * default_an_inband to true tells phylink to turn it off only 1039 * if necessary (e.g. a fixed link or a PHY that doesn't support 1040 * inband). 1041 */ 1042 bp->phylink_config.default_an_inband = true; 1043 } 1044 1045 bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | 1046 MAC_10 | MAC_100; 1047 1048 __set_bit(PHY_INTERFACE_MODE_MII, 1049 bp->phylink_config.supported_interfaces); 1050 __set_bit(PHY_INTERFACE_MODE_RMII, 1051 bp->phylink_config.supported_interfaces); 1052 1053 /* Determine what modes are supported */ 1054 if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { 1055 bp->phylink_config.mac_capabilities |= MAC_1000FD; 1056 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) 1057 bp->phylink_config.mac_capabilities |= MAC_1000HD; 1058 1059 __set_bit(PHY_INTERFACE_MODE_GMII, 1060 bp->phylink_config.supported_interfaces); 1061 phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); 1062 1063 if (bp->caps & MACB_CAPS_PCS) 1064 __set_bit(PHY_INTERFACE_MODE_SGMII, 1065 bp->phylink_config.supported_interfaces); 1066 1067 if (bp->caps & MACB_CAPS_HIGH_SPEED) { 1068 __set_bit(PHY_INTERFACE_MODE_10GBASER, 1069 bp->phylink_config.supported_interfaces); 1070 bp->phylink_config.mac_capabilities |= MAC_10000FD; 1071 } 1072 } 1073 1074 /* Configure EEE LPI if supported */ 1075 if (bp->caps & MACB_CAPS_EEE) { 1076 __set_bit(PHY_INTERFACE_MODE_MII, 1077 bp->phylink_config.lpi_interfaces); 1078 __set_bit(PHY_INTERFACE_MODE_GMII, 1079 bp->phylink_config.lpi_interfaces); 1080 phy_interface_set_rgmii(bp->phylink_config.lpi_interfaces); 1081 bp->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD; 1082 bp->phylink_config.lpi_timer_default = 250000; 1083 bp->phylink_config.eee_enabled_default = true; 1084 } 1085 1086 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, 1087 bp->phy_interface, &macb_phylink_ops); 1088 if (IS_ERR(bp->phylink)) { 1089 netdev_err(netdev, "Could not create a phylink instance (%ld)\n", 1090 PTR_ERR(bp->phylink)); 1091 return PTR_ERR(bp->phylink); 1092 } 1093 1094 return 0; 1095 } 1096 1097 static int macb_mdiobus_register(struct macb *bp, struct device_node *mdio_np) 1098 { 1099 struct device_node *child, *np = bp->pdev->dev.of_node; 1100 1101 /* If we have a child named mdio, probe it instead of looking for PHYs 1102 * directly under the MAC node 1103 */ 1104 if (mdio_np) 1105 return of_mdiobus_register(bp->mii_bus, mdio_np); 1106 1107 /* Only create the PHY from the device tree if at least one PHY is 1108 * described. Otherwise scan the entire MDIO bus. We do this to support 1109 * old device tree that did not follow the best practices and did not 1110 * describe their network PHYs. 1111 */ 1112 for_each_available_child_of_node(np, child) 1113 if (of_mdiobus_child_is_phy(child)) { 1114 /* The loop increments the child refcount, 1115 * decrement it before returning. 1116 */ 1117 of_node_put(child); 1118 1119 return of_mdiobus_register(bp->mii_bus, np); 1120 } 1121 1122 return mdiobus_register(bp->mii_bus); 1123 } 1124 1125 static int macb_mii_init(struct macb *bp) 1126 { 1127 struct device_node *mdio_np, *np = bp->pdev->dev.of_node; 1128 int err = -ENXIO; 1129 1130 /* With fixed-link, we don't need to register the MDIO bus, 1131 * except if we have a child named "mdio" in the device tree. 1132 * In that case, some devices may be attached to the MACB's MDIO bus. 1133 */ 1134 mdio_np = of_get_child_by_name(np, "mdio"); 1135 if (!mdio_np && of_phy_is_fixed_link(np)) 1136 return macb_mii_probe(bp->netdev); 1137 1138 /* Enable management port */ 1139 macb_writel(bp, NCR, MACB_BIT(MPE)); 1140 1141 bp->mii_bus = mdiobus_alloc(); 1142 if (!bp->mii_bus) { 1143 err = -ENOMEM; 1144 goto err_out; 1145 } 1146 1147 bp->mii_bus->name = "MACB_mii_bus"; 1148 bp->mii_bus->read = &macb_mdio_read_c22; 1149 bp->mii_bus->write = &macb_mdio_write_c22; 1150 bp->mii_bus->read_c45 = &macb_mdio_read_c45; 1151 bp->mii_bus->write_c45 = &macb_mdio_write_c45; 1152 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 1153 bp->pdev->name, bp->pdev->id); 1154 bp->mii_bus->priv = bp; 1155 bp->mii_bus->parent = &bp->pdev->dev; 1156 1157 dev_set_drvdata(&bp->netdev->dev, bp->mii_bus); 1158 1159 err = macb_mdiobus_register(bp, mdio_np); 1160 if (err) 1161 goto err_out_free_mdiobus; 1162 1163 err = macb_mii_probe(bp->netdev); 1164 if (err) 1165 goto err_out_unregister_bus; 1166 1167 of_node_put(mdio_np); 1168 1169 return 0; 1170 1171 err_out_unregister_bus: 1172 mdiobus_unregister(bp->mii_bus); 1173 err_out_free_mdiobus: 1174 mdiobus_free(bp->mii_bus); 1175 err_out: 1176 of_node_put(mdio_np); 1177 1178 return err; 1179 } 1180 1181 static void macb_update_stats(struct macb *bp) 1182 { 1183 u64 *p = &bp->hw_stats.macb.rx_pause_frames; 1184 u64 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 1185 int offset = MACB_PFR; 1186 1187 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 1188 1189 for (; p < end; p++, offset += 4) 1190 *p += bp->macb_reg_readl(bp, offset); 1191 } 1192 1193 static int macb_halt_tx(struct macb *bp) 1194 { 1195 u32 status; 1196 1197 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 1198 1199 /* Poll TSR until TGO is cleared or timeout. */ 1200 return read_poll_timeout_atomic(macb_readl, status, 1201 !(status & MACB_BIT(TGO)), 1202 250, MACB_HALT_TIMEOUT, false, 1203 bp, TSR); 1204 } 1205 1206 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget) 1207 { 1208 if (tx_skb->mapping) { 1209 if (tx_skb->mapped_as_page) 1210 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 1211 tx_skb->size, DMA_TO_DEVICE); 1212 else 1213 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 1214 tx_skb->size, DMA_TO_DEVICE); 1215 tx_skb->mapping = 0; 1216 } 1217 1218 if (tx_skb->skb) { 1219 dev_consume_skb_any(tx_skb->skb); 1220 tx_skb->skb = NULL; 1221 } 1222 } 1223 1224 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 1225 { 1226 if (macb_dma64(bp)) { 1227 struct macb_dma_desc_64 *desc_64; 1228 1229 desc_64 = macb_64b_desc(bp, desc); 1230 desc_64->addrh = upper_32_bits(addr); 1231 /* The low bits of RX address contain the RX_USED bit, clearing 1232 * of which allows packet RX. Make sure the high bits are also 1233 * visible to HW at that point. 1234 */ 1235 dma_wmb(); 1236 } 1237 1238 desc->addr = lower_32_bits(addr); 1239 } 1240 1241 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 1242 { 1243 dma_addr_t addr = 0; 1244 1245 if (macb_dma64(bp)) { 1246 struct macb_dma_desc_64 *desc_64; 1247 1248 desc_64 = macb_64b_desc(bp, desc); 1249 addr = ((u64)(desc_64->addrh) << 32); 1250 } 1251 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 1252 if (macb_dma_ptp(bp)) 1253 addr &= ~GEM_BIT(DMA_RXVALID); 1254 return addr; 1255 } 1256 1257 static void macb_tx_error_task(struct work_struct *work) 1258 { 1259 struct macb_queue *queue = container_of(work, struct macb_queue, 1260 tx_error_task); 1261 unsigned int q = queue - queue->bp->queues; 1262 struct macb *bp = queue->bp; 1263 struct macb_tx_skb *tx_skb; 1264 struct macb_dma_desc *desc; 1265 bool halt_timeout = false; 1266 struct sk_buff *skb; 1267 unsigned long flags; 1268 unsigned int tail; 1269 u32 packets = 0; 1270 u32 bytes = 0; 1271 1272 netdev_vdbg(bp->netdev, "%s: q = %u, t = %u, h = %u\n", 1273 __func__, q, queue->tx_tail, queue->tx_head); 1274 1275 /* Prevent the queue NAPI TX poll from running, as it calls 1276 * macb_tx_complete(), which in turn may call netif_wake_subqueue(). 1277 * As explained below, we have to halt the transmission before updating 1278 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 1279 * network engine about the macb/gem being halted. 1280 */ 1281 napi_disable(&queue->napi_tx); 1282 spin_lock_irqsave(&bp->lock, flags); 1283 1284 /* Make sure nobody is trying to queue up new packets */ 1285 netif_tx_stop_all_queues(bp->netdev); 1286 1287 /* Stop transmission now 1288 * (in case we have just queued new packets) 1289 * macb/gem must be halted to write TBQP register 1290 */ 1291 if (macb_halt_tx(bp)) { 1292 netdev_err(bp->netdev, "BUG: halt tx timed out\n"); 1293 macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE))); 1294 halt_timeout = true; 1295 } 1296 1297 /* Treat frames in TX queue including the ones that caused the error. 1298 * Free transmit buffers in upper layer. 1299 */ 1300 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 1301 u32 ctrl; 1302 1303 desc = macb_tx_desc(queue, tail); 1304 ctrl = desc->ctrl; 1305 tx_skb = macb_tx_skb(queue, tail); 1306 skb = tx_skb->skb; 1307 1308 if (ctrl & MACB_BIT(TX_USED)) { 1309 /* skb is set for the last buffer of the frame */ 1310 while (!skb) { 1311 macb_tx_unmap(bp, tx_skb, 0); 1312 tail++; 1313 tx_skb = macb_tx_skb(queue, tail); 1314 skb = tx_skb->skb; 1315 } 1316 1317 /* ctrl still refers to the first buffer descriptor 1318 * since it's the only one written back by the hardware 1319 */ 1320 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 1321 netdev_vdbg(bp->netdev, "txerr skb %u (data %p) TX complete\n", 1322 macb_tx_ring_wrap(bp, tail), 1323 skb->data); 1324 bp->netdev->stats.tx_packets++; 1325 queue->stats.tx_packets++; 1326 packets++; 1327 bp->netdev->stats.tx_bytes += skb->len - tx_skb->fcs_len; 1328 queue->stats.tx_bytes += skb->len - tx_skb->fcs_len; 1329 bytes += skb->len; 1330 } 1331 } else { 1332 /* "Buffers exhausted mid-frame" errors may only happen 1333 * if the driver is buggy, so complain loudly about 1334 * those. Statistics are updated by hardware. 1335 */ 1336 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 1337 netdev_err(bp->netdev, 1338 "BUG: TX buffers exhausted mid-frame\n"); 1339 1340 desc->ctrl = ctrl | MACB_BIT(TX_USED); 1341 } 1342 1343 macb_tx_unmap(bp, tx_skb, 0); 1344 } 1345 1346 netdev_tx_completed_queue(netdev_get_tx_queue(bp->netdev, q), 1347 packets, bytes); 1348 1349 /* Set end of TX queue */ 1350 desc = macb_tx_desc(queue, 0); 1351 macb_set_addr(bp, desc, 0); 1352 desc->ctrl = MACB_BIT(TX_USED); 1353 1354 /* Make descriptor updates visible to hardware */ 1355 wmb(); 1356 1357 /* Reinitialize the TX desc queue */ 1358 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1359 /* Make TX ring reflect state of hardware */ 1360 queue->tx_head = 0; 1361 queue->tx_tail = 0; 1362 1363 /* Housework before enabling TX IRQ */ 1364 macb_writel(bp, TSR, macb_readl(bp, TSR)); 1365 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 1366 1367 if (halt_timeout) 1368 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE)); 1369 1370 /* Now we are ready to start transmission again */ 1371 netif_tx_start_all_queues(bp->netdev); 1372 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1373 1374 spin_unlock_irqrestore(&bp->lock, flags); 1375 napi_enable(&queue->napi_tx); 1376 } 1377 1378 static bool ptp_one_step_sync(struct sk_buff *skb) 1379 { 1380 struct ptp_header *hdr; 1381 unsigned int ptp_class; 1382 u8 msgtype; 1383 1384 /* No need to parse packet if PTP TS is not involved */ 1385 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) 1386 goto not_oss; 1387 1388 /* Identify and return whether PTP one step sync is being processed */ 1389 ptp_class = ptp_classify_raw(skb); 1390 if (ptp_class == PTP_CLASS_NONE) 1391 goto not_oss; 1392 1393 hdr = ptp_parse_header(skb, ptp_class); 1394 if (!hdr) 1395 goto not_oss; 1396 1397 if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP) 1398 goto not_oss; 1399 1400 msgtype = ptp_get_msgtype(hdr, ptp_class); 1401 if (msgtype == PTP_MSGTYPE_SYNC) 1402 return true; 1403 1404 not_oss: 1405 return false; 1406 } 1407 1408 static int macb_tx_complete(struct macb_queue *queue, int budget) 1409 { 1410 struct macb *bp = queue->bp; 1411 unsigned int q = queue - bp->queues; 1412 unsigned long flags; 1413 unsigned int tail; 1414 unsigned int head; 1415 int packets = 0; 1416 u32 bytes = 0; 1417 1418 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 1419 head = queue->tx_head; 1420 for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { 1421 struct macb_tx_skb *tx_skb; 1422 struct sk_buff *skb; 1423 struct macb_dma_desc *desc; 1424 u32 ctrl; 1425 1426 desc = macb_tx_desc(queue, tail); 1427 1428 /* Make hw descriptor updates visible to CPU */ 1429 rmb(); 1430 1431 ctrl = desc->ctrl; 1432 1433 /* TX_USED bit is only set by hardware on the very first buffer 1434 * descriptor of the transmitted frame. 1435 */ 1436 if (!(ctrl & MACB_BIT(TX_USED))) 1437 break; 1438 1439 /* Process all buffers of the current transmitted frame */ 1440 for (;; tail++) { 1441 tx_skb = macb_tx_skb(queue, tail); 1442 skb = tx_skb->skb; 1443 1444 /* First, update TX stats if needed */ 1445 if (skb) { 1446 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 1447 !ptp_one_step_sync(skb)) 1448 gem_ptp_do_txstamp(bp, skb, desc); 1449 1450 netdev_vdbg(bp->netdev, "skb %u (data %p) TX complete\n", 1451 macb_tx_ring_wrap(bp, tail), 1452 skb->data); 1453 bp->netdev->stats.tx_packets++; 1454 queue->stats.tx_packets++; 1455 bp->netdev->stats.tx_bytes += skb->len - tx_skb->fcs_len; 1456 queue->stats.tx_bytes += skb->len - tx_skb->fcs_len; 1457 packets++; 1458 bytes += skb->len; 1459 } 1460 1461 /* Now we can safely release resources */ 1462 macb_tx_unmap(bp, tx_skb, budget); 1463 1464 /* skb is set only for the last buffer of the frame. 1465 * WARNING: at this point skb has been freed by 1466 * macb_tx_unmap(). 1467 */ 1468 if (skb) 1469 break; 1470 } 1471 } 1472 1473 netdev_tx_completed_queue(netdev_get_tx_queue(bp->netdev, q), 1474 packets, bytes); 1475 1476 queue->tx_tail = tail; 1477 if (__netif_subqueue_stopped(bp->netdev, q) && 1478 CIRC_CNT(queue->tx_head, queue->tx_tail, 1479 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 1480 netif_wake_subqueue(bp->netdev, q); 1481 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 1482 1483 if (packets) 1484 macb_tx_lpi_schedule(bp); 1485 1486 return packets; 1487 } 1488 1489 static void gem_rx_refill(struct macb_queue *queue) 1490 { 1491 struct macb *bp = queue->bp; 1492 struct macb_dma_desc *desc; 1493 struct sk_buff *skb; 1494 unsigned int entry; 1495 dma_addr_t paddr; 1496 1497 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 1498 bp->rx_ring_size) > 0) { 1499 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 1500 1501 /* Make hw descriptor updates visible to CPU */ 1502 rmb(); 1503 1504 desc = macb_rx_desc(queue, entry); 1505 1506 if (!queue->rx_skbuff[entry]) { 1507 /* allocate sk_buff for this free entry in ring */ 1508 skb = netdev_alloc_skb(bp->netdev, bp->rx_buffer_size); 1509 if (unlikely(!skb)) { 1510 netdev_err(bp->netdev, 1511 "Unable to allocate sk_buff\n"); 1512 break; 1513 } 1514 1515 /* now fill corresponding descriptor entry */ 1516 paddr = dma_map_single(&bp->pdev->dev, skb->data, 1517 bp->rx_buffer_size, 1518 DMA_FROM_DEVICE); 1519 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 1520 dev_kfree_skb(skb); 1521 break; 1522 } 1523 1524 queue->rx_skbuff[entry] = skb; 1525 1526 if (entry == bp->rx_ring_size - 1) 1527 paddr |= MACB_BIT(RX_WRAP); 1528 desc->ctrl = 0; 1529 /* Setting addr clears RX_USED and allows reception, 1530 * make sure ctrl is cleared first to avoid a race. 1531 */ 1532 dma_wmb(); 1533 macb_set_addr(bp, desc, paddr); 1534 1535 /* Properly align Ethernet header. 1536 * 1537 * Hardware can add dummy bytes if asked using the RBOF 1538 * field inside the NCFGR register. That feature isn't 1539 * available if hardware is RSC capable. 1540 * 1541 * We cannot fallback to doing the 2-byte shift before 1542 * DMA mapping because the address field does not allow 1543 * setting the low 2/3 bits. 1544 * It is 3 bits if HW_DMA_CAP_PTP, else 2 bits. 1545 */ 1546 if (!(bp->caps & MACB_CAPS_RSC)) 1547 skb_reserve(skb, NET_IP_ALIGN); 1548 } else { 1549 desc->ctrl = 0; 1550 dma_wmb(); 1551 desc->addr &= ~MACB_BIT(RX_USED); 1552 } 1553 queue->rx_prepared_head++; 1554 } 1555 1556 /* Make descriptor updates visible to hardware */ 1557 wmb(); 1558 1559 netdev_vdbg(bp->netdev, "rx ring: queue: %p, prepared head %d, tail %d\n", 1560 queue, queue->rx_prepared_head, queue->rx_tail); 1561 } 1562 1563 /* Mark DMA descriptors from begin up to and not including end as unused */ 1564 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 1565 unsigned int end) 1566 { 1567 unsigned int frag; 1568 1569 for (frag = begin; frag != end; frag++) { 1570 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 1571 1572 desc->addr &= ~MACB_BIT(RX_USED); 1573 } 1574 1575 /* Make descriptor updates visible to hardware */ 1576 wmb(); 1577 1578 /* When this happens, the hardware stats registers for 1579 * whatever caused this is updated, so we don't have to record 1580 * anything. 1581 */ 1582 } 1583 1584 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 1585 int budget) 1586 { 1587 struct macb *bp = queue->bp; 1588 struct macb_dma_desc *desc; 1589 struct sk_buff *skb; 1590 unsigned int entry; 1591 unsigned int len; 1592 int count = 0; 1593 1594 while (count < budget) { 1595 u32 ctrl; 1596 dma_addr_t addr; 1597 bool rxused; 1598 1599 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1600 desc = macb_rx_desc(queue, entry); 1601 1602 /* Make hw descriptor updates visible to CPU */ 1603 rmb(); 1604 1605 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1606 addr = macb_get_addr(bp, desc); 1607 1608 if (!rxused) 1609 break; 1610 1611 /* Ensure ctrl is at least as up-to-date as rxused */ 1612 dma_rmb(); 1613 1614 ctrl = desc->ctrl; 1615 1616 queue->rx_tail++; 1617 count++; 1618 1619 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1620 netdev_err(bp->netdev, 1621 "not whole frame pointed by descriptor\n"); 1622 bp->netdev->stats.rx_dropped++; 1623 queue->stats.rx_dropped++; 1624 break; 1625 } 1626 skb = queue->rx_skbuff[entry]; 1627 if (unlikely(!skb)) { 1628 netdev_err(bp->netdev, 1629 "inconsistent Rx descriptor chain\n"); 1630 bp->netdev->stats.rx_dropped++; 1631 queue->stats.rx_dropped++; 1632 break; 1633 } 1634 /* now everything is ready for receiving packet */ 1635 queue->rx_skbuff[entry] = NULL; 1636 len = ctrl & bp->rx_frm_len_mask; 1637 1638 netdev_vdbg(bp->netdev, "%s %u (len %u)\n", 1639 __func__, entry, len); 1640 1641 skb_put(skb, len); 1642 dma_unmap_single(&bp->pdev->dev, addr, 1643 bp->rx_buffer_size, DMA_FROM_DEVICE); 1644 1645 skb->protocol = eth_type_trans(skb, bp->netdev); 1646 skb_checksum_none_assert(skb); 1647 if (bp->netdev->features & NETIF_F_RXCSUM && 1648 !(bp->netdev->flags & IFF_PROMISC) && 1649 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1650 skb->ip_summed = CHECKSUM_UNNECESSARY; 1651 1652 bp->netdev->stats.rx_packets++; 1653 queue->stats.rx_packets++; 1654 bp->netdev->stats.rx_bytes += skb->len; 1655 queue->stats.rx_bytes += skb->len; 1656 1657 gem_ptp_do_rxstamp(bp, skb, desc); 1658 1659 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1660 netdev_vdbg(bp->netdev, "received skb of length %u, csum: %08x\n", 1661 skb->len, skb->csum); 1662 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1663 skb_mac_header(skb), 16, true); 1664 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1665 skb->data, 32, true); 1666 #endif 1667 1668 napi_gro_receive(napi, skb); 1669 } 1670 1671 gem_rx_refill(queue); 1672 1673 return count; 1674 } 1675 1676 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1677 unsigned int first_frag, unsigned int last_frag) 1678 { 1679 struct macb *bp = queue->bp; 1680 struct macb_dma_desc *desc; 1681 unsigned int offset; 1682 struct sk_buff *skb; 1683 unsigned int frag; 1684 unsigned int len; 1685 1686 desc = macb_rx_desc(queue, last_frag); 1687 len = desc->ctrl & bp->rx_frm_len_mask; 1688 1689 netdev_vdbg(bp->netdev, "%s frags %u - %u (len %u)\n", 1690 __func__, 1691 macb_rx_ring_wrap(bp, first_frag), 1692 macb_rx_ring_wrap(bp, last_frag), len); 1693 1694 /* The ethernet header starts NET_IP_ALIGN bytes into the 1695 * first buffer. Since the header is 14 bytes, this makes the 1696 * payload word-aligned. 1697 * 1698 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1699 * the two padding bytes into the skb so that we avoid hitting 1700 * the slowpath in memcpy(), and pull them off afterwards. 1701 */ 1702 skb = netdev_alloc_skb(bp->netdev, len + NET_IP_ALIGN); 1703 if (!skb) { 1704 bp->netdev->stats.rx_dropped++; 1705 for (frag = first_frag; ; frag++) { 1706 desc = macb_rx_desc(queue, frag); 1707 desc->addr &= ~MACB_BIT(RX_USED); 1708 if (frag == last_frag) 1709 break; 1710 } 1711 1712 /* Make descriptor updates visible to hardware */ 1713 wmb(); 1714 1715 return 1; 1716 } 1717 1718 offset = 0; 1719 len += NET_IP_ALIGN; 1720 skb_checksum_none_assert(skb); 1721 skb_put(skb, len); 1722 1723 for (frag = first_frag; ; frag++) { 1724 unsigned int frag_len = bp->rx_buffer_size; 1725 1726 if (offset + frag_len > len) { 1727 if (unlikely(frag != last_frag)) { 1728 dev_kfree_skb_any(skb); 1729 return -1; 1730 } 1731 frag_len = len - offset; 1732 } 1733 skb_copy_to_linear_data_offset(skb, offset, 1734 macb_rx_buffer(queue, frag), 1735 frag_len); 1736 offset += bp->rx_buffer_size; 1737 desc = macb_rx_desc(queue, frag); 1738 desc->addr &= ~MACB_BIT(RX_USED); 1739 1740 if (frag == last_frag) 1741 break; 1742 } 1743 1744 /* Make descriptor updates visible to hardware */ 1745 wmb(); 1746 1747 __skb_pull(skb, NET_IP_ALIGN); 1748 skb->protocol = eth_type_trans(skb, bp->netdev); 1749 1750 bp->netdev->stats.rx_packets++; 1751 bp->netdev->stats.rx_bytes += skb->len; 1752 netdev_vdbg(bp->netdev, "received skb of length %u, csum: %08x\n", 1753 skb->len, skb->csum); 1754 napi_gro_receive(napi, skb); 1755 1756 return 0; 1757 } 1758 1759 static inline void macb_init_rx_ring(struct macb_queue *queue) 1760 { 1761 struct macb_dma_desc *desc = NULL; 1762 struct macb *bp = queue->bp; 1763 dma_addr_t addr; 1764 int i; 1765 1766 addr = queue->rx_buffers_dma; 1767 for (i = 0; i < bp->rx_ring_size; i++) { 1768 desc = macb_rx_desc(queue, i); 1769 macb_set_addr(bp, desc, addr); 1770 desc->ctrl = 0; 1771 addr += bp->rx_buffer_size; 1772 } 1773 desc->addr |= MACB_BIT(RX_WRAP); 1774 queue->rx_tail = 0; 1775 } 1776 1777 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1778 int budget) 1779 { 1780 struct macb *bp = queue->bp; 1781 bool reset_rx_queue = false; 1782 int first_frag = -1; 1783 unsigned int tail; 1784 int received = 0; 1785 1786 for (tail = queue->rx_tail; budget > 0; tail++) { 1787 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1788 u32 ctrl; 1789 1790 /* Make hw descriptor updates visible to CPU */ 1791 rmb(); 1792 1793 if (!(desc->addr & MACB_BIT(RX_USED))) 1794 break; 1795 1796 /* Ensure ctrl is at least as up-to-date as addr */ 1797 dma_rmb(); 1798 1799 ctrl = desc->ctrl; 1800 1801 if (ctrl & MACB_BIT(RX_SOF)) { 1802 if (first_frag != -1) 1803 discard_partial_frame(queue, first_frag, tail); 1804 first_frag = tail; 1805 } 1806 1807 if (ctrl & MACB_BIT(RX_EOF)) { 1808 int dropped; 1809 1810 if (unlikely(first_frag == -1)) { 1811 reset_rx_queue = true; 1812 continue; 1813 } 1814 1815 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1816 first_frag = -1; 1817 if (unlikely(dropped < 0)) { 1818 reset_rx_queue = true; 1819 continue; 1820 } 1821 if (!dropped) { 1822 received++; 1823 budget--; 1824 } 1825 } 1826 } 1827 1828 if (unlikely(reset_rx_queue)) { 1829 unsigned long flags; 1830 u32 ctrl; 1831 1832 netdev_err(bp->netdev, "RX queue corruption: reset it\n"); 1833 1834 spin_lock_irqsave(&bp->lock, flags); 1835 1836 ctrl = macb_readl(bp, NCR); 1837 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1838 1839 macb_init_rx_ring(queue); 1840 queue_writel(queue, RBQP, queue->rx_ring_dma); 1841 1842 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1843 1844 spin_unlock_irqrestore(&bp->lock, flags); 1845 return received; 1846 } 1847 1848 if (first_frag != -1) 1849 queue->rx_tail = first_frag; 1850 else 1851 queue->rx_tail = tail; 1852 1853 return received; 1854 } 1855 1856 static bool macb_rx_pending(struct macb_queue *queue) 1857 { 1858 struct macb *bp = queue->bp; 1859 struct macb_dma_desc *desc; 1860 unsigned int entry; 1861 1862 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1863 desc = macb_rx_desc(queue, entry); 1864 1865 /* Make hw descriptor updates visible to CPU */ 1866 rmb(); 1867 1868 return (desc->addr & MACB_BIT(RX_USED)) != 0; 1869 } 1870 1871 static int macb_rx_poll(struct napi_struct *napi, int budget) 1872 { 1873 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx); 1874 struct macb *bp = queue->bp; 1875 int work_done; 1876 1877 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 1878 1879 netdev_vdbg(bp->netdev, "RX poll: queue = %u, work_done = %d, budget = %d\n", 1880 (unsigned int)(queue - bp->queues), work_done, budget); 1881 1882 if (work_done < budget && napi_complete_done(napi, work_done)) { 1883 queue_writel(queue, IER, bp->rx_intr_mask); 1884 1885 /* Packet completions only seem to propagate to raise 1886 * interrupts when interrupts are enabled at the time, so if 1887 * packets were received while interrupts were disabled, 1888 * they will not cause another interrupt to be generated when 1889 * interrupts are re-enabled. 1890 * Check for this case here to avoid losing a wakeup. This can 1891 * potentially race with the interrupt handler doing the same 1892 * actions if an interrupt is raised just after enabling them, 1893 * but this should be harmless. 1894 */ 1895 if (macb_rx_pending(queue)) { 1896 queue_writel(queue, IDR, bp->rx_intr_mask); 1897 macb_queue_isr_clear(bp, queue, MACB_BIT(RCOMP)); 1898 netdev_vdbg(bp->netdev, "poll: packets pending, reschedule\n"); 1899 napi_schedule(napi); 1900 } 1901 } 1902 1903 /* TODO: Handle errors */ 1904 1905 return work_done; 1906 } 1907 1908 static void macb_tx_restart(struct macb_queue *queue) 1909 { 1910 struct macb *bp = queue->bp; 1911 unsigned int head_idx, tbqp; 1912 unsigned long flags; 1913 1914 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 1915 1916 if (queue->tx_head == queue->tx_tail) 1917 goto out_tx_ptr_unlock; 1918 1919 tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp); 1920 tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp)); 1921 head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head)); 1922 1923 if (tbqp == head_idx) 1924 goto out_tx_ptr_unlock; 1925 1926 spin_lock(&bp->lock); 1927 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1928 spin_unlock(&bp->lock); 1929 1930 out_tx_ptr_unlock: 1931 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 1932 } 1933 1934 static bool macb_tx_complete_pending(struct macb_queue *queue) 1935 { 1936 bool retval = false; 1937 unsigned long flags; 1938 1939 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 1940 if (queue->tx_head != queue->tx_tail) { 1941 /* Make hw descriptor updates visible to CPU */ 1942 rmb(); 1943 1944 if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED)) 1945 retval = true; 1946 } 1947 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 1948 return retval; 1949 } 1950 1951 static int macb_tx_poll(struct napi_struct *napi, int budget) 1952 { 1953 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx); 1954 struct macb *bp = queue->bp; 1955 int work_done; 1956 1957 work_done = macb_tx_complete(queue, budget); 1958 1959 rmb(); // ensure txubr_pending is up to date 1960 if (queue->txubr_pending) { 1961 queue->txubr_pending = false; 1962 netdev_vdbg(bp->netdev, "poll: tx restart\n"); 1963 macb_tx_restart(queue); 1964 } 1965 1966 netdev_vdbg(bp->netdev, "TX poll: queue = %u, work_done = %d, budget = %d\n", 1967 (unsigned int)(queue - bp->queues), work_done, budget); 1968 1969 if (work_done < budget && napi_complete_done(napi, work_done)) { 1970 queue_writel(queue, IER, MACB_BIT(TCOMP)); 1971 1972 /* Packet completions only seem to propagate to raise 1973 * interrupts when interrupts are enabled at the time, so if 1974 * packets were sent while interrupts were disabled, 1975 * they will not cause another interrupt to be generated when 1976 * interrupts are re-enabled. 1977 * Check for this case here to avoid losing a wakeup. This can 1978 * potentially race with the interrupt handler doing the same 1979 * actions if an interrupt is raised just after enabling them, 1980 * but this should be harmless. 1981 */ 1982 if (macb_tx_complete_pending(queue)) { 1983 queue_writel(queue, IDR, MACB_BIT(TCOMP)); 1984 macb_queue_isr_clear(bp, queue, MACB_BIT(TCOMP)); 1985 netdev_vdbg(bp->netdev, "TX poll: packets pending, reschedule\n"); 1986 napi_schedule(napi); 1987 } 1988 } 1989 1990 return work_done; 1991 } 1992 1993 static void macb_hresp_error_task(struct work_struct *work) 1994 { 1995 struct macb *bp = from_work(bp, work, hresp_err_bh_work); 1996 struct net_device *netdev = bp->netdev; 1997 struct macb_queue *queue; 1998 unsigned int q; 1999 u32 ctrl; 2000 2001 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2002 queue_writel(queue, IDR, bp->rx_intr_mask | 2003 MACB_TX_INT_FLAGS | 2004 MACB_BIT(HRESP)); 2005 } 2006 ctrl = macb_readl(bp, NCR); 2007 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2008 macb_writel(bp, NCR, ctrl); 2009 2010 netif_tx_stop_all_queues(netdev); 2011 netif_carrier_off(netdev); 2012 2013 bp->macbgem_ops.mog_init_rings(bp); 2014 2015 /* Initialize TX and RX buffers */ 2016 macb_init_buffers(bp); 2017 2018 /* Enable interrupts */ 2019 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2020 queue_writel(queue, IER, 2021 bp->rx_intr_mask | 2022 MACB_TX_INT_FLAGS | 2023 MACB_BIT(HRESP)); 2024 2025 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 2026 macb_writel(bp, NCR, ctrl); 2027 2028 netif_carrier_on(netdev); 2029 netif_tx_start_all_queues(netdev); 2030 } 2031 2032 static void macb_wol_interrupt(struct macb_queue *queue, u32 status) 2033 { 2034 struct macb *bp = queue->bp; 2035 2036 queue_writel(queue, IDR, MACB_BIT(WOL)); 2037 macb_writel(bp, WOL, 0); 2038 netdev_vdbg(bp->netdev, "MACB WoL: queue = %u, isr = 0x%08lx\n", 2039 (unsigned int)(queue - bp->queues), 2040 (unsigned long)status); 2041 macb_queue_isr_clear(bp, queue, MACB_BIT(WOL)); 2042 pm_wakeup_event(&bp->pdev->dev, 0); 2043 } 2044 2045 static void gem_wol_interrupt(struct macb_queue *queue, u32 status) 2046 { 2047 struct macb *bp = queue->bp; 2048 2049 queue_writel(queue, IDR, GEM_BIT(WOL)); 2050 gem_writel(bp, WOL, 0); 2051 netdev_vdbg(bp->netdev, "GEM WoL: queue = %u, isr = 0x%08lx\n", 2052 (unsigned int)(queue - bp->queues), 2053 (unsigned long)status); 2054 macb_queue_isr_clear(bp, queue, GEM_BIT(WOL)); 2055 pm_wakeup_event(&bp->pdev->dev, 0); 2056 } 2057 2058 static int macb_interrupt_misc(struct macb_queue *queue, u32 status) 2059 { 2060 struct macb *bp = queue->bp; 2061 struct net_device *netdev; 2062 u32 ctrl; 2063 2064 netdev = bp->netdev; 2065 2066 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 2067 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 2068 schedule_work(&queue->tx_error_task); 2069 macb_queue_isr_clear(bp, queue, MACB_TX_ERR_FLAGS); 2070 return -1; 2071 } 2072 2073 /* Link change detection isn't possible with RMII, so we'll 2074 * add that if/when we get our hands on a full-blown MII PHY. 2075 */ 2076 2077 /* There is a hardware issue under heavy load where DMA can 2078 * stop, this causes endless "used buffer descriptor read" 2079 * interrupts but it can be cleared by re-enabling RX. See 2080 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 2081 * section 16.7.4 for details. RXUBR is only enabled for 2082 * these two versions. 2083 */ 2084 if (status & MACB_BIT(RXUBR)) { 2085 ctrl = macb_readl(bp, NCR); 2086 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 2087 wmb(); 2088 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 2089 macb_queue_isr_clear(bp, queue, MACB_BIT(RXUBR)); 2090 } 2091 2092 if (status & MACB_BIT(ISR_ROVR)) { 2093 /* We missed at least one packet */ 2094 spin_lock(&bp->stats_lock); 2095 if (macb_is_gem(bp)) 2096 bp->hw_stats.gem.rx_overruns++; 2097 else 2098 bp->hw_stats.macb.rx_overruns++; 2099 spin_unlock(&bp->stats_lock); 2100 macb_queue_isr_clear(bp, queue, MACB_BIT(ISR_ROVR)); 2101 } 2102 2103 if (status & MACB_BIT(HRESP)) { 2104 queue_work(system_bh_wq, &bp->hresp_err_bh_work); 2105 netdev_err(netdev, "DMA bus error: HRESP not OK\n"); 2106 macb_queue_isr_clear(bp, queue, MACB_BIT(HRESP)); 2107 } 2108 2109 if (macb_is_gem(bp)) { 2110 if (status & GEM_BIT(WOL)) 2111 gem_wol_interrupt(queue, status); 2112 } else { 2113 if (status & MACB_BIT(WOL)) 2114 macb_wol_interrupt(queue, status); 2115 } 2116 2117 return 0; 2118 } 2119 2120 static irqreturn_t macb_interrupt(int irq, void *dev_id) 2121 { 2122 struct macb_queue *queue = dev_id; 2123 struct macb *bp = queue->bp; 2124 struct net_device *netdev = bp->netdev; 2125 u32 status; 2126 2127 status = queue_readl(queue, ISR); 2128 2129 if (unlikely(!status)) 2130 return IRQ_NONE; 2131 2132 spin_lock(&bp->lock); 2133 2134 while (status) { 2135 /* close possible race with dev_close */ 2136 if (unlikely(!netif_running(netdev))) { 2137 queue_writel(queue, IDR, -1); 2138 macb_queue_isr_clear(bp, queue, -1); 2139 break; 2140 } 2141 2142 netdev_vdbg(netdev, "queue = %u, isr = 0x%08lx\n", 2143 (unsigned int)(queue - bp->queues), 2144 (unsigned long)status); 2145 2146 if (status & bp->rx_intr_mask) { 2147 /* There's no point taking any more interrupts 2148 * until we have processed the buffers. The 2149 * scheduling call may fail if the poll routine 2150 * is already scheduled, so disable interrupts 2151 * now. 2152 */ 2153 queue_writel(queue, IDR, bp->rx_intr_mask); 2154 macb_queue_isr_clear(bp, queue, MACB_BIT(RCOMP)); 2155 napi_schedule_irqoff(&queue->napi_rx); 2156 } 2157 2158 if (status & (MACB_BIT(TCOMP) | 2159 MACB_BIT(TXUBR))) { 2160 queue_writel(queue, IDR, MACB_BIT(TCOMP)); 2161 macb_queue_isr_clear(bp, queue, MACB_BIT(TCOMP) | 2162 MACB_BIT(TXUBR)); 2163 if (status & MACB_BIT(TXUBR)) { 2164 queue->txubr_pending = true; 2165 wmb(); // ensure softirq can see update 2166 } 2167 2168 napi_schedule_irqoff(&queue->napi_tx); 2169 } 2170 2171 if (unlikely(status & MACB_INT_MISC_FLAGS)) 2172 if (macb_interrupt_misc(queue, status)) 2173 break; 2174 2175 status = queue_readl(queue, ISR); 2176 } 2177 2178 spin_unlock(&bp->lock); 2179 2180 return IRQ_HANDLED; 2181 } 2182 2183 #ifdef CONFIG_NET_POLL_CONTROLLER 2184 /* Polling receive - used by netconsole and other diagnostic tools 2185 * to allow network i/o with interrupts disabled. 2186 */ 2187 static void macb_poll_controller(struct net_device *netdev) 2188 { 2189 struct macb *bp = netdev_priv(netdev); 2190 struct macb_queue *queue; 2191 unsigned long flags; 2192 unsigned int q; 2193 2194 local_irq_save(flags); 2195 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2196 macb_interrupt(netdev->irq, queue); 2197 local_irq_restore(flags); 2198 } 2199 #endif 2200 2201 static unsigned int macb_tx_map(struct macb *bp, 2202 struct macb_queue *queue, 2203 struct sk_buff *skb, 2204 unsigned int hdrlen, 2205 u8 fcs_len) 2206 { 2207 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 2208 unsigned int len, i, tx_head = queue->tx_head; 2209 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 2210 unsigned int eof = 1, mss_mfs = 0; 2211 struct macb_tx_skb *tx_skb = NULL; 2212 struct macb_dma_desc *desc; 2213 unsigned int offset, size; 2214 dma_addr_t mapping; 2215 2216 /* LSO */ 2217 if (skb_shinfo(skb)->gso_size != 0) { 2218 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2219 /* UDP - UFO */ 2220 lso_ctrl = MACB_LSO_UFO_ENABLE; 2221 else 2222 /* TCP - TSO */ 2223 lso_ctrl = MACB_LSO_TSO_ENABLE; 2224 } 2225 2226 /* First, map non-paged data */ 2227 len = skb_headlen(skb); 2228 2229 /* first buffer length */ 2230 size = hdrlen; 2231 2232 offset = 0; 2233 while (len) { 2234 tx_skb = macb_tx_skb(queue, tx_head); 2235 2236 mapping = dma_map_single(&bp->pdev->dev, 2237 skb->data + offset, 2238 size, DMA_TO_DEVICE); 2239 if (dma_mapping_error(&bp->pdev->dev, mapping)) 2240 goto dma_error; 2241 2242 /* Save info to properly release resources */ 2243 tx_skb->skb = NULL; 2244 tx_skb->mapping = mapping; 2245 tx_skb->size = size; 2246 tx_skb->mapped_as_page = false; 2247 2248 len -= size; 2249 offset += size; 2250 tx_head++; 2251 2252 size = umin(len, bp->max_tx_length); 2253 } 2254 2255 /* Then, map paged data from fragments */ 2256 for (f = 0; f < nr_frags; f++) { 2257 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2258 2259 len = skb_frag_size(frag); 2260 offset = 0; 2261 while (len) { 2262 size = umin(len, bp->max_tx_length); 2263 tx_skb = macb_tx_skb(queue, tx_head); 2264 2265 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 2266 offset, size, DMA_TO_DEVICE); 2267 if (dma_mapping_error(&bp->pdev->dev, mapping)) 2268 goto dma_error; 2269 2270 /* Save info to properly release resources */ 2271 tx_skb->skb = NULL; 2272 tx_skb->mapping = mapping; 2273 tx_skb->size = size; 2274 tx_skb->mapped_as_page = true; 2275 2276 len -= size; 2277 offset += size; 2278 tx_head++; 2279 } 2280 } 2281 2282 /* Should never happen */ 2283 if (unlikely(!tx_skb)) { 2284 netdev_err(bp->netdev, "BUG! empty skb!\n"); 2285 return 0; 2286 } 2287 2288 /* This is the last buffer of the frame: save socket buffer */ 2289 tx_skb->skb = skb; 2290 tx_skb->fcs_len = fcs_len; 2291 2292 /* Update TX ring: update buffer descriptors in reverse order 2293 * to avoid race condition 2294 */ 2295 2296 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 2297 * to set the end of TX queue 2298 */ 2299 i = tx_head; 2300 ctrl = MACB_BIT(TX_USED); 2301 desc = macb_tx_desc(queue, i); 2302 desc->ctrl = ctrl; 2303 2304 if (lso_ctrl) { 2305 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 2306 /* include header and FCS in value given to h/w */ 2307 mss_mfs = skb_shinfo(skb)->gso_size + 2308 skb_transport_offset(skb) + 2309 ETH_FCS_LEN; 2310 else /* TSO */ { 2311 mss_mfs = skb_shinfo(skb)->gso_size; 2312 /* TCP Sequence Number Source Select 2313 * can be set only for TSO 2314 */ 2315 seq_ctrl = 0; 2316 } 2317 } 2318 2319 do { 2320 i--; 2321 tx_skb = macb_tx_skb(queue, i); 2322 desc = macb_tx_desc(queue, i); 2323 2324 ctrl = (u32)tx_skb->size; 2325 if (eof) { 2326 ctrl |= MACB_BIT(TX_LAST); 2327 eof = 0; 2328 } 2329 if (unlikely(macb_tx_ring_wrap(bp, i) == bp->tx_ring_size - 1)) 2330 ctrl |= MACB_BIT(TX_WRAP); 2331 2332 /* First descriptor is header descriptor */ 2333 if (i == queue->tx_head) { 2334 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 2335 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 2336 if ((bp->netdev->features & NETIF_F_HW_CSUM) && 2337 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && 2338 !ptp_one_step_sync(skb)) 2339 ctrl |= MACB_BIT(TX_NOCRC); 2340 } else 2341 /* Only set MSS/MFS on payload descriptors 2342 * (second or later descriptor) 2343 */ 2344 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 2345 2346 /* Set TX buffer descriptor */ 2347 macb_set_addr(bp, desc, tx_skb->mapping); 2348 /* desc->addr must be visible to hardware before clearing 2349 * 'TX_USED' bit in desc->ctrl. 2350 */ 2351 wmb(); 2352 desc->ctrl = ctrl; 2353 } while (i != queue->tx_head); 2354 2355 queue->tx_head = tx_head; 2356 2357 return 0; 2358 2359 dma_error: 2360 netdev_err(bp->netdev, "TX DMA map failed\n"); 2361 2362 for (i = queue->tx_head; i != tx_head; i++) { 2363 tx_skb = macb_tx_skb(queue, i); 2364 2365 macb_tx_unmap(bp, tx_skb, 0); 2366 } 2367 2368 return -ENOMEM; 2369 } 2370 2371 static netdev_features_t macb_features_check(struct sk_buff *skb, 2372 struct net_device *netdev, 2373 netdev_features_t features) 2374 { 2375 unsigned int nr_frags, f; 2376 unsigned int hdrlen; 2377 2378 /* Validate LSO compatibility */ 2379 2380 /* there is only one buffer or protocol is not UDP */ 2381 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) 2382 return features; 2383 2384 /* length of header */ 2385 hdrlen = skb_transport_offset(skb); 2386 2387 /* For UFO only: 2388 * When software supplies two or more payload buffers all payload buffers 2389 * apart from the last must be a multiple of 8 bytes in size. 2390 */ 2391 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 2392 return features & ~MACB_NETIF_LSO; 2393 2394 nr_frags = skb_shinfo(skb)->nr_frags; 2395 /* No need to check last fragment */ 2396 nr_frags--; 2397 for (f = 0; f < nr_frags; f++) { 2398 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2399 2400 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 2401 return features & ~MACB_NETIF_LSO; 2402 } 2403 return features; 2404 } 2405 2406 static inline int macb_clear_csum(struct sk_buff *skb) 2407 { 2408 /* no change for packets without checksum offloading */ 2409 if (skb->ip_summed != CHECKSUM_PARTIAL) 2410 return 0; 2411 2412 /* make sure we can modify the header */ 2413 if (unlikely(skb_cow_head(skb, 0))) 2414 return -1; 2415 2416 /* initialize checksum field 2417 * This is required - at least for Zynq, which otherwise calculates 2418 * wrong UDP header checksums for UDP packets with UDP data len <=2 2419 */ 2420 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 2421 return 0; 2422 } 2423 2424 static bool macb_needs_sw_fcs(struct sk_buff *skb, struct net_device *netdev) 2425 { 2426 return netdev->features & NETIF_F_HW_CSUM && 2427 skb->ip_summed != CHECKSUM_PARTIAL && 2428 !skb_shinfo(skb)->gso_size && !ptp_one_step_sync(skb); 2429 } 2430 2431 /* Returns a negative errno, or the FCS bytes appended (0 or ETH_FCS_LEN). */ 2432 static int macb_pad_and_fcs(struct sk_buff **skb, bool add_fcs) 2433 { 2434 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || 2435 skb_is_nonlinear(*skb); 2436 int padlen = ETH_ZLEN - (*skb)->len; 2437 int tailroom = skb_tailroom(*skb); 2438 bool shared = skb_shared(*skb); 2439 struct sk_buff *nskb; 2440 u32 fcs; 2441 2442 if (!add_fcs) 2443 return 0; 2444 2445 if (padlen <= 0) { 2446 /* FCS could be appended to tailroom. */ 2447 if (!shared && !skb_is_nonlinear(*skb) && 2448 tailroom >= ETH_FCS_LEN) 2449 goto add_fcs; 2450 /* Reallocate with room for the FCS. */ 2451 padlen = ETH_FCS_LEN; 2452 } else { 2453 /* Add room for FCS. */ 2454 padlen += ETH_FCS_LEN; 2455 } 2456 2457 if (shared || cloned || tailroom < padlen) { 2458 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 2459 if (!nskb) 2460 return -ENOMEM; 2461 2462 dev_consume_skb_any(*skb); 2463 *skb = nskb; 2464 } 2465 2466 if (padlen > ETH_FCS_LEN) 2467 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 2468 2469 add_fcs: 2470 /* set FCS to packet */ 2471 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 2472 fcs = ~fcs; 2473 2474 skb_put_u8(*skb, fcs & 0xff); 2475 skb_put_u8(*skb, (fcs >> 8) & 0xff); 2476 skb_put_u8(*skb, (fcs >> 16) & 0xff); 2477 skb_put_u8(*skb, (fcs >> 24) & 0xff); 2478 2479 return ETH_FCS_LEN; 2480 } 2481 2482 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, 2483 struct net_device *netdev) 2484 { 2485 struct macb *bp = netdev_priv(netdev); 2486 unsigned int q = skb_get_queue_mapping(skb); 2487 unsigned int desc_cnt, nr_frags, frag_size, f; 2488 struct macb_queue *queue = &bp->queues[q]; 2489 netdev_tx_t ret = NETDEV_TX_OK; 2490 unsigned int hdrlen, tx_len; 2491 bool add_fcs, is_lso; 2492 unsigned long flags; 2493 int fcs_len; 2494 2495 add_fcs = macb_needs_sw_fcs(skb, netdev); 2496 is_lso = (skb_shinfo(skb)->gso_size != 0); 2497 tx_len = add_fcs ? max_t(unsigned int, skb->len, ETH_ZLEN) + 2498 ETH_FCS_LEN : skb->len; 2499 2500 if (is_lso) { 2501 /* length of headers */ 2502 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2503 /* only queue eth + ip headers separately for UDP */ 2504 hdrlen = skb_transport_offset(skb); 2505 else 2506 hdrlen = skb_tcp_all_headers(skb); 2507 if (skb_headlen(skb) < hdrlen) { 2508 netdev_err(bp->netdev, "Error - LSO headers fragmented!!!\n"); 2509 /* if this is required, would need to copy to single buffer */ 2510 return NETDEV_TX_BUSY; 2511 } 2512 } else if (add_fcs) { 2513 hdrlen = umin(tx_len, bp->max_tx_length); 2514 } else { 2515 hdrlen = umin(skb_headlen(skb), bp->max_tx_length); 2516 } 2517 2518 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 2519 netdev_vdbg(bp->netdev, 2520 "start_xmit: queue %u len %u head %p data %p tail %p end %p\n", 2521 q, skb->len, skb->head, skb->data, 2522 skb_tail_pointer(skb), skb_end_pointer(skb)); 2523 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 2524 skb->data, 16, true); 2525 #endif 2526 2527 /* Count how many TX buffer descriptors are needed to send this 2528 * socket buffer: skb fragments of jumbo frames may need to be 2529 * split into many buffer descriptors. 2530 */ 2531 if (add_fcs) { 2532 /* macb_pad_and_fcs() linearizes the skb before adding the FCS. */ 2533 desc_cnt = DIV_ROUND_UP(tx_len, bp->max_tx_length); 2534 nr_frags = 0; 2535 } else if (is_lso && (skb_headlen(skb) > hdrlen)) { 2536 /* extra header descriptor if also payload in first buffer */ 2537 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 2538 nr_frags = skb_shinfo(skb)->nr_frags; 2539 } else { 2540 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 2541 nr_frags = skb_shinfo(skb)->nr_frags; 2542 } 2543 for (f = 0; f < nr_frags; f++) { 2544 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 2545 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 2546 } 2547 2548 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 2549 2550 /* This is a hard error, log it. */ 2551 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 2552 bp->tx_ring_size) < desc_cnt) { 2553 netif_stop_subqueue(netdev, q); 2554 netdev_dbg(netdev, "tx_head = %u, tx_tail = %u\n", 2555 queue->tx_head, queue->tx_tail); 2556 ret = NETDEV_TX_BUSY; 2557 goto unlock; 2558 } 2559 2560 if (macb_clear_csum(skb)) { 2561 dev_kfree_skb_any(skb); 2562 goto unlock; 2563 } 2564 2565 fcs_len = macb_pad_and_fcs(&skb, add_fcs); 2566 if (fcs_len < 0) { 2567 dev_kfree_skb_any(skb); 2568 goto unlock; 2569 } 2570 2571 if (macb_dma_ptp(bp) && 2572 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 2573 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2574 2575 /* Map socket buffer for DMA transfer */ 2576 if (macb_tx_map(bp, queue, skb, hdrlen, fcs_len)) { 2577 dev_kfree_skb_any(skb); 2578 goto unlock; 2579 } 2580 2581 /* Make newly initialized descriptor visible to hardware */ 2582 wmb(); 2583 skb_tx_timestamp(skb); 2584 netdev_tx_sent_queue(netdev_get_tx_queue(bp->netdev, q), 2585 skb->len); 2586 2587 spin_lock(&bp->lock); 2588 macb_tx_lpi_wake(bp); 2589 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 2590 spin_unlock(&bp->lock); 2591 2592 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 2593 netif_stop_subqueue(netdev, q); 2594 2595 unlock: 2596 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 2597 2598 return ret; 2599 } 2600 2601 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 2602 { 2603 if (!macb_is_gem(bp)) { 2604 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 2605 } else { 2606 bp->rx_buffer_size = MIN(size, RX_BUFFER_MAX); 2607 2608 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 2609 netdev_dbg(bp->netdev, 2610 "RX buffer must be multiple of %d bytes, expanding\n", 2611 RX_BUFFER_MULTIPLE); 2612 bp->rx_buffer_size = 2613 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 2614 } 2615 } 2616 2617 netdev_dbg(bp->netdev, "mtu [%u] rx_buffer_size [%zu]\n", 2618 bp->netdev->mtu, bp->rx_buffer_size); 2619 } 2620 2621 static void gem_free_rx_buffers(struct macb *bp) 2622 { 2623 struct sk_buff *skb; 2624 struct macb_dma_desc *desc; 2625 struct macb_queue *queue; 2626 dma_addr_t addr; 2627 unsigned int q; 2628 int i; 2629 2630 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2631 if (!queue->rx_skbuff) 2632 continue; 2633 2634 for (i = 0; i < bp->rx_ring_size; i++) { 2635 skb = queue->rx_skbuff[i]; 2636 2637 if (!skb) 2638 continue; 2639 2640 desc = macb_rx_desc(queue, i); 2641 addr = macb_get_addr(bp, desc); 2642 2643 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 2644 DMA_FROM_DEVICE); 2645 dev_kfree_skb_any(skb); 2646 skb = NULL; 2647 } 2648 2649 kfree(queue->rx_skbuff); 2650 queue->rx_skbuff = NULL; 2651 } 2652 } 2653 2654 static void macb_free_rx_buffers(struct macb *bp) 2655 { 2656 struct macb_queue *queue = &bp->queues[0]; 2657 2658 if (queue->rx_buffers) { 2659 dma_free_coherent(&bp->pdev->dev, 2660 bp->rx_ring_size * bp->rx_buffer_size, 2661 queue->rx_buffers, queue->rx_buffers_dma); 2662 queue->rx_buffers = NULL; 2663 } 2664 } 2665 2666 static unsigned int macb_tx_ring_size_per_queue(struct macb *bp) 2667 { 2668 return macb_dma_desc_get_size(bp) * bp->tx_ring_size + bp->tx_bd_rd_prefetch; 2669 } 2670 2671 static unsigned int macb_rx_ring_size_per_queue(struct macb *bp) 2672 { 2673 return macb_dma_desc_get_size(bp) * bp->rx_ring_size + bp->rx_bd_rd_prefetch; 2674 } 2675 2676 static void macb_free(struct macb *bp) 2677 { 2678 struct device *dev = &bp->pdev->dev; 2679 struct macb_queue *queue; 2680 unsigned int q; 2681 size_t size; 2682 2683 bp->macbgem_ops.mog_free_rx_buffers(bp); 2684 2685 size = bp->num_queues * macb_tx_ring_size_per_queue(bp); 2686 dma_free_coherent(dev, size, bp->queues[0].tx_ring, bp->queues[0].tx_ring_dma); 2687 2688 size = bp->num_queues * macb_rx_ring_size_per_queue(bp); 2689 dma_free_coherent(dev, size, bp->queues[0].rx_ring, bp->queues[0].rx_ring_dma); 2690 2691 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2692 if (queue->tx_skb) { 2693 unsigned int dropped = 0, tail; 2694 2695 for (tail = queue->tx_tail; tail != queue->tx_head; 2696 tail++) { 2697 if (macb_tx_skb(queue, tail)->skb) 2698 dropped++; 2699 macb_tx_unmap(bp, macb_tx_skb(queue, tail), 0); 2700 } 2701 2702 queue->stats.tx_dropped += dropped; 2703 bp->netdev->stats.tx_dropped += dropped; 2704 2705 kfree(queue->tx_skb); 2706 queue->tx_skb = NULL; 2707 } 2708 2709 queue->tx_head = 0; 2710 queue->tx_tail = 0; 2711 queue->tx_ring = NULL; 2712 queue->rx_ring = NULL; 2713 } 2714 } 2715 2716 static int gem_alloc_rx_buffers(struct macb *bp) 2717 { 2718 struct macb_queue *queue; 2719 unsigned int q; 2720 int size; 2721 2722 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2723 size = bp->rx_ring_size * sizeof(struct sk_buff *); 2724 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 2725 if (!queue->rx_skbuff) 2726 return -ENOMEM; 2727 else 2728 netdev_dbg(bp->netdev, 2729 "Allocated %d RX struct sk_buff entries at %p\n", 2730 bp->rx_ring_size, queue->rx_skbuff); 2731 } 2732 return 0; 2733 } 2734 2735 static int macb_alloc_rx_buffers(struct macb *bp) 2736 { 2737 struct macb_queue *queue = &bp->queues[0]; 2738 int size; 2739 2740 size = bp->rx_ring_size * bp->rx_buffer_size; 2741 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 2742 &queue->rx_buffers_dma, GFP_KERNEL); 2743 if (!queue->rx_buffers) 2744 return -ENOMEM; 2745 2746 netdev_dbg(bp->netdev, 2747 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 2748 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 2749 return 0; 2750 } 2751 2752 static int macb_alloc(struct macb *bp) 2753 { 2754 struct device *dev = &bp->pdev->dev; 2755 dma_addr_t tx_dma, rx_dma; 2756 struct macb_queue *queue; 2757 unsigned int q; 2758 void *tx, *rx; 2759 size_t size; 2760 2761 /* 2762 * Upper 32-bits of Tx/Rx DMA descriptor for each queues much match! 2763 * We cannot enforce this guarantee, the best we can do is do a single 2764 * allocation and hope it will land into alloc_pages() that guarantees 2765 * natural alignment of physical addresses. 2766 */ 2767 2768 size = bp->num_queues * macb_tx_ring_size_per_queue(bp); 2769 tx = dma_alloc_coherent(dev, size, &tx_dma, GFP_KERNEL); 2770 if (!tx) 2771 goto out_err; 2772 /* Record the buffer so that the error path frees it. */ 2773 bp->queues[0].tx_ring = tx; 2774 bp->queues[0].tx_ring_dma = tx_dma; 2775 if (upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1)) 2776 goto out_err; 2777 netdev_dbg(bp->netdev, "Allocated %zu bytes for %u TX rings at %08lx (mapped %p)\n", 2778 size, bp->num_queues, (unsigned long)tx_dma, tx); 2779 2780 size = bp->num_queues * macb_rx_ring_size_per_queue(bp); 2781 rx = dma_alloc_coherent(dev, size, &rx_dma, GFP_KERNEL); 2782 if (!rx) 2783 goto out_err; 2784 /* Record the buffer so that the error path frees it. */ 2785 bp->queues[0].rx_ring = rx; 2786 bp->queues[0].rx_ring_dma = rx_dma; 2787 if (upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1)) 2788 goto out_err; 2789 netdev_dbg(bp->netdev, "Allocated %zu bytes for %u RX rings at %08lx (mapped %p)\n", 2790 size, bp->num_queues, (unsigned long)rx_dma, rx); 2791 2792 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2793 queue->tx_ring = tx + macb_tx_ring_size_per_queue(bp) * q; 2794 queue->tx_ring_dma = tx_dma + macb_tx_ring_size_per_queue(bp) * q; 2795 2796 queue->rx_ring = rx + macb_rx_ring_size_per_queue(bp) * q; 2797 queue->rx_ring_dma = rx_dma + macb_rx_ring_size_per_queue(bp) * q; 2798 2799 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2800 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2801 if (!queue->tx_skb) 2802 goto out_err; 2803 } 2804 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2805 goto out_err; 2806 2807 return 0; 2808 2809 out_err: 2810 macb_free(bp); 2811 return -ENOMEM; 2812 } 2813 2814 static void gem_init_rx_ring(struct macb_queue *queue) 2815 { 2816 queue->rx_tail = 0; 2817 queue->rx_prepared_head = 0; 2818 2819 gem_rx_refill(queue); 2820 } 2821 2822 static void gem_init_rings(struct macb *bp) 2823 { 2824 struct macb_queue *queue; 2825 struct macb_dma_desc *desc = NULL; 2826 unsigned int q; 2827 int i; 2828 2829 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2830 for (i = 0; i < bp->tx_ring_size; i++) { 2831 desc = macb_tx_desc(queue, i); 2832 macb_set_addr(bp, desc, 0); 2833 desc->ctrl = MACB_BIT(TX_USED); 2834 } 2835 desc->ctrl |= MACB_BIT(TX_WRAP); 2836 queue->tx_head = 0; 2837 queue->tx_tail = 0; 2838 2839 gem_init_rx_ring(queue); 2840 } 2841 } 2842 2843 static void macb_init_rings(struct macb *bp) 2844 { 2845 int i; 2846 struct macb_dma_desc *desc = NULL; 2847 2848 macb_init_rx_ring(&bp->queues[0]); 2849 2850 for (i = 0; i < bp->tx_ring_size; i++) { 2851 desc = macb_tx_desc(&bp->queues[0], i); 2852 macb_set_addr(bp, desc, 0); 2853 desc->ctrl = MACB_BIT(TX_USED); 2854 } 2855 bp->queues[0].tx_head = 0; 2856 bp->queues[0].tx_tail = 0; 2857 desc->ctrl |= MACB_BIT(TX_WRAP); 2858 } 2859 2860 static void macb_reset_hw(struct macb *bp) 2861 { 2862 struct macb_queue *queue; 2863 unsigned int q; 2864 u32 ctrl = macb_readl(bp, NCR); 2865 2866 /* Disable RX and TX (XXX: Should we halt the transmission 2867 * more gracefully?) 2868 */ 2869 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2870 2871 /* Clear the stats registers (XXX: Update stats first?) */ 2872 ctrl |= MACB_BIT(CLRSTAT); 2873 2874 macb_writel(bp, NCR, ctrl); 2875 2876 /* Clear all status flags */ 2877 macb_writel(bp, TSR, -1); 2878 macb_writel(bp, RSR, -1); 2879 2880 /* Disable RX partial store and forward and reset watermark value */ 2881 gem_writel(bp, PBUFRXCUT, 0); 2882 2883 /* Disable all interrupts */ 2884 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2885 queue_writel(queue, IDR, -1); 2886 queue_readl(queue, ISR); 2887 macb_queue_isr_clear(bp, queue, -1); 2888 } 2889 } 2890 2891 static u32 gem_mdc_clk_div(struct macb *bp) 2892 { 2893 u32 config; 2894 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2895 2896 if (pclk_hz <= 20000000) 2897 config = GEM_BF(CLK, GEM_CLK_DIV8); 2898 else if (pclk_hz <= 40000000) 2899 config = GEM_BF(CLK, GEM_CLK_DIV16); 2900 else if (pclk_hz <= 80000000) 2901 config = GEM_BF(CLK, GEM_CLK_DIV32); 2902 else if (pclk_hz <= 120000000) 2903 config = GEM_BF(CLK, GEM_CLK_DIV48); 2904 else if (pclk_hz <= 160000000) 2905 config = GEM_BF(CLK, GEM_CLK_DIV64); 2906 else if (pclk_hz <= 240000000) 2907 config = GEM_BF(CLK, GEM_CLK_DIV96); 2908 else if (pclk_hz <= 320000000) 2909 config = GEM_BF(CLK, GEM_CLK_DIV128); 2910 else 2911 config = GEM_BF(CLK, GEM_CLK_DIV224); 2912 2913 return config; 2914 } 2915 2916 static u32 macb_mdc_clk_div(struct macb *bp) 2917 { 2918 u32 config; 2919 unsigned long pclk_hz; 2920 2921 if (macb_is_gem(bp)) 2922 return gem_mdc_clk_div(bp); 2923 2924 pclk_hz = clk_get_rate(bp->pclk); 2925 if (pclk_hz <= 20000000) 2926 config = MACB_BF(CLK, MACB_CLK_DIV8); 2927 else if (pclk_hz <= 40000000) 2928 config = MACB_BF(CLK, MACB_CLK_DIV16); 2929 else if (pclk_hz <= 80000000) 2930 config = MACB_BF(CLK, MACB_CLK_DIV32); 2931 else 2932 config = MACB_BF(CLK, MACB_CLK_DIV64); 2933 2934 return config; 2935 } 2936 2937 /* Get the DMA bus width field of the network configuration register that we 2938 * should program. We find the width from decoding the design configuration 2939 * register to find the maximum supported data bus width. 2940 */ 2941 static u32 macb_dbw(struct macb *bp) 2942 { 2943 if (!macb_is_gem(bp)) 2944 return 0; 2945 2946 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2947 case 4: 2948 return GEM_BF(DBW, GEM_DBW128); 2949 case 2: 2950 return GEM_BF(DBW, GEM_DBW64); 2951 case 1: 2952 default: 2953 return GEM_BF(DBW, GEM_DBW32); 2954 } 2955 } 2956 2957 /* Configure the receive DMA engine 2958 * - use the correct receive buffer size 2959 * - set best burst length for DMA operations 2960 * (if not supported by FIFO, it will fallback to default) 2961 * - set both rx/tx packet buffers to full memory size 2962 * These are configurable parameters for GEM. 2963 */ 2964 static void macb_configure_dma(struct macb *bp) 2965 { 2966 struct macb_queue *queue; 2967 u32 buffer_size; 2968 unsigned int q; 2969 u32 dmacfg; 2970 2971 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2972 if (macb_is_gem(bp)) { 2973 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2974 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2975 if (q) 2976 queue_writel(queue, RBQS, buffer_size); 2977 else 2978 dmacfg |= GEM_BF(RXBS, buffer_size); 2979 } 2980 if (bp->dma_burst_length) 2981 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2982 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2983 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2984 2985 if (bp->native_io) 2986 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2987 else 2988 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2989 2990 if (bp->netdev->features & NETIF_F_HW_CSUM) 2991 dmacfg |= GEM_BIT(TXCOEN); 2992 else 2993 dmacfg &= ~GEM_BIT(TXCOEN); 2994 2995 dmacfg &= ~GEM_BIT(ADDR64); 2996 if (macb_dma64(bp)) 2997 dmacfg |= GEM_BIT(ADDR64); 2998 if (macb_dma_ptp(bp)) 2999 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 3000 netdev_dbg(bp->netdev, "Cadence configure DMA with 0x%08x\n", 3001 dmacfg); 3002 gem_writel(bp, DMACFG, dmacfg); 3003 } 3004 } 3005 3006 static void macb_init_hw(struct macb *bp) 3007 { 3008 u32 config; 3009 3010 macb_reset_hw(bp); 3011 macb_set_hwaddr(bp); 3012 3013 config = macb_mdc_clk_div(bp); 3014 /* Make eth data aligned. 3015 * If RSC capable, that offset is ignored by HW. 3016 */ 3017 if (!(bp->caps & MACB_CAPS_RSC)) 3018 config |= MACB_BF(RBOF, NET_IP_ALIGN); 3019 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 3020 if (bp->caps & MACB_CAPS_JUMBO) 3021 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 3022 else 3023 config |= MACB_BIT(BIG); /* Receive oversized frames */ 3024 if (bp->netdev->flags & IFF_PROMISC) 3025 config |= MACB_BIT(CAF); /* Copy All Frames */ 3026 else if (macb_is_gem(bp) && bp->netdev->features & NETIF_F_RXCSUM) 3027 config |= GEM_BIT(RXCOEN); 3028 if (!(bp->netdev->flags & IFF_BROADCAST)) 3029 config |= MACB_BIT(NBC); /* No BroadCast */ 3030 config |= macb_dbw(bp); 3031 macb_writel(bp, NCFGR, config); 3032 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 3033 gem_writel(bp, JML, bp->jumbo_max_len); 3034 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 3035 if (bp->caps & MACB_CAPS_JUMBO) 3036 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 3037 3038 macb_configure_dma(bp); 3039 3040 /* Enable RX partial store and forward and set watermark */ 3041 if (bp->rx_watermark) 3042 gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU))); 3043 } 3044 3045 /* The hash address register is 64 bits long and takes up two 3046 * locations in the memory map. The least significant bits are stored 3047 * in EMAC_HSL and the most significant bits in EMAC_HSH. 3048 * 3049 * The unicast hash enable and the multicast hash enable bits in the 3050 * network configuration register enable the reception of hash matched 3051 * frames. The destination address is reduced to a 6 bit index into 3052 * the 64 bit hash register using the following hash function. The 3053 * hash function is an exclusive or of every sixth bit of the 3054 * destination address. 3055 * 3056 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 3057 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 3058 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 3059 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 3060 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 3061 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 3062 * 3063 * da[0] represents the least significant bit of the first byte 3064 * received, that is, the multicast/unicast indicator, and da[47] 3065 * represents the most significant bit of the last byte received. If 3066 * the hash index, hi[n], points to a bit that is set in the hash 3067 * register then the frame will be matched according to whether the 3068 * frame is multicast or unicast. A multicast match will be signalled 3069 * if the multicast hash enable bit is set, da[0] is 1 and the hash 3070 * index points to a bit set in the hash register. A unicast match 3071 * will be signalled if the unicast hash enable bit is set, da[0] is 0 3072 * and the hash index points to a bit set in the hash register. To 3073 * receive all multicast frames, the hash register should be set with 3074 * all ones and the multicast hash enable bit should be set in the 3075 * network configuration register. 3076 */ 3077 3078 static inline int hash_bit_value(int bitnr, __u8 *addr) 3079 { 3080 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 3081 return 1; 3082 return 0; 3083 } 3084 3085 /* Return the hash index value for the specified address. */ 3086 static int hash_get_index(__u8 *addr) 3087 { 3088 int i, j, bitval; 3089 int hash_index = 0; 3090 3091 for (j = 0; j < 6; j++) { 3092 for (i = 0, bitval = 0; i < 8; i++) 3093 bitval ^= hash_bit_value(i * 6 + j, addr); 3094 3095 hash_index |= (bitval << j); 3096 } 3097 3098 return hash_index; 3099 } 3100 3101 /* Add multicast addresses to the internal multicast-hash table. */ 3102 static void macb_sethashtable(struct net_device *netdev) 3103 { 3104 struct netdev_hw_addr *ha; 3105 unsigned long mc_filter[2]; 3106 unsigned int bitnr; 3107 struct macb *bp = netdev_priv(netdev); 3108 3109 mc_filter[0] = 0; 3110 mc_filter[1] = 0; 3111 3112 netdev_for_each_mc_addr(ha, netdev) { 3113 bitnr = hash_get_index(ha->addr); 3114 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 3115 } 3116 3117 macb_or_gem_writel(bp, HRB, mc_filter[0]); 3118 macb_or_gem_writel(bp, HRT, mc_filter[1]); 3119 } 3120 3121 /* Enable/Disable promiscuous and multicast modes. */ 3122 static void macb_set_rx_mode(struct net_device *netdev) 3123 { 3124 unsigned long cfg; 3125 struct macb *bp = netdev_priv(netdev); 3126 3127 cfg = macb_readl(bp, NCFGR); 3128 3129 if (netdev->flags & IFF_PROMISC) { 3130 /* Enable promiscuous mode */ 3131 cfg |= MACB_BIT(CAF); 3132 3133 /* Disable RX checksum offload */ 3134 if (macb_is_gem(bp)) 3135 cfg &= ~GEM_BIT(RXCOEN); 3136 } else { 3137 /* Disable promiscuous mode */ 3138 cfg &= ~MACB_BIT(CAF); 3139 3140 /* Enable RX checksum offload only if requested */ 3141 if (macb_is_gem(bp) && netdev->features & NETIF_F_RXCSUM) 3142 cfg |= GEM_BIT(RXCOEN); 3143 } 3144 3145 if (netdev->flags & IFF_ALLMULTI) { 3146 /* Enable all multicast mode */ 3147 macb_or_gem_writel(bp, HRB, -1); 3148 macb_or_gem_writel(bp, HRT, -1); 3149 cfg |= MACB_BIT(NCFGR_MTI); 3150 } else if (!netdev_mc_empty(netdev)) { 3151 /* Enable specific multicasts */ 3152 macb_sethashtable(netdev); 3153 cfg |= MACB_BIT(NCFGR_MTI); 3154 } else if (netdev->flags & (~IFF_ALLMULTI)) { 3155 /* Disable all multicast mode */ 3156 macb_or_gem_writel(bp, HRB, 0); 3157 macb_or_gem_writel(bp, HRT, 0); 3158 cfg &= ~MACB_BIT(NCFGR_MTI); 3159 } 3160 3161 macb_writel(bp, NCFGR, cfg); 3162 } 3163 3164 static int macb_open(struct net_device *netdev) 3165 { 3166 size_t bufsz = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 3167 struct macb *bp = netdev_priv(netdev); 3168 struct macb_queue *queue; 3169 unsigned int q; 3170 int err; 3171 3172 netdev_dbg(bp->netdev, "open\n"); 3173 3174 err = pm_runtime_resume_and_get(&bp->pdev->dev); 3175 if (err < 0) 3176 return err; 3177 3178 /* RX buffers initialization */ 3179 macb_init_rx_buffer_size(bp, bufsz); 3180 3181 err = macb_alloc(bp); 3182 if (err) { 3183 netdev_err(netdev, "Unable to allocate DMA memory (error %d)\n", 3184 err); 3185 goto pm_exit; 3186 } 3187 3188 bp->macbgem_ops.mog_init_rings(bp); 3189 macb_init_buffers(bp); 3190 3191 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3192 napi_enable(&queue->napi_rx); 3193 napi_enable(&queue->napi_tx); 3194 } 3195 3196 macb_init_hw(bp); 3197 3198 err = phy_set_mode_ext(bp->phy, PHY_MODE_ETHERNET, bp->phy_interface); 3199 if (err) 3200 goto reset_hw; 3201 3202 err = phy_power_on(bp->phy); 3203 if (err) 3204 goto reset_hw; 3205 3206 err = macb_phylink_connect(bp); 3207 if (err) 3208 goto phy_off; 3209 3210 netif_tx_start_all_queues(netdev); 3211 3212 if (bp->ptp_info) 3213 bp->ptp_info->ptp_init(netdev); 3214 3215 return 0; 3216 3217 phy_off: 3218 phy_power_off(bp->phy); 3219 3220 reset_hw: 3221 macb_reset_hw(bp); 3222 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3223 napi_disable(&queue->napi_rx); 3224 napi_disable(&queue->napi_tx); 3225 } 3226 macb_free(bp); 3227 pm_exit: 3228 pm_runtime_put_sync(&bp->pdev->dev); 3229 return err; 3230 } 3231 3232 static int macb_close(struct net_device *netdev) 3233 { 3234 struct macb *bp = netdev_priv(netdev); 3235 struct macb_queue *queue; 3236 unsigned long flags; 3237 unsigned int q; 3238 3239 netif_tx_stop_all_queues(netdev); 3240 3241 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3242 napi_disable(&queue->napi_rx); 3243 napi_disable(&queue->napi_tx); 3244 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, q)); 3245 } 3246 3247 cancel_delayed_work_sync(&bp->tx_lpi_work); 3248 3249 phylink_stop(bp->phylink); 3250 phylink_disconnect_phy(bp->phylink); 3251 3252 phy_power_off(bp->phy); 3253 3254 spin_lock_irqsave(&bp->lock, flags); 3255 macb_reset_hw(bp); 3256 netif_carrier_off(netdev); 3257 spin_unlock_irqrestore(&bp->lock, flags); 3258 3259 macb_free(bp); 3260 3261 if (bp->ptp_info) 3262 bp->ptp_info->ptp_remove(netdev); 3263 3264 pm_runtime_put(&bp->pdev->dev); 3265 3266 return 0; 3267 } 3268 3269 static int macb_change_mtu(struct net_device *netdev, int new_mtu) 3270 { 3271 if (netif_running(netdev)) 3272 return -EBUSY; 3273 3274 WRITE_ONCE(netdev->mtu, new_mtu); 3275 3276 return 0; 3277 } 3278 3279 static int macb_set_mac_addr(struct net_device *netdev, void *addr) 3280 { 3281 int err; 3282 3283 err = eth_mac_addr(netdev, addr); 3284 if (err < 0) 3285 return err; 3286 3287 macb_set_hwaddr(netdev_priv(netdev)); 3288 return 0; 3289 } 3290 3291 static void gem_update_stats(struct macb *bp) 3292 { 3293 struct macb_queue *queue; 3294 unsigned int i, q, idx; 3295 unsigned long *stat; 3296 3297 u64 *p = &bp->hw_stats.gem.tx_octets; 3298 3299 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 3300 u32 offset = gem_statistics[i].offset; 3301 u64 val = bp->macb_reg_readl(bp, offset); 3302 3303 bp->ethtool_stats[i] += val; 3304 *p += val; 3305 3306 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 3307 /* Add GEM_OCTTXH, GEM_OCTRXH */ 3308 val = bp->macb_reg_readl(bp, offset + 4); 3309 bp->ethtool_stats[i] += ((u64)val) << 32; 3310 *p += ((u64)val) << 32; 3311 } 3312 } 3313 3314 idx = GEM_STATS_LEN; 3315 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 3316 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 3317 bp->ethtool_stats[idx++] = *stat; 3318 } 3319 3320 static void gem_get_stats(struct macb *bp, struct rtnl_link_stats64 *nstat) 3321 { 3322 struct gem_stats *hwstat = &bp->hw_stats.gem; 3323 3324 spin_lock_irq(&bp->stats_lock); 3325 if (netif_running(bp->netdev)) 3326 gem_update_stats(bp); 3327 3328 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 3329 hwstat->rx_alignment_errors + 3330 hwstat->rx_resource_errors + 3331 hwstat->rx_overruns + 3332 hwstat->rx_oversize_frames + 3333 hwstat->rx_jabbers + 3334 hwstat->rx_undersized_frames + 3335 hwstat->rx_length_field_frame_errors); 3336 nstat->tx_errors = (hwstat->tx_late_collisions + 3337 hwstat->tx_excessive_collisions + 3338 hwstat->tx_underrun + 3339 hwstat->tx_carrier_sense_errors); 3340 nstat->multicast = hwstat->rx_multicast_frames; 3341 nstat->collisions = (hwstat->tx_single_collision_frames + 3342 hwstat->tx_multiple_collision_frames + 3343 hwstat->tx_excessive_collisions); 3344 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 3345 hwstat->rx_jabbers + 3346 hwstat->rx_undersized_frames + 3347 hwstat->rx_length_field_frame_errors); 3348 nstat->rx_over_errors = hwstat->rx_resource_errors; 3349 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 3350 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 3351 nstat->rx_fifo_errors = hwstat->rx_overruns; 3352 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 3353 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 3354 nstat->tx_fifo_errors = hwstat->tx_underrun; 3355 spin_unlock_irq(&bp->stats_lock); 3356 } 3357 3358 static void gem_get_ethtool_stats(struct net_device *netdev, 3359 struct ethtool_stats *stats, u64 *data) 3360 { 3361 struct macb *bp = netdev_priv(netdev); 3362 3363 spin_lock_irq(&bp->stats_lock); 3364 gem_update_stats(bp); 3365 memcpy(data, &bp->ethtool_stats, sizeof(u64) 3366 * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); 3367 spin_unlock_irq(&bp->stats_lock); 3368 } 3369 3370 static int gem_get_sset_count(struct net_device *netdev, int sset) 3371 { 3372 struct macb *bp = netdev_priv(netdev); 3373 3374 switch (sset) { 3375 case ETH_SS_STATS: 3376 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 3377 default: 3378 return -EOPNOTSUPP; 3379 } 3380 } 3381 3382 static void gem_get_ethtool_strings(struct net_device *netdev, u32 sset, u8 *p) 3383 { 3384 struct macb *bp = netdev_priv(netdev); 3385 struct macb_queue *queue; 3386 unsigned int i; 3387 unsigned int q; 3388 3389 switch (sset) { 3390 case ETH_SS_STATS: 3391 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 3392 memcpy(p, gem_statistics[i].stat_string, 3393 ETH_GSTRING_LEN); 3394 3395 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3396 for (i = 0; i < QUEUE_STATS_LEN; i++) 3397 ethtool_sprintf(&p, "q%u_%s", q, queue_statistics[i].stat_string); 3398 } 3399 break; 3400 } 3401 } 3402 3403 static void macb_get_stats(struct net_device *netdev, 3404 struct rtnl_link_stats64 *nstat) 3405 { 3406 struct macb *bp = netdev_priv(netdev); 3407 struct macb_stats *hwstat = &bp->hw_stats.macb; 3408 3409 netdev_stats_to_stats64(nstat, &bp->netdev->stats); 3410 if (macb_is_gem(bp)) { 3411 gem_get_stats(bp, nstat); 3412 return; 3413 } 3414 3415 /* read stats from hardware */ 3416 spin_lock_irq(&bp->stats_lock); 3417 macb_update_stats(bp); 3418 3419 /* Convert HW stats into netdevice stats */ 3420 nstat->rx_errors = (hwstat->rx_fcs_errors + 3421 hwstat->rx_align_errors + 3422 hwstat->rx_resource_errors + 3423 hwstat->rx_overruns + 3424 hwstat->rx_oversize_pkts + 3425 hwstat->rx_jabbers + 3426 hwstat->rx_undersize_pkts + 3427 hwstat->rx_length_mismatch); 3428 nstat->tx_errors = (hwstat->tx_late_cols + 3429 hwstat->tx_excessive_cols + 3430 hwstat->tx_underruns + 3431 hwstat->tx_carrier_errors + 3432 hwstat->sqe_test_errors); 3433 nstat->collisions = (hwstat->tx_single_cols + 3434 hwstat->tx_multiple_cols + 3435 hwstat->tx_excessive_cols); 3436 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 3437 hwstat->rx_jabbers + 3438 hwstat->rx_undersize_pkts + 3439 hwstat->rx_length_mismatch); 3440 nstat->rx_over_errors = hwstat->rx_resource_errors + 3441 hwstat->rx_overruns; 3442 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 3443 nstat->rx_frame_errors = hwstat->rx_align_errors; 3444 nstat->rx_fifo_errors = hwstat->rx_overruns; 3445 /* XXX: What does "missed" mean? */ 3446 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 3447 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 3448 nstat->tx_fifo_errors = hwstat->tx_underruns; 3449 /* Don't know about heartbeat or window errors... */ 3450 spin_unlock_irq(&bp->stats_lock); 3451 } 3452 3453 static void macb_get_pause_stats(struct net_device *netdev, 3454 struct ethtool_pause_stats *pause_stats) 3455 { 3456 struct macb *bp = netdev_priv(netdev); 3457 struct macb_stats *hwstat = &bp->hw_stats.macb; 3458 3459 spin_lock_irq(&bp->stats_lock); 3460 macb_update_stats(bp); 3461 pause_stats->tx_pause_frames = hwstat->tx_pause_frames; 3462 pause_stats->rx_pause_frames = hwstat->rx_pause_frames; 3463 spin_unlock_irq(&bp->stats_lock); 3464 } 3465 3466 static void gem_get_pause_stats(struct net_device *netdev, 3467 struct ethtool_pause_stats *pause_stats) 3468 { 3469 struct macb *bp = netdev_priv(netdev); 3470 struct gem_stats *hwstat = &bp->hw_stats.gem; 3471 3472 spin_lock_irq(&bp->stats_lock); 3473 gem_update_stats(bp); 3474 pause_stats->tx_pause_frames = hwstat->tx_pause_frames; 3475 pause_stats->rx_pause_frames = hwstat->rx_pause_frames; 3476 spin_unlock_irq(&bp->stats_lock); 3477 } 3478 3479 static void macb_get_eth_mac_stats(struct net_device *netdev, 3480 struct ethtool_eth_mac_stats *mac_stats) 3481 { 3482 struct macb *bp = netdev_priv(netdev); 3483 struct macb_stats *hwstat = &bp->hw_stats.macb; 3484 3485 spin_lock_irq(&bp->stats_lock); 3486 macb_update_stats(bp); 3487 mac_stats->FramesTransmittedOK = hwstat->tx_ok; 3488 mac_stats->SingleCollisionFrames = hwstat->tx_single_cols; 3489 mac_stats->MultipleCollisionFrames = hwstat->tx_multiple_cols; 3490 mac_stats->FramesReceivedOK = hwstat->rx_ok; 3491 mac_stats->FrameCheckSequenceErrors = hwstat->rx_fcs_errors; 3492 mac_stats->AlignmentErrors = hwstat->rx_align_errors; 3493 mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred; 3494 mac_stats->LateCollisions = hwstat->tx_late_cols; 3495 mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_cols; 3496 mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underruns; 3497 mac_stats->CarrierSenseErrors = hwstat->tx_carrier_errors; 3498 mac_stats->FramesLostDueToIntMACRcvError = hwstat->rx_overruns; 3499 mac_stats->InRangeLengthErrors = hwstat->rx_length_mismatch; 3500 mac_stats->FrameTooLongErrors = hwstat->rx_oversize_pkts; 3501 spin_unlock_irq(&bp->stats_lock); 3502 } 3503 3504 static void gem_get_eth_mac_stats(struct net_device *netdev, 3505 struct ethtool_eth_mac_stats *mac_stats) 3506 { 3507 struct macb *bp = netdev_priv(netdev); 3508 struct gem_stats *hwstat = &bp->hw_stats.gem; 3509 3510 spin_lock_irq(&bp->stats_lock); 3511 gem_update_stats(bp); 3512 mac_stats->FramesTransmittedOK = hwstat->tx_frames; 3513 mac_stats->SingleCollisionFrames = hwstat->tx_single_collision_frames; 3514 mac_stats->MultipleCollisionFrames = 3515 hwstat->tx_multiple_collision_frames; 3516 mac_stats->FramesReceivedOK = hwstat->rx_frames; 3517 mac_stats->FrameCheckSequenceErrors = 3518 hwstat->rx_frame_check_sequence_errors; 3519 mac_stats->AlignmentErrors = hwstat->rx_alignment_errors; 3520 mac_stats->OctetsTransmittedOK = hwstat->tx_octets; 3521 mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred_frames; 3522 mac_stats->LateCollisions = hwstat->tx_late_collisions; 3523 mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_collisions; 3524 mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underrun; 3525 mac_stats->CarrierSenseErrors = hwstat->tx_carrier_sense_errors; 3526 mac_stats->OctetsReceivedOK = hwstat->rx_octets; 3527 mac_stats->MulticastFramesXmittedOK = hwstat->tx_multicast_frames; 3528 mac_stats->BroadcastFramesXmittedOK = hwstat->tx_broadcast_frames; 3529 mac_stats->MulticastFramesReceivedOK = hwstat->rx_multicast_frames; 3530 mac_stats->BroadcastFramesReceivedOK = hwstat->rx_broadcast_frames; 3531 mac_stats->InRangeLengthErrors = hwstat->rx_length_field_frame_errors; 3532 mac_stats->FrameTooLongErrors = hwstat->rx_oversize_frames; 3533 spin_unlock_irq(&bp->stats_lock); 3534 } 3535 3536 /* TODO: Report SQE test errors when added to phy_stats */ 3537 static void macb_get_eth_phy_stats(struct net_device *netdev, 3538 struct ethtool_eth_phy_stats *phy_stats) 3539 { 3540 struct macb *bp = netdev_priv(netdev); 3541 struct macb_stats *hwstat = &bp->hw_stats.macb; 3542 3543 spin_lock_irq(&bp->stats_lock); 3544 macb_update_stats(bp); 3545 phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; 3546 spin_unlock_irq(&bp->stats_lock); 3547 } 3548 3549 static void gem_get_eth_phy_stats(struct net_device *netdev, 3550 struct ethtool_eth_phy_stats *phy_stats) 3551 { 3552 struct macb *bp = netdev_priv(netdev); 3553 struct gem_stats *hwstat = &bp->hw_stats.gem; 3554 3555 spin_lock_irq(&bp->stats_lock); 3556 gem_update_stats(bp); 3557 phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; 3558 spin_unlock_irq(&bp->stats_lock); 3559 } 3560 3561 static void macb_get_rmon_stats(struct net_device *netdev, 3562 struct ethtool_rmon_stats *rmon_stats, 3563 const struct ethtool_rmon_hist_range **ranges) 3564 { 3565 struct macb *bp = netdev_priv(netdev); 3566 struct macb_stats *hwstat = &bp->hw_stats.macb; 3567 3568 spin_lock_irq(&bp->stats_lock); 3569 macb_update_stats(bp); 3570 rmon_stats->undersize_pkts = hwstat->rx_undersize_pkts; 3571 rmon_stats->oversize_pkts = hwstat->rx_oversize_pkts; 3572 rmon_stats->jabbers = hwstat->rx_jabbers; 3573 spin_unlock_irq(&bp->stats_lock); 3574 } 3575 3576 static const struct ethtool_rmon_hist_range gem_rmon_ranges[] = { 3577 { 64, 64 }, 3578 { 65, 127 }, 3579 { 128, 255 }, 3580 { 256, 511 }, 3581 { 512, 1023 }, 3582 { 1024, 1518 }, 3583 { 1519, 16384 }, 3584 { }, 3585 }; 3586 3587 static void gem_get_rmon_stats(struct net_device *netdev, 3588 struct ethtool_rmon_stats *rmon_stats, 3589 const struct ethtool_rmon_hist_range **ranges) 3590 { 3591 struct macb *bp = netdev_priv(netdev); 3592 struct gem_stats *hwstat = &bp->hw_stats.gem; 3593 3594 spin_lock_irq(&bp->stats_lock); 3595 gem_update_stats(bp); 3596 rmon_stats->undersize_pkts = hwstat->rx_undersized_frames; 3597 rmon_stats->oversize_pkts = hwstat->rx_oversize_frames; 3598 rmon_stats->jabbers = hwstat->rx_jabbers; 3599 rmon_stats->hist[0] = hwstat->rx_64_byte_frames; 3600 rmon_stats->hist[1] = hwstat->rx_65_127_byte_frames; 3601 rmon_stats->hist[2] = hwstat->rx_128_255_byte_frames; 3602 rmon_stats->hist[3] = hwstat->rx_256_511_byte_frames; 3603 rmon_stats->hist[4] = hwstat->rx_512_1023_byte_frames; 3604 rmon_stats->hist[5] = hwstat->rx_1024_1518_byte_frames; 3605 rmon_stats->hist[6] = hwstat->rx_greater_than_1518_byte_frames; 3606 rmon_stats->hist_tx[0] = hwstat->tx_64_byte_frames; 3607 rmon_stats->hist_tx[1] = hwstat->tx_65_127_byte_frames; 3608 rmon_stats->hist_tx[2] = hwstat->tx_128_255_byte_frames; 3609 rmon_stats->hist_tx[3] = hwstat->tx_256_511_byte_frames; 3610 rmon_stats->hist_tx[4] = hwstat->tx_512_1023_byte_frames; 3611 rmon_stats->hist_tx[5] = hwstat->tx_1024_1518_byte_frames; 3612 rmon_stats->hist_tx[6] = hwstat->tx_greater_than_1518_byte_frames; 3613 spin_unlock_irq(&bp->stats_lock); 3614 *ranges = gem_rmon_ranges; 3615 } 3616 3617 static int macb_get_regs_len(struct net_device *netdev) 3618 { 3619 return MACB_GREGS_NBR * sizeof(u32); 3620 } 3621 3622 static void macb_get_regs(struct net_device *netdev, struct ethtool_regs *regs, 3623 void *p) 3624 { 3625 struct macb *bp = netdev_priv(netdev); 3626 unsigned int tail, head; 3627 u32 *regs_buff = p; 3628 3629 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 3630 | MACB_GREGS_VERSION; 3631 3632 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 3633 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 3634 3635 regs_buff[0] = macb_readl(bp, NCR); 3636 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 3637 regs_buff[2] = macb_readl(bp, NSR); 3638 regs_buff[3] = macb_readl(bp, TSR); 3639 regs_buff[4] = macb_readl(bp, RBQP); 3640 regs_buff[5] = macb_readl(bp, TBQP); 3641 regs_buff[6] = macb_readl(bp, RSR); 3642 regs_buff[7] = macb_readl(bp, IMR); 3643 3644 regs_buff[8] = tail; 3645 regs_buff[9] = head; 3646 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 3647 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 3648 3649 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 3650 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 3651 if (macb_is_gem(bp)) 3652 regs_buff[13] = gem_readl(bp, DMACFG); 3653 } 3654 3655 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 3656 { 3657 struct macb *bp = netdev_priv(netdev); 3658 3659 phylink_ethtool_get_wol(bp->phylink, wol); 3660 wol->supported |= (WAKE_MAGIC | WAKE_ARP); 3661 3662 /* Add macb wolopts to phy wolopts */ 3663 wol->wolopts |= bp->wolopts; 3664 } 3665 3666 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 3667 { 3668 struct macb *bp = netdev_priv(netdev); 3669 int ret; 3670 3671 /* Pass the order to phylink layer */ 3672 ret = phylink_ethtool_set_wol(bp->phylink, wol); 3673 /* Don't manage WoL on MAC, if PHY set_wol() fails */ 3674 if (ret && ret != -EOPNOTSUPP) 3675 return ret; 3676 3677 bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0; 3678 bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0; 3679 bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0; 3680 3681 device_set_wakeup_enable(&bp->pdev->dev, bp->wol); 3682 3683 return 0; 3684 } 3685 3686 static int macb_get_link_ksettings(struct net_device *netdev, 3687 struct ethtool_link_ksettings *kset) 3688 { 3689 struct macb *bp = netdev_priv(netdev); 3690 3691 return phylink_ethtool_ksettings_get(bp->phylink, kset); 3692 } 3693 3694 static int macb_set_link_ksettings(struct net_device *netdev, 3695 const struct ethtool_link_ksettings *kset) 3696 { 3697 struct macb *bp = netdev_priv(netdev); 3698 3699 return phylink_ethtool_ksettings_set(bp->phylink, kset); 3700 } 3701 3702 static void macb_get_ringparam(struct net_device *netdev, 3703 struct ethtool_ringparam *ring, 3704 struct kernel_ethtool_ringparam *kernel_ring, 3705 struct netlink_ext_ack *extack) 3706 { 3707 struct macb *bp = netdev_priv(netdev); 3708 3709 ring->rx_max_pending = MAX_RX_RING_SIZE; 3710 ring->tx_max_pending = MAX_TX_RING_SIZE; 3711 3712 ring->rx_pending = bp->rx_ring_size; 3713 ring->tx_pending = bp->tx_ring_size; 3714 } 3715 3716 static int macb_set_ringparam(struct net_device *netdev, 3717 struct ethtool_ringparam *ring, 3718 struct kernel_ethtool_ringparam *kernel_ring, 3719 struct netlink_ext_ack *extack) 3720 { 3721 struct macb *bp = netdev_priv(netdev); 3722 u32 new_rx_size, new_tx_size; 3723 unsigned int reset = 0; 3724 3725 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) 3726 return -EOPNOTSUPP; 3727 3728 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 3729 return -EINVAL; 3730 3731 new_rx_size = clamp_t(u32, ring->rx_pending, 3732 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 3733 new_rx_size = roundup_pow_of_two(new_rx_size); 3734 3735 new_tx_size = clamp_t(u32, ring->tx_pending, 3736 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 3737 new_tx_size = roundup_pow_of_two(new_tx_size); 3738 3739 if ((new_tx_size == bp->tx_ring_size) && 3740 (new_rx_size == bp->rx_ring_size)) { 3741 /* nothing to do */ 3742 return 0; 3743 } 3744 3745 if (netif_running(bp->netdev)) { 3746 reset = 1; 3747 macb_close(bp->netdev); 3748 } 3749 3750 bp->rx_ring_size = new_rx_size; 3751 bp->tx_ring_size = new_tx_size; 3752 3753 if (reset) 3754 macb_open(bp->netdev); 3755 3756 return 0; 3757 } 3758 3759 #ifdef CONFIG_MACB_USE_HWSTAMP 3760 static unsigned int gem_get_tsu_rate(struct macb *bp) 3761 { 3762 struct clk *tsu_clk; 3763 unsigned int tsu_rate; 3764 3765 if (!IS_ERR_OR_NULL(bp->tsu_clk)) { 3766 tsu_rate = clk_get_rate(bp->tsu_clk); 3767 } else { 3768 tsu_clk = bp->pclk; 3769 tsu_rate = clk_get_rate(tsu_clk); 3770 dev_warn(&bp->pdev->dev, "devicetree missing tsu_clk, using pclk as fallback\n"); 3771 } 3772 3773 return tsu_rate; 3774 } 3775 3776 static s32 gem_get_ptp_max_adj(void) 3777 { 3778 return 64000000; 3779 } 3780 3781 static int gem_get_ts_info(struct net_device *netdev, 3782 struct kernel_ethtool_ts_info *info) 3783 { 3784 struct macb *bp = netdev_priv(netdev); 3785 3786 if (!macb_dma_ptp(bp)) { 3787 ethtool_op_get_ts_info(netdev, info); 3788 return 0; 3789 } 3790 3791 info->so_timestamping = 3792 SOF_TIMESTAMPING_TX_SOFTWARE | 3793 SOF_TIMESTAMPING_TX_HARDWARE | 3794 SOF_TIMESTAMPING_RX_HARDWARE | 3795 SOF_TIMESTAMPING_RAW_HARDWARE; 3796 info->tx_types = 3797 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 3798 (1 << HWTSTAMP_TX_OFF) | 3799 (1 << HWTSTAMP_TX_ON); 3800 info->rx_filters = 3801 (1 << HWTSTAMP_FILTER_NONE) | 3802 (1 << HWTSTAMP_FILTER_ALL); 3803 3804 if (bp->ptp_clock) 3805 info->phc_index = ptp_clock_index(bp->ptp_clock); 3806 3807 return 0; 3808 } 3809 3810 static struct macb_ptp_info gem_ptp_info = { 3811 .ptp_init = gem_ptp_init, 3812 .ptp_remove = gem_ptp_remove, 3813 .get_ptp_max_adj = gem_get_ptp_max_adj, 3814 .get_tsu_rate = gem_get_tsu_rate, 3815 .get_ts_info = gem_get_ts_info, 3816 .get_hwtst = gem_get_hwtst, 3817 .set_hwtst = gem_set_hwtst, 3818 }; 3819 #endif 3820 3821 static int macb_get_ts_info(struct net_device *netdev, 3822 struct kernel_ethtool_ts_info *info) 3823 { 3824 struct macb *bp = netdev_priv(netdev); 3825 3826 if (bp->ptp_info) 3827 return bp->ptp_info->get_ts_info(netdev, info); 3828 3829 return ethtool_op_get_ts_info(netdev, info); 3830 } 3831 3832 static void gem_enable_flow_filters(struct macb *bp, bool enable) 3833 { 3834 struct net_device *netdev = bp->netdev; 3835 struct ethtool_rx_fs_item *item; 3836 u32 t2_scr; 3837 int num_t2_scr; 3838 3839 if (!(netdev->features & NETIF_F_NTUPLE)) 3840 return; 3841 3842 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 3843 3844 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3845 struct ethtool_rx_flow_spec *fs = &item->fs; 3846 struct ethtool_tcpip4_spec *tp4sp_m; 3847 3848 if (fs->location >= num_t2_scr) 3849 continue; 3850 3851 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 3852 3853 /* enable/disable screener regs for the flow entry */ 3854 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 3855 3856 /* only enable fields with no masking */ 3857 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3858 3859 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 3860 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 3861 else 3862 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 3863 3864 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 3865 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 3866 else 3867 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 3868 3869 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 3870 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 3871 else 3872 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 3873 3874 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 3875 } 3876 } 3877 3878 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 3879 { 3880 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 3881 uint16_t index = fs->location; 3882 u32 w0, w1, t2_scr; 3883 bool cmp_a = false; 3884 bool cmp_b = false; 3885 bool cmp_c = false; 3886 3887 if (!macb_is_gem(bp)) 3888 return; 3889 3890 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 3891 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3892 3893 /* ignore field if any masking set */ 3894 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 3895 /* 1st compare reg - IP source address */ 3896 w0 = 0; 3897 w1 = 0; 3898 w0 = tp4sp_v->ip4src; 3899 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3900 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3901 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 3902 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 3903 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 3904 cmp_a = true; 3905 } 3906 3907 /* ignore field if any masking set */ 3908 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 3909 /* 2nd compare reg - IP destination address */ 3910 w0 = 0; 3911 w1 = 0; 3912 w0 = tp4sp_v->ip4dst; 3913 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3914 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3915 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 3916 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 3917 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 3918 cmp_b = true; 3919 } 3920 3921 /* ignore both port fields if masking set in both */ 3922 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 3923 /* 3rd compare reg - source port, destination port */ 3924 w0 = 0; 3925 w1 = 0; 3926 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 3927 if (tp4sp_m->psrc == tp4sp_m->pdst) { 3928 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 3929 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3930 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3931 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3932 } else { 3933 /* only one port definition */ 3934 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 3935 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 3936 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 3937 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 3938 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3939 } else { /* dst port */ 3940 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3941 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 3942 } 3943 } 3944 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 3945 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 3946 cmp_c = true; 3947 } 3948 3949 t2_scr = 0; 3950 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 3951 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 3952 if (cmp_a) 3953 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 3954 if (cmp_b) 3955 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 3956 if (cmp_c) 3957 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 3958 gem_writel_n(bp, SCRT2, index, t2_scr); 3959 } 3960 3961 static int gem_add_flow_filter(struct net_device *netdev, 3962 struct ethtool_rxnfc *cmd) 3963 { 3964 struct macb *bp = netdev_priv(netdev); 3965 struct ethtool_rx_flow_spec *fs = &cmd->fs; 3966 struct ethtool_rx_fs_item *item, *newfs; 3967 unsigned long flags; 3968 int ret = -EINVAL; 3969 bool added = false; 3970 3971 newfs = kmalloc_obj(*newfs); 3972 if (newfs == NULL) 3973 return -ENOMEM; 3974 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 3975 3976 netdev_dbg(netdev, 3977 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3978 fs->flow_type, (int)fs->ring_cookie, fs->location, 3979 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3980 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3981 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), 3982 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); 3983 3984 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3985 3986 /* find correct place to add in list */ 3987 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3988 if (item->fs.location > newfs->fs.location) { 3989 list_add_tail(&newfs->list, &item->list); 3990 added = true; 3991 break; 3992 } else if (item->fs.location == fs->location) { 3993 netdev_err(netdev, "Rule not added: location %d not free!\n", 3994 fs->location); 3995 ret = -EBUSY; 3996 goto err; 3997 } 3998 } 3999 if (!added) 4000 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 4001 4002 gem_prog_cmp_regs(bp, fs); 4003 bp->rx_fs_list.count++; 4004 /* enable filtering if NTUPLE on */ 4005 gem_enable_flow_filters(bp, 1); 4006 4007 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 4008 return 0; 4009 4010 err: 4011 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 4012 kfree(newfs); 4013 return ret; 4014 } 4015 4016 static int gem_del_flow_filter(struct net_device *netdev, 4017 struct ethtool_rxnfc *cmd) 4018 { 4019 struct macb *bp = netdev_priv(netdev); 4020 struct ethtool_rx_fs_item *item; 4021 struct ethtool_rx_flow_spec *fs; 4022 unsigned long flags; 4023 4024 spin_lock_irqsave(&bp->rx_fs_lock, flags); 4025 4026 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 4027 if (item->fs.location == cmd->fs.location) { 4028 /* disable screener regs for the flow entry */ 4029 fs = &(item->fs); 4030 netdev_dbg(netdev, 4031 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 4032 fs->flow_type, (int)fs->ring_cookie, fs->location, 4033 htonl(fs->h_u.tcp_ip4_spec.ip4src), 4034 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 4035 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), 4036 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); 4037 4038 gem_writel_n(bp, SCRT2, fs->location, 0); 4039 4040 list_del(&item->list); 4041 bp->rx_fs_list.count--; 4042 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 4043 kfree(item); 4044 return 0; 4045 } 4046 } 4047 4048 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 4049 return -EINVAL; 4050 } 4051 4052 static int gem_get_flow_entry(struct net_device *netdev, 4053 struct ethtool_rxnfc *cmd) 4054 { 4055 struct macb *bp = netdev_priv(netdev); 4056 struct ethtool_rx_fs_item *item; 4057 4058 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 4059 if (item->fs.location == cmd->fs.location) { 4060 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 4061 return 0; 4062 } 4063 } 4064 return -EINVAL; 4065 } 4066 4067 static int gem_get_all_flow_entries(struct net_device *netdev, 4068 struct ethtool_rxnfc *cmd, u32 *rule_locs) 4069 { 4070 struct macb *bp = netdev_priv(netdev); 4071 struct ethtool_rx_fs_item *item; 4072 uint32_t cnt = 0; 4073 4074 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 4075 if (cnt == cmd->rule_cnt) 4076 return -EMSGSIZE; 4077 rule_locs[cnt] = item->fs.location; 4078 cnt++; 4079 } 4080 cmd->data = bp->max_tuples; 4081 cmd->rule_cnt = cnt; 4082 4083 return 0; 4084 } 4085 4086 static u32 gem_get_rx_ring_count(struct net_device *netdev) 4087 { 4088 struct macb *bp = netdev_priv(netdev); 4089 4090 return bp->num_queues; 4091 } 4092 4093 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 4094 u32 *rule_locs) 4095 { 4096 struct macb *bp = netdev_priv(netdev); 4097 int ret = 0; 4098 4099 switch (cmd->cmd) { 4100 case ETHTOOL_GRXCLSRLCNT: 4101 cmd->rule_cnt = bp->rx_fs_list.count; 4102 break; 4103 case ETHTOOL_GRXCLSRULE: 4104 ret = gem_get_flow_entry(netdev, cmd); 4105 break; 4106 case ETHTOOL_GRXCLSRLALL: 4107 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 4108 break; 4109 default: 4110 netdev_err(netdev, 4111 "Command parameter %d is not supported\n", cmd->cmd); 4112 ret = -EOPNOTSUPP; 4113 } 4114 4115 return ret; 4116 } 4117 4118 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 4119 { 4120 struct macb *bp = netdev_priv(netdev); 4121 int ret; 4122 4123 if (!(netdev->hw_features & NETIF_F_NTUPLE)) 4124 return -EOPNOTSUPP; 4125 4126 switch (cmd->cmd) { 4127 case ETHTOOL_SRXCLSRLINS: 4128 if ((cmd->fs.location >= bp->max_tuples) 4129 || (cmd->fs.ring_cookie >= bp->num_queues)) { 4130 ret = -EINVAL; 4131 break; 4132 } 4133 ret = gem_add_flow_filter(netdev, cmd); 4134 break; 4135 case ETHTOOL_SRXCLSRLDEL: 4136 ret = gem_del_flow_filter(netdev, cmd); 4137 break; 4138 default: 4139 netdev_err(netdev, 4140 "Command parameter %d is not supported\n", cmd->cmd); 4141 ret = -EOPNOTSUPP; 4142 } 4143 4144 return ret; 4145 } 4146 4147 static const struct ethtool_ops macb_ethtool_ops = { 4148 .get_regs_len = macb_get_regs_len, 4149 .get_regs = macb_get_regs, 4150 .get_link = ethtool_op_get_link, 4151 .get_ts_info = ethtool_op_get_ts_info, 4152 .get_pause_stats = macb_get_pause_stats, 4153 .get_eth_mac_stats = macb_get_eth_mac_stats, 4154 .get_eth_phy_stats = macb_get_eth_phy_stats, 4155 .get_rmon_stats = macb_get_rmon_stats, 4156 .get_wol = macb_get_wol, 4157 .set_wol = macb_set_wol, 4158 .get_link_ksettings = macb_get_link_ksettings, 4159 .set_link_ksettings = macb_set_link_ksettings, 4160 .get_ringparam = macb_get_ringparam, 4161 .set_ringparam = macb_set_ringparam, 4162 }; 4163 4164 static int macb_get_eee(struct net_device *netdev, struct ethtool_keee *eee) 4165 { 4166 struct macb *bp = netdev_priv(netdev); 4167 4168 return phylink_ethtool_get_eee(bp->phylink, eee); 4169 } 4170 4171 static int macb_set_eee(struct net_device *netdev, struct ethtool_keee *eee) 4172 { 4173 struct macb *bp = netdev_priv(netdev); 4174 4175 return phylink_ethtool_set_eee(bp->phylink, eee); 4176 } 4177 4178 static const struct ethtool_ops gem_ethtool_ops = { 4179 .get_regs_len = macb_get_regs_len, 4180 .get_regs = macb_get_regs, 4181 .get_wol = macb_get_wol, 4182 .set_wol = macb_set_wol, 4183 .get_link = ethtool_op_get_link, 4184 .get_ts_info = macb_get_ts_info, 4185 .get_ethtool_stats = gem_get_ethtool_stats, 4186 .get_strings = gem_get_ethtool_strings, 4187 .get_sset_count = gem_get_sset_count, 4188 .get_pause_stats = gem_get_pause_stats, 4189 .get_eth_mac_stats = gem_get_eth_mac_stats, 4190 .get_eth_phy_stats = gem_get_eth_phy_stats, 4191 .get_rmon_stats = gem_get_rmon_stats, 4192 .get_link_ksettings = macb_get_link_ksettings, 4193 .set_link_ksettings = macb_set_link_ksettings, 4194 .get_ringparam = macb_get_ringparam, 4195 .set_ringparam = macb_set_ringparam, 4196 .get_rxnfc = gem_get_rxnfc, 4197 .set_rxnfc = gem_set_rxnfc, 4198 .get_rx_ring_count = gem_get_rx_ring_count, 4199 .nway_reset = phy_ethtool_nway_reset, 4200 .get_eee = macb_get_eee, 4201 .set_eee = macb_set_eee, 4202 }; 4203 4204 static int macb_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd) 4205 { 4206 struct macb *bp = netdev_priv(netdev); 4207 4208 if (!netif_running(netdev)) 4209 return -EINVAL; 4210 4211 return phylink_mii_ioctl(bp->phylink, rq, cmd); 4212 } 4213 4214 static int macb_hwtstamp_get(struct net_device *netdev, 4215 struct kernel_hwtstamp_config *cfg) 4216 { 4217 struct macb *bp = netdev_priv(netdev); 4218 4219 if (!netif_running(netdev)) 4220 return -EINVAL; 4221 4222 if (!bp->ptp_info) 4223 return -EOPNOTSUPP; 4224 4225 return bp->ptp_info->get_hwtst(netdev, cfg); 4226 } 4227 4228 static int macb_hwtstamp_set(struct net_device *netdev, 4229 struct kernel_hwtstamp_config *cfg, 4230 struct netlink_ext_ack *extack) 4231 { 4232 struct macb *bp = netdev_priv(netdev); 4233 4234 if (!netif_running(netdev)) 4235 return -EINVAL; 4236 4237 if (!bp->ptp_info) 4238 return -EOPNOTSUPP; 4239 4240 return bp->ptp_info->set_hwtst(netdev, cfg, extack); 4241 } 4242 4243 static inline void macb_set_txcsum_feature(struct macb *bp, 4244 netdev_features_t features) 4245 { 4246 u32 val; 4247 4248 if (!macb_is_gem(bp)) 4249 return; 4250 4251 val = gem_readl(bp, DMACFG); 4252 if (features & NETIF_F_HW_CSUM) 4253 val |= GEM_BIT(TXCOEN); 4254 else 4255 val &= ~GEM_BIT(TXCOEN); 4256 4257 gem_writel(bp, DMACFG, val); 4258 } 4259 4260 static inline void macb_set_rxcsum_feature(struct macb *bp, 4261 netdev_features_t features) 4262 { 4263 struct net_device *netdev = bp->netdev; 4264 u32 val; 4265 4266 if (!macb_is_gem(bp)) 4267 return; 4268 4269 val = gem_readl(bp, NCFGR); 4270 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 4271 val |= GEM_BIT(RXCOEN); 4272 else 4273 val &= ~GEM_BIT(RXCOEN); 4274 4275 gem_writel(bp, NCFGR, val); 4276 } 4277 4278 static inline void macb_set_rxflow_feature(struct macb *bp, 4279 netdev_features_t features) 4280 { 4281 if (!macb_is_gem(bp)) 4282 return; 4283 4284 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 4285 } 4286 4287 static int macb_set_features(struct net_device *netdev, 4288 netdev_features_t features) 4289 { 4290 struct macb *bp = netdev_priv(netdev); 4291 netdev_features_t changed = features ^ netdev->features; 4292 4293 /* TX checksum offload */ 4294 if (changed & NETIF_F_HW_CSUM) 4295 macb_set_txcsum_feature(bp, features); 4296 4297 /* RX checksum offload */ 4298 if (changed & NETIF_F_RXCSUM) 4299 macb_set_rxcsum_feature(bp, features); 4300 4301 /* RX Flow Filters */ 4302 if (changed & NETIF_F_NTUPLE) 4303 macb_set_rxflow_feature(bp, features); 4304 4305 return 0; 4306 } 4307 4308 static void macb_restore_features(struct macb *bp) 4309 { 4310 struct net_device *netdev = bp->netdev; 4311 netdev_features_t features = netdev->features; 4312 struct ethtool_rx_fs_item *item; 4313 4314 /* TX checksum offload */ 4315 macb_set_txcsum_feature(bp, features); 4316 4317 /* RX checksum offload */ 4318 macb_set_rxcsum_feature(bp, features); 4319 4320 /* RX Flow Filters */ 4321 list_for_each_entry(item, &bp->rx_fs_list.list, list) 4322 gem_prog_cmp_regs(bp, &item->fs); 4323 4324 macb_set_rxflow_feature(bp, features); 4325 } 4326 4327 static int macb_taprio_setup_replace(struct net_device *netdev, 4328 struct tc_taprio_qopt_offload *conf) 4329 { 4330 u64 total_on_time = 0, start_time_sec = 0, start_time = conf->base_time; 4331 u32 configured_queues = 0, speed = 0, start_time_nsec; 4332 struct macb_queue_enst_config *enst_queue; 4333 struct ethtool_link_ksettings kset = {}; 4334 struct macb *bp = netdev_priv(netdev); 4335 struct tc_taprio_sched_entry *entry; 4336 struct macb_queue *queue; 4337 u32 queue_mask; 4338 u8 queue_id; 4339 size_t i; 4340 int err; 4341 4342 if (conf->num_entries > bp->num_queues) { 4343 netdev_err(netdev, "Too many TAPRIO entries: %zu > %d queues\n", 4344 conf->num_entries, bp->num_queues); 4345 return -EINVAL; 4346 } 4347 4348 if (conf->base_time < 0) { 4349 netdev_err(netdev, "Invalid base_time: must be 0 or positive, got %lld\n", 4350 conf->base_time); 4351 return -ERANGE; 4352 } 4353 4354 /* Get the current link speed */ 4355 err = phylink_ethtool_ksettings_get(bp->phylink, &kset); 4356 if (unlikely(err)) { 4357 netdev_err(netdev, "Failed to get link settings: %d\n", err); 4358 return err; 4359 } 4360 4361 speed = kset.base.speed; 4362 if (unlikely(speed == SPEED_UNKNOWN || !speed)) { 4363 netdev_err(netdev, "Invalid speed %d, link-down?\n", speed); 4364 return -EINVAL; 4365 } 4366 4367 enst_queue = kzalloc_objs(*enst_queue, conf->num_entries); 4368 if (unlikely(!enst_queue)) 4369 return -ENOMEM; 4370 4371 /* Pre-validate all entries before making any hardware changes */ 4372 for (i = 0; i < conf->num_entries; i++) { 4373 entry = &conf->entries[i]; 4374 4375 if (entry->command != TC_TAPRIO_CMD_SET_GATES) { 4376 netdev_err(netdev, "Entry %zu: unsupported command %d\n", 4377 i, entry->command); 4378 err = -EOPNOTSUPP; 4379 goto cleanup; 4380 } 4381 4382 /* Validate gate_mask: must be nonzero, single queue, and within range */ 4383 if (!is_power_of_2(entry->gate_mask)) { 4384 netdev_err(netdev, "Entry %zu: gate_mask 0x%x is not a power of 2 (only one queue per entry allowed)\n", 4385 i, entry->gate_mask); 4386 err = -EINVAL; 4387 goto cleanup; 4388 } 4389 4390 /* gate_mask must not select queues outside the valid queues */ 4391 queue_id = order_base_2(entry->gate_mask); 4392 if (queue_id >= bp->num_queues) { 4393 netdev_err(netdev, "Entry %zu: gate_mask 0x%x exceeds queue range (max_queues=%d)\n", 4394 i, entry->gate_mask, bp->num_queues); 4395 err = -EINVAL; 4396 goto cleanup; 4397 } 4398 4399 /* Check for start time limits */ 4400 start_time_sec = start_time; 4401 start_time_nsec = do_div(start_time_sec, NSEC_PER_SEC); 4402 if (start_time_sec > GENMASK(GEM_START_TIME_SEC_SIZE - 1, 0)) { 4403 netdev_err(netdev, "Entry %zu: Start time %llu s exceeds hardware limit\n", 4404 i, start_time_sec); 4405 err = -ERANGE; 4406 goto cleanup; 4407 } 4408 4409 /* Check for on time limit */ 4410 if (entry->interval > enst_max_hw_interval(speed)) { 4411 netdev_err(netdev, "Entry %zu: interval %u ns exceeds hardware limit %llu ns\n", 4412 i, entry->interval, enst_max_hw_interval(speed)); 4413 err = -ERANGE; 4414 goto cleanup; 4415 } 4416 4417 /* Check for off time limit*/ 4418 if ((conf->cycle_time - entry->interval) > enst_max_hw_interval(speed)) { 4419 netdev_err(netdev, "Entry %zu: off_time %llu ns exceeds hardware limit %llu ns\n", 4420 i, conf->cycle_time - entry->interval, 4421 enst_max_hw_interval(speed)); 4422 err = -ERANGE; 4423 goto cleanup; 4424 } 4425 4426 enst_queue[i].queue_id = queue_id; 4427 enst_queue[i].start_time_mask = 4428 (start_time_sec << GEM_START_TIME_SEC_OFFSET) | 4429 start_time_nsec; 4430 enst_queue[i].on_time_bytes = 4431 enst_ns_to_hw_units(entry->interval, speed); 4432 enst_queue[i].off_time_bytes = 4433 enst_ns_to_hw_units(conf->cycle_time - entry->interval, speed); 4434 4435 configured_queues |= entry->gate_mask; 4436 total_on_time += entry->interval; 4437 start_time += entry->interval; 4438 } 4439 4440 /* Check total interval doesn't exceed cycle time */ 4441 if (total_on_time > conf->cycle_time) { 4442 netdev_err(netdev, "Total ON %llu ns exceeds cycle time %llu ns\n", 4443 total_on_time, conf->cycle_time); 4444 err = -EINVAL; 4445 goto cleanup; 4446 } 4447 4448 netdev_dbg(netdev, "TAPRIO setup: %zu entries, base_time=%lld ns, cycle_time=%llu ns\n", 4449 conf->num_entries, conf->base_time, conf->cycle_time); 4450 4451 /* All validations passed - proceed with hardware configuration */ 4452 scoped_guard(spinlock_irqsave, &bp->lock) { 4453 /* Disable ENST queues if running before configuring */ 4454 queue_mask = BIT_U32(bp->num_queues) - 1; 4455 gem_writel(bp, ENST_CONTROL, 4456 queue_mask << GEM_ENST_DISABLE_QUEUE_OFFSET); 4457 4458 for (i = 0; i < conf->num_entries; i++) { 4459 queue = &bp->queues[enst_queue[i].queue_id]; 4460 /* Configure queue timing registers */ 4461 queue_writel(queue, ENST_START_TIME, 4462 enst_queue[i].start_time_mask); 4463 queue_writel(queue, ENST_ON_TIME, 4464 enst_queue[i].on_time_bytes); 4465 queue_writel(queue, ENST_OFF_TIME, 4466 enst_queue[i].off_time_bytes); 4467 } 4468 4469 /* Enable ENST for all configured queues in one write */ 4470 gem_writel(bp, ENST_CONTROL, configured_queues); 4471 } 4472 4473 netdev_info(netdev, "TAPRIO configuration completed successfully: %zu entries, %d queues configured\n", 4474 conf->num_entries, hweight32(configured_queues)); 4475 4476 cleanup: 4477 kfree(enst_queue); 4478 return err; 4479 } 4480 4481 static void macb_taprio_destroy(struct net_device *netdev) 4482 { 4483 struct macb *bp = netdev_priv(netdev); 4484 struct macb_queue *queue; 4485 u32 queue_mask; 4486 unsigned int q; 4487 4488 netdev_reset_tc(netdev); 4489 queue_mask = BIT_U32(bp->num_queues) - 1; 4490 4491 scoped_guard(spinlock_irqsave, &bp->lock) { 4492 /* Single disable command for all queues */ 4493 gem_writel(bp, ENST_CONTROL, 4494 queue_mask << GEM_ENST_DISABLE_QUEUE_OFFSET); 4495 4496 /* Clear all queue ENST registers in batch */ 4497 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 4498 queue_writel(queue, ENST_START_TIME, 0); 4499 queue_writel(queue, ENST_ON_TIME, 0); 4500 queue_writel(queue, ENST_OFF_TIME, 0); 4501 } 4502 } 4503 netdev_info(netdev, "TAPRIO destroy: All gates disabled\n"); 4504 } 4505 4506 static int macb_setup_taprio(struct net_device *netdev, 4507 struct tc_taprio_qopt_offload *taprio) 4508 { 4509 struct macb *bp = netdev_priv(netdev); 4510 int err = 0; 4511 4512 if (unlikely(!(netdev->hw_features & NETIF_F_HW_TC))) 4513 return -EOPNOTSUPP; 4514 4515 /* Check if Device is in runtime suspend */ 4516 if (unlikely(pm_runtime_suspended(&bp->pdev->dev))) { 4517 netdev_err(netdev, "Device is in runtime suspend\n"); 4518 return -EOPNOTSUPP; 4519 } 4520 4521 switch (taprio->cmd) { 4522 case TAPRIO_CMD_REPLACE: 4523 err = macb_taprio_setup_replace(netdev, taprio); 4524 break; 4525 case TAPRIO_CMD_DESTROY: 4526 macb_taprio_destroy(netdev); 4527 break; 4528 default: 4529 err = -EOPNOTSUPP; 4530 } 4531 4532 return err; 4533 } 4534 4535 static int macb_setup_tc(struct net_device *netdev, enum tc_setup_type type, 4536 void *type_data) 4537 { 4538 if (!netdev || !type_data) 4539 return -EINVAL; 4540 4541 switch (type) { 4542 case TC_SETUP_QDISC_TAPRIO: 4543 return macb_setup_taprio(netdev, type_data); 4544 default: 4545 return -EOPNOTSUPP; 4546 } 4547 } 4548 4549 static void macb_tx_timeout(struct net_device *netdev, unsigned int q) 4550 { 4551 struct macb *bp = netdev_priv(netdev); 4552 4553 macb_tx_restart(&bp->queues[q]); 4554 } 4555 4556 static const struct net_device_ops macb_netdev_ops = { 4557 .ndo_open = macb_open, 4558 .ndo_stop = macb_close, 4559 .ndo_start_xmit = macb_start_xmit, 4560 .ndo_set_rx_mode = macb_set_rx_mode, 4561 .ndo_get_stats64 = macb_get_stats, 4562 .ndo_eth_ioctl = macb_ioctl, 4563 .ndo_validate_addr = eth_validate_addr, 4564 .ndo_change_mtu = macb_change_mtu, 4565 .ndo_set_mac_address = macb_set_mac_addr, 4566 #ifdef CONFIG_NET_POLL_CONTROLLER 4567 .ndo_poll_controller = macb_poll_controller, 4568 #endif 4569 .ndo_set_features = macb_set_features, 4570 .ndo_features_check = macb_features_check, 4571 .ndo_hwtstamp_set = macb_hwtstamp_set, 4572 .ndo_hwtstamp_get = macb_hwtstamp_get, 4573 .ndo_setup_tc = macb_setup_tc, 4574 .ndo_tx_timeout = macb_tx_timeout, 4575 }; 4576 4577 /* Configure peripheral capabilities according to device tree 4578 * and integration options used 4579 */ 4580 static void macb_configure_caps(struct macb *bp, 4581 const struct macb_config *dt_conf) 4582 { 4583 u32 dcfg; 4584 4585 bp->caps = dt_conf->caps; 4586 4587 if (!dt_conf->usrio) 4588 bp->caps |= MACB_CAPS_USRIO_DISABLED; 4589 4590 if (hw_is_gem(bp->regs, bp->native_io)) { 4591 bp->caps |= MACB_CAPS_MACB_IS_GEM; 4592 4593 dcfg = gem_readl(bp, DCFG1); 4594 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 4595 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 4596 if (GEM_BFEXT(NO_PCS, dcfg) == 0) 4597 bp->caps |= MACB_CAPS_PCS; 4598 if (!(dcfg & GEM_BIT(USERIO))) 4599 bp->caps |= MACB_CAPS_USRIO_DISABLED; 4600 dcfg = gem_readl(bp, DCFG12); 4601 if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) 4602 bp->caps |= MACB_CAPS_HIGH_SPEED; 4603 dcfg = gem_readl(bp, DCFG2); 4604 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 4605 bp->caps |= MACB_CAPS_FIFO_MODE; 4606 if (GEM_BFEXT(PBUF_RSC, gem_readl(bp, DCFG6))) 4607 bp->caps |= MACB_CAPS_RSC; 4608 if (gem_has_ptp(bp)) { 4609 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 4610 dev_err(&bp->pdev->dev, 4611 "GEM doesn't support hardware ptp.\n"); 4612 else { 4613 #ifdef CONFIG_MACB_USE_HWSTAMP 4614 bp->caps |= MACB_CAPS_DMA_PTP; 4615 bp->ptp_info = &gem_ptp_info; 4616 #endif 4617 } 4618 } 4619 } 4620 4621 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 4622 } 4623 4624 static int macb_probe_queues(struct device *dev, void __iomem *mem, bool native_io) 4625 { 4626 /* BIT(0) is never set but queue 0 always exists. */ 4627 unsigned int queue_mask = 0x1; 4628 4629 /* Use hw_is_gem() as MACB_CAPS_MACB_IS_GEM is not yet positioned. */ 4630 if (hw_is_gem(mem, native_io)) { 4631 if (native_io) 4632 queue_mask |= __raw_readl(mem + GEM_DCFG6) & 0xFF; 4633 else 4634 queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xFF; 4635 4636 if (fls(queue_mask) != ffz(queue_mask)) { 4637 dev_err(dev, "queue mask %#x has a hole\n", queue_mask); 4638 return -EINVAL; 4639 } 4640 } 4641 4642 return hweight32(queue_mask); 4643 } 4644 4645 static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk, 4646 struct clk *rx_clk, struct clk *tsu_clk) 4647 { 4648 struct clk_bulk_data clks[] = { 4649 { .clk = tsu_clk, }, 4650 { .clk = rx_clk, }, 4651 { .clk = pclk, }, 4652 { .clk = hclk, }, 4653 { .clk = tx_clk }, 4654 }; 4655 4656 clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); 4657 } 4658 4659 static int macb_clk_init_dflt(struct platform_device *pdev, struct clk **pclk, 4660 struct clk **hclk, struct clk **tx_clk, 4661 struct clk **rx_clk, struct clk **tsu_clk) 4662 { 4663 struct macb_platform_data *pdata; 4664 int err; 4665 4666 pdata = dev_get_platdata(&pdev->dev); 4667 if (pdata) { 4668 *pclk = pdata->pclk; 4669 *hclk = pdata->hclk; 4670 } else { 4671 *pclk = devm_clk_get(&pdev->dev, "pclk"); 4672 *hclk = devm_clk_get(&pdev->dev, "hclk"); 4673 } 4674 4675 if (IS_ERR_OR_NULL(*pclk)) 4676 return dev_err_probe(&pdev->dev, 4677 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, 4678 "failed to get pclk\n"); 4679 4680 if (IS_ERR_OR_NULL(*hclk)) 4681 return dev_err_probe(&pdev->dev, 4682 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, 4683 "failed to get hclk\n"); 4684 4685 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); 4686 if (IS_ERR(*tx_clk)) 4687 return PTR_ERR(*tx_clk); 4688 4689 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); 4690 if (IS_ERR(*rx_clk)) 4691 return PTR_ERR(*rx_clk); 4692 4693 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); 4694 if (IS_ERR(*tsu_clk)) 4695 return PTR_ERR(*tsu_clk); 4696 4697 err = clk_prepare_enable(*pclk); 4698 if (err) { 4699 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 4700 return err; 4701 } 4702 4703 err = clk_prepare_enable(*hclk); 4704 if (err) { 4705 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 4706 goto err_disable_pclk; 4707 } 4708 4709 err = clk_prepare_enable(*tx_clk); 4710 if (err) { 4711 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 4712 goto err_disable_hclk; 4713 } 4714 4715 err = clk_prepare_enable(*rx_clk); 4716 if (err) { 4717 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 4718 goto err_disable_txclk; 4719 } 4720 4721 err = clk_prepare_enable(*tsu_clk); 4722 if (err) { 4723 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 4724 goto err_disable_rxclk; 4725 } 4726 4727 return 0; 4728 4729 err_disable_rxclk: 4730 clk_disable_unprepare(*rx_clk); 4731 4732 err_disable_txclk: 4733 clk_disable_unprepare(*tx_clk); 4734 4735 err_disable_hclk: 4736 clk_disable_unprepare(*hclk); 4737 4738 err_disable_pclk: 4739 clk_disable_unprepare(*pclk); 4740 4741 return err; 4742 } 4743 4744 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 4745 struct clk **hclk, struct clk **tx_clk, 4746 struct clk **rx_clk, struct clk **tsu_clk, 4747 const struct macb_config *config) 4748 { 4749 if (config->clk_init) 4750 return config->clk_init(pdev, pclk, hclk, tx_clk, rx_clk, 4751 tsu_clk); 4752 else 4753 return macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk, 4754 tsu_clk); 4755 } 4756 4757 static int macb_init_dflt(struct platform_device *pdev) 4758 { 4759 struct net_device *netdev = platform_get_drvdata(pdev); 4760 unsigned int hw_q, q; 4761 struct macb *bp = netdev_priv(netdev); 4762 struct macb_queue *queue; 4763 int err; 4764 u32 val, reg; 4765 4766 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 4767 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 4768 4769 /* set the queue register mapping once for all: queue0 has a special 4770 * register mapping but we don't want to test the queue index then 4771 * compute the corresponding register offset at run time. 4772 */ 4773 for (hw_q = 0, q = 0; hw_q < bp->num_queues; ++hw_q) { 4774 queue = &bp->queues[q]; 4775 queue->bp = bp; 4776 spin_lock_init(&queue->tx_ptr_lock); 4777 netif_napi_add(netdev, &queue->napi_rx, macb_rx_poll); 4778 netif_napi_add_tx(netdev, &queue->napi_tx, macb_tx_poll); 4779 if (hw_q) { 4780 queue->ISR = GEM_ISR(hw_q - 1); 4781 queue->IER = GEM_IER(hw_q - 1); 4782 queue->IDR = GEM_IDR(hw_q - 1); 4783 queue->IMR = GEM_IMR(hw_q - 1); 4784 queue->TBQP = GEM_TBQP(hw_q - 1); 4785 queue->RBQP = GEM_RBQP(hw_q - 1); 4786 queue->RBQS = GEM_RBQS(hw_q - 1); 4787 } else { 4788 /* queue0 uses legacy registers */ 4789 queue->ISR = MACB_ISR; 4790 queue->IER = MACB_IER; 4791 queue->IDR = MACB_IDR; 4792 queue->IMR = MACB_IMR; 4793 queue->TBQP = MACB_TBQP; 4794 queue->RBQP = MACB_RBQP; 4795 } 4796 4797 queue->ENST_START_TIME = GEM_ENST_START_TIME(hw_q); 4798 queue->ENST_ON_TIME = GEM_ENST_ON_TIME(hw_q); 4799 queue->ENST_OFF_TIME = GEM_ENST_OFF_TIME(hw_q); 4800 4801 /* get irq: here we use the linux queue index, not the hardware 4802 * queue index. the queue irq definitions in the device tree 4803 * must remove the optional gaps that could exist in the 4804 * hardware queue mask. 4805 */ 4806 queue->irq = platform_get_irq(pdev, q); 4807 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 4808 IRQF_SHARED, netdev->name, queue); 4809 if (err) { 4810 dev_err(&pdev->dev, 4811 "Unable to request IRQ %d (error %d)\n", 4812 queue->irq, err); 4813 return err; 4814 } 4815 4816 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 4817 q++; 4818 } 4819 4820 netdev->netdev_ops = &macb_netdev_ops; 4821 4822 /* setup appropriated routines according to adapter type */ 4823 if (macb_is_gem(bp)) { 4824 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 4825 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 4826 bp->macbgem_ops.mog_init_rings = gem_init_rings; 4827 bp->macbgem_ops.mog_rx = gem_rx; 4828 netdev->ethtool_ops = &gem_ethtool_ops; 4829 } else { 4830 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 4831 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 4832 bp->macbgem_ops.mog_init_rings = macb_init_rings; 4833 bp->macbgem_ops.mog_rx = macb_rx; 4834 netdev->ethtool_ops = &macb_ethtool_ops; 4835 } 4836 4837 netdev_sw_irq_coalesce_default_on(netdev); 4838 4839 netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 4840 4841 /* Set features */ 4842 netdev->hw_features = NETIF_F_SG; 4843 4844 /* Check LSO capability; runtime detection can be overridden by a cap 4845 * flag if the hardware is known to be buggy 4846 */ 4847 if (!(bp->caps & MACB_CAPS_NO_LSO) && 4848 GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 4849 netdev->hw_features |= MACB_NETIF_LSO; 4850 4851 /* Checksum offload is only available on gem with packet buffer */ 4852 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 4853 netdev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 4854 if (bp->caps & MACB_CAPS_SG_DISABLED) 4855 netdev->hw_features &= ~NETIF_F_SG; 4856 /* Enable HW_TC if hardware supports QBV */ 4857 if (bp->caps & MACB_CAPS_QBV) 4858 netdev->hw_features |= NETIF_F_HW_TC; 4859 4860 netdev->features = netdev->hw_features; 4861 4862 /* Check RX Flow Filters support. 4863 * Max Rx flows set by availability of screeners & compare regs: 4864 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 4865 */ 4866 reg = gem_readl(bp, DCFG8); 4867 bp->max_tuples = umin((GEM_BFEXT(SCR2CMP, reg) / 3), 4868 GEM_BFEXT(T2SCR, reg)); 4869 INIT_LIST_HEAD(&bp->rx_fs_list.list); 4870 if (bp->max_tuples > 0) { 4871 /* also needs one ethtype match to check IPv4 */ 4872 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 4873 /* program this reg now */ 4874 reg = 0; 4875 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 4876 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 4877 /* Filtering is supported in hw but don't enable it in kernel now */ 4878 netdev->hw_features |= NETIF_F_NTUPLE; 4879 /* init Rx flow definitions */ 4880 bp->rx_fs_list.count = 0; 4881 spin_lock_init(&bp->rx_fs_lock); 4882 } else 4883 bp->max_tuples = 0; 4884 } 4885 4886 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 4887 val = 0; 4888 if (bp->caps & MACB_CAPS_USRIO_HAS_MII) { 4889 if (phy_interface_mode_is_rgmii(bp->phy_interface)) 4890 val = bp->usrio->rgmii; 4891 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 4892 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 4893 val = bp->usrio->rmii; 4894 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 4895 val = bp->usrio->mii; 4896 } 4897 4898 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 4899 val |= bp->usrio->clken; 4900 4901 if (bp->caps & MACB_CAPS_USRIO_HAS_REFCLK_SOURCE) { 4902 const char *prop; 4903 bool refclk_ext; 4904 int ret; 4905 4906 /* Default to whatever was set in the match data for 4907 * this device. There's two properties for refclk 4908 * control, but the boolean one is deprecated so is 4909 * a lower priority to check, no device should have 4910 * both. 4911 */ 4912 refclk_ext = bp->usrio->refclk_default_external; 4913 4914 ret = of_property_read_string(pdev->dev.of_node, 4915 "cdns,refclk-source", &prop); 4916 if (!ret) { 4917 if (!strcmp(prop, "external")) 4918 refclk_ext = true; 4919 else 4920 refclk_ext = false; 4921 } else { 4922 ret = of_property_read_bool(pdev->dev.of_node, 4923 "cdns,refclk-ext"); 4924 if (ret) 4925 refclk_ext = true; 4926 } 4927 4928 if (refclk_ext) 4929 val |= bp->usrio->refclk; 4930 } 4931 4932 if (bp->caps & MACB_CAPS_USRIO_HAS_TSUCLK_SOURCE) 4933 val |= bp->usrio->tsu_source; 4934 4935 macb_or_gem_writel(bp, USRIO, val); 4936 } 4937 4938 /* Set MII management clock divider */ 4939 val = macb_mdc_clk_div(bp); 4940 val |= macb_dbw(bp); 4941 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 4942 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 4943 macb_writel(bp, NCFGR, val); 4944 4945 return 0; 4946 } 4947 4948 static int macb_init(struct platform_device *pdev, 4949 const struct macb_config *config) 4950 { 4951 if (config->init) 4952 return config->init(pdev); 4953 else 4954 return macb_init_dflt(pdev); 4955 } 4956 4957 static const struct macb_usrio_config at91_default_usrio = { 4958 .mii = MACB_BIT(MII), 4959 .rmii = MACB_BIT(RMII), 4960 .rgmii = GEM_BIT(RGMII), 4961 .clken = MACB_BIT(CLKEN), 4962 }; 4963 4964 /* 1518 rounded up */ 4965 #define AT91ETHER_MAX_RBUFF_SZ 0x600 4966 /* max number of receive buffers */ 4967 #define AT91ETHER_MAX_RX_DESCR 9 4968 4969 static struct sifive_fu540_macb_mgmt *mgmt; 4970 4971 static int at91ether_alloc_coherent(struct macb *bp) 4972 { 4973 struct macb_queue *queue = &bp->queues[0]; 4974 4975 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, 4976 (AT91ETHER_MAX_RX_DESCR * 4977 macb_dma_desc_get_size(bp)), 4978 &queue->rx_ring_dma, GFP_KERNEL); 4979 if (!queue->rx_ring) 4980 return -ENOMEM; 4981 4982 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, 4983 AT91ETHER_MAX_RX_DESCR * 4984 AT91ETHER_MAX_RBUFF_SZ, 4985 &queue->rx_buffers_dma, 4986 GFP_KERNEL); 4987 if (!queue->rx_buffers) { 4988 dma_free_coherent(&bp->pdev->dev, 4989 AT91ETHER_MAX_RX_DESCR * 4990 macb_dma_desc_get_size(bp), 4991 queue->rx_ring, queue->rx_ring_dma); 4992 queue->rx_ring = NULL; 4993 return -ENOMEM; 4994 } 4995 4996 return 0; 4997 } 4998 4999 static void at91ether_free_coherent(struct macb *bp) 5000 { 5001 struct macb_queue *queue = &bp->queues[0]; 5002 5003 if (queue->rx_ring) { 5004 dma_free_coherent(&bp->pdev->dev, 5005 AT91ETHER_MAX_RX_DESCR * 5006 macb_dma_desc_get_size(bp), 5007 queue->rx_ring, queue->rx_ring_dma); 5008 queue->rx_ring = NULL; 5009 } 5010 5011 if (queue->rx_buffers) { 5012 dma_free_coherent(&bp->pdev->dev, 5013 AT91ETHER_MAX_RX_DESCR * 5014 AT91ETHER_MAX_RBUFF_SZ, 5015 queue->rx_buffers, queue->rx_buffers_dma); 5016 queue->rx_buffers = NULL; 5017 } 5018 } 5019 5020 /* Initialize and start the Receiver and Transmit subsystems */ 5021 static int at91ether_start(struct macb *bp) 5022 { 5023 struct macb_queue *queue = &bp->queues[0]; 5024 struct macb_dma_desc *desc; 5025 dma_addr_t addr; 5026 u32 ctl; 5027 int i, ret; 5028 5029 ret = at91ether_alloc_coherent(bp); 5030 if (ret) 5031 return ret; 5032 5033 addr = queue->rx_buffers_dma; 5034 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 5035 desc = macb_rx_desc(queue, i); 5036 macb_set_addr(bp, desc, addr); 5037 desc->ctrl = 0; 5038 addr += AT91ETHER_MAX_RBUFF_SZ; 5039 } 5040 5041 /* Set the Wrap bit on the last descriptor */ 5042 desc->addr |= MACB_BIT(RX_WRAP); 5043 5044 /* Reset buffer index */ 5045 queue->rx_tail = 0; 5046 5047 /* Program address of descriptor list in Rx Buffer Queue register */ 5048 macb_writel(bp, RBQP, queue->rx_ring_dma); 5049 5050 /* Enable Receive and Transmit */ 5051 ctl = macb_readl(bp, NCR); 5052 macb_writel(bp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 5053 5054 /* Enable MAC interrupts */ 5055 macb_writel(bp, IER, MACB_BIT(RCOMP) | 5056 MACB_BIT(RXUBR) | 5057 MACB_BIT(ISR_TUND) | 5058 MACB_BIT(ISR_RLE) | 5059 MACB_BIT(TCOMP) | 5060 MACB_BIT(ISR_ROVR) | 5061 MACB_BIT(HRESP)); 5062 5063 return 0; 5064 } 5065 5066 static void at91ether_stop(struct macb *bp) 5067 { 5068 u32 ctl; 5069 5070 /* Disable MAC interrupts */ 5071 macb_writel(bp, IDR, MACB_BIT(RCOMP) | 5072 MACB_BIT(RXUBR) | 5073 MACB_BIT(ISR_TUND) | 5074 MACB_BIT(ISR_RLE) | 5075 MACB_BIT(TCOMP) | 5076 MACB_BIT(ISR_ROVR) | 5077 MACB_BIT(HRESP)); 5078 5079 /* Disable Receiver and Transmitter */ 5080 ctl = macb_readl(bp, NCR); 5081 macb_writel(bp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 5082 5083 /* Free resources. */ 5084 at91ether_free_coherent(bp); 5085 } 5086 5087 /* Open the ethernet interface */ 5088 static int at91ether_open(struct net_device *netdev) 5089 { 5090 struct macb *bp = netdev_priv(netdev); 5091 u32 ctl; 5092 int ret; 5093 5094 ret = pm_runtime_resume_and_get(&bp->pdev->dev); 5095 if (ret < 0) 5096 return ret; 5097 5098 /* Clear internal statistics */ 5099 ctl = macb_readl(bp, NCR); 5100 macb_writel(bp, NCR, ctl | MACB_BIT(CLRSTAT)); 5101 5102 macb_set_hwaddr(bp); 5103 5104 ret = at91ether_start(bp); 5105 if (ret) 5106 goto pm_exit; 5107 5108 ret = macb_phylink_connect(bp); 5109 if (ret) 5110 goto stop; 5111 5112 netif_start_queue(netdev); 5113 5114 return 0; 5115 5116 stop: 5117 at91ether_stop(bp); 5118 pm_exit: 5119 pm_runtime_put_sync(&bp->pdev->dev); 5120 return ret; 5121 } 5122 5123 /* Close the interface */ 5124 static int at91ether_close(struct net_device *netdev) 5125 { 5126 struct macb *bp = netdev_priv(netdev); 5127 5128 netif_stop_queue(netdev); 5129 5130 phylink_stop(bp->phylink); 5131 phylink_disconnect_phy(bp->phylink); 5132 5133 at91ether_stop(bp); 5134 5135 pm_runtime_put(&bp->pdev->dev); 5136 5137 return 0; 5138 } 5139 5140 /* Transmit packet */ 5141 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 5142 struct net_device *netdev) 5143 { 5144 struct macb *bp = netdev_priv(netdev); 5145 struct device *dev = &bp->pdev->dev; 5146 5147 if (macb_readl(bp, TSR) & MACB_BIT(RM9200_BNQ)) { 5148 int desc = 0; 5149 5150 netif_stop_queue(netdev); 5151 5152 /* Store packet information (to free when Tx completed) */ 5153 bp->rm9200_txq[desc].skb = skb; 5154 bp->rm9200_txq[desc].size = skb->len; 5155 bp->rm9200_txq[desc].mapping = dma_map_single(dev, skb->data, 5156 skb->len, 5157 DMA_TO_DEVICE); 5158 if (dma_mapping_error(dev, bp->rm9200_txq[desc].mapping)) { 5159 dev_kfree_skb_any(skb); 5160 netdev->stats.tx_dropped++; 5161 netdev_err(netdev, "%s: DMA mapping error\n", __func__); 5162 return NETDEV_TX_OK; 5163 } 5164 5165 /* Set address of the data in the Transmit Address register */ 5166 macb_writel(bp, TAR, bp->rm9200_txq[desc].mapping); 5167 /* Set length of the packet in the Transmit Control register */ 5168 macb_writel(bp, TCR, skb->len); 5169 5170 } else { 5171 netdev_err(netdev, "%s called, but device is busy!\n", 5172 __func__); 5173 return NETDEV_TX_BUSY; 5174 } 5175 5176 return NETDEV_TX_OK; 5177 } 5178 5179 /* Extract received frame from buffer descriptors and sent to upper layers. 5180 * (Called from interrupt context) 5181 */ 5182 static void at91ether_rx(struct net_device *netdev) 5183 { 5184 struct macb *bp = netdev_priv(netdev); 5185 struct macb_queue *queue = &bp->queues[0]; 5186 struct macb_dma_desc *desc; 5187 unsigned char *p_recv; 5188 struct sk_buff *skb; 5189 unsigned int pktlen; 5190 5191 desc = macb_rx_desc(queue, queue->rx_tail); 5192 while (desc->addr & MACB_BIT(RX_USED)) { 5193 p_recv = queue->rx_buffers + 5194 queue->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 5195 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 5196 skb = netdev_alloc_skb(netdev, pktlen + 2); 5197 if (skb) { 5198 skb_reserve(skb, 2); 5199 skb_put_data(skb, p_recv, pktlen); 5200 5201 skb->protocol = eth_type_trans(skb, netdev); 5202 netdev->stats.rx_packets++; 5203 netdev->stats.rx_bytes += pktlen; 5204 netif_rx(skb); 5205 } else { 5206 netdev->stats.rx_dropped++; 5207 } 5208 5209 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 5210 netdev->stats.multicast++; 5211 5212 /* reset ownership bit */ 5213 desc->addr &= ~MACB_BIT(RX_USED); 5214 5215 /* wrap after last buffer */ 5216 if (queue->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 5217 queue->rx_tail = 0; 5218 else 5219 queue->rx_tail++; 5220 5221 desc = macb_rx_desc(queue, queue->rx_tail); 5222 } 5223 } 5224 5225 /* MAC interrupt handler */ 5226 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 5227 { 5228 struct net_device *netdev = dev_id; 5229 struct macb *bp = netdev_priv(netdev); 5230 u32 intstatus, ctl; 5231 unsigned int desc; 5232 5233 /* MAC Interrupt Status register indicates what interrupts are pending. 5234 * It is automatically cleared once read. 5235 */ 5236 intstatus = macb_readl(bp, ISR); 5237 5238 /* Receive complete */ 5239 if (intstatus & MACB_BIT(RCOMP)) 5240 at91ether_rx(netdev); 5241 5242 /* Transmit complete */ 5243 if (intstatus & MACB_BIT(TCOMP)) { 5244 /* The TCOM bit is set even if the transmission failed */ 5245 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 5246 netdev->stats.tx_errors++; 5247 5248 desc = 0; 5249 if (bp->rm9200_txq[desc].skb) { 5250 dev_consume_skb_irq(bp->rm9200_txq[desc].skb); 5251 bp->rm9200_txq[desc].skb = NULL; 5252 dma_unmap_single(&bp->pdev->dev, 5253 bp->rm9200_txq[desc].mapping, 5254 bp->rm9200_txq[desc].size, 5255 DMA_TO_DEVICE); 5256 netdev->stats.tx_packets++; 5257 netdev->stats.tx_bytes += bp->rm9200_txq[desc].size; 5258 } 5259 netif_wake_queue(netdev); 5260 } 5261 5262 /* Work-around for EMAC Errata section 41.3.1 */ 5263 if (intstatus & MACB_BIT(RXUBR)) { 5264 ctl = macb_readl(bp, NCR); 5265 macb_writel(bp, NCR, ctl & ~MACB_BIT(RE)); 5266 wmb(); 5267 macb_writel(bp, NCR, ctl | MACB_BIT(RE)); 5268 } 5269 5270 if (intstatus & MACB_BIT(ISR_ROVR)) 5271 netdev_err(netdev, "ROVR error\n"); 5272 5273 return IRQ_HANDLED; 5274 } 5275 5276 #ifdef CONFIG_NET_POLL_CONTROLLER 5277 static void at91ether_poll_controller(struct net_device *netdev) 5278 { 5279 unsigned long flags; 5280 5281 local_irq_save(flags); 5282 at91ether_interrupt(netdev->irq, netdev); 5283 local_irq_restore(flags); 5284 } 5285 #endif 5286 5287 static const struct net_device_ops at91ether_netdev_ops = { 5288 .ndo_open = at91ether_open, 5289 .ndo_stop = at91ether_close, 5290 .ndo_start_xmit = at91ether_start_xmit, 5291 .ndo_get_stats64 = macb_get_stats, 5292 .ndo_set_rx_mode = macb_set_rx_mode, 5293 .ndo_set_mac_address = eth_mac_addr, 5294 .ndo_eth_ioctl = macb_ioctl, 5295 .ndo_validate_addr = eth_validate_addr, 5296 #ifdef CONFIG_NET_POLL_CONTROLLER 5297 .ndo_poll_controller = at91ether_poll_controller, 5298 #endif 5299 .ndo_hwtstamp_set = macb_hwtstamp_set, 5300 .ndo_hwtstamp_get = macb_hwtstamp_get, 5301 }; 5302 5303 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 5304 struct clk **hclk, struct clk **tx_clk, 5305 struct clk **rx_clk, struct clk **tsu_clk) 5306 { 5307 int err; 5308 5309 *hclk = NULL; 5310 *tx_clk = NULL; 5311 *rx_clk = NULL; 5312 *tsu_clk = NULL; 5313 5314 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 5315 if (IS_ERR(*pclk)) 5316 return PTR_ERR(*pclk); 5317 5318 err = clk_prepare_enable(*pclk); 5319 if (err) { 5320 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 5321 return err; 5322 } 5323 5324 return 0; 5325 } 5326 5327 static int at91ether_init(struct platform_device *pdev) 5328 { 5329 struct net_device *netdev = platform_get_drvdata(pdev); 5330 struct macb *bp = netdev_priv(netdev); 5331 int err; 5332 5333 bp->queues[0].bp = bp; 5334 5335 netdev->netdev_ops = &at91ether_netdev_ops; 5336 netdev->ethtool_ops = &macb_ethtool_ops; 5337 5338 err = devm_request_irq(&pdev->dev, netdev->irq, at91ether_interrupt, 5339 0, netdev->name, netdev); 5340 if (err) 5341 return err; 5342 5343 macb_writel(bp, NCR, 0); 5344 5345 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); 5346 5347 return 0; 5348 } 5349 5350 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 5351 unsigned long parent_rate) 5352 { 5353 return mgmt->rate; 5354 } 5355 5356 static int fu540_macb_tx_determine_rate(struct clk_hw *hw, 5357 struct clk_rate_request *req) 5358 { 5359 if (WARN_ON(req->rate < 2500000)) 5360 req->rate = 2500000; 5361 else if (req->rate == 2500000) 5362 req->rate = 2500000; 5363 else if (WARN_ON(req->rate < 13750000)) 5364 req->rate = 2500000; 5365 else if (WARN_ON(req->rate < 25000000)) 5366 req->rate = 25000000; 5367 else if (req->rate == 25000000) 5368 req->rate = 25000000; 5369 else if (WARN_ON(req->rate < 75000000)) 5370 req->rate = 25000000; 5371 else if (WARN_ON(req->rate < 125000000)) 5372 req->rate = 125000000; 5373 else if (req->rate == 125000000) 5374 req->rate = 125000000; 5375 else if (WARN_ON(req->rate > 125000000)) 5376 req->rate = 125000000; 5377 else 5378 req->rate = 125000000; 5379 5380 return 0; 5381 } 5382 5383 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 5384 unsigned long parent_rate) 5385 { 5386 struct clk_rate_request req; 5387 int ret; 5388 5389 clk_hw_init_rate_request(hw, &req, rate); 5390 ret = fu540_macb_tx_determine_rate(hw, &req); 5391 if (ret != 0) 5392 return ret; 5393 5394 if (req.rate != 125000000) 5395 iowrite32(1, mgmt->reg); 5396 else 5397 iowrite32(0, mgmt->reg); 5398 mgmt->rate = rate; 5399 5400 return 0; 5401 } 5402 5403 static const struct clk_ops fu540_c000_ops = { 5404 .recalc_rate = fu540_macb_tx_recalc_rate, 5405 .determine_rate = fu540_macb_tx_determine_rate, 5406 .set_rate = fu540_macb_tx_set_rate, 5407 }; 5408 5409 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 5410 struct clk **hclk, struct clk **tx_clk, 5411 struct clk **rx_clk, struct clk **tsu_clk) 5412 { 5413 struct clk_init_data init; 5414 int err = 0; 5415 5416 err = macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 5417 if (err) 5418 return err; 5419 5420 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 5421 if (!mgmt) { 5422 err = -ENOMEM; 5423 goto err_disable_clks; 5424 } 5425 5426 init.name = "sifive-gemgxl-mgmt"; 5427 init.ops = &fu540_c000_ops; 5428 init.flags = 0; 5429 init.num_parents = 0; 5430 5431 mgmt->rate = 0; 5432 mgmt->hw.init = &init; 5433 5434 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); 5435 if (IS_ERR(*tx_clk)) { 5436 err = PTR_ERR(*tx_clk); 5437 goto err_disable_clks; 5438 } 5439 5440 err = clk_prepare_enable(*tx_clk); 5441 if (err) { 5442 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 5443 *tx_clk = NULL; 5444 goto err_disable_clks; 5445 } else { 5446 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 5447 } 5448 5449 return 0; 5450 5451 err_disable_clks: 5452 macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk); 5453 5454 return err; 5455 } 5456 5457 static int fu540_c000_init(struct platform_device *pdev) 5458 { 5459 mgmt->reg = devm_platform_ioremap_resource(pdev, 1); 5460 if (IS_ERR(mgmt->reg)) 5461 return PTR_ERR(mgmt->reg); 5462 5463 return macb_init_dflt(pdev); 5464 } 5465 5466 static int init_reset_optional(struct platform_device *pdev) 5467 { 5468 struct net_device *netdev = platform_get_drvdata(pdev); 5469 struct macb *bp = netdev_priv(netdev); 5470 int ret; 5471 5472 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { 5473 /* Ensure PHY device used in SGMII mode is ready */ 5474 bp->phy = devm_phy_optional_get(&pdev->dev, NULL); 5475 5476 if (IS_ERR(bp->phy)) 5477 return dev_err_probe(&pdev->dev, PTR_ERR(bp->phy), 5478 "failed to get SGMII PHY\n"); 5479 5480 ret = phy_init(bp->phy); 5481 if (ret) 5482 return dev_err_probe(&pdev->dev, ret, 5483 "failed to init SGMII PHY\n"); 5484 5485 ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG); 5486 if (!ret) { 5487 u32 pm_info[2]; 5488 5489 ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", 5490 pm_info, ARRAY_SIZE(pm_info)); 5491 if (ret) { 5492 dev_err(&pdev->dev, "Failed to read power management information\n"); 5493 goto err_out_phy_exit; 5494 } 5495 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0); 5496 if (ret) 5497 goto err_out_phy_exit; 5498 5499 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1); 5500 if (ret) 5501 goto err_out_phy_exit; 5502 } 5503 5504 } 5505 5506 /* Fully reset controller at hardware level if mapped in device tree */ 5507 ret = device_reset_optional(&pdev->dev); 5508 if (ret) { 5509 phy_exit(bp->phy); 5510 return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); 5511 } 5512 5513 ret = macb_init_dflt(pdev); 5514 5515 err_out_phy_exit: 5516 if (ret) 5517 phy_exit(bp->phy); 5518 5519 return ret; 5520 } 5521 5522 static int eyeq5_init(struct platform_device *pdev) 5523 { 5524 struct net_device *netdev = platform_get_drvdata(pdev); 5525 struct macb *bp = netdev_priv(netdev); 5526 struct device *dev = &pdev->dev; 5527 int ret; 5528 5529 bp->phy = devm_phy_get(dev, NULL); 5530 if (IS_ERR(bp->phy)) 5531 return dev_err_probe(dev, PTR_ERR(bp->phy), 5532 "failed to get PHY\n"); 5533 5534 ret = phy_init(bp->phy); 5535 if (ret) 5536 return dev_err_probe(dev, ret, "failed to init PHY\n"); 5537 5538 ret = macb_init_dflt(pdev); 5539 if (ret) 5540 phy_exit(bp->phy); 5541 return ret; 5542 } 5543 5544 static int macb_alloc_tieoff(struct macb *bp) 5545 { 5546 /* Tieoff is a workaround in case HW cannot disable queues, for PM. */ 5547 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) 5548 return 0; 5549 5550 bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev, 5551 macb_dma_desc_get_size(bp), 5552 &bp->rx_ring_tieoff_dma, 5553 GFP_KERNEL); 5554 if (!bp->rx_ring_tieoff) 5555 return -ENOMEM; 5556 5557 macb_set_addr(bp, bp->rx_ring_tieoff, 5558 MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED)); 5559 5560 bp->rx_ring_tieoff->ctrl = 0; 5561 5562 return 0; 5563 } 5564 5565 static void macb_free_tieoff(struct macb *bp) 5566 { 5567 if (!bp->rx_ring_tieoff) 5568 return; 5569 5570 dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(bp), 5571 bp->rx_ring_tieoff, 5572 bp->rx_ring_tieoff_dma); 5573 bp->rx_ring_tieoff = NULL; 5574 } 5575 5576 static const struct macb_usrio_config mpfs_usrio = { 5577 .tsu_source = 0, 5578 }; 5579 5580 static const struct macb_usrio_config sama7g5_gem_usrio = { 5581 .mii = 0, 5582 .rmii = 1, 5583 .rgmii = 2, 5584 .refclk = BIT(2), 5585 .refclk_default_external = false, 5586 .hdfctlen = BIT(6), 5587 }; 5588 5589 static const struct macb_usrio_config sama7g5_emac_usrio = { 5590 .mii = 0, 5591 .rmii = 1, 5592 .rgmii = 2, 5593 .refclk = BIT(2), 5594 .refclk_default_external = true, 5595 .hdfctlen = BIT(6), 5596 }; 5597 5598 static const struct macb_config fu540_c000_config = { 5599 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 5600 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_USRIO_HAS_MII, 5601 .dma_burst_length = 16, 5602 .clk_init = fu540_c000_clk_init, 5603 .init = fu540_c000_init, 5604 .jumbo_max_len = 10240, 5605 .usrio = &at91_default_usrio, 5606 }; 5607 5608 static const struct macb_config at91sam9260_config = { 5609 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 5610 MACB_CAPS_USRIO_HAS_MII, 5611 .usrio = &at91_default_usrio, 5612 }; 5613 5614 static const struct macb_config sama5d3macb_config = { 5615 .caps = MACB_CAPS_SG_DISABLED | 5616 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 5617 MACB_CAPS_USRIO_HAS_MII, 5618 .usrio = &at91_default_usrio, 5619 }; 5620 5621 static const struct macb_config pc302gem_config = { 5622 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5623 MACB_CAPS_USRIO_HAS_MII, 5624 .dma_burst_length = 16, 5625 .usrio = &at91_default_usrio, 5626 }; 5627 5628 static const struct macb_config sama5d2_config = { 5629 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO | 5630 MACB_CAPS_USRIO_HAS_MII, 5631 .dma_burst_length = 16, 5632 .jumbo_max_len = 10240, 5633 .usrio = &at91_default_usrio, 5634 }; 5635 5636 static const struct macb_config sama5d29_config = { 5637 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP | 5638 MACB_CAPS_USRIO_HAS_MII, 5639 .dma_burst_length = 16, 5640 .usrio = &at91_default_usrio, 5641 }; 5642 5643 static const struct macb_config sama5d3_config = { 5644 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5645 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO | 5646 MACB_CAPS_USRIO_HAS_MII, 5647 .dma_burst_length = 16, 5648 .jumbo_max_len = 10240, 5649 .usrio = &at91_default_usrio, 5650 }; 5651 5652 static const struct macb_config sama5d4_config = { 5653 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 5654 MACB_CAPS_USRIO_HAS_MII, 5655 .dma_burst_length = 4, 5656 .usrio = &at91_default_usrio, 5657 }; 5658 5659 static const struct macb_config emac_config = { 5660 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC | 5661 MACB_CAPS_USRIO_HAS_MII, 5662 .clk_init = at91ether_clk_init, 5663 .init = at91ether_init, 5664 .usrio = &at91_default_usrio, 5665 }; 5666 5667 static const struct macb_config np4_config = { 5668 .caps = MACB_CAPS_USRIO_DISABLED, 5669 }; 5670 5671 static const struct macb_config zynqmp_config = { 5672 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5673 MACB_CAPS_JUMBO | 5674 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | 5675 MACB_CAPS_USRIO_HAS_MII, 5676 .dma_burst_length = 16, 5677 .init = init_reset_optional, 5678 .jumbo_max_len = 10240, 5679 .usrio = &at91_default_usrio, 5680 }; 5681 5682 static const struct macb_config zynq_config = { 5683 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 5684 MACB_CAPS_NEEDS_RSTONUBR | 5685 MACB_CAPS_USRIO_HAS_MII, 5686 .dma_burst_length = 16, 5687 .usrio = &at91_default_usrio, 5688 }; 5689 5690 static const struct macb_config mpfs_config = { 5691 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5692 MACB_CAPS_JUMBO | 5693 MACB_CAPS_GEM_HAS_PTP | 5694 MACB_CAPS_USRIO_HAS_TSUCLK_SOURCE, 5695 .dma_burst_length = 16, 5696 .init = init_reset_optional, 5697 .usrio = &mpfs_usrio, 5698 .max_tx_length = 4040, /* Cadence Erratum 1686 */ 5699 .jumbo_max_len = 4040, 5700 }; 5701 5702 static const struct macb_config sama7g5_gem_config = { 5703 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | 5704 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 5705 MACB_CAPS_USRIO_HAS_REFCLK_SOURCE | 5706 MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP | 5707 MACB_CAPS_USRIO_HAS_MII, 5708 .dma_burst_length = 16, 5709 .usrio = &sama7g5_gem_usrio, 5710 }; 5711 5712 static const struct macb_config sama7g5_emac_config = { 5713 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 5714 MACB_CAPS_MIIONRGMII | 5715 MACB_CAPS_USRIO_HAS_REFCLK_SOURCE | 5716 MACB_CAPS_GEM_HAS_PTP | 5717 MACB_CAPS_USRIO_HAS_MII, 5718 .dma_burst_length = 16, 5719 .usrio = &sama7g5_emac_usrio, 5720 }; 5721 5722 static const struct macb_config versal_config = { 5723 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 5724 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | 5725 MACB_CAPS_NEED_TSUCLK | MACB_CAPS_QUEUE_DISABLE | 5726 MACB_CAPS_QBV | 5727 MACB_CAPS_USRIO_HAS_MII, 5728 .dma_burst_length = 16, 5729 .init = init_reset_optional, 5730 .jumbo_max_len = 10240, 5731 .usrio = &at91_default_usrio, 5732 }; 5733 5734 static const struct macb_config eyeq5_config = { 5735 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 5736 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_QUEUE_DISABLE | 5737 MACB_CAPS_NO_LSO | MACB_CAPS_EEE, 5738 .dma_burst_length = 16, 5739 .init = eyeq5_init, 5740 .jumbo_max_len = 10240, 5741 }; 5742 5743 static const struct macb_config raspberrypi_rp1_config = { 5744 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | 5745 MACB_CAPS_JUMBO | 5746 MACB_CAPS_GEM_HAS_PTP | 5747 MACB_CAPS_EEE | 5748 MACB_CAPS_USRIO_HAS_MII, 5749 .dma_burst_length = 16, 5750 .usrio = &at91_default_usrio, 5751 .jumbo_max_len = 10240, 5752 }; 5753 5754 static const struct macb_config pic64hpsc_config = { 5755 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 5756 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_USRIO_DISABLED, 5757 .dma_burst_length = 16, 5758 .init = init_reset_optional, 5759 .jumbo_max_len = 16383, 5760 }; 5761 5762 static const struct of_device_id macb_dt_ids[] = { 5763 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 5764 { .compatible = "cdns,macb" }, 5765 { .compatible = "cdns,np4-macb", .data = &np4_config }, 5766 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 5767 { .compatible = "cdns,gem", .data = &pc302gem_config }, 5768 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 5769 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 5770 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config }, 5771 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 5772 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 5773 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 5774 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 5775 { .compatible = "cdns,emac", .data = &emac_config }, 5776 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */ 5777 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */ 5778 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 5779 { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, 5780 { .compatible = "microchip,pic64hpsc-gem", .data = &pic64hpsc_config}, 5781 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, 5782 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, 5783 { .compatible = "mobileye,eyeq5-gem", .data = &eyeq5_config }, 5784 { .compatible = "raspberrypi,rp1-gem", .data = &raspberrypi_rp1_config }, 5785 { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config}, 5786 { .compatible = "xlnx,zynq-gem", .data = &zynq_config }, 5787 { .compatible = "xlnx,versal-gem", .data = &versal_config}, 5788 { /* sentinel */ } 5789 }; 5790 MODULE_DEVICE_TABLE(of, macb_dt_ids); 5791 5792 static const struct macb_config default_gem_config = { 5793 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5794 MACB_CAPS_JUMBO | 5795 MACB_CAPS_GEM_HAS_PTP, 5796 .dma_burst_length = 16, 5797 .usrio = &at91_default_usrio, 5798 .jumbo_max_len = 10240, 5799 }; 5800 5801 static int macb_probe(struct platform_device *pdev) 5802 { 5803 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 5804 struct device_node *np = pdev->dev.of_node; 5805 const struct macb_config *macb_config; 5806 struct clk *tsu_clk = NULL; 5807 phy_interface_t interface; 5808 struct net_device *netdev; 5809 struct resource *regs; 5810 u32 wtrmrk_rst_val; 5811 void __iomem *mem; 5812 struct macb *bp; 5813 int num_queues; 5814 bool native_io; 5815 int err, val; 5816 5817 mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); 5818 if (IS_ERR(mem)) 5819 return PTR_ERR(mem); 5820 5821 macb_config = of_device_get_match_data(&pdev->dev); 5822 if (!macb_config) 5823 macb_config = &default_gem_config; 5824 5825 err = macb_clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk, 5826 macb_config); 5827 if (err) 5828 return err; 5829 5830 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 5831 pm_runtime_use_autosuspend(&pdev->dev); 5832 pm_runtime_get_noresume(&pdev->dev); 5833 pm_runtime_set_active(&pdev->dev); 5834 pm_runtime_enable(&pdev->dev); 5835 native_io = hw_is_native_io(mem); 5836 5837 num_queues = macb_probe_queues(&pdev->dev, mem, native_io); 5838 if (num_queues < 0) { 5839 err = num_queues; 5840 goto err_disable_clocks; 5841 } 5842 5843 netdev = alloc_etherdev_mq(sizeof(*bp), num_queues); 5844 if (!netdev) { 5845 err = -ENOMEM; 5846 goto err_disable_clocks; 5847 } 5848 5849 netdev->base_addr = regs->start; 5850 5851 SET_NETDEV_DEV(netdev, &pdev->dev); 5852 5853 bp = netdev_priv(netdev); 5854 bp->pdev = pdev; 5855 bp->netdev = netdev; 5856 bp->regs = mem; 5857 bp->native_io = native_io; 5858 if (native_io) { 5859 bp->macb_reg_readl = hw_readl_native; 5860 bp->macb_reg_writel = hw_writel_native; 5861 } else { 5862 bp->macb_reg_readl = hw_readl; 5863 bp->macb_reg_writel = hw_writel; 5864 } 5865 bp->num_queues = num_queues; 5866 bp->dma_burst_length = macb_config->dma_burst_length; 5867 bp->pclk = pclk; 5868 bp->hclk = hclk; 5869 bp->tx_clk = tx_clk; 5870 bp->rx_clk = rx_clk; 5871 bp->tsu_clk = tsu_clk; 5872 bp->jumbo_max_len = macb_config->jumbo_max_len; 5873 5874 if (!hw_is_gem(bp->regs, bp->native_io)) 5875 bp->max_tx_length = MACB_MAX_TX_LEN; 5876 else if (macb_config->max_tx_length) 5877 bp->max_tx_length = macb_config->max_tx_length; 5878 else 5879 bp->max_tx_length = GEM_MAX_TX_LEN; 5880 5881 bp->wol = 0; 5882 device_set_wakeup_capable(&pdev->dev, 1); 5883 5884 bp->usrio = macb_config->usrio; 5885 5886 if (of_property_read_bool(bp->pdev->dev.of_node, "cdns,timer-adjust") && 5887 IS_ENABLED(CONFIG_MACB_USE_HWSTAMP)) { 5888 dev_err(&pdev->dev, "Timer adjust mode is not supported\n"); 5889 err = -EINVAL; 5890 goto err_out_free_netdev; 5891 } 5892 5893 /* By default we set to partial store and forward mode for zynqmp. 5894 * Disable if not set in devicetree. 5895 */ 5896 if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) { 5897 err = of_property_read_u32(bp->pdev->dev.of_node, 5898 "cdns,rx-watermark", 5899 &bp->rx_watermark); 5900 5901 if (!err) { 5902 /* Disable partial store and forward in case of error or 5903 * invalid watermark value 5904 */ 5905 wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1; 5906 if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) { 5907 dev_info(&bp->pdev->dev, "Invalid watermark value\n"); 5908 bp->rx_watermark = 0; 5909 } 5910 } 5911 } 5912 spin_lock_init(&bp->lock); 5913 spin_lock_init(&bp->stats_lock); 5914 spin_lock_init(&bp->tsu_clk_lock); 5915 5916 /* setup capabilities */ 5917 macb_configure_caps(bp, macb_config); 5918 5919 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 5920 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 5921 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); 5922 if (err) { 5923 dev_err(&pdev->dev, "failed to set DMA mask\n"); 5924 goto err_out_free_netdev; 5925 } 5926 bp->caps |= MACB_CAPS_DMA_64B; 5927 } 5928 #endif 5929 platform_set_drvdata(pdev, netdev); 5930 5931 netdev->irq = platform_get_irq(pdev, 0); 5932 if (netdev->irq < 0) { 5933 err = netdev->irq; 5934 goto err_out_free_netdev; 5935 } 5936 5937 /* MTU range: 68 - 1518 or 10240 */ 5938 netdev->min_mtu = GEM_MTU_MIN_SIZE; 5939 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 5940 netdev->max_mtu = MIN(bp->jumbo_max_len, RX_BUFFER_MAX) - 5941 ETH_HLEN - ETH_FCS_LEN; 5942 else 5943 netdev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN; 5944 5945 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 5946 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 5947 if (val) 5948 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 5949 macb_dma_desc_get_size(bp); 5950 5951 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 5952 if (val) 5953 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 5954 macb_dma_desc_get_size(bp); 5955 } 5956 5957 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 5958 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 5959 bp->rx_intr_mask |= MACB_BIT(RXUBR); 5960 5961 err = of_get_ethdev_address(np, bp->netdev); 5962 if (err == -EPROBE_DEFER) 5963 goto err_out_free_netdev; 5964 else if (err) 5965 macb_get_hwaddr(bp); 5966 5967 err = of_get_phy_mode(np, &interface); 5968 if (err) 5969 /* not found in DT, MII by default */ 5970 bp->phy_interface = PHY_INTERFACE_MODE_MII; 5971 else 5972 bp->phy_interface = interface; 5973 5974 /* IP specific init */ 5975 err = macb_init(pdev, macb_config); 5976 if (err) 5977 goto err_out_free_netdev; 5978 5979 err = macb_mii_init(bp); 5980 if (err) 5981 goto err_out_phy_exit; 5982 5983 netif_carrier_off(netdev); 5984 5985 err = macb_alloc_tieoff(bp); 5986 if (err) 5987 goto err_out_unregister_mdio; 5988 5989 err = register_netdev(netdev); 5990 if (err) { 5991 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 5992 goto err_out_free_tieoff; 5993 } 5994 5995 INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task); 5996 INIT_DELAYED_WORK(&bp->tx_lpi_work, macb_tx_lpi_work_fn); 5997 5998 netdev_info(netdev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 5999 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 6000 netdev->base_addr, netdev->irq, netdev->dev_addr); 6001 6002 pm_runtime_put_autosuspend(&bp->pdev->dev); 6003 6004 return 0; 6005 6006 err_out_free_tieoff: 6007 macb_free_tieoff(bp); 6008 6009 err_out_unregister_mdio: 6010 if (bp->mii_bus) { 6011 mdiobus_unregister(bp->mii_bus); 6012 mdiobus_free(bp->mii_bus); 6013 } 6014 phylink_destroy(bp->phylink); 6015 6016 err_out_phy_exit: 6017 phy_exit(bp->phy); 6018 6019 err_out_free_netdev: 6020 free_netdev(netdev); 6021 6022 err_disable_clocks: 6023 macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); 6024 pm_runtime_disable(&pdev->dev); 6025 pm_runtime_set_suspended(&pdev->dev); 6026 pm_runtime_dont_use_autosuspend(&pdev->dev); 6027 6028 return err; 6029 } 6030 6031 static void macb_remove(struct platform_device *pdev) 6032 { 6033 struct net_device *netdev; 6034 struct macb *bp; 6035 6036 netdev = platform_get_drvdata(pdev); 6037 6038 if (netdev) { 6039 bp = netdev_priv(netdev); 6040 unregister_netdev(netdev); 6041 macb_free_tieoff(bp); 6042 phy_exit(bp->phy); 6043 if (bp->mii_bus) { 6044 mdiobus_unregister(bp->mii_bus); 6045 mdiobus_free(bp->mii_bus); 6046 } 6047 6048 device_set_wakeup_enable(&bp->pdev->dev, 0); 6049 cancel_delayed_work_sync(&bp->tx_lpi_work); 6050 cancel_work_sync(&bp->hresp_err_bh_work); 6051 pm_runtime_disable(&pdev->dev); 6052 pm_runtime_dont_use_autosuspend(&pdev->dev); 6053 pm_runtime_set_suspended(&pdev->dev); 6054 phylink_destroy(bp->phylink); 6055 free_netdev(netdev); 6056 } 6057 } 6058 6059 static int __maybe_unused macb_suspend(struct device *dev) 6060 { 6061 struct net_device *netdev = dev_get_drvdata(dev); 6062 struct macb *bp = netdev_priv(netdev); 6063 struct in_ifaddr *ifa = NULL; 6064 struct macb_queue *queue; 6065 struct in_device *idev; 6066 unsigned long flags; 6067 u32 tmp, ifa_local; 6068 unsigned int q; 6069 6070 if (!device_may_wakeup(&bp->netdev->dev)) 6071 phy_exit(bp->phy); 6072 6073 if (!netif_running(netdev)) 6074 return 0; 6075 6076 if (bp->wol & MACB_WOL_ENABLED) { 6077 if (bp->wolopts & WAKE_ARP) { 6078 /* Check for IP address in WOL ARP mode */ 6079 rcu_read_lock(); 6080 idev = __in_dev_get_rcu(bp->netdev); 6081 if (idev) 6082 ifa = rcu_dereference(idev->ifa_list); 6083 if (!ifa) { 6084 rcu_read_unlock(); 6085 netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); 6086 return -EOPNOTSUPP; 6087 } 6088 ifa_local = be32_to_cpu(ifa->ifa_local); 6089 rcu_read_unlock(); 6090 } 6091 6092 spin_lock_irqsave(&bp->lock, flags); 6093 6094 /* Disable Tx and Rx engines before disabling the queues, 6095 * this is mandatory as per the IP spec sheet 6096 */ 6097 tmp = macb_readl(bp, NCR); 6098 macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE))); 6099 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 6100 if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) 6101 macb_writel(bp, RBQPH, 6102 upper_32_bits(bp->rx_ring_tieoff_dma)); 6103 #endif 6104 for (q = 0, queue = bp->queues; q < bp->num_queues; 6105 ++q, ++queue) { 6106 /* Disable RX queues */ 6107 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) { 6108 queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE)); 6109 } else { 6110 /* Tie off RX queues */ 6111 queue_writel(queue, RBQP, 6112 lower_32_bits(bp->rx_ring_tieoff_dma)); 6113 } 6114 /* Disable all interrupts */ 6115 queue_writel(queue, IDR, -1); 6116 queue_readl(queue, ISR); 6117 macb_queue_isr_clear(bp, queue, -1); 6118 } 6119 /* Enable Receive engine */ 6120 macb_writel(bp, NCR, tmp | MACB_BIT(RE)); 6121 /* Flush all status bits */ 6122 macb_writel(bp, TSR, -1); 6123 macb_writel(bp, RSR, -1); 6124 6125 tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0; 6126 if (bp->wolopts & WAKE_ARP) { 6127 tmp |= MACB_BIT(ARP); 6128 /* write IP address into register */ 6129 tmp |= MACB_BFEXT(IP, ifa_local); 6130 } 6131 6132 if (macb_is_gem(bp)) { 6133 queue_writel(bp->queues, IER, GEM_BIT(WOL)); 6134 gem_writel(bp, WOL, tmp); 6135 } else { 6136 queue_writel(bp->queues, IER, MACB_BIT(WOL)); 6137 macb_writel(bp, WOL, tmp); 6138 } 6139 spin_unlock_irqrestore(&bp->lock, flags); 6140 6141 enable_irq_wake(bp->queues[0].irq); 6142 } 6143 6144 netif_device_detach(netdev); 6145 for (q = 0, queue = bp->queues; q < bp->num_queues; 6146 ++q, ++queue) { 6147 napi_disable(&queue->napi_rx); 6148 napi_disable(&queue->napi_tx); 6149 } 6150 6151 if (!(bp->wol & MACB_WOL_ENABLED)) { 6152 rtnl_lock(); 6153 phylink_stop(bp->phylink); 6154 rtnl_unlock(); 6155 spin_lock_irqsave(&bp->lock, flags); 6156 macb_reset_hw(bp); 6157 spin_unlock_irqrestore(&bp->lock, flags); 6158 } 6159 6160 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 6161 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 6162 6163 if (netdev->hw_features & NETIF_F_NTUPLE) 6164 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 6165 6166 if (bp->ptp_info) 6167 bp->ptp_info->ptp_remove(netdev); 6168 if (!device_may_wakeup(dev)) 6169 pm_runtime_force_suspend(dev); 6170 6171 return 0; 6172 } 6173 6174 static int __maybe_unused macb_resume(struct device *dev) 6175 { 6176 struct net_device *netdev = dev_get_drvdata(dev); 6177 struct macb *bp = netdev_priv(netdev); 6178 struct macb_queue *queue; 6179 unsigned long flags; 6180 unsigned int q; 6181 6182 if (!device_may_wakeup(&bp->netdev->dev)) 6183 phy_init(bp->phy); 6184 6185 if (!netif_running(netdev)) 6186 return 0; 6187 6188 if (!device_may_wakeup(dev)) 6189 pm_runtime_force_resume(dev); 6190 6191 if (bp->wol & MACB_WOL_ENABLED) { 6192 spin_lock_irqsave(&bp->lock, flags); 6193 /* Disable WoL */ 6194 if (macb_is_gem(bp)) { 6195 queue_writel(bp->queues, IDR, GEM_BIT(WOL)); 6196 gem_writel(bp, WOL, 0); 6197 } else { 6198 queue_writel(bp->queues, IDR, MACB_BIT(WOL)); 6199 macb_writel(bp, WOL, 0); 6200 } 6201 /* Clear ISR on queue 0 */ 6202 queue_readl(bp->queues, ISR); 6203 macb_queue_isr_clear(bp, bp->queues, -1); 6204 spin_unlock_irqrestore(&bp->lock, flags); 6205 6206 disable_irq_wake(bp->queues[0].irq); 6207 6208 /* Now make sure we disable phy before moving 6209 * to common restore path 6210 */ 6211 rtnl_lock(); 6212 phylink_stop(bp->phylink); 6213 rtnl_unlock(); 6214 } 6215 6216 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 6217 macb_init_buffers(bp); 6218 6219 for (q = 0, queue = bp->queues; q < bp->num_queues; 6220 ++q, ++queue) { 6221 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 6222 if (macb_is_gem(bp)) 6223 gem_init_rx_ring(queue); 6224 else 6225 macb_init_rx_ring(queue); 6226 } 6227 6228 napi_enable(&queue->napi_rx); 6229 napi_enable(&queue->napi_tx); 6230 } 6231 6232 if (netdev->hw_features & NETIF_F_NTUPLE) 6233 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 6234 6235 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 6236 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 6237 6238 macb_writel(bp, NCR, MACB_BIT(MPE)); 6239 macb_init_hw(bp); 6240 macb_set_rx_mode(netdev); 6241 macb_restore_features(bp); 6242 rtnl_lock(); 6243 6244 phylink_start(bp->phylink); 6245 rtnl_unlock(); 6246 6247 netif_device_attach(netdev); 6248 if (bp->ptp_info) 6249 bp->ptp_info->ptp_init(netdev); 6250 6251 return 0; 6252 } 6253 6254 static int __maybe_unused macb_runtime_suspend(struct device *dev) 6255 { 6256 struct net_device *netdev = dev_get_drvdata(dev); 6257 struct macb *bp = netdev_priv(netdev); 6258 6259 if (!(device_may_wakeup(dev))) 6260 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); 6261 else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) 6262 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); 6263 6264 return 0; 6265 } 6266 6267 static int __maybe_unused macb_runtime_resume(struct device *dev) 6268 { 6269 struct net_device *netdev = dev_get_drvdata(dev); 6270 struct macb *bp = netdev_priv(netdev); 6271 6272 if (!(device_may_wakeup(dev))) { 6273 clk_prepare_enable(bp->pclk); 6274 clk_prepare_enable(bp->hclk); 6275 clk_prepare_enable(bp->tx_clk); 6276 clk_prepare_enable(bp->rx_clk); 6277 clk_prepare_enable(bp->tsu_clk); 6278 } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { 6279 clk_prepare_enable(bp->tsu_clk); 6280 } 6281 6282 return 0; 6283 } 6284 6285 static void macb_shutdown(struct platform_device *pdev) 6286 { 6287 struct net_device *netdev = platform_get_drvdata(pdev); 6288 6289 rtnl_lock(); 6290 6291 if (netif_running(netdev)) 6292 dev_close(netdev); 6293 6294 netif_device_detach(netdev); 6295 6296 rtnl_unlock(); 6297 } 6298 6299 static const struct dev_pm_ops macb_pm_ops = { 6300 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 6301 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 6302 }; 6303 6304 static struct platform_driver macb_driver = { 6305 .probe = macb_probe, 6306 .remove = macb_remove, 6307 .driver = { 6308 .name = "macb", 6309 .of_match_table = macb_dt_ids, 6310 .pm = &macb_pm_ops, 6311 }, 6312 .shutdown = macb_shutdown, 6313 }; 6314 6315 module_platform_driver(macb_driver); 6316 6317 MODULE_LICENSE("GPL"); 6318 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 6319 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 6320 MODULE_ALIAS("platform:macb"); 6321